2024-09-21T09:19:15.515 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T09:19:15.548 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:19:15.638 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913647 branch: main description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' first_in_suite: false job_id: '7913647' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - PG_AVAILABILITY sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 5357 sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 targets: smithi017.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDtbptxes1X1RyVnooqWzI3H3PkFcPrM4fQSE7O2Mbqb2B/ivAoPe9l2BV8SAeisY/11r/guEd3BnXjGq5TP4DY= smithi134.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFGFo92CacxaDaLFkicQX58PzPG4yotJEzS3YtiXzFAsys6ZcSbHrB/73FAsvSK7tzzUQGBvCR+MuZFuFXtqEec= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_03:08:26 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 2024-09-21T09:19:15.638 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa; will attempt to use it 2024-09-21T09:19:15.638 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks 2024-09-21T09:19:15.639 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T09:19:15.640 INFO:teuthology.task.internal:Checking packages... 2024-09-21T09:19:15.660 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '62da54b4b83a96ca9976b7e1d0c8272a564a2208' 2024-09-21T09:19:15.660 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:19:15.660 INFO:teuthology.packaging:ref: None 2024-09-21T09:19:15.660 INFO:teuthology.packaging:tag: None 2024-09-21T09:19:15.660 INFO:teuthology.packaging:branch: main 2024-09-21T09:19:15.660 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:19:15.660 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-21T09:19:15.889 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5092.g62da54b4 2024-09-21T09:19:15.890 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T09:19:15.897 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T09:19:15.897 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T09:19:15.904 INFO:teuthology.task.internal:Saving configuration 2024-09-21T09:19:15.926 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T09:19:15.933 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T09:19:15.958 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi017.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913647', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:14:27.322103', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDtbptxes1X1RyVnooqWzI3H3PkFcPrM4fQSE7O2Mbqb2B/ivAoPe9l2BV8SAeisY/11r/guEd3BnXjGq5TP4DY='} 2024-09-21T09:19:15.978 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi134.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913647', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:14:27.320605', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFGFo92CacxaDaLFkicQX58PzPG4yotJEzS3YtiXzFAsys6ZcSbHrB/73FAsvSK7tzzUQGBvCR+MuZFuFXtqEec='} 2024-09-21T09:19:15.979 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T09:19:15.984 INFO:teuthology.task.internal:roles: ubuntu@smithi017.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-21T09:19:15.985 INFO:teuthology.task.internal:roles: ubuntu@smithi134.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-21T09:19:15.985 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T09:19:16.049 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f3d0add8160>, signals=[15]) 2024-09-21T09:19:16.049 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T09:19:16.055 INFO:teuthology.task.internal:Opening connections... 2024-09-21T09:19:16.055 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi017.front.sepia.ceph.com 2024-09-21T09:19:16.057 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:19:16.134 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi134.front.sepia.ceph.com 2024-09-21T09:19:16.135 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:19:16.238 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T09:19:16.245 DEBUG:teuthology.orchestra.run.smithi017:> uname -m 2024-09-21T09:19:16.263 INFO:teuthology.orchestra.run.smithi017.stdout:x86_64 2024-09-21T09:19:16.264 DEBUG:teuthology.orchestra.run.smithi017:> cat /etc/os-release 2024-09-21T09:19:16.321 INFO:teuthology.orchestra.run.smithi017.stdout:NAME="CentOS Stream" 2024-09-21T09:19:16.321 INFO:teuthology.orchestra.run.smithi017.stdout:VERSION="9" 2024-09-21T09:19:16.321 INFO:teuthology.orchestra.run.smithi017.stdout:ID="centos" 2024-09-21T09:19:16.321 INFO:teuthology.orchestra.run.smithi017.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:19:16.321 INFO:teuthology.orchestra.run.smithi017.stdout:VERSION_ID="9" 2024-09-21T09:19:16.321 INFO:teuthology.orchestra.run.smithi017.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:19:16.322 INFO:teuthology.orchestra.run.smithi017.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:19:16.322 INFO:teuthology.orchestra.run.smithi017.stdout:ANSI_COLOR="0;31" 2024-09-21T09:19:16.322 INFO:teuthology.orchestra.run.smithi017.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:19:16.322 INFO:teuthology.orchestra.run.smithi017.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:19:16.322 INFO:teuthology.orchestra.run.smithi017.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:19:16.322 INFO:teuthology.orchestra.run.smithi017.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:19:16.322 INFO:teuthology.orchestra.run.smithi017.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:19:16.322 INFO:teuthology.orchestra.run.smithi017.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:19:16.323 INFO:teuthology.lock.ops:Updating smithi017.front.sepia.ceph.com on lock server 2024-09-21T09:19:16.347 DEBUG:teuthology.orchestra.run.smithi134:> uname -m 2024-09-21T09:19:16.366 INFO:teuthology.orchestra.run.smithi134.stdout:x86_64 2024-09-21T09:19:16.366 DEBUG:teuthology.orchestra.run.smithi134:> cat /etc/os-release 2024-09-21T09:19:16.424 INFO:teuthology.orchestra.run.smithi134.stdout:NAME="CentOS Stream" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION="9" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:ID="centos" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION_ID="9" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:ANSI_COLOR="0;31" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:19:16.425 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:19:16.426 INFO:teuthology.lock.ops:Updating smithi134.front.sepia.ceph.com on lock server 2024-09-21T09:19:16.623 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T09:19:16.633 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T09:19:16.639 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T09:19:16.639 DEBUG:teuthology.orchestra.run.smithi017:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:19:16.642 DEBUG:teuthology.orchestra.run.smithi134:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:19:16.659 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T09:19:16.666 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T09:19:16.666 DEBUG:teuthology.orchestra.run.smithi017:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:19:16.699 DEBUG:teuthology.orchestra.run.smithi134:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:19:16.725 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T09:19:16.775 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T09:19:16.795 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T09:19:16.796 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T09:19:16.796 DEBUG:teuthology.orchestra.run.smithi017:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:19:16.797 DEBUG:teuthology.orchestra.run.smithi134:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:19:16.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:19:16.813 DEBUG:teuthology.orchestra.run.smithi017:> uname -r 2024-09-21T09:19:16.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:19:16.814 DEBUG:teuthology.orchestra.run.smithi134:> uname -r 2024-09-21T09:19:16.870 INFO:teuthology.orchestra.run.smithi017.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:19:16.871 INFO:teuthology.task.kernel:Running kernel on smithi017: 5.14.0-503.el9.x86_64 2024-09-21T09:19:16.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum install -y kernel 2024-09-21T09:19:16.872 INFO:teuthology.orchestra.run.smithi134.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:19:16.872 INFO:teuthology.task.kernel:Running kernel on smithi134: 5.14.0-503.el9.x86_64 2024-09-21T09:19:16.872 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum install -y kernel 2024-09-21T09:19:19.171 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.3 MB 00:00 2024-09-21T09:19:19.275 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.3 MB 00:00 2024-09-21T09:19:20.989 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - AppStream 30 MB/s | 20 MB 00:00 2024-09-21T09:19:22.576 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - AppStream 9.1 MB/s | 20 MB 00:02 2024-09-21T09:19:26.581 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.5 MB 00:01 2024-09-21T09:19:26.859 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-21T09:19:28.098 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - Extras packages 85 kB/s | 19 kB 00:00 2024-09-21T09:19:28.752 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - Extras packages 28 kB/s | 19 kB 00:00 2024-09-21T09:19:28.994 INFO:teuthology.orchestra.run.smithi017.stdout:Extra Packages for Enterprise Linux 29 MB/s | 23 MB 00:00 2024-09-21T09:19:29.539 INFO:teuthology.orchestra.run.smithi134.stdout:Extra Packages for Enterprise Linux 34 MB/s | 23 MB 00:00 2024-09-21T09:19:34.609 INFO:teuthology.orchestra.run.smithi017.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-21T09:19:34.878 INFO:teuthology.orchestra.run.smithi134.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-21T09:19:36.732 INFO:teuthology.orchestra.run.smithi017.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:19:36.732 INFO:teuthology.orchestra.run.smithi017.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:19:36.793 INFO:teuthology.orchestra.run.smithi134.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:19:36.793 INFO:teuthology.orchestra.run.smithi134.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:19:36.840 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:19:36.851 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:19:36.851 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2024-09-21T09:19:36.851 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:19:36.851 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout:Install 4 Packages 2024-09-21T09:19:36.852 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:19:36.853 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 79 M 2024-09-21T09:19:36.853 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 126 M 2024-09-21T09:19:36.853 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:19:36.872 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:19:36.876 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:19:36.877 INFO:teuthology.orchestra.run.smithi134.stdout: Package Architecture Version Repository Size 2024-09-21T09:19:36.877 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:19:36.877 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:19:36.877 INFO:teuthology.orchestra.run.smithi134.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:19:36.877 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:19:36.877 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:19:36.877 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:19:36.877 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:19:36.877 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:19:36.878 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:19:36.878 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:19:36.878 INFO:teuthology.orchestra.run.smithi134.stdout:Install 4 Packages 2024-09-21T09:19:36.878 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:19:36.879 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 79 M 2024-09-21T09:19:36.879 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 126 M 2024-09-21T09:19:36.879 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:19:36.988 INFO:teuthology.orchestra.run.smithi134.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 402 kB/s | 36 kB 00:00 2024-09-21T09:19:36.993 INFO:teuthology.orchestra.run.smithi017.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 303 kB/s | 36 kB 00:00 2024-09-21T09:19:37.255 INFO:teuthology.orchestra.run.smithi134.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 44 MB/s | 16 MB 00:00 2024-09-21T09:19:37.605 INFO:teuthology.orchestra.run.smithi134.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 46 MB/s | 29 MB 00:00 2024-09-21T09:19:38.106 INFO:teuthology.orchestra.run.smithi134.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 29 MB/s | 35 MB 00:01 2024-09-21T09:19:38.106 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:38.106 INFO:teuthology.orchestra.run.smithi134.stdout:Total 64 MB/s | 79 MB 00:01 2024-09-21T09:19:38.235 INFO:teuthology.orchestra.run.smithi017.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 12 MB/s | 16 MB 00:01 2024-09-21T09:19:38.782 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:19:38.827 INFO:teuthology.orchestra.run.smithi017.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 16 MB/s | 29 MB 00:01 2024-09-21T09:19:39.132 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:19:39.132 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:19:39.152 INFO:teuthology.orchestra.run.smithi017.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 15 MB/s | 35 MB 00:02 2024-09-21T09:19:39.152 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:39.153 INFO:teuthology.orchestra.run.smithi017.stdout:Total 34 MB/s | 79 MB 00:02 2024-09-21T09:19:39.865 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:19:40.107 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:19:40.107 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:19:40.239 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:19:40.240 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:19:41.231 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:19:41.231 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:19:42.572 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:19:43.237 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:19:43.308 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:19:43.802 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:19:44.466 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:19:44.571 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:19:46.049 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:19:46.178 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:19:47.320 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:19:47.446 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:19:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:19:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:19:52.571 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:19:52.712 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:19:55.864 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:19:57.856 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:34.217 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:34.257 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:34.549 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:34.550 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:20:34.550 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:20:34.550 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:20:35.404 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:35.404 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:20:35.404 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:20:35.404 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:35.404 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:20:35.404 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:20:35.404 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:20:35.404 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:20:35.404 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:20:35.792 DEBUG:teuthology.orchestra.run.smithi134:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:20:35.877 INFO:teuthology.orchestra.run.smithi134.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:35.877 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:20:35.877 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:20:35.877 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:20:35.877 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:20:35.877 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-21T09:20:35.877 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi134.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:20:35.878 DEBUG:teuthology.orchestra.run.smithi134:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:20:35.936 INFO:teuthology.orchestra.run.smithi134.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:35.936 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:20:35.937 DEBUG:teuthology.orchestra.run.smithi134:> sudo rpm -qi grub2-tools 2024-09-21T09:20:35.968 INFO:teuthology.orchestra.run.smithi134.stdout:Name : grub2-tools 2024-09-21T09:20:35.968 INFO:teuthology.orchestra.run.smithi134.stdout:Epoch : 1 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Version : 2.06 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Release : 82.el9 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Architecture: x86_64 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Group : Unspecified 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Size : 8274670 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:License : GPLv3+ 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:20:35.969 INFO:teuthology.orchestra.run.smithi134.stdout:Packager : builder@centos.org 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout:Vendor : CentOS 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout:Summary : Support tools for GRUB. 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout:Description : 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout:hardware devices. 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:20:35.970 INFO:teuthology.orchestra.run.smithi134.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:20:35.971 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:20:35.971 INFO:teuthology.task.kernel:Updating grub on smithi134 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:20:35.971 DEBUG:teuthology.orchestra.run.smithi134:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:20:36.554 INFO:teuthology.orchestra.run.smithi134.stderr:Generating grub configuration file ... 2024-09-21T09:20:37.262 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:37.829 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:37.829 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:20:37.829 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:20:37.829 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:20:37.923 INFO:teuthology.orchestra.run.smithi134.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:20:37.965 INFO:teuthology.orchestra.run.smithi134.stderr:done 2024-09-21T09:20:37.967 DEBUG:teuthology.orchestra.run.smithi134:> mktemp 2024-09-21T09:20:37.983 INFO:teuthology.orchestra.run.smithi134.stdout:/tmp/tmp.CsxOu97xry 2024-09-21T09:20:37.983 DEBUG:teuthology.orchestra.run.smithi134:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.CsxOu97xry 2024-09-21T09:20:38.051 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 0666 /tmp/tmp.CsxOu97xry 2024-09-21T09:20:38.197 DEBUG:teuthology.orchestra.remote:smithi134:/tmp/tmp.CsxOu97xry is 6KB 2024-09-21T09:20:38.218 DEBUG:teuthology.orchestra.run.smithi134:> rm -fr /tmp/tmp.CsxOu97xry 2024-09-21T09:20:38.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:20:38.295 INFO:teuthology.orchestra.run.smithi134.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:20:38.295 INFO:teuthology.orchestra.run.smithi134.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:20:38.295 INFO:teuthology.orchestra.run.smithi134.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:20:38.296 INFO:teuthology.orchestra.run.smithi134.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:20:38.297 DEBUG:teuthology.orchestra.run.smithi134:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:20:38.455 DEBUG:teuthology.orchestra.run.smithi134:> sudo shutdown -r now 2024-09-21T09:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:20:39.005 DEBUG:teuthology.orchestra.run.smithi017:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:20:39.081 INFO:teuthology.orchestra.run.smithi017.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:39.081 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:20:39.082 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:20:39.082 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:20:39.082 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:20:39.082 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-21T09:20:39.082 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi017.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:20:39.082 DEBUG:teuthology.orchestra.run.smithi017:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:20:39.141 INFO:teuthology.orchestra.run.smithi017.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:39.141 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:20:39.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo rpm -qi grub2-tools 2024-09-21T09:20:39.174 INFO:teuthology.orchestra.run.smithi017.stdout:Name : grub2-tools 2024-09-21T09:20:39.174 INFO:teuthology.orchestra.run.smithi017.stdout:Epoch : 1 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Version : 2.06 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Release : 82.el9 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Architecture: x86_64 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Group : Unspecified 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Size : 8274670 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:License : GPLv3+ 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Packager : builder@centos.org 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Vendor : CentOS 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Summary : Support tools for GRUB. 2024-09-21T09:20:39.175 INFO:teuthology.orchestra.run.smithi017.stdout:Description : 2024-09-21T09:20:39.176 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:20:39.176 INFO:teuthology.orchestra.run.smithi017.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:20:39.176 INFO:teuthology.orchestra.run.smithi017.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:20:39.176 INFO:teuthology.orchestra.run.smithi017.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:20:39.176 INFO:teuthology.orchestra.run.smithi017.stdout:hardware devices. 2024-09-21T09:20:39.176 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:20:39.176 INFO:teuthology.orchestra.run.smithi017.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:20:39.177 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:20:39.177 INFO:teuthology.task.kernel:Updating grub on smithi017 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:20:39.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:20:39.851 INFO:teuthology.orchestra.run.smithi017.stderr:Generating grub configuration file ... 2024-09-21T09:20:41.151 INFO:teuthology.orchestra.run.smithi017.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:20:41.166 INFO:teuthology.orchestra.run.smithi017.stderr:done 2024-09-21T09:20:41.170 DEBUG:teuthology.orchestra.run.smithi017:> mktemp 2024-09-21T09:20:41.185 INFO:teuthology.orchestra.run.smithi017.stdout:/tmp/tmp.NPumv2UHc1 2024-09-21T09:20:41.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.NPumv2UHc1 2024-09-21T09:20:41.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod 0666 /tmp/tmp.NPumv2UHc1 2024-09-21T09:20:41.395 DEBUG:teuthology.orchestra.remote:smithi017:/tmp/tmp.NPumv2UHc1 is 6KB 2024-09-21T09:20:41.443 DEBUG:teuthology.orchestra.run.smithi017:> rm -fr /tmp/tmp.NPumv2UHc1 2024-09-21T09:20:41.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:20:41.520 INFO:teuthology.orchestra.run.smithi017.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:20:41.520 INFO:teuthology.orchestra.run.smithi017.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:20:41.520 INFO:teuthology.orchestra.run.smithi017.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:20:41.520 INFO:teuthology.orchestra.run.smithi017.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:20:41.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:20:41.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo shutdown -r now 2024-09-21T09:21:08.482 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-21T09:21:08.483 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-09-21T09:21:08.483 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:21:11.670 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-21T09:21:11.670 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-21T09:21:11.671 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:21:27.023 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.134 2024-09-21T09:21:30.095 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.17 2024-09-21T09:21:36.026 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-09-21T09:21:36.027 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:21:39.091 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.134 2024-09-21T09:21:39.095 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-21T09:21:39.095 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:21:51.100 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-09-21T09:21:51.101 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:22:22.914 DEBUG:teuthology.orchestra.run.smithi134:> true 2024-09-21T09:22:23.243 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-09-21T09:22:23.243 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:22:23.243 DEBUG:teuthology.orchestra.run.smithi134:> uname -r 2024-09-21T09:22:23.258 INFO:teuthology.orchestra.run.smithi134.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:22:23.258 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:22:23.258 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:22:23.258 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:22:24.259 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-21T09:22:24.259 DEBUG:teuthology.orchestra.run.smithi134:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:22:24.321 INFO:teuthology.orchestra.run.smithi134.stdout:ttyS1 2024-09-21T09:22:24.346 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:39.098 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T09:22:51.099 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-21T09:22:51.099 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:22:51.399 DEBUG:teuthology.orchestra.run.smithi017:> true 2024-09-21T09:22:51.766 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-21T09:22:51.766 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:22:51.767 DEBUG:teuthology.orchestra.run.smithi017:> uname -r 2024-09-21T09:22:51.783 INFO:teuthology.orchestra.run.smithi017.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:22:51.783 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:22:51.783 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:22:51.784 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:22:52.785 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-21T09:22:52.785 DEBUG:teuthology.orchestra.run.smithi017:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:22:52.853 INFO:teuthology.orchestra.run.smithi017.stdout:ttyS1 2024-09-21T09:22:52.876 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:52.876 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T09:22:52.883 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T09:22:52.883 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:22:52.885 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:22:52.904 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T09:22:52.911 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T09:22:52.918 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T09:22:52.918 DEBUG:teuthology.orchestra.run.smithi017:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:22:52.945 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:22:53.024 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T09:22:53.031 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T09:22:53.031 DEBUG:teuthology.orchestra.run.smithi017:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:22:53.033 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:22:53.067 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:22:53.068 INFO:teuthology.orchestra.run.smithi017.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:22:53.079 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:22:53.080 INFO:teuthology.orchestra.run.smithi017.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:22:53.081 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T09:22:53.088 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T09:22:53.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:22:53.124 DEBUG:teuthology.orchestra.run.smithi134:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:22:53.151 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T09:22:53.159 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T09:22:53.159 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:22:53.193 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:22:53.209 DEBUG:teuthology.orchestra.run.smithi017:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:22:53.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:22:53.347 DEBUG:teuthology.orchestra.run.smithi017:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:22:53.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:22:53.475 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:22:53.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:22:53.544 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:22:53.588 DEBUG:teuthology.orchestra.run.smithi134:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:22:53.645 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:22:53.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:22:53.771 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:22:53.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:22:53.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo service rsyslog restart 2024-09-21T09:22:53.839 DEBUG:teuthology.orchestra.run.smithi134:> sudo service rsyslog restart 2024-09-21T09:22:53.887 INFO:teuthology.orchestra.run.smithi017.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:22:53.925 INFO:teuthology.orchestra.run.smithi134.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:22:54.276 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T09:22:54.284 INFO:teuthology.task.internal:Starting timer... 2024-09-21T09:22:54.284 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T09:22:54.319 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T09:22:54.327 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T09:22:54.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo service auditd rotate 2024-09-21T09:22:54.387 INFO:teuthology.orchestra.run.smithi017.stdout:Rotating logs: 2024-09-21T09:22:54.389 DEBUG:teuthology.orchestra.run.smithi134:> sudo service auditd rotate 2024-09-21T09:22:54.434 INFO:teuthology.orchestra.run.smithi134.stdout:Rotating logs: 2024-09-21T09:22:54.436 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T09:22:54.436 DEBUG:teuthology.orchestra.run.smithi017:> /usr/sbin/getenforce 2024-09-21T09:22:54.463 INFO:teuthology.orchestra.run.smithi017.stdout:Permissive 2024-09-21T09:22:54.464 DEBUG:teuthology.orchestra.run.smithi134:> /usr/sbin/getenforce 2024-09-21T09:22:54.487 INFO:teuthology.orchestra.run.smithi134.stdout:Permissive 2024-09-21T09:22:54.487 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi017.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi134.front.sepia.ceph.com': 'permissive'} 2024-09-21T09:22:54.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:22:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:22:54.533 DEBUG:teuthology.orchestra.run.smithi134:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:22:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:22:54.561 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T09:22:54.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:22:54.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:22:54.656 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T09:22:54.671 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T09:22:54.680 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T09:22:54.681 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi017.front.sepia.ceph.com,smithi134.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T09:26:42.921 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi017.front.sepia.ceph.com'), Remote(name='ubuntu@smithi134.front.sepia.ceph.com')] 2024-09-21T09:26:42.922 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-21T09:26:42.922 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:26:42.996 DEBUG:teuthology.orchestra.run.smithi017:> true 2024-09-21T09:26:43.074 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi017.front.sepia.ceph.com' 2024-09-21T09:26:43.075 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-09-21T09:26:43.075 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:26:43.151 DEBUG:teuthology.orchestra.run.smithi134:> true 2024-09-21T09:26:43.231 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-09-21T09:26:43.231 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T09:26:43.240 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T09:26:43.240 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:26:43.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:26:43.243 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:26:43.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:26:43.273 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:26:43.288 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:26:43.301 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:26:43.315 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:26:43.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: ntpd: command not found 2024-09-21T09:26:43.327 INFO:teuthology.orchestra.run.smithi017.stdout:506 Cannot talk to daemon 2024-09-21T09:26:43.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: ntpd: command not found 2024-09-21T09:26:43.342 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:26:43.352 INFO:teuthology.orchestra.run.smithi134.stdout:506 Cannot talk to daemon 2024-09-21T09:26:43.357 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:26:43.366 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:26:43.381 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:26:43.415 INFO:teuthology.orchestra.run.smithi017.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:26:43.419 INFO:teuthology.orchestra.run.smithi017.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:26:43.420 INFO:teuthology.orchestra.run.smithi017.stdout:=============================================================================== 2024-09-21T09:26:43.420 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:43.420 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:43.420 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:43.420 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:43.434 INFO:teuthology.orchestra.run.smithi134.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:26:43.437 INFO:teuthology.orchestra.run.smithi134.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:26:43.437 INFO:teuthology.orchestra.run.smithi134.stdout:=============================================================================== 2024-09-21T09:26:43.438 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:43.438 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:43.438 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:43.438 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:43.439 INFO:teuthology.run_tasks:Running task pexec... 2024-09-21T09:26:43.449 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-21T09:26:43.449 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi017.front.sepia.ceph.com 2024-09-21T09:26:43.449 DEBUG:teuthology.orchestra.run.smithi017:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:26:43.450 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi134.front.sepia.ceph.com 2024-09-21T09:26:43.450 DEBUG:teuthology.orchestra.run.smithi134:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:26:43.829 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:26:43.829 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:26:43.833 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:26:43.833 INFO:teuthology.orchestra.run.smithi134.stdout: Package Architecture Version Repository Size 2024-09-21T09:26:43.833 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:26:43.833 INFO:teuthology.orchestra.run.smithi134.stdout:Removing: 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout:Removing unused dependencies: 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout:Remove 2 Packages 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout:Freed space: 5.5 M 2024-09-21T09:26:43.834 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:26:43.838 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:26:43.838 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:26:43.847 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:26:43.847 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2024-09-21T09:26:43.847 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:26:43.847 INFO:teuthology.orchestra.run.smithi017.stdout:Removing: 2024-09-21T09:26:43.847 INFO:teuthology.orchestra.run.smithi017.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:26:43.848 INFO:teuthology.orchestra.run.smithi017.stdout:Removing unused dependencies: 2024-09-21T09:26:43.848 INFO:teuthology.orchestra.run.smithi017.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:26:43.848 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:26:43.848 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:26:43.848 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:26:43.848 INFO:teuthology.orchestra.run.smithi017.stdout:Remove 2 Packages 2024-09-21T09:26:43.848 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:26:43.848 INFO:teuthology.orchestra.run.smithi017.stdout:Freed space: 5.5 M 2024-09-21T09:26:43.848 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:26:43.852 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:26:43.852 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:26:43.892 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:26:43.893 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:26:43.922 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:26:43.922 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:26:44.033 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:26:44.061 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:26:44.115 INFO:teuthology.orchestra.run.smithi134.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:26:44.123 INFO:teuthology.orchestra.run.smithi017.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:26:44.185 INFO:teuthology.orchestra.run.smithi134.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:26:44.191 INFO:teuthology.orchestra.run.smithi017.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:26:44.657 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:26:44.657 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:26:44.736 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:26:44.736 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:26:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:26:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:26:44.916 INFO:teuthology.orchestra.run.smithi017.stdout:Removed: 2024-09-21T09:26:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:26:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:26:44.916 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:26:44.988 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:26:44.988 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:26:44.988 INFO:teuthology.orchestra.run.smithi134.stdout:Removed: 2024-09-21T09:26:44.988 INFO:teuthology.orchestra.run.smithi134.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:26:44.988 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:26:44.988 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:26:45.769 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:49 ago on Sat 21 Sep 2024 09:25:56 AM UTC. 2024-09-21T09:26:45.852 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:00:55 ago on Sat 21 Sep 2024 09:25:50 AM UTC. 2024-09-21T09:26:45.915 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:26:45.916 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:26:45.917 INFO:teuthology.orchestra.run.smithi017.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:26:45.917 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:26:45.917 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:26:45.917 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:26:45.917 INFO:teuthology.orchestra.run.smithi017.stdout:Install 7 Packages 2024-09-21T09:26:45.917 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:26:45.927 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 5.2 M 2024-09-21T09:26:45.927 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 19 M 2024-09-21T09:26:45.927 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:26:45.988 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:26:45.989 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:26:45.989 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:26:45.989 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:26:45.989 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:26:45.989 INFO:teuthology.orchestra.run.smithi134.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:26:45.989 INFO:teuthology.orchestra.run.smithi134.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:26:45.989 INFO:teuthology.orchestra.run.smithi134.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-21T09:26:45.989 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:26:45.990 INFO:teuthology.orchestra.run.smithi134.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:26:45.990 INFO:teuthology.orchestra.run.smithi134.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:26:45.990 INFO:teuthology.orchestra.run.smithi134.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:26:45.990 INFO:teuthology.orchestra.run.smithi134.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:26:45.990 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:26:45.990 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:26:45.990 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:26:45.990 INFO:teuthology.orchestra.run.smithi134.stdout:Install 7 Packages 2024-09-21T09:26:45.990 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:26:46.001 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 5.2 M 2024-09-21T09:26:46.001 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 19 M 2024-09-21T09:26:46.001 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:26:46.279 INFO:teuthology.orchestra.run.smithi017.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 471 kB/s | 44 kB 00:00 2024-09-21T09:26:46.312 INFO:teuthology.orchestra.run.smithi017.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 829 kB/s | 106 kB 00:00 2024-09-21T09:26:46.337 INFO:teuthology.orchestra.run.smithi017.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 1.2 MB/s | 72 kB 00:00 2024-09-21T09:26:46.371 INFO:teuthology.orchestra.run.smithi017.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.4 MB/s | 84 kB 00:00 2024-09-21T09:26:46.405 INFO:teuthology.orchestra.run.smithi017.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 4.0 MB/s | 906 kB 00:00 2024-09-21T09:26:46.455 INFO:teuthology.orchestra.run.smithi017.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 7.0 MB/s | 837 kB 00:00 2024-09-21T09:26:46.523 INFO:teuthology.orchestra.run.smithi134.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 246 kB/s | 44 kB 00:00 2024-09-21T09:26:46.548 INFO:teuthology.orchestra.run.smithi134.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 515 kB/s | 106 kB 00:00 2024-09-21T09:26:46.607 INFO:teuthology.orchestra.run.smithi134.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 857 kB/s | 72 kB 00:00 2024-09-21T09:26:46.632 INFO:teuthology.orchestra.run.smithi134.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.0 MB/s | 84 kB 00:00 2024-09-21T09:26:46.699 INFO:teuthology.orchestra.run.smithi134.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 2.5 MB/s | 906 kB 00:00 2024-09-21T09:26:46.791 INFO:teuthology.orchestra.run.smithi134.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 4.4 MB/s | 837 kB 00:00 2024-09-21T09:26:47.400 INFO:teuthology.orchestra.run.smithi134.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 4.1 MB/s | 3.2 MB 00:00 2024-09-21T09:26:47.400 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:26:47.400 INFO:teuthology.orchestra.run.smithi134.stdout:Total 3.7 MB/s | 5.2 MB 00:01 2024-09-21T09:26:47.447 INFO:teuthology.orchestra.run.smithi017.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 2.9 MB/s | 3.2 MB 00:01 2024-09-21T09:26:47.447 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:26:47.447 INFO:teuthology.orchestra.run.smithi017.stdout:Total 3.4 MB/s | 5.2 MB 00:01 2024-09-21T09:26:47.516 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:26:47.531 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:26:47.531 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:26:47.564 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:26:47.580 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:26:47.580 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:26:47.874 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:26:47.874 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:26:47.879 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:26:47.880 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:26:48.334 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:26:48.360 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:26:48.418 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-21T09:26:48.453 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-21T09:26:48.468 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-21T09:26:48.482 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-21T09:26:48.501 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-21T09:26:48.530 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-21T09:26:48.686 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-21T09:26:48.721 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:26:48.763 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-21T09:26:48.794 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:26:49.163 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:26:49.199 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:26:49.279 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:26:49.305 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:26:49.305 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:26:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:49.431 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:26:49.494 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:50.311 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:50.311 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-21T09:26:50.311 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-21T09:26:50.311 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-21T09:26:50.311 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-21T09:26:50.311 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-21T09:26:50.312 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-21T09:26:50.526 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:50.526 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-21T09:26:50.526 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-21T09:26:50.526 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-21T09:26:50.526 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-21T09:26:50.527 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-21T09:26:50.527 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-21T09:26:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:26:50.673 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:26:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:26:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:26:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:26:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-21T09:26:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:26:50.674 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:26:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:26:50.874 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:26:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:26:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:26:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:26:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-21T09:26:50.874 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:26:50.874 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:26:51.050 DEBUG:teuthology.parallel:result is None 2024-09-21T09:26:51.218 DEBUG:teuthology.parallel:result is None 2024-09-21T09:26:51.218 INFO:teuthology.run_tasks:Running task install... 2024-09-21T09:26:51.228 DEBUG:teuthology.task.install:project ceph 2024-09-21T09:26:51.229 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:26:51.229 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:26:51.229 INFO:teuthology.task.install:Using flavor: default 2024-09-21T09:26:51.234 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T09:26:51.234 INFO:teuthology.task.install:extra packages: [] 2024-09-21T09:26:51.234 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:26:51.234 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:26:51.235 INFO:teuthology.packaging:ref: None 2024-09-21T09:26:51.235 INFO:teuthology.packaging:tag: None 2024-09-21T09:26:51.235 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:26:51.235 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:26:51.235 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:26:51.236 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:26:51.236 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:26:51.237 INFO:teuthology.packaging:ref: None 2024-09-21T09:26:51.237 INFO:teuthology.packaging:tag: None 2024-09-21T09:26:51.237 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:26:51.237 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:26:51.237 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:26:51.442 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/ 2024-09-21T09:26:51.442 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1649.gfbd1a6ab 2024-09-21T09:26:51.466 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/ 2024-09-21T09:26:51.467 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1649.gfbd1a6ab 2024-09-21T09:26:51.555 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:26:51.555 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:26:51.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:26:51.572 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:26:51.572 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:26:51.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:26:51.581 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-21T09:26:51.582 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:26:51.582 INFO:teuthology.packaging:ref: None 2024-09-21T09:26:51.582 INFO:teuthology.packaging:tag: None 2024-09-21T09:26:51.582 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:26:51.582 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:26:51.582 DEBUG:teuthology.orchestra.run.smithi134:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:26:51.600 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-21T09:26:51.600 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:26:51.600 INFO:teuthology.packaging:ref: None 2024-09-21T09:26:51.600 INFO:teuthology.packaging:tag: None 2024-09-21T09:26:51.600 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:26:51.600 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:26:51.600 DEBUG:teuthology.orchestra.run.smithi017:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:26:51.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:26:51.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:26:51.724 DEBUG:teuthology.orchestra.run.smithi134:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:26:51.746 DEBUG:teuthology.orchestra.run.smithi017:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:26:51.788 INFO:teuthology.orchestra.run.smithi134.stdout:check_obsoletes = 1 2024-09-21T09:26:51.789 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum clean all 2024-09-21T09:26:51.812 INFO:teuthology.orchestra.run.smithi017.stdout:check_obsoletes = 1 2024-09-21T09:26:51.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum clean all 2024-09-21T09:26:52.026 INFO:teuthology.orchestra.run.smithi134.stdout:45 files removed 2024-09-21T09:26:52.050 INFO:teuthology.orchestra.run.smithi017.stdout:45 files removed 2024-09-21T09:26:52.053 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-radosgw 2024-09-21T09:26:52.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-radosgw 2024-09-21T09:26:52.605 INFO:teuthology.orchestra.run.smithi134.stdout:ceph packages for x86_64 289 kB/s | 77 kB 00:00 2024-09-21T09:26:52.637 INFO:teuthology.orchestra.run.smithi017.stdout:ceph packages for x86_64 295 kB/s | 77 kB 00:00 2024-09-21T09:26:52.897 INFO:teuthology.orchestra.run.smithi134.stdout:ceph noarch packages 43 kB/s | 11 kB 00:00 2024-09-21T09:26:52.913 INFO:teuthology.orchestra.run.smithi017.stdout:ceph noarch packages 47 kB/s | 11 kB 00:00 2024-09-21T09:26:53.173 INFO:teuthology.orchestra.run.smithi134.stdout:ceph source packages 7.4 kB/s | 1.8 kB 00:00 2024-09-21T09:26:53.180 INFO:teuthology.orchestra.run.smithi017.stdout:ceph source packages 7.7 kB/s | 1.8 kB 00:00 2024-09-21T09:26:53.614 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - BaseOS 20 MB/s | 8.3 MB 00:00 2024-09-21T09:26:55.232 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - AppStream 37 MB/s | 20 MB 00:00 2024-09-21T09:26:56.613 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - BaseOS 2.4 MB/s | 8.3 MB 00:03 2024-09-21T09:26:58.373 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - AppStream 30 MB/s | 20 MB 00:00 2024-09-21T09:26:59.716 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - CRB 9.2 MB/s | 6.5 MB 00:00 2024-09-21T09:27:01.100 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - Extras packages 122 kB/s | 19 kB 00:00 2024-09-21T09:27:02.088 INFO:teuthology.orchestra.run.smithi134.stdout:Extra Packages for Enterprise Linux 26 MB/s | 23 MB 00:00 2024-09-21T09:27:02.833 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-21T09:27:04.642 INFO:teuthology.orchestra.run.smithi017.stdout:CentOS Stream 9 - Extras packages 46 kB/s | 19 kB 00:00 2024-09-21T09:27:05.813 INFO:teuthology.orchestra.run.smithi017.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-21T09:27:07.477 INFO:teuthology.orchestra.run.smithi134.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-21T09:27:09.754 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:27:09.756 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:09.756 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-radosgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 14 M 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout:Upgrading: 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout: librados2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.1 M 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout: librbd1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-base x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 5.0 M 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 22 M 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-selinux x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout: libcephfs2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 636 k 2024-09-21T09:27:09.757 INFO:teuthology.orchestra.run.smithi134.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: libradosstriper1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 457 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: librgw2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 4.9 M 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ceph-argparse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 45 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 120 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cephfs x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 156 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rados x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 320 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 295 k 2024-09-21T09:27:09.758 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 99 k 2024-09-21T09:27:09.759 INFO:teuthology.orchestra.run.smithi134.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:27:09.759 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:09.759 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:27:09.759 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:09.759 INFO:teuthology.orchestra.run.smithi134.stdout:Install 22 Packages 2024-09-21T09:27:09.759 INFO:teuthology.orchestra.run.smithi134.stdout:Upgrade 2 Packages 2024-09-21T09:27:09.759 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:09.759 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 58 M 2024-09-21T09:27:09.759 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:27:10.803 INFO:teuthology.orchestra.run.smithi134.stdout:(1/24): ceph-base-17.2.7-1649.gfbd1a6ab.el9.x86 7.9 MB/s | 5.0 MB 00:00 2024-09-21T09:27:10.878 INFO:teuthology.orchestra.run.smithi134.stdout:(2/24): ceph-selinux-17.2.7-1649.gfbd1a6ab.el9. 337 kB/s | 25 kB 00:00 2024-09-21T09:27:11.120 INFO:teuthology.orchestra.run.smithi134.stdout:(3/24): libcephfs2-17.2.7-1649.gfbd1a6ab.el9.x8 2.6 MB/s | 636 kB 00:00 2024-09-21T09:27:11.212 INFO:teuthology.orchestra.run.smithi134.stdout:(4/24): libradosstriper1-17.2.7-1649.gfbd1a6ab. 4.9 MB/s | 457 kB 00:00 2024-09-21T09:27:11.488 INFO:teuthology.orchestra.run.smithi134.stdout:(5/24): ceph-radosgw-17.2.7-1649.gfbd1a6ab.el9. 11 MB/s | 14 MB 00:01 2024-09-21T09:27:11.546 INFO:teuthology.orchestra.run.smithi134.stdout:(6/24): python3-ceph-argparse-17.2.7-1649.gfbd1 778 kB/s | 45 kB 00:00 2024-09-21T09:27:11.610 INFO:teuthology.orchestra.run.smithi017.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-21T09:27:11.621 INFO:teuthology.orchestra.run.smithi134.stdout:(7/24): librgw2-17.2.7-1649.gfbd1a6ab.el9.x86_6 12 MB/s | 4.9 MB 00:00 2024-09-21T09:27:11.647 INFO:teuthology.orchestra.run.smithi134.stdout:(8/24): python3-ceph-common-17.2.7-1649.gfbd1a6 1.2 MB/s | 120 kB 00:00 2024-09-21T09:27:11.689 INFO:teuthology.orchestra.run.smithi134.stdout:(9/24): python3-cephfs-17.2.7-1649.gfbd1a6ab.el 2.3 MB/s | 156 kB 00:00 2024-09-21T09:27:11.914 INFO:teuthology.orchestra.run.smithi134.stdout:(10/24): ceph-common-17.2.7-1649.gfbd1a6ab.el9. 13 MB/s | 22 MB 00:01 2024-09-21T09:27:11.939 INFO:teuthology.orchestra.run.smithi134.stdout:(11/24): python3-rbd-17.2.7-1649.gfbd1a6ab.el9. 1.2 MB/s | 295 kB 00:00 2024-09-21T09:27:11.965 INFO:teuthology.orchestra.run.smithi134.stdout:(12/24): python3-rados-17.2.7-1649.gfbd1a6ab.el 1.0 MB/s | 320 kB 00:00 2024-09-21T09:27:11.990 INFO:teuthology.orchestra.run.smithi134.stdout:(13/24): python3-rgw-17.2.7-1649.gfbd1a6ab.el9. 1.3 MB/s | 99 kB 00:00 2024-09-21T09:27:12.065 INFO:teuthology.orchestra.run.smithi134.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 452 kB/s | 45 kB 00:00 2024-09-21T09:27:12.107 INFO:teuthology.orchestra.run.smithi134.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 957 kB/s | 160 kB 00:00 2024-09-21T09:27:12.149 INFO:teuthology.orchestra.run.smithi134.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 1.0 MB/s | 42 kB 00:00 2024-09-21T09:27:12.174 INFO:teuthology.orchestra.run.smithi134.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.6 MB/s | 292 kB 00:00 2024-09-21T09:27:12.199 INFO:teuthology.orchestra.run.smithi134.stdout:(18/24): librdkafka-1.6.1-102.el9.x86_64.rpm 3.1 MB/s | 662 kB 00:00 2024-09-21T09:27:12.242 INFO:teuthology.orchestra.run.smithi134.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 727 kB/s | 49 kB 00:00 2024-09-21T09:27:12.266 INFO:teuthology.orchestra.run.smithi134.stdout:(20/24): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.6 MB/s | 308 kB 00:00 2024-09-21T09:27:12.292 INFO:teuthology.orchestra.run.smithi134.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 871 kB/s | 67 kB 00:00 2024-09-21T09:27:12.359 INFO:teuthology.orchestra.run.smithi134.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 14 MB/s | 1.6 MB 00:00 2024-09-21T09:27:12.484 INFO:teuthology.orchestra.run.smithi134.stdout:(23/24): librbd1-17.2.7-1649.gfbd1a6ab.el9.x86_ 15 MB/s | 2.9 MB 00:00 2024-09-21T09:27:12.543 INFO:teuthology.orchestra.run.smithi134.stdout:(24/24): librados2-17.2.7-1649.gfbd1a6ab.el9.x8 11 MB/s | 3.1 MB 00:00 2024-09-21T09:27:12.548 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:12.548 INFO:teuthology.orchestra.run.smithi134.stdout:Total 21 MB/s | 58 MB 00:02 2024-09-21T09:27:12.611 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:27:12.645 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:27:12.645 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:27:13.119 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:27:13.120 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:27:13.948 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:27:13.950 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:27:13.950 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:27:13.950 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:27:13.950 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:27:13.950 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-radosgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 14 M 2024-09-21T09:27:13.950 INFO:teuthology.orchestra.run.smithi017.stdout:Upgrading: 2024-09-21T09:27:13.950 INFO:teuthology.orchestra.run.smithi017.stdout: librados2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.1 M 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: librbd1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-base x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 5.0 M 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 22 M 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-selinux x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 636 k 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: libradosstriper1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 457 k 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:27:13.951 INFO:teuthology.orchestra.run.smithi017.stdout: librgw2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 4.9 M 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-argparse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 45 k 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 120 k 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cephfs x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 156 k 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rados x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 320 k 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 295 k 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 99 k 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:27:13.952 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:27:13.953 INFO:teuthology.orchestra.run.smithi017.stdout:Install 22 Packages 2024-09-21T09:27:13.953 INFO:teuthology.orchestra.run.smithi017.stdout:Upgrade 2 Packages 2024-09-21T09:27:13.953 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:13.953 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 58 M 2024-09-21T09:27:13.953 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:27:14.287 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:27:14.612 INFO:teuthology.orchestra.run.smithi017.stdout:(1/24): ceph-base-17.2.7-1649.gfbd1a6ab.el9.x86 9.8 MB/s | 5.0 MB 00:00 2024-09-21T09:27:14.664 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-09-21T09:27:14.670 INFO:teuthology.orchestra.run.smithi017.stdout:(2/24): ceph-selinux-17.2.7-1649.gfbd1a6ab.el9. 433 kB/s | 25 kB 00:00 2024-09-21T09:27:14.728 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/26 2024-09-21T09:27:14.762 INFO:teuthology.orchestra.run.smithi017.stdout:(3/24): libcephfs2-17.2.7-1649.gfbd1a6ab.el9.x8 6.8 MB/s | 636 kB 00:00 2024-09-21T09:27:14.844 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/26 2024-09-21T09:27:14.862 INFO:teuthology.orchestra.run.smithi017.stdout:(4/24): libradosstriper1-17.2.7-1649.gfbd1a6ab. 4.5 MB/s | 457 kB 00:00 2024-09-21T09:27:14.922 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/26 2024-09-21T09:27:14.971 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/26 2024-09-21T09:27:15.067 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/26 2024-09-21T09:27:15.096 INFO:teuthology.orchestra.run.smithi017.stdout:(5/24): ceph-radosgw-17.2.7-1649.gfbd1a6ab.el9. 15 MB/s | 14 MB 00:00 2024-09-21T09:27:15.129 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-09-21T09:27:15.130 INFO:teuthology.orchestra.run.smithi017.stdout:(6/24): python3-ceph-argparse-17.2.7-1649.gfbd1 1.3 MB/s | 45 kB 00:00 2024-09-21T09:27:15.167 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-09-21T09:27:15.197 INFO:teuthology.orchestra.run.smithi017.stdout:(7/24): librgw2-17.2.7-1649.gfbd1a6ab.el9.x86_6 15 MB/s | 4.9 MB 00:00 2024-09-21T09:27:15.222 INFO:teuthology.orchestra.run.smithi017.stdout:(8/24): python3-ceph-common-17.2.7-1649.gfbd1a6 1.3 MB/s | 120 kB 00:00 2024-09-21T09:27:15.247 INFO:teuthology.orchestra.run.smithi017.stdout:(9/24): python3-cephfs-17.2.7-1649.gfbd1a6ab.el 3.0 MB/s | 156 kB 00:00 2024-09-21T09:27:15.272 INFO:teuthology.orchestra.run.smithi017.stdout:(10/24): python3-rados-17.2.7-1649.gfbd1a6ab.el 6.2 MB/s | 320 kB 00:00 2024-09-21T09:27:15.298 INFO:teuthology.orchestra.run.smithi017.stdout:(11/24): python3-rbd-17.2.7-1649.gfbd1a6ab.el9. 5.7 MB/s | 295 kB 00:00 2024-09-21T09:27:15.323 INFO:teuthology.orchestra.run.smithi017.stdout:(12/24): python3-rgw-17.2.7-1649.gfbd1a6ab.el9. 1.9 MB/s | 99 kB 00:00 2024-09-21T09:27:15.549 INFO:teuthology.orchestra.run.smithi017.stdout:(13/24): ceph-common-17.2.7-1649.gfbd1a6ab.el9. 16 MB/s | 22 MB 00:01 2024-09-21T09:27:15.612 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-09-21T09:27:15.641 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 8/26 2024-09-21T09:27:15.687 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 8/26 2024-09-21T09:27:15.723 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-09-21T09:27:15.776 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9. 10/26 2024-09-21T09:27:15.812 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/26 2024-09-21T09:27:15.873 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 12/26 2024-09-21T09:27:15.902 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 13/26 2024-09-21T09:27:15.958 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 13/26 2024-09-21T09:27:15.997 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-09-21T09:27:16.042 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/26 2024-09-21T09:27:16.084 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-09-21T09:27:16.099 INFO:teuthology.orchestra.run.smithi017.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 58 kB/s | 45 kB 00:00 2024-09-21T09:27:16.199 INFO:teuthology.orchestra.run.smithi017.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 178 kB/s | 160 kB 00:00 2024-09-21T09:27:16.405 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-09-21T09:27:16.446 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 18/26 2024-09-21T09:27:16.483 INFO:teuthology.orchestra.run.smithi017.stdout:(16/24): librdkafka-1.6.1-102.el9.x86_64.rpm 710 kB/s | 662 kB 00:00 2024-09-21T09:27:16.533 INFO:teuthology.orchestra.run.smithi017.stdout:(17/24): gperftools-libs-2.9.1-3.el9.x86_64.rpm 6.1 MB/s | 308 kB 00:00 2024-09-21T09:27:16.535 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 18/26 2024-09-21T09:27:16.558 INFO:teuthology.orchestra.run.smithi017.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-21T09:27:16.583 INFO:teuthology.orchestra.run.smithi017.stdout:(19/24): libunwind-1.6.2-1.el9.x86_64.rpm 2.6 MB/s | 67 kB 00:00 2024-09-21T09:27:16.608 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 19/26 2024-09-21T09:27:16.634 INFO:teuthology.orchestra.run.smithi017.stdout:(20/24): thrift-0.15.0-2.el9.x86_64.rpm 32 MB/s | 1.6 MB 00:00 2024-09-21T09:27:16.640 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x8 20/26 2024-09-21T09:27:16.792 INFO:teuthology.orchestra.run.smithi017.stdout:(21/24): python3-prettytable-0.7.2-27.el9.noarc 71 kB/s | 42 kB 00:00 2024-09-21T09:27:16.842 INFO:teuthology.orchestra.run.smithi017.stdout:(22/24): librados2-17.2.7-1649.gfbd1a6ab.el9.x8 15 MB/s | 3.1 MB 00:00 2024-09-21T09:27:16.935 INFO:teuthology.orchestra.run.smithi017.stdout:(23/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 350 kB/s | 292 kB 00:00 2024-09-21T09:27:17.052 INFO:teuthology.orchestra.run.smithi017.stdout:(24/24): librbd1-17.2.7-1649.gfbd1a6ab.el9.x86_ 11 MB/s | 2.9 MB 00:00 2024-09-21T09:27:17.056 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:17.056 INFO:teuthology.orchestra.run.smithi017.stdout:Total 19 MB/s | 58 MB 00:03 2024-09-21T09:27:17.124 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:27:17.162 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:27:17.163 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:27:17.687 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:27:17.687 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:27:18.905 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:27:19.091 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 21/26 2024-09-21T09:27:19.128 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 21/26 2024-09-21T09:27:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-09-21T09:27:19.345 INFO:teuthology.orchestra.run.smithi017.stdout: Upgrading : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/26 2024-09-21T09:27:19.439 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/26 2024-09-21T09:27:19.519 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/26 2024-09-21T09:27:19.547 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/26 2024-09-21T09:27:19.637 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/26 2024-09-21T09:27:19.646 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 21/26 2024-09-21T09:27:19.681 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 22/26 2024-09-21T09:27:19.691 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-09-21T09:27:19.719 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-09-21T09:27:19.770 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 22/26 2024-09-21T09:27:19.770 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:27:19.770 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:27:19.771 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:19.799 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/26 2024-09-21T09:27:20.191 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-09-21T09:27:20.233 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 8/26 2024-09-21T09:27:20.299 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 8/26 2024-09-21T09:27:20.335 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-09-21T09:27:20.383 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9. 10/26 2024-09-21T09:27:20.416 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/26 2024-09-21T09:27:20.479 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 12/26 2024-09-21T09:27:20.505 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 13/26 2024-09-21T09:27:20.554 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 13/26 2024-09-21T09:27:20.594 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-09-21T09:27:20.639 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/26 2024-09-21T09:27:20.688 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-09-21T09:27:21.004 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-09-21T09:27:21.042 INFO:teuthology.orchestra.run.smithi017.stdout: Upgrading : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 18/26 2024-09-21T09:27:21.133 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 18/26 2024-09-21T09:27:21.210 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 19/26 2024-09-21T09:27:21.245 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x8 20/26 2024-09-21T09:27:23.840 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 21/26 2024-09-21T09:27:23.875 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 21/26 2024-09-21T09:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 21/26 2024-09-21T09:27:24.470 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 22/26 2024-09-21T09:27:24.574 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 22/26 2024-09-21T09:27:24.574 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:27:24.574 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:27:24.574 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:24.613 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/26 2024-09-21T09:27:29.517 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/26 2024-09-21T09:27:29.517 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /sys 2024-09-21T09:27:29.517 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /proc 2024-09-21T09:27:29.517 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /mnt 2024-09-21T09:27:29.517 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /var/tmp 2024-09-21T09:27:29.517 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /home 2024-09-21T09:27:29.517 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /root 2024-09-21T09:27:29.517 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /tmp 2024-09-21T09:27:29.517 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:30.865 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 24/26 2024-09-21T09:27:30.943 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 24/26 2024-09-21T09:27:30.943 INFO:teuthology.orchestra.run.smithi134.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:30.943 INFO:teuthology.orchestra.run.smithi134.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:27:30.943 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:27:30.943 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:27:30.943 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:30.945 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-21T09:27:31.025 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-21T09:27:31.026 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-21T09:27:32.277 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 6/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9. 8/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x8 9/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/26 2024-09-21T09:27:32.278 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 12/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 13/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 19/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-09-21T09:27:32.279 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 25/26 2024-09-21T09:27:33.406 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-09-21T09:27:33.406 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:33.406 INFO:teuthology.orchestra.run.smithi134.stdout:Upgraded: 2024-09-21T09:27:33.406 INFO:teuthology.orchestra.run.smithi134.stdout: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.406 INFO:teuthology.orchestra.run.smithi134.stdout: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.406 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:27:33.406 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.406 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.407 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.408 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.408 INFO:teuthology.orchestra.run.smithi134.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:27:33.408 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.408 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.408 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:33.408 INFO:teuthology.orchestra.run.smithi134.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:27:33.408 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:33.408 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:27:33.804 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-test 2024-09-21T09:27:34.385 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:00:27 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:27:34.517 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:27:34.517 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:34.517 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:27:34.517 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:34.517 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-test x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 48 M 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout: libcephsqlite x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 163 k 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout:Install 4 Packages 2024-09-21T09:27:34.518 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:34.519 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 48 M 2024-09-21T09:27:34.519 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 193 M 2024-09-21T09:27:34.519 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:27:34.779 INFO:teuthology.orchestra.run.smithi134.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 2.1 MB/s | 304 kB 00:00 2024-09-21T09:27:34.812 INFO:teuthology.orchestra.run.smithi134.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.9 MB/s | 64 kB 00:00 2024-09-21T09:27:34.987 INFO:teuthology.orchestra.run.smithi134.stdout:(3/4): libcephsqlite-17.2.7-1649.gfbd1a6ab.el9. 464 kB/s | 163 kB 00:00 2024-09-21T09:27:34.998 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/26 2024-09-21T09:27:34.998 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /sys 2024-09-21T09:27:34.998 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /proc 2024-09-21T09:27:34.998 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /mnt 2024-09-21T09:27:34.998 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /var/tmp 2024-09-21T09:27:34.998 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /home 2024-09-21T09:27:34.999 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /root 2024-09-21T09:27:34.999 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /tmp 2024-09-21T09:27:34.999 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:36.278 INFO:teuthology.orchestra.run.smithi134.stdout:(4/4): ceph-test-17.2.7-1649.gfbd1a6ab.el9.x86_ 29 MB/s | 48 MB 00:01 2024-09-21T09:27:36.280 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:36.281 INFO:teuthology.orchestra.run.smithi134.stdout:Total 27 MB/s | 48 MB 00:01 2024-09-21T09:27:36.308 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:27:36.327 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:27:36.327 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:27:36.345 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 24/26 2024-09-21T09:27:36.441 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 24/26 2024-09-21T09:27:36.441 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:36.441 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:27:36.441 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:27:36.441 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:27:36.441 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:36.443 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-21T09:27:36.540 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-09-21T09:27:36.540 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-21T09:27:36.694 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:27:36.694 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:27:37.226 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:27:37.324 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-21T09:27:37.370 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-21T09:27:37.398 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/4 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/26 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/26 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/26 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/26 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/26 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 6/26 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/26 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9. 8/26 2024-09-21T09:27:37.824 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x8 9/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 12/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 13/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 19/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-09-21T09:27:37.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-09-21T09:27:37.826 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/26 2024-09-21T09:27:37.826 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-09-21T09:27:37.826 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 25/26 2024-09-21T09:27:38.829 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-09-21T09:27:38.829 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:38.829 INFO:teuthology.orchestra.run.smithi017.stdout:Upgraded: 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:27:38.830 INFO:teuthology.orchestra.run.smithi017.stdout: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:38.831 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:27:39.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-test 2024-09-21T09:27:39.808 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:28 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:27:39.947 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:27:39.947 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-test x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 48 M 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout: libcephsqlite x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 163 k 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:27:39.948 INFO:teuthology.orchestra.run.smithi017.stdout:Install 4 Packages 2024-09-21T09:27:39.949 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:39.949 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 48 M 2024-09-21T09:27:39.949 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 193 M 2024-09-21T09:27:39.949 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:27:40.279 INFO:teuthology.orchestra.run.smithi017.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.4 MB/s | 304 kB 00:00 2024-09-21T09:27:40.305 INFO:teuthology.orchestra.run.smithi017.stdout:(2/4): libcephsqlite-17.2.7-1649.gfbd1a6ab.el9. 693 kB/s | 163 kB 00:00 2024-09-21T09:27:40.321 INFO:teuthology.orchestra.run.smithi017.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-09-21T09:27:41.589 INFO:teuthology.orchestra.run.smithi017.stdout:(4/4): ceph-test-17.2.7-1649.gfbd1a6ab.el9.x86_ 32 MB/s | 48 MB 00:01 2024-09-21T09:27:41.590 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:41.591 INFO:teuthology.orchestra.run.smithi017.stdout:Total 30 MB/s | 48 MB 00:01 2024-09-21T09:27:41.618 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:27:41.640 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:27:41.640 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:27:41.808 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/4 2024-09-21T09:27:41.890 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/4 2024-09-21T09:27:42.026 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:27:42.027 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:27:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:27:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-21T09:27:42.709 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-21T09:27:42.736 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/4 2024-09-21T09:27:44.045 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/4 2024-09-21T09:27:44.046 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/4 2024-09-21T09:27:44.046 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/4 2024-09-21T09:27:44.046 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-21T09:27:44.352 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-21T09:27:44.353 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:44.353 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:27:44.353 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:44.353 INFO:teuthology.orchestra.run.smithi134.stdout: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:44.353 INFO:teuthology.orchestra.run.smithi134.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:27:44.353 INFO:teuthology.orchestra.run.smithi134.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:27:44.353 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:44.353 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:27:44.676 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph 2024-09-21T09:27:45.260 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:00:38 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:27:45.395 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:27:45.397 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: ceph x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 6.5 k 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mds x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.1 M 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 1.4 M 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 240 k 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mon x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.9 M 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-osd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 15 M 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:27:45.398 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-collections 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:27:45.399 INFO:teuthology.orchestra.run.smithi134.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:27:45.400 INFO:teuthology.orchestra.run.smithi134.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:27:45.400 INFO:teuthology.orchestra.run.smithi134.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:27:45.400 INFO:teuthology.orchestra.run.smithi134.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-21T09:27:45.400 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:27:45.400 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:27:45.400 INFO:teuthology.orchestra.run.smithi134.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:27:45.400 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:27:45.400 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:27:45.400 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout:Install 34 Packages 2024-09-21T09:27:45.401 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:45.402 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 27 M 2024-09-21T09:27:45.402 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 96 M 2024-09-21T09:27:45.402 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:27:45.782 INFO:teuthology.orchestra.run.smithi134.stdout:(1/34): ceph-17.2.7-1649.gfbd1a6ab.el9.x86_64.r 53 kB/s | 6.5 kB 00:00 2024-09-21T09:27:45.966 INFO:teuthology.orchestra.run.smithi134.stdout:(2/34): ceph-mgr-17.2.7-1649.gfbd1a6ab.el9.x86_ 4.6 MB/s | 1.4 MB 00:00 2024-09-21T09:27:46.016 INFO:teuthology.orchestra.run.smithi134.stdout:(3/34): ceph-mds-17.2.7-1649.gfbd1a6ab.el9.x86_ 5.8 MB/s | 2.1 MB 00:00 2024-09-21T09:27:46.108 INFO:teuthology.orchestra.run.smithi134.stdout:(4/34): ceph-mon-17.2.7-1649.gfbd1a6ab.el9.x86_ 12 MB/s | 3.9 MB 00:00 2024-09-21T09:27:46.141 INFO:teuthology.orchestra.run.smithi134.stdout:(5/34): ceph-mgr-modules-core-17.2.7-1649.gfbd1 1.9 MB/s | 240 kB 00:00 2024-09-21T09:27:46.259 INFO:teuthology.orchestra.run.smithi134.stdout:(6/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.6 MB/s | 253 kB 00:00 2024-09-21T09:27:46.309 INFO:teuthology.orchestra.run.smithi134.stdout:(7/34): python3-cryptography-36.0.1-4.el9.x86_6 7.3 MB/s | 1.2 MB 00:00 2024-09-21T09:27:46.334 INFO:teuthology.orchestra.run.smithi134.stdout:(8/34): python3-ply-3.11-14.el9.noarch.rpm 1.4 MB/s | 106 kB 00:00 2024-09-21T09:27:46.501 INFO:teuthology.orchestra.run.smithi134.stdout:(9/34): ceph-osd-17.2.7-1649.gfbd1a6ab.el9.x86_ 29 MB/s | 15 MB 00:00 2024-09-21T09:27:46.526 INFO:teuthology.orchestra.run.smithi134.stdout:(10/34): python3-requests-2.25.1-8.el9.noarch.r 650 kB/s | 125 kB 00:00 2024-09-21T09:27:46.551 INFO:teuthology.orchestra.run.smithi134.stdout:(11/34): python3-pycparser-2.20-6.el9.noarch.rp 558 kB/s | 135 kB 00:00 2024-09-21T09:27:46.625 INFO:teuthology.orchestra.run.smithi134.stdout:(12/34): python3-markupsafe-1.1.1-12.el9.x86_64 471 kB/s | 35 kB 00:00 2024-09-21T09:27:46.651 INFO:teuthology.orchestra.run.smithi134.stdout:(13/34): python3-urllib3-1.26.5-6.el9.noarch.rp 1.4 MB/s | 215 kB 00:00 2024-09-21T09:27:46.676 INFO:teuthology.orchestra.run.smithi134.stdout:(14/34): python3-mako-1.1.4-6.el9.noarch.rpm 1.1 MB/s | 172 kB 00:00 2024-09-21T09:27:46.702 INFO:teuthology.orchestra.run.smithi134.stdout:(15/34): python3-packaging-20.9-5.el9.noarch.rp 1.0 MB/s | 77 kB 00:00 2024-09-21T09:27:46.727 INFO:teuthology.orchestra.run.smithi134.stdout:(16/34): python3-toml-0.10.2-6.el9.noarch.rpm 830 kB/s | 42 kB 00:00 2024-09-21T09:27:46.752 INFO:teuthology.orchestra.run.smithi134.stdout:(17/34): python3-pytz-2021.1-5.el9.noarch.rpm 505 kB/s | 51 kB 00:00 2024-09-21T09:27:46.802 INFO:teuthology.orchestra.run.smithi134.stdout:(18/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 434 kB/s | 43 kB 00:00 2024-09-21T09:27:46.827 INFO:teuthology.orchestra.run.smithi134.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 459 kB/s | 11 kB 00:00 2024-09-21T09:27:46.852 INFO:teuthology.orchestra.run.smithi134.stdout:(20/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-09-21T09:27:46.878 INFO:teuthology.orchestra.run.smithi134.stdout:(21/34): python3-jaraco-classes-3.2.1-5.el9.noa 352 kB/s | 18 kB 00:00 2024-09-21T09:27:46.903 INFO:teuthology.orchestra.run.smithi134.stdout:(22/34): python3-jaraco-functools-3.5.0-2.el9.n 786 kB/s | 19 kB 00:00 2024-09-21T09:27:46.928 INFO:teuthology.orchestra.run.smithi134.stdout:(23/34): python3-jaraco-collections-3.0.0-8.el9 463 kB/s | 23 kB 00:00 2024-09-21T09:27:46.953 INFO:teuthology.orchestra.run.smithi134.stdout:(24/34): python3-jaraco-text-3.2.0-6.el9.noarch 393 kB/s | 20 kB 00:00 2024-09-21T09:27:46.979 INFO:teuthology.orchestra.run.smithi134.stdout:(25/34): python3-logutils-0.3.5-21.el9.noarch.r 922 kB/s | 46 kB 00:00 2024-09-21T09:27:47.004 INFO:teuthology.orchestra.run.smithi134.stdout:(26/34): python3-cherrypy-18.6.1-2.el9.noarch.r 1.5 MB/s | 358 kB 00:00 2024-09-21T09:27:47.029 INFO:teuthology.orchestra.run.smithi134.stdout:(27/34): python3-more-itertools-8.12.0-2.el9.no 1.0 MB/s | 79 kB 00:00 2024-09-21T09:27:47.055 INFO:teuthology.orchestra.run.smithi134.stdout:(28/34): python3-portend-3.1.0-2.el9.noarch.rpm 324 kB/s | 16 kB 00:00 2024-09-21T09:27:47.080 INFO:teuthology.orchestra.run.smithi134.stdout:(29/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-09-21T09:27:47.105 INFO:teuthology.orchestra.run.smithi134.stdout:(30/34): python3-tempora-5.0.0-2.el9.noarch.rpm 716 kB/s | 36 kB 00:00 2024-09-21T09:27:47.130 INFO:teuthology.orchestra.run.smithi134.stdout:(31/34): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-09-21T09:27:47.164 INFO:teuthology.orchestra.run.smithi134.stdout:(32/34): python3-webob-1.8.8-2.el9.noarch.rpm 3.9 MB/s | 230 kB 00:00 2024-09-21T09:27:47.189 INFO:teuthology.orchestra.run.smithi134.stdout:(33/34): python3-zc-lockfile-2.0-10.el9.noarch. 341 kB/s | 20 kB 00:00 2024-09-21T09:27:47.248 INFO:teuthology.orchestra.run.smithi134.stdout:(34/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.9 MB/s | 427 kB 00:00 2024-09-21T09:27:47.252 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:47.252 INFO:teuthology.orchestra.run.smithi134.stdout:Total 15 MB/s | 27 MB 00:01 2024-09-21T09:27:47.375 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/4 2024-09-21T09:27:47.378 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:27:47.414 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:27:47.414 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:27:47.462 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/4 2024-09-21T09:27:47.795 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:27:47.796 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:27:49.038 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:27:49.092 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-09-21T09:27:49.124 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-09-21T09:27:49.166 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-09-21T09:27:49.273 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-09-21T09:27:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-09-21T09:27:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/34 2024-09-21T09:27:49.453 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-09-21T09:27:49.484 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-09-21T09:27:49.517 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-09-21T09:27:49.551 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-09-21T09:27:49.593 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-09-21T09:27:49.628 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-09-21T09:27:49.663 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-09-21T09:27:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/4 2024-09-21T09:27:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/4 2024-09-21T09:27:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/4 2024-09-21T09:27:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-21T09:27:49.711 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-09-21T09:27:49.743 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-21T09:27:49.824 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-09-21T09:27:49.970 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-09-21T09:27:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-21T09:27:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:49.977 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:27:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:27:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:27:49.977 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:49.977 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:27:50.050 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-09-21T09:27:50.104 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-09-21T09:27:50.142 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-09-21T09:27:50.191 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-09-21T09:27:50.246 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-09-21T09:27:50.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph 2024-09-21T09:27:50.439 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-09-21T09:27:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-09-21T09:27:50.557 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-09-21T09:27:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-09-21T09:27:50.703 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-09-21T09:27:50.818 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-09-21T09:27:50.891 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:39 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:27:50.966 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9. 29/34 2024-09-21T09:27:50.989 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 30/34 2024-09-21T09:27:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 30/34 2024-09-21T09:27:51.015 INFO:teuthology.orchestra.run.smithi134.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:51.015 INFO:teuthology.orchestra.run.smithi134.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:27:51.015 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:27:51.015 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:27:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:51.030 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:27:51.032 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:27:51.032 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:27:51.032 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:27:51.032 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:27:51.032 INFO:teuthology.orchestra.run.smithi017.stdout: ceph x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 6.5 k 2024-09-21T09:27:51.032 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mds x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.1 M 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 1.4 M 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 240 k 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mon x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.9 M 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-osd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 15 M 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-collections 2024-09-21T09:27:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:27:51.034 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout:Install 34 Packages 2024-09-21T09:27:51.035 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:51.036 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 27 M 2024-09-21T09:27:51.036 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 96 M 2024-09-21T09:27:51.036 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:27:51.260 INFO:teuthology.orchestra.run.smithi017.stdout:(1/34): ceph-17.2.7-1649.gfbd1a6ab.el9.x86_64.r 56 kB/s | 6.5 kB 00:00 2024-09-21T09:27:51.435 INFO:teuthology.orchestra.run.smithi017.stdout:(2/34): ceph-mds-17.2.7-1649.gfbd1a6ab.el9.x86_ 7.1 MB/s | 2.1 MB 00:00 2024-09-21T09:27:51.479 INFO:teuthology.orchestra.run.smithi017.stdout:(3/34): ceph-mgr-17.2.7-1649.gfbd1a6ab.el9.x86_ 4.2 MB/s | 1.4 MB 00:00 2024-09-21T09:27:51.563 INFO:teuthology.orchestra.run.smithi017.stdout:(4/34): ceph-mgr-modules-core-17.2.7-1649.gfbd1 2.8 MB/s | 240 kB 00:00 2024-09-21T09:27:51.621 INFO:teuthology.orchestra.run.smithi017.stdout:(5/34): ceph-mon-17.2.7-1649.gfbd1a6ab.el9.x86_ 11 MB/s | 3.9 MB 00:00 2024-09-21T09:27:51.730 INFO:teuthology.orchestra.run.smithi017.stdout:(6/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.5 MB/s | 253 kB 00:00 2024-09-21T09:27:51.789 INFO:teuthology.orchestra.run.smithi017.stdout:(7/34): python3-cryptography-36.0.1-4.el9.x86_6 7.3 MB/s | 1.2 MB 00:00 2024-09-21T09:27:51.823 INFO:teuthology.orchestra.run.smithi017.stdout:(8/34): python3-ply-3.11-14.el9.noarch.rpm 1.1 MB/s | 106 kB 00:00 2024-09-21T09:27:51.856 INFO:teuthology.orchestra.run.smithi017.stdout:(9/34): python3-pycparser-2.20-6.el9.noarch.rpm 2.0 MB/s | 135 kB 00:00 2024-09-21T09:27:52.015 INFO:teuthology.orchestra.run.smithi017.stdout:(10/34): ceph-osd-17.2.7-1649.gfbd1a6ab.el9.x86 26 MB/s | 15 MB 00:00 2024-09-21T09:27:52.040 INFO:teuthology.orchestra.run.smithi017.stdout:(11/34): python3-requests-2.25.1-8.el9.noarch.r 575 kB/s | 125 kB 00:00 2024-09-21T09:27:52.065 INFO:teuthology.orchestra.run.smithi017.stdout:(12/34): python3-urllib3-1.26.5-6.el9.noarch.rp 1.0 MB/s | 215 kB 00:00 2024-09-21T09:27:52.107 INFO:teuthology.orchestra.run.smithi017.stdout:(13/34): python3-markupsafe-1.1.1-12.el9.x86_64 519 kB/s | 35 kB 00:00 2024-09-21T09:27:52.132 INFO:teuthology.orchestra.run.smithi017.stdout:(14/34): python3-packaging-20.9-5.el9.noarch.rp 1.1 MB/s | 77 kB 00:00 2024-09-21T09:27:52.166 INFO:teuthology.orchestra.run.smithi017.stdout:(15/34): python3-pytz-2021.1-5.el9.noarch.rpm 869 kB/s | 51 kB 00:00 2024-09-21T09:27:52.191 INFO:teuthology.orchestra.run.smithi017.stdout:(16/34): python3-mako-1.1.4-6.el9.noarch.rpm 977 kB/s | 172 kB 00:00 2024-09-21T09:27:52.217 INFO:teuthology.orchestra.run.smithi017.stdout:(17/34): python3-toml-0.10.2-6.el9.noarch.rpm 495 kB/s | 42 kB 00:00 2024-09-21T09:27:52.267 INFO:teuthology.orchestra.run.smithi017.stdout:(18/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 432 kB/s | 43 kB 00:00 2024-09-21T09:27:52.292 INFO:teuthology.orchestra.run.smithi017.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 464 kB/s | 11 kB 00:00 2024-09-21T09:27:52.317 INFO:teuthology.orchestra.run.smithi017.stdout:(20/34): python3-jaraco-classes-3.2.1-5.el9.noa 707 kB/s | 18 kB 00:00 2024-09-21T09:27:52.343 INFO:teuthology.orchestra.run.smithi017.stdout:(21/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.1 MB/s | 172 kB 00:00 2024-09-21T09:27:52.366 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 31/34 2024-09-21T09:27:52.368 INFO:teuthology.orchestra.run.smithi017.stdout:(22/34): python3-jaraco-collections-3.0.0-8.el9 460 kB/s | 23 kB 00:00 2024-09-21T09:27:52.393 INFO:teuthology.orchestra.run.smithi017.stdout:(23/34): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-09-21T09:27:52.396 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 31/34 2024-09-21T09:27:52.396 INFO:teuthology.orchestra.run.smithi134.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:52.396 INFO:teuthology.orchestra.run.smithi134.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:27:52.396 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:27:52.396 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:27:52.396 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:52.419 INFO:teuthology.orchestra.run.smithi017.stdout:(24/34): python3-jaraco-text-3.2.0-6.el9.noarch 390 kB/s | 20 kB 00:00 2024-09-21T09:27:52.452 INFO:teuthology.orchestra.run.smithi017.stdout:(25/34): python3-logutils-0.3.5-21.el9.noarch.r 787 kB/s | 46 kB 00:00 2024-09-21T09:27:52.478 INFO:teuthology.orchestra.run.smithi017.stdout:(26/34): python3-more-itertools-8.12.0-2.el9.no 1.3 MB/s | 79 kB 00:00 2024-09-21T09:27:52.503 INFO:teuthology.orchestra.run.smithi017.stdout:(27/34): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-09-21T09:27:52.528 INFO:teuthology.orchestra.run.smithi017.stdout:(28/34): python3-portend-3.1.0-2.el9.noarch.rpm 328 kB/s | 16 kB 00:00 2024-09-21T09:27:52.562 INFO:teuthology.orchestra.run.smithi017.stdout:(29/34): python3-tempora-5.0.0-2.el9.noarch.rpm 1.0 MB/s | 36 kB 00:00 2024-09-21T09:27:52.587 INFO:teuthology.orchestra.run.smithi017.stdout:(30/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.0 MB/s | 90 kB 00:00 2024-09-21T09:27:52.621 INFO:teuthology.orchestra.run.smithi017.stdout:(31/34): python3-pecan-1.4.2-3.el9.noarch.rpm 1.6 MB/s | 272 kB 00:00 2024-09-21T09:27:52.646 INFO:teuthology.orchestra.run.smithi017.stdout:(32/34): python3-zc-lockfile-2.0-10.el9.noarch. 797 kB/s | 20 kB 00:00 2024-09-21T09:27:52.671 INFO:teuthology.orchestra.run.smithi017.stdout:(33/34): python3-webob-1.8.8-2.el9.noarch.rpm 2.1 MB/s | 230 kB 00:00 2024-09-21T09:27:52.696 INFO:teuthology.orchestra.run.smithi017.stdout:(34/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-09-21T09:27:52.700 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:52.700 INFO:teuthology.orchestra.run.smithi017.stdout:Total 16 MB/s | 27 MB 00:01 2024-09-21T09:27:52.775 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 32/34 2024-09-21T09:27:52.799 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 32/34 2024-09-21T09:27:52.799 INFO:teuthology.orchestra.run.smithi134.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:52.799 INFO:teuthology.orchestra.run.smithi134.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:27:52.799 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:27:52.799 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:27:52.799 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:52.828 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:27:52.870 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:27:52.871 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:27:53.009 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 33/34 2024-09-21T09:27:53.032 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 33/34 2024-09-21T09:27:53.032 INFO:teuthology.orchestra.run.smithi134.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:53.032 INFO:teuthology.orchestra.run.smithi134.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:27:53.032 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:27:53.032 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:27:53.032 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:53.098 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 34/34 2024-09-21T09:27:53.272 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:27:53.272 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:27:54.552 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:27:54.608 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-09-21T09:27:54.641 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-09-21T09:27:54.683 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-09-21T09:27:54.778 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-09-21T09:27:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-09-21T09:27:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/34 2024-09-21T09:27:54.937 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 34/34 2024-09-21T09:27:54.937 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/34 2024-09-21T09:27:54.937 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/34 2024-09-21T09:27:54.937 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/34 2024-09-21T09:27:54.937 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/34 2024-09-21T09:27:54.937 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/34 2024-09-21T09:27:54.937 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9. 6/34 2024-09-21T09:27:54.937 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-09-21T09:27:54.938 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-09-21T09:27:54.939 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-09-21T09:27:54.939 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-09-21T09:27:54.939 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-09-21T09:27:54.939 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-09-21T09:27:54.939 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-09-21T09:27:54.939 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-09-21T09:27:54.939 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-09-21T09:27:54.940 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-09-21T09:27:54.940 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-09-21T09:27:54.940 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-09-21T09:27:54.940 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 32/34 2024-09-21T09:27:54.940 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-09-21T09:27:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-09-21T09:27:54.983 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-09-21T09:27:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-09-21T09:27:55.051 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-09-21T09:27:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-09-21T09:27:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-09-21T09:27:55.177 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-09-21T09:27:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-09-21T09:27:55.268 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-21T09:27:55.354 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-09-21T09:27:55.507 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-09-21T09:27:55.589 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-09-21T09:27:55.648 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-09-21T09:27:55.685 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:27:55.692 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:27:55.693 INFO:teuthology.orchestra.run.smithi134.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:27:55.694 INFO:teuthology.orchestra.run.smithi134.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:27:55.695 INFO:teuthology.orchestra.run.smithi134.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:27:55.695 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:55.695 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:27:55.745 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-09-21T09:27:55.807 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-09-21T09:27:56.013 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-09-21T09:27:56.044 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-base 2024-09-21T09:27:56.065 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-09-21T09:27:56.146 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-09-21T09:27:56.244 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-09-21T09:27:56.280 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-09-21T09:27:56.405 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-09-21T09:27:56.566 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9. 29/34 2024-09-21T09:27:56.598 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 30/34 2024-09-21T09:27:56.627 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 30/34 2024-09-21T09:27:56.627 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:56.627 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:27:56.627 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:27:56.627 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:27:56.627 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:56.633 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:00:49 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:27:56.721 INFO:teuthology.orchestra.run.smithi134.stdout:Package ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:27:56.764 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:27:56.765 INFO:teuthology.orchestra.run.smithi134.stdout:Nothing to do. 2024-09-21T09:27:56.765 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:27:56.846 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install cephadm 2024-09-21T09:27:57.423 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:00:50 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:27:57.556 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:27:57.556 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:57.556 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout: cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 83 k 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout:Install 1 Package 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 83 k 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 375 k 2024-09-21T09:27:57.557 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:27:57.764 INFO:teuthology.orchestra.run.smithi134.stdout:cephadm-17.2.7-1649.gfbd1a6ab.el9.noarch.rpm 400 kB/s | 83 kB 00:00 2024-09-21T09:27:57.764 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:57.765 INFO:teuthology.orchestra.run.smithi134.stdout:Total 397 kB/s | 83 kB 00:00 2024-09-21T09:27:57.765 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:27:57.770 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:27:57.770 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:27:57.772 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:27:57.772 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:27:57.848 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:27:58.016 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 31/34 2024-09-21T09:27:58.046 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 31/34 2024-09-21T09:27:58.046 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:58.046 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:27:58.046 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:27:58.046 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:27:58.046 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:58.322 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/1 2024-09-21T09:27:58.427 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/1 2024-09-21T09:27:58.442 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 32/34 2024-09-21T09:27:58.468 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 32/34 2024-09-21T09:27:58.468 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:58.468 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:27:58.468 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:27:58.468 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:27:58.468 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:58.684 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 33/34 2024-09-21T09:27:58.710 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 33/34 2024-09-21T09:27:58.710 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:58.710 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:27:58.710 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:27:58.710 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:27:58.710 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:27:58.776 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 34/34 2024-09-21T09:27:58.891 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/1 2024-09-21T09:27:59.132 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/1 2024-09-21T09:27:59.132 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:59.132 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:27:59.132 INFO:teuthology.orchestra.run.smithi134.stdout: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:27:59.133 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:27:59.133 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:27:59.352 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-immutable-object-cache 2024-09-21T09:27:59.946 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:00:52 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:00.082 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:00.082 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:00.082 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repo Size 2024-09-21T09:28:00.083 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:00.083 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:00.083 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 137 k 2024-09-21T09:28:00.083 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:00.083 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:00.083 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:00.083 INFO:teuthology.orchestra.run.smithi134.stdout:Install 1 Package 2024-09-21T09:28:00.083 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:00.084 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 137 k 2024-09-21T09:28:00.084 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 379 k 2024-09-21T09:28:00.084 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:00.297 INFO:teuthology.orchestra.run.smithi134.stdout:ceph-immutable-object-cache-17.2.7-1649.gfbd1a6 652 kB/s | 137 kB 00:00 2024-09-21T09:28:00.298 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:00.298 INFO:teuthology.orchestra.run.smithi134.stdout:Total 648 kB/s | 137 kB 00:00 2024-09-21T09:28:00.298 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:00.303 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:00.303 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:00.359 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:00.360 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:00.503 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:00.549 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 34/34 2024-09-21T09:28:00.549 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/34 2024-09-21T09:28:00.549 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/34 2024-09-21T09:28:00.549 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/34 2024-09-21T09:28:00.549 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/34 2024-09-21T09:28:00.549 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/34 2024-09-21T09:28:00.549 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9. 6/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-09-21T09:28:00.550 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-09-21T09:28:00.551 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-09-21T09:28:00.551 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-09-21T09:28:00.551 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-09-21T09:28:00.551 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-09-21T09:28:00.551 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-09-21T09:28:00.551 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-09-21T09:28:00.551 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-09-21T09:28:00.551 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-09-21T09:28:00.552 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-09-21T09:28:00.552 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-09-21T09:28:00.552 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-09-21T09:28:00.552 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 32/34 2024-09-21T09:28:00.552 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-09-21T09:28:00.562 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab. 1/1 2024-09-21T09:28:00.587 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab. 1/1 2024-09-21T09:28:00.587 INFO:teuthology.orchestra.run.smithi134.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:28:00.587 INFO:teuthology.orchestra.run.smithi134.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:28:00.588 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:01.480 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:28:01.481 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:28:01.482 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:01.483 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:01.524 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab. 1/1 2024-09-21T09:28:01.524 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:01.524 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:01.524 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:01.524 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:01.524 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:01.774 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-mgr 2024-09-21T09:28:01.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-base 2024-09-21T09:28:02.377 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:00:55 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:02.462 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:51 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:02.467 INFO:teuthology.orchestra.run.smithi134.stdout:Package ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:02.509 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:02.510 INFO:teuthology.orchestra.run.smithi134.stdout:Nothing to do. 2024-09-21T09:28:02.510 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:02.555 INFO:teuthology.orchestra.run.smithi017.stdout:Package ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:02.594 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-mgr-dashboard 2024-09-21T09:28:02.600 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:02.601 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-21T09:28:02.601 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:02.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install cephadm 2024-09-21T09:28:03.182 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:00:56 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:03.256 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:52 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:03.314 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 3.4 M 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 24 k 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 15 k 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:28:03.315 INFO:teuthology.orchestra.run.smithi134.stdout:Installing weak dependencies: 2024-09-21T09:28:03.316 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-21T09:28:03.316 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:03.316 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:03.316 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:03.316 INFO:teuthology.orchestra.run.smithi134.stdout:Install 7 Packages 2024-09-21T09:28:03.316 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:03.316 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 3.7 M 2024-09-21T09:28:03.316 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 80 M 2024-09-21T09:28:03.316 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:03.393 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:03.394 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:03.394 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:28:03.394 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:03.394 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:03.394 INFO:teuthology.orchestra.run.smithi017.stdout: cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 83 k 2024-09-21T09:28:03.394 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:03.394 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:03.395 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:03.395 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-21T09:28:03.395 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:03.395 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 83 k 2024-09-21T09:28:03.395 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 375 k 2024-09-21T09:28:03.395 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:03.522 INFO:teuthology.orchestra.run.smithi134.stdout:(1/7): ceph-grafana-dashboards-17.2.7-1649.gfbd 169 kB/s | 24 kB 00:00 2024-09-21T09:28:03.548 INFO:teuthology.orchestra.run.smithi134.stdout:(2/7): ceph-prometheus-alerts-17.2.7-1649.gfbd1 86 kB/s | 15 kB 00:00 2024-09-21T09:28:03.583 INFO:teuthology.orchestra.run.smithi017.stdout:cephadm-17.2.7-1649.gfbd1a6ab.el9.noarch.rpm 440 kB/s | 83 kB 00:00 2024-09-21T09:28:03.584 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:03.584 INFO:teuthology.orchestra.run.smithi017.stdout:Total 437 kB/s | 83 kB 00:00 2024-09-21T09:28:03.584 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:03.590 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:03.590 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:03.590 INFO:teuthology.orchestra.run.smithi134.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 135 kB/s | 9.0 kB 00:00 2024-09-21T09:28:03.591 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:03.592 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:03.615 INFO:teuthology.orchestra.run.smithi134.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 1.3 MB/s | 31 kB 00:00 2024-09-21T09:28:03.640 INFO:teuthology.orchestra.run.smithi134.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 444 kB/s | 41 kB 00:00 2024-09-21T09:28:03.665 INFO:teuthology.orchestra.run.smithi134.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 3.7 MB/s | 188 kB 00:00 2024-09-21T09:28:03.670 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:03.774 INFO:teuthology.orchestra.run.smithi134.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1649.gfbd1a6ab 8.5 MB/s | 3.4 MB 00:00 2024-09-21T09:28:03.777 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:03.778 INFO:teuthology.orchestra.run.smithi134.stdout:Total 7.9 MB/s | 3.7 MB 00:00 2024-09-21T09:28:03.778 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:03.786 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:03.786 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:03.876 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:03.876 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/1 2024-09-21T09:28:04.185 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:04.223 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/1 2024-09-21T09:28:04.258 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-09-21T09:28:04.292 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-09-21T09:28:04.348 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-09-21T09:28:04.388 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-09-21T09:28:04.416 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.n 5/7 2024-09-21T09:28:04.576 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/1 2024-09-21T09:28:04.823 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/1 2024-09-21T09:28:04.823 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:04.823 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:04.823 INFO:teuthology.orchestra.run.smithi017.stdout: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:04.823 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:04.824 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:05.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-immutable-object-cache 2024-09-21T09:28:05.411 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9. 6/7 2024-09-21T09:28:05.462 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarc 7/7 2024-09-21T09:28:05.631 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:54 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:05.768 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:05.769 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:05.769 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2024-09-21T09:28:05.769 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:05.769 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:05.769 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 137 k 2024-09-21T09:28:05.769 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:05.769 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:05.769 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:05.769 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-21T09:28:05.770 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:05.770 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 137 k 2024-09-21T09:28:05.770 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 379 k 2024-09-21T09:28:05.770 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:05.973 INFO:teuthology.orchestra.run.smithi017.stdout:ceph-immutable-object-cache-17.2.7-1649.gfbd1a6 679 kB/s | 137 kB 00:00 2024-09-21T09:28:05.973 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:05.973 INFO:teuthology.orchestra.run.smithi017.stdout:Total 673 kB/s | 137 kB 00:00 2024-09-21T09:28:05.974 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:05.982 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:05.982 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:06.043 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:06.043 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:06.265 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab. 1/1 2024-09-21T09:28:06.292 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab. 1/1 2024-09-21T09:28:06.292 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:28:06.292 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:28:06.293 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:06.558 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarc 7/7 2024-09-21T09:28:06.559 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9. 1/7 2024-09-21T09:28:06.559 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarc 2/7 2024-09-21T09:28:06.559 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.n 3/7 2024-09-21T09:28:06.559 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-09-21T09:28:06.559 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-09-21T09:28:06.559 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-09-21T09:28:06.846 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:06.847 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:07.085 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-21T09:28:07.224 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab. 1/1 2024-09-21T09:28:07.224 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:07.224 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:07.224 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:07.224 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:07.224 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:07.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr 2024-09-21T09:28:07.672 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:00 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:07.805 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:07.806 INFO:teuthology.orchestra.run.smithi134.stdout:======================================================================================= 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout:======================================================================================= 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 7.4 M 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:28:07.807 INFO:teuthology.orchestra.run.smithi134.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:28:07.808 INFO:teuthology.orchestra.run.smithi134.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:28:07.808 INFO:teuthology.orchestra.run.smithi134.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:28:07.808 INFO:teuthology.orchestra.run.smithi134.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:28:07.808 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:07.808 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:07.808 INFO:teuthology.orchestra.run.smithi134.stdout:======================================================================================= 2024-09-21T09:28:07.808 INFO:teuthology.orchestra.run.smithi134.stdout:Install 12 Packages 2024-09-21T09:28:07.808 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:07.809 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 38 M 2024-09-21T09:28:07.809 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 212 M 2024-09-21T09:28:07.809 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:08.046 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:57 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:08.098 INFO:teuthology.orchestra.run.smithi134.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.5 MB/s | 191 kB 00:00 2024-09-21T09:28:08.139 INFO:teuthology.orchestra.run.smithi017.stdout:Package ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:08.149 INFO:teuthology.orchestra.run.smithi134.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 4.6 MB/s | 801 kB 00:00 2024-09-21T09:28:08.183 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:08.184 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-21T09:28:08.184 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:08.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-dashboard 2024-09-21T09:28:08.424 INFO:teuthology.orchestra.run.smithi134.stdout:(3/12): ceph-mgr-diskprediction-local-17.2.7-16 17 MB/s | 7.4 MB 00:00 2024-09-21T09:28:08.674 INFO:teuthology.orchestra.run.smithi134.stdout:(4/12): flexiblas-3.0.4-8.el9.x86_64.rpm 57 kB/s | 33 kB 00:00 2024-09-21T09:28:08.854 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:57 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:08.899 INFO:teuthology.orchestra.run.smithi134.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 39 kB/s | 18 kB 00:00 2024-09-21T09:28:08.993 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:08.993 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:08.993 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 3.4 M 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 24 k 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 15 k 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout:Installing weak dependencies: 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:08.994 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:08.995 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:08.995 INFO:teuthology.orchestra.run.smithi017.stdout:Install 7 Packages 2024-09-21T09:28:08.995 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:08.995 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 3.7 M 2024-09-21T09:28:08.995 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 80 M 2024-09-21T09:28:08.995 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:09.185 INFO:teuthology.orchestra.run.smithi017.stdout:(1/7): ceph-grafana-dashboards-17.2.7-1649.gfbd 153 kB/s | 24 kB 00:00 2024-09-21T09:28:09.210 INFO:teuthology.orchestra.run.smithi017.stdout:(2/7): ceph-prometheus-alerts-17.2.7-1649.gfbd1 79 kB/s | 15 kB 00:00 2024-09-21T09:28:09.252 INFO:teuthology.orchestra.run.smithi017.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 136 kB/s | 9.0 kB 00:00 2024-09-21T09:28:09.277 INFO:teuthology.orchestra.run.smithi017.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 1.3 MB/s | 31 kB 00:00 2024-09-21T09:28:09.283 INFO:teuthology.orchestra.run.smithi134.stdout:(6/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.6 MB/s | 3.0 MB 00:01 2024-09-21T09:28:09.303 INFO:teuthology.orchestra.run.smithi017.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 444 kB/s | 41 kB 00:00 2024-09-21T09:28:09.308 INFO:teuthology.orchestra.run.smithi134.stdout:(7/12): openblas-0.3.26-2.el9.x86_64.rpm 61 kB/s | 38 kB 00:00 2024-09-21T09:28:09.328 INFO:teuthology.orchestra.run.smithi017.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 3.7 MB/s | 188 kB 00:00 2024-09-21T09:28:09.445 INFO:teuthology.orchestra.run.smithi017.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1649.gfbd1a6ab 8.0 MB/s | 3.4 MB 00:00 2024-09-21T09:28:09.448 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:09.448 INFO:teuthology.orchestra.run.smithi017.stdout:Total 8.1 MB/s | 3.7 MB 00:00 2024-09-21T09:28:09.449 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:09.457 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:09.457 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:09.550 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:09.550 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:09.871 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:09.929 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-09-21T09:28:09.964 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-09-21T09:28:10.020 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-09-21T09:28:10.025 INFO:teuthology.orchestra.run.smithi134.stdout:(8/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 4.4 MB/s | 4.9 MB 00:01 2024-09-21T09:28:10.050 INFO:teuthology.orchestra.run.smithi134.stdout:(9/12): python3-devel-3.9.19-8.el9.x86_64.rpm 319 kB/s | 245 kB 00:00 2024-09-21T09:28:10.058 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-09-21T09:28:10.088 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.n 5/7 2024-09-21T09:28:10.475 INFO:teuthology.orchestra.run.smithi134.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 4.4 MB/s | 5.1 MB 00:01 2024-09-21T09:28:10.809 INFO:teuthology.orchestra.run.smithi134.stdout:(11/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 320 kB/s | 250 kB 00:00 2024-09-21T09:28:11.147 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9. 6/7 2024-09-21T09:28:11.191 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarc 7/7 2024-09-21T09:28:11.551 INFO:teuthology.orchestra.run.smithi134.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-09-21T09:28:11.552 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:11.552 INFO:teuthology.orchestra.run.smithi134.stdout:Total 10 MB/s | 38 MB 00:03 2024-09-21T09:28:11.911 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:11.938 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:11.938 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:12.229 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarc 7/7 2024-09-21T09:28:12.229 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9. 1/7 2024-09-21T09:28:12.229 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarc 2/7 2024-09-21T09:28:12.229 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.n 3/7 2024-09-21T09:28:12.229 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-09-21T09:28:12.229 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-09-21T09:28:12.229 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-09-21T09:28:12.450 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:12.450 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:12.509 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-09-21T09:28:12.509 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:12.510 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:12.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-21T09:28:13.236 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:13.311 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-21T09:28:13.340 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:02 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:13.349 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:28:13.448 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:28:13.471 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:13.472 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================= 2024-09-21T09:28:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:28:13.472 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================= 2024-09-21T09:28:13.472 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 7.4 M 2024-09-21T09:28:13.472 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:28:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:28:13.472 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================= 2024-09-21T09:28:13.473 INFO:teuthology.orchestra.run.smithi017.stdout:Install 12 Packages 2024-09-21T09:28:13.474 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:13.474 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 38 M 2024-09-21T09:28:13.474 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 212 M 2024-09-21T09:28:13.474 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:13.483 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-21T09:28:13.668 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-21T09:28:13.699 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-21T09:28:13.784 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-21T09:28:14.303 INFO:teuthology.orchestra.run.smithi017.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 649 kB/s | 191 kB 00:00 2024-09-21T09:28:14.378 INFO:teuthology.orchestra.run.smithi017.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.1 MB/s | 801 kB 00:00 2024-09-21T09:28:14.440 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-21T09:28:14.479 INFO:teuthology.orchestra.run.smithi017.stdout:(3/12): ceph-mgr-diskprediction-local-17.2.7-16 16 MB/s | 7.4 MB 00:00 2024-09-21T09:28:14.504 INFO:teuthology.orchestra.run.smithi017.stdout:(4/12): flexiblas-3.0.4-8.el9.x86_64.rpm 163 kB/s | 33 kB 00:00 2024-09-21T09:28:14.529 INFO:teuthology.orchestra.run.smithi017.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 363 kB/s | 18 kB 00:00 2024-09-21T09:28:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-21T09:28:14.613 INFO:teuthology.orchestra.run.smithi017.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 354 kB/s | 38 kB 00:00 2024-09-21T09:28:14.755 INFO:teuthology.orchestra.run.smithi017.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 1.7 MB/s | 245 kB 00:00 2024-09-21T09:28:15.238 INFO:teuthology.orchestra.run.smithi017.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 3.5 MB/s | 3.0 MB 00:00 2024-09-21T09:28:15.322 INFO:teuthology.orchestra.run.smithi017.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.9 MB/s | 250 kB 00:00 2024-09-21T09:28:15.480 INFO:teuthology.orchestra.run.smithi017.stdout:(10/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 5.2 MB/s | 4.9 MB 00:00 2024-09-21T09:28:15.832 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:28:15.997 INFO:teuthology.orchestra.run.smithi017.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 4.1 MB/s | 5.1 MB 00:01 2024-09-21T09:28:16.862 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-21T09:28:16.897 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a 12/12 2024-09-21T09:28:18.114 INFO:teuthology.orchestra.run.smithi017.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 5.8 MB/s | 16 MB 00:02 2024-09-21T09:28:18.115 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:18.116 INFO:teuthology.orchestra.run.smithi017.stdout:Total 8.2 MB/s | 38 MB 00:04 2024-09-21T09:28:18.485 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:18.512 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:18.512 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:19.047 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:19.048 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a 12/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a 1/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-21T09:28:19.598 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:28:19.599 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-21T09:28:19.839 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:19.900 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-21T09:28:19.938 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:28:20.043 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:28:20.047 INFO:teuthology.orchestra.run.smithi134.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:28:20.048 INFO:teuthology.orchestra.run.smithi134.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:28:20.048 INFO:teuthology.orchestra.run.smithi134.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:28:20.048 INFO:teuthology.orchestra.run.smithi134.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:28:20.048 INFO:teuthology.orchestra.run.smithi134.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:28:20.048 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:20.048 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:20.072 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-21T09:28:20.267 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-21T09:28:20.296 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-21T09:28:20.386 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-21T09:28:20.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-mgr-rook 2024-09-21T09:28:20.999 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:13 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:21.086 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-21T09:28:21.130 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-rook noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 47 k 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:28:21.132 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout:Install 13 Packages 2024-09-21T09:28:21.133 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:21.134 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 2.2 M 2024-09-21T09:28:21.134 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 26 M 2024-09-21T09:28:21.134 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:21.206 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-21T09:28:21.354 INFO:teuthology.orchestra.run.smithi134.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 132 kB/s | 19 kB 00:00 2024-09-21T09:28:21.379 INFO:teuthology.orchestra.run.smithi134.stdout:(2/13): ceph-mgr-rook-17.2.7-1649.gfbd1a6ab.el9 279 kB/s | 47 kB 00:00 2024-09-21T09:28:21.413 INFO:teuthology.orchestra.run.smithi134.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 126 kB/s | 26 kB 00:00 2024-09-21T09:28:21.530 INFO:teuthology.orchestra.run.smithi134.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.2 MB/s | 222 kB 00:00 2024-09-21T09:28:21.555 INFO:teuthology.orchestra.run.smithi134.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.1 MB/s | 159 kB 00:00 2024-09-21T09:28:21.589 INFO:teuthology.orchestra.run.smithi134.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 918 kB/s | 54 kB 00:00 2024-09-21T09:28:21.622 INFO:teuthology.orchestra.run.smithi134.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 480 kB/s | 32 kB 00:00 2024-09-21T09:28:21.648 INFO:teuthology.orchestra.run.smithi134.stdout:(8/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.2 MB/s | 279 kB 00:00 2024-09-21T09:28:21.673 INFO:teuthology.orchestra.run.smithi134.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 172 kB/s | 14 kB 00:00 2024-09-21T09:28:21.711 INFO:teuthology.orchestra.run.smithi134.stdout:(10/13): python3-google-auth-2.34.0-1.el9.noarc 2.5 MB/s | 220 kB 00:00 2024-09-21T09:28:21.732 INFO:teuthology.orchestra.run.smithi134.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-09-21T09:28:21.765 INFO:teuthology.orchestra.run.smithi134.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 2.8 MB/s | 90 kB 00:00 2024-09-21T09:28:21.807 INFO:teuthology.orchestra.run.smithi134.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.4 MB/s | 1.0 MB 00:00 2024-09-21T09:28:21.811 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:21.811 INFO:teuthology.orchestra.run.smithi134.stdout:Total 3.3 MB/s | 2.2 MB 00:00 2024-09-21T09:28:21.879 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:21.895 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:21.895 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:21.984 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:21.984 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:22.488 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:22.538 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:28:22.661 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-21T09:28:22.725 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-21T09:28:22.780 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-21T09:28:22.826 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-21T09:28:22.876 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-21T09:28:22.985 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-21T09:28:23.095 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-21T09:28:23.160 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-21T09:28:23.618 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-21T09:28:23.645 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a 12/12 2024-09-21T09:28:23.885 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-21T09:28:23.941 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-21T09:28:23.975 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-21T09:28:24.020 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-21T09:28:24.044 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 13/13 2024-09-21T09:28:24.878 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 13/13 2024-09-21T09:28:24.878 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-21T09:28:24.879 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:28:25.220 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:28:25.221 INFO:teuthology.orchestra.run.smithi134.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:28:25.221 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:28:25.221 INFO:teuthology.orchestra.run.smithi134.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:28:25.221 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:25.221 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:25.479 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-mgr-cephadm 2024-09-21T09:28:26.056 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:19 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:26.142 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a 12/12 2024-09-21T09:28:26.142 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a 1/12 2024-09-21T09:28:26.142 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:28:26.142 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:28:26.142 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-21T09:28:26.142 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-21T09:28:26.142 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-21T09:28:26.143 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-21T09:28:26.143 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-21T09:28:26.143 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-21T09:28:26.143 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:28:26.143 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-21T09:28:26.187 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:26.188 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:26.188 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:28:26.188 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 124 k 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout:Install 6 Packages 2024-09-21T09:28:26.189 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:26.190 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 7.0 M 2024-09-21T09:28:26.190 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 33 M 2024-09-21T09:28:26.190 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:26.522 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-21T09:28:26.522 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:26.522 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:26.522 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:26.522 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:28:26.522 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:28:26.522 INFO:teuthology.orchestra.run.smithi017.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:28:26.522 INFO:teuthology.orchestra.run.smithi017.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:28:26.523 INFO:teuthology.orchestra.run.smithi017.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:28:26.523 INFO:teuthology.orchestra.run.smithi017.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:28:26.523 INFO:teuthology.orchestra.run.smithi017.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:28:26.523 INFO:teuthology.orchestra.run.smithi017.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:28:26.523 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:28:26.523 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:28:26.523 INFO:teuthology.orchestra.run.smithi017.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:28:26.523 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:26.523 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:26.525 INFO:teuthology.orchestra.run.smithi134.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.4 MB/s | 248 kB 00:00 2024-09-21T09:28:26.550 INFO:teuthology.orchestra.run.smithi134.stdout:(2/6): ceph-mgr-cephadm-17.2.7-1649.gfbd1a6ab.e 624 kB/s | 124 kB 00:00 2024-09-21T09:28:26.601 INFO:teuthology.orchestra.run.smithi134.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 7.2 MB/s | 548 kB 00:00 2024-09-21T09:28:26.618 INFO:teuthology.orchestra.run.smithi134.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 861 kB/s | 58 kB 00:00 2024-09-21T09:28:26.643 INFO:teuthology.orchestra.run.smithi134.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.7 MB/s | 74 kB 00:00 2024-09-21T09:28:26.776 INFO:teuthology.orchestra.run.smithi134.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 14 MB/s | 6.0 MB 00:00 2024-09-21T09:28:26.779 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:26.779 INFO:teuthology.orchestra.run.smithi134.stdout:Total 12 MB/s | 7.0 MB 00:00 2024-09-21T09:28:26.850 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:26.861 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:26.861 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:26.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-rook 2024-09-21T09:28:26.960 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:26.962 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:27.263 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-21T09:28:27.469 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-21T09:28:27.501 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:16 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:27.639 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:27.640 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:27.640 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:28:27.640 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:27.640 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:27.640 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-rook noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 47 k 2024-09-21T09:28:27.640 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:27.641 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:27.642 INFO:teuthology.orchestra.run.smithi017.stdout:Install 13 Packages 2024-09-21T09:28:27.642 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:27.642 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 2.2 M 2024-09-21T09:28:27.642 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 26 M 2024-09-21T09:28:27.642 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:27.984 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-21T09:28:28.030 INFO:teuthology.orchestra.run.smithi017.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 175 kB/s | 19 kB 00:00 2024-09-21T09:28:28.055 INFO:teuthology.orchestra.run.smithi017.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 190 kB/s | 26 kB 00:00 2024-09-21T09:28:28.071 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-21T09:28:28.114 INFO:teuthology.orchestra.run.smithi017.stdout:(3/13): ceph-mgr-rook-17.2.7-1649.gfbd1a6ab.el9 246 kB/s | 47 kB 00:00 2024-09-21T09:28:28.136 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-21T09:28:28.165 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 6/6 2024-09-21T09:28:28.172 INFO:teuthology.orchestra.run.smithi017.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.5 MB/s | 222 kB 00:00 2024-09-21T09:28:28.198 INFO:teuthology.orchestra.run.smithi017.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.1 MB/s | 159 kB 00:00 2024-09-21T09:28:28.223 INFO:teuthology.orchestra.run.smithi017.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-09-21T09:28:28.273 INFO:teuthology.orchestra.run.smithi017.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 428 kB/s | 32 kB 00:00 2024-09-21T09:28:28.299 INFO:teuthology.orchestra.run.smithi017.stdout:(8/13): python3-certifi-2023.05.07-4.el9.noarch 186 kB/s | 14 kB 00:00 2024-09-21T09:28:28.326 INFO:teuthology.orchestra.run.smithi017.stdout:(9/13): python3-google-auth-2.34.0-1.el9.noarch 4.4 MB/s | 220 kB 00:00 2024-09-21T09:28:28.350 INFO:teuthology.orchestra.run.smithi017.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2024-09-21T09:28:28.375 INFO:teuthology.orchestra.run.smithi017.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 3.5 MB/s | 90 kB 00:00 2024-09-21T09:28:28.417 INFO:teuthology.orchestra.run.smithi017.stdout:(12/13): python3-kubernetes-26.1.0-2.el9.noarch 8.7 MB/s | 1.0 MB 00:00 2024-09-21T09:28:28.450 INFO:teuthology.orchestra.run.smithi017.stdout:(13/13): python3-pyasn1-modules-0.4.8-6.el9.noa 829 kB/s | 279 kB 00:00 2024-09-21T09:28:28.452 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:28.452 INFO:teuthology.orchestra.run.smithi017.stdout:Total 2.7 MB/s | 2.2 MB 00:00 2024-09-21T09:28:28.522 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:28.539 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:28.539 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:28.637 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:28.638 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:28.955 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 6/6 2024-09-21T09:28:28.955 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/6 2024-09-21T09:28:28.955 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-21T09:28:28.955 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-21T09:28:28.955 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-21T09:28:28.955 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-21T09:28:29.164 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:29.226 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:29.227 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:29.340 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-21T09:28:29.404 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-21T09:28:29.459 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-21T09:28:29.490 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-fuse 2024-09-21T09:28:29.494 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-21T09:28:29.545 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-21T09:28:29.660 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-21T09:28:29.771 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-21T09:28:29.822 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-21T09:28:30.069 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:23 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:30.203 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:30.203 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:30.203 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-09-21T09:28:30.203 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:30.203 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:30.203 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 813 k 2024-09-21T09:28:30.203 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-09-21T09:28:30.203 INFO:teuthology.orchestra.run.smithi134.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:28:30.204 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:30.204 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:30.204 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:30.204 INFO:teuthology.orchestra.run.smithi134.stdout:Install 2 Packages 2024-09-21T09:28:30.204 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:30.204 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 893 k 2024-09-21T09:28:30.204 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 2.6 M 2024-09-21T09:28:30.205 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:30.386 INFO:teuthology.orchestra.run.smithi134.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 496 kB/s | 80 kB 00:00 2024-09-21T09:28:30.527 INFO:teuthology.orchestra.run.smithi134.stdout:(2/2): ceph-fuse-17.2.7-1649.gfbd1a6ab.el9.x86_ 2.6 MB/s | 813 kB 00:00 2024-09-21T09:28:30.528 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:30.529 INFO:teuthology.orchestra.run.smithi134.stdout:Total 2.7 MB/s | 893 kB 00:00 2024-09-21T09:28:30.540 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:30.561 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:30.561 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-21T09:28:30.641 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:30.641 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-21T09:28:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-21T09:28:30.741 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-21T09:28:30.763 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 13/13 2024-09-21T09:28:30.824 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:30.961 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-21T09:28:31.027 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/2 2024-09-21T09:28:31.497 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 13/13 2024-09-21T09:28:31.497 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/13 2024-09-21T09:28:31.497 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-21T09:28:31.497 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-21T09:28:31.498 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-21T09:28:31.498 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-21T09:28:31.498 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-21T09:28:31.498 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-21T09:28:31.498 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-21T09:28:31.498 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-21T09:28:31.498 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-21T09:28:31.498 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-21T09:28:31.498 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-21T09:28:31.721 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/2 2024-09-21T09:28:31.722 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/2 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:28:31.838 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:28:31.839 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:28:31.839 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:28:31.839 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:28:31.839 INFO:teuthology.orchestra.run.smithi017.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:28:31.839 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:31.839 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:32.028 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-21T09:28:32.028 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:32.028 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:32.028 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-21T09:28:32.028 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:32.028 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:32.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-cephadm 2024-09-21T09:28:32.327 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install librados-devel 2024-09-21T09:28:32.694 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:21 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:32.832 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:32.833 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:32.833 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:28:32.833 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:32.833 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:32.833 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 124 k 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout:Install 6 Packages 2024-09-21T09:28:32.834 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:32.835 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 7.0 M 2024-09-21T09:28:32.835 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 33 M 2024-09-21T09:28:32.835 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:32.929 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:25 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:33.065 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repo Size 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout: librados-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 126 k 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout:Install 1 Package 2024-09-21T09:28:33.066 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:33.067 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 126 k 2024-09-21T09:28:33.067 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 448 k 2024-09-21T09:28:33.067 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:33.069 INFO:teuthology.orchestra.run.smithi017.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.3 MB/s | 248 kB 00:00 2024-09-21T09:28:33.103 INFO:teuthology.orchestra.run.smithi017.stdout:(2/6): ceph-mgr-cephadm-17.2.7-1649.gfbd1a6ab.e 575 kB/s | 124 kB 00:00 2024-09-21T09:28:33.153 INFO:teuthology.orchestra.run.smithi017.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 6.4 MB/s | 548 kB 00:00 2024-09-21T09:28:33.170 INFO:teuthology.orchestra.run.smithi017.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 864 kB/s | 58 kB 00:00 2024-09-21T09:28:33.195 INFO:teuthology.orchestra.run.smithi017.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.7 MB/s | 74 kB 00:00 2024-09-21T09:28:33.282 INFO:teuthology.orchestra.run.smithi134.stdout:librados-devel-17.2.7-1649.gfbd1a6ab.el9.x86_64 590 kB/s | 126 kB 00:00 2024-09-21T09:28:33.282 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:33.282 INFO:teuthology.orchestra.run.smithi134.stdout:Total 585 kB/s | 126 kB 00:00 2024-09-21T09:28:33.283 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:33.288 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:33.288 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:33.351 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:33.351 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:33.497 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:33.587 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:34.089 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:34.360 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:34.360 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:34.360 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:34.360 INFO:teuthology.orchestra.run.smithi134.stdout: librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:34.360 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:34.360 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:34.537 INFO:teuthology.orchestra.run.smithi017.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.6 MB/s | 6.0 MB 00:01 2024-09-21T09:28:34.540 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:34.540 INFO:teuthology.orchestra.run.smithi017.stdout:Total 4.1 MB/s | 7.0 MB 00:01 2024-09-21T09:28:34.614 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:34.626 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:34.626 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:34.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install libcephfs2 2024-09-21T09:28:34.735 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:34.736 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:35.053 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:35.220 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-21T09:28:35.231 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:28 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:35.303 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-21T09:28:35.324 INFO:teuthology.orchestra.run.smithi134.stdout:Package libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:35.368 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:35.369 INFO:teuthology.orchestra.run.smithi134.stdout:Nothing to do. 2024-09-21T09:28:35.369 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:35.457 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install libcephfs-devel 2024-09-21T09:28:35.868 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-21T09:28:35.953 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-21T09:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-21T09:28:36.040 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:29 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 6/6 2024-09-21T09:28:36.173 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:36.173 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:36.173 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repo Size 2024-09-21T09:28:36.173 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:36.173 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:36.174 INFO:teuthology.orchestra.run.smithi134.stdout: libcephfs-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:28:36.174 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:36.174 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:36.174 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:36.174 INFO:teuthology.orchestra.run.smithi134.stdout:Install 1 Package 2024-09-21T09:28:36.174 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:36.174 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 25 k 2024-09-21T09:28:36.175 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 110 k 2024-09-21T09:28:36.175 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:36.318 INFO:teuthology.orchestra.run.smithi134.stdout:libcephfs-devel-17.2.7-1649.gfbd1a6ab.el9.x86_6 174 kB/s | 25 kB 00:00 2024-09-21T09:28:36.318 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:36.318 INFO:teuthology.orchestra.run.smithi134.stdout:Total 173 kB/s | 25 kB 00:00 2024-09-21T09:28:36.319 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:36.322 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:36.322 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:36.323 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:36.324 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:36.401 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:36.506 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:36.785 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 6/6 2024-09-21T09:28:36.785 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 1/6 2024-09-21T09:28:36.785 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-21T09:28:36.785 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-21T09:28:36.785 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-21T09:28:36.785 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-21T09:28:36.885 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:37.056 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-21T09:28:37.056 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:37.056 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:37.056 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:28:37.056 INFO:teuthology.orchestra.run.smithi017.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:28:37.057 INFO:teuthology.orchestra.run.smithi017.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:28:37.057 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:28:37.057 INFO:teuthology.orchestra.run.smithi017.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:28:37.057 INFO:teuthology.orchestra.run.smithi017.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:28:37.057 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:37.057 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:37.108 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:37.108 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:37.108 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:37.108 INFO:teuthology.orchestra.run.smithi134.stdout: libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:37.108 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:37.108 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:37.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-fuse 2024-09-21T09:28:37.326 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install librados2 2024-09-21T09:28:37.906 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:30 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:37.920 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:26 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:37.995 INFO:teuthology.orchestra.run.smithi134.stdout:Package librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:38.038 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:38.039 INFO:teuthology.orchestra.run.smithi134.stdout:Nothing to do. 2024-09-21T09:28:38.040 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:38.059 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 813 k 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout:Install 2 Packages 2024-09-21T09:28:38.060 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:38.061 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 893 k 2024-09-21T09:28:38.061 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 2.6 M 2024-09-21T09:28:38.061 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:38.119 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install librbd1 2024-09-21T09:28:38.390 INFO:teuthology.orchestra.run.smithi017.stdout:(1/2): ceph-fuse-17.2.7-1649.gfbd1a6ab.el9.x86_ 2.8 MB/s | 813 kB 00:00 2024-09-21T09:28:38.699 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:31 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:38.787 INFO:teuthology.orchestra.run.smithi134.stdout:Package librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:38.790 INFO:teuthology.orchestra.run.smithi017.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 117 kB/s | 80 kB 00:00 2024-09-21T09:28:38.790 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:38.791 INFO:teuthology.orchestra.run.smithi017.stdout:Total 1.2 MB/s | 893 kB 00:00 2024-09-21T09:28:38.803 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:38.827 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:38.827 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:38.830 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:38.831 INFO:teuthology.orchestra.run.smithi134.stdout:Nothing to do. 2024-09-21T09:28:38.831 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:38.909 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install python3-rados 2024-09-21T09:28:38.913 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:38.913 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:39.103 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:39.250 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-21T09:28:39.325 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/2 2024-09-21T09:28:39.488 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:32 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:39.579 INFO:teuthology.orchestra.run.smithi134.stdout:Package python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:39.622 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:39.623 INFO:teuthology.orchestra.run.smithi134.stdout:Nothing to do. 2024-09-21T09:28:39.623 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:39.700 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install python3-rgw 2024-09-21T09:28:39.975 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/2 2024-09-21T09:28:39.976 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/2 2024-09-21T09:28:40.262 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-21T09:28:40.262 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:40.262 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:40.262 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-21T09:28:40.262 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:40.262 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:40.286 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:33 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:40.372 INFO:teuthology.orchestra.run.smithi134.stdout:Package python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:40.415 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:40.416 INFO:teuthology.orchestra.run.smithi134.stdout:Nothing to do. 2024-09-21T09:28:40.416 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:40.497 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install python3-cephfs 2024-09-21T09:28:40.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install librados-devel 2024-09-21T09:28:41.070 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:34 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:41.120 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:30 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:41.159 INFO:teuthology.orchestra.run.smithi134.stdout:Package python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:41.201 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:41.202 INFO:teuthology.orchestra.run.smithi134.stdout:Nothing to do. 2024-09-21T09:28:41.203 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:41.259 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:41.259 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:41.259 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2024-09-21T09:28:41.259 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:41.260 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:41.260 INFO:teuthology.orchestra.run.smithi017.stdout: librados-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 126 k 2024-09-21T09:28:41.260 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:41.260 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:41.260 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:41.260 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-21T09:28:41.260 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:41.260 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 126 k 2024-09-21T09:28:41.261 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 448 k 2024-09-21T09:28:41.261 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:41.288 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install python3-rbd 2024-09-21T09:28:41.480 INFO:teuthology.orchestra.run.smithi017.stdout:librados-devel-17.2.7-1649.gfbd1a6ab.el9.x86_64 575 kB/s | 126 kB 00:00 2024-09-21T09:28:41.481 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:41.481 INFO:teuthology.orchestra.run.smithi017.stdout:Total 571 kB/s | 126 kB 00:00 2024-09-21T09:28:41.481 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:41.487 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:41.487 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:41.554 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:41.554 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:41.708 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:41.821 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:41.873 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:34 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:41.962 INFO:teuthology.orchestra.run.smithi134.stdout:Package python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:42.005 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:42.006 INFO:teuthology.orchestra.run.smithi134.stdout:Nothing to do. 2024-09-21T09:28:42.006 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:42.085 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install rbd-fuse 2024-09-21T09:28:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:42.547 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:42.547 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:42.547 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:42.547 INFO:teuthology.orchestra.run.smithi017.stdout: librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:42.547 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:42.547 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:42.670 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:35 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:42.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install libcephfs2 2024-09-21T09:28:42.803 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout: Package Architecture Version Repository Size 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 82 k 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout:Install 1 Package 2024-09-21T09:28:42.804 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:42.805 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 82 k 2024-09-21T09:28:42.805 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 222 k 2024-09-21T09:28:42.805 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:42.995 INFO:teuthology.orchestra.run.smithi134.stdout:rbd-fuse-17.2.7-1649.gfbd1a6ab.el9.x86_64.rpm 430 kB/s | 82 kB 00:00 2024-09-21T09:28:42.996 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:42.996 INFO:teuthology.orchestra.run.smithi134.stdout:Total 427 kB/s | 82 kB 00:00 2024-09-21T09:28:42.996 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:43.003 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:43.003 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:43.067 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:43.067 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:43.207 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:43.301 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:43.369 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:32 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:43.463 INFO:teuthology.orchestra.run.smithi017.stdout:Package libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:43.508 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:43.509 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-21T09:28:43.509 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:43.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install libcephfs-devel 2024-09-21T09:28:43.692 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:43.942 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:43.942 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:43.942 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:43.942 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:43.942 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:43.942 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:44.173 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install rbd-mirror 2024-09-21T09:28:44.176 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:33 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:44.313 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:44.314 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:44.314 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2024-09-21T09:28:44.315 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:44.315 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:44.315 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:28:44.315 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:44.315 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:44.315 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:44.315 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-21T09:28:44.315 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:44.316 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 25 k 2024-09-21T09:28:44.316 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 110 k 2024-09-21T09:28:44.316 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:44.482 INFO:teuthology.orchestra.run.smithi017.stdout:libcephfs-devel-17.2.7-1649.gfbd1a6ab.el9.x86_6 149 kB/s | 25 kB 00:00 2024-09-21T09:28:44.483 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:44.483 INFO:teuthology.orchestra.run.smithi017.stdout:Total 148 kB/s | 25 kB 00:00 2024-09-21T09:28:44.483 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:44.487 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:44.487 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:44.488 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:44.488 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:44.584 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:44.708 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:44.765 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:37 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:44.900 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:44.901 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:44.901 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repo Size 2024-09-21T09:28:44.901 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:44.901 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:44.901 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-mirror x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:28:44.901 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:44.901 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:44.901 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:44.902 INFO:teuthology.orchestra.run.smithi134.stdout:Install 1 Package 2024-09-21T09:28:44.902 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:44.902 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 2.9 M 2024-09-21T09:28:44.902 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 12 M 2024-09-21T09:28:44.902 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:45.055 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:45.240 INFO:teuthology.orchestra.run.smithi134.stdout:rbd-mirror-17.2.7-1649.gfbd1a6ab.el9.x86_64.rpm 8.7 MB/s | 2.9 MB 00:00 2024-09-21T09:28:45.240 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:45.240 INFO:teuthology.orchestra.run.smithi134.stdout:Total 8.7 MB/s | 2.9 MB 00:00 2024-09-21T09:28:45.241 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:45.250 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:45.250 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:45.280 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:45.280 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:45.280 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:45.280 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:45.281 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:45.281 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:45.329 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:45.330 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:45.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install librados2 2024-09-21T09:28:45.743 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:45.807 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:45.832 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:45.832 INFO:teuthology.orchestra.run.smithi134.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:28:45.832 INFO:teuthology.orchestra.run.smithi134.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:28:45.832 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:28:45.832 INFO:teuthology.orchestra.run.smithi134.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:28:45.832 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:46.080 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:35 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:46.176 INFO:teuthology.orchestra.run.smithi017.stdout:Package librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:46.221 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:46.222 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-21T09:28:46.222 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:46.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install librbd1 2024-09-21T09:28:46.890 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:35 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:46.924 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:46.924 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:46.924 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:46.924 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:46.924 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:46.924 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:46.983 INFO:teuthology.orchestra.run.smithi017.stdout:Package librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:47.028 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:47.029 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-21T09:28:47.030 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:47.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-rados 2024-09-21T09:28:47.142 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install rbd-nbd 2024-09-21T09:28:47.698 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:36 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:47.737 INFO:teuthology.orchestra.run.smithi134.stdout:Last metadata expiration check: 0:01:40 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:47.790 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:47.835 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:47.837 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-21T09:28:47.837 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:47.871 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-09-21T09:28:47.871 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:47.872 INFO:teuthology.orchestra.run.smithi134.stdout: Package Architecture Version Repository Size 2024-09-21T09:28:47.872 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:47.872 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-09-21T09:28:47.872 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-nbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 165 k 2024-09-21T09:28:47.872 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:47.872 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-09-21T09:28:47.872 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-09-21T09:28:47.872 INFO:teuthology.orchestra.run.smithi134.stdout:Install 1 Package 2024-09-21T09:28:47.872 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:47.873 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 165 k 2024-09-21T09:28:47.873 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 477 k 2024-09-21T09:28:47.873 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-09-21T09:28:47.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-rgw 2024-09-21T09:28:48.092 INFO:teuthology.orchestra.run.smithi134.stdout:rbd-nbd-17.2.7-1649.gfbd1a6ab.el9.x86_64.rpm 753 kB/s | 165 kB 00:00 2024-09-21T09:28:48.093 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:48.093 INFO:teuthology.orchestra.run.smithi134.stdout:Total 748 kB/s | 165 kB 00:00 2024-09-21T09:28:48.093 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-09-21T09:28:48.101 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-09-21T09:28:48.101 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-09-21T09:28:48.164 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-09-21T09:28:48.165 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-09-21T09:28:48.316 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-09-21T09:28:48.423 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:48.507 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:37 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:48.600 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:48.646 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:48.647 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-21T09:28:48.647 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:48.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-cephfs 2024-09-21T09:28:48.868 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:49.116 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:49.116 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:49.116 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-09-21T09:28:49.116 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:49.116 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:28:49.116 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-09-21T09:28:49.313 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:38 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:49.337 DEBUG:teuthology.parallel:result is None 2024-09-21T09:28:49.404 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:49.449 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:49.450 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-21T09:28:49.450 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:49.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-rbd 2024-09-21T09:28:50.112 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:39 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:50.205 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 is already installed. 2024-09-21T09:28:50.251 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:50.252 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2024-09-21T09:28:50.252 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:50.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install rbd-fuse 2024-09-21T09:28:50.920 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:39 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:51.059 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:51.059 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 82 k 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 82 k 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 222 k 2024-09-21T09:28:51.060 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:51.252 INFO:teuthology.orchestra.run.smithi017.stdout:rbd-fuse-17.2.7-1649.gfbd1a6ab.el9.x86_64.rpm 427 kB/s | 82 kB 00:00 2024-09-21T09:28:51.253 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:51.253 INFO:teuthology.orchestra.run.smithi017.stdout:Total 423 kB/s | 82 kB 00:00 2024-09-21T09:28:51.254 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:51.261 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:51.261 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:51.329 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:51.330 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:51.477 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:51.593 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:51.976 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:52.254 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:52.254 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:52.254 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:52.254 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:52.254 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:52.254 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:52.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install rbd-mirror 2024-09-21T09:28:53.075 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:42 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:53.219 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:53.219 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:53.219 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2024-09-21T09:28:53.219 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-mirror x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 2.9 M 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 12 M 2024-09-21T09:28:53.220 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout:rbd-mirror-17.2.7-1649.gfbd1a6ab.el9.x86_64.rpm 5.9 MB/s | 2.9 MB 00:00 2024-09-21T09:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout:Total 5.9 MB/s | 2.9 MB 00:00 2024-09-21T09:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:53.725 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:53.726 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:53.811 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:53.812 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:54.333 INFO:teuthology.orchestra.run.smithi017.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:28:54.333 INFO:teuthology.orchestra.run.smithi017.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:28:54.334 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:28:54.334 INFO:teuthology.orchestra.run.smithi017.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:28:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:55.459 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:55.459 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:55.459 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:55.459 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:55.460 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:55.460 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:55.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install rbd-nbd 2024-09-21T09:28:56.312 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:45 ago on Sat 21 Sep 2024 09:27:11 AM UTC. 2024-09-21T09:28:56.450 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-nbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 165 k 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:56.451 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 165 k 2024-09-21T09:28:56.452 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 477 k 2024-09-21T09:28:56.452 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2024-09-21T09:28:56.652 INFO:teuthology.orchestra.run.smithi017.stdout:rbd-nbd-17.2.7-1649.gfbd1a6ab.el9.x86_64.rpm 826 kB/s | 165 kB 00:00 2024-09-21T09:28:56.668 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:56.668 INFO:teuthology.orchestra.run.smithi017.stdout:Total 820 kB/s | 165 kB 00:00 2024-09-21T09:28:56.668 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2024-09-21T09:28:56.668 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2024-09-21T09:28:56.668 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2024-09-21T09:28:56.729 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2024-09-21T09:28:56.729 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2024-09-21T09:28:56.894 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2024-09-21T09:28:57.002 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:57.436 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:57.691 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/1 2024-09-21T09:28:57.692 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:57.692 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2024-09-21T09:28:57.692 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:28:57.692 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:28:57.692 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2024-09-21T09:28:57.905 DEBUG:teuthology.parallel:result is None 2024-09-21T09:28:57.905 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:28:57.906 INFO:teuthology.packaging:ref: None 2024-09-21T09:28:57.906 INFO:teuthology.packaging:tag: None 2024-09-21T09:28:57.906 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:28:57.906 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:28:57.906 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:28:58.111 DEBUG:teuthology.orchestra.run.smithi017:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:28:58.134 INFO:teuthology.orchestra.run.smithi017.stdout:17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:28:58.134 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:28:58.135 INFO:teuthology.task.install:The correct ceph version 17.2.7-1649.gfbd1a6ab is installed. 2024-09-21T09:28:58.136 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:28:58.136 INFO:teuthology.packaging:ref: None 2024-09-21T09:28:58.137 INFO:teuthology.packaging:tag: None 2024-09-21T09:28:58.137 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:28:58.137 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:28:58.137 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:28:58.340 DEBUG:teuthology.orchestra.run.smithi134:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:28:58.366 INFO:teuthology.orchestra.run.smithi134.stdout:17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:28:58.366 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:28:58.366 INFO:teuthology.task.install:The correct ceph version 17.2.7-1649.gfbd1a6ab is installed. 2024-09-21T09:28:58.368 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-21T09:28:58.368 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:28:58.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:28:58.396 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:28:58.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:28:58.437 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-21T09:28:58.437 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:28:58.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:28:58.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:28:58.536 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:28:58.536 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:28:58.565 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:28:58.636 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-21T09:28:58.636 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:28:58.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:28:58.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:28:58.730 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:28:58.730 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:28:58.755 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:28:58.820 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-21T09:28:58.821 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:28:58.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:28:58.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:28:58.913 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:28:58.913 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:28:58.938 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:28:59.003 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-21T09:28:59.087 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:28:59.088 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:28:59.088 INFO:tasks.cephadm:Cluster fsid is ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:28:59.088 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-21T09:28:59.088 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.17', 'mon.c': '[v2:172.21.15.17:3301,v1:172.21.15.17:6790]', 'mon.b': '172.21.15.134'} 2024-09-21T09:28:59.088 INFO:tasks.cephadm:First mon is mon.a on smithi017 2024-09-21T09:28:59.088 INFO:tasks.cephadm:First mgr is y 2024-09-21T09:28:59.088 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-21T09:28:59.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo hostname $(hostname -s) 2024-09-21T09:28:59.115 DEBUG:teuthology.orchestra.run.smithi134:> sudo hostname $(hostname -s) 2024-09-21T09:28:59.139 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-21T09:28:59.140 DEBUG:teuthology.orchestra.run.smithi017:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:59.280 INFO:teuthology.orchestra.run.smithi017.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 21 09:28 /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:59.280 DEBUG:teuthology.orchestra.run.smithi134:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:59.406 INFO:teuthology.orchestra.run.smithi134.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 21 09:28 /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:59.406 DEBUG:teuthology.orchestra.run.smithi017:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:59.423 DEBUG:teuthology.orchestra.run.smithi134:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:59.449 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-21T09:28:59.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-21T09:28:59.465 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-21T09:29:00.954 INFO:teuthology.orchestra.run.smithi134.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:29:01.024 INFO:teuthology.orchestra.run.smithi017.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:29:38.143 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:29:38.143 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph_version": "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)", 2024-09-21T09:29:38.143 INFO:teuthology.orchestra.run.smithi017.stdout: "image_id": "fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9", 2024-09-21T09:29:38.143 INFO:teuthology.orchestra.run.smithi017.stdout: "repo_digests": [ 2024-09-21T09:29:38.143 INFO:teuthology.orchestra.run.smithi017.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2" 2024-09-21T09:29:38.143 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:29:38.143 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:29:38.434 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:29:38.434 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_version": "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)", 2024-09-21T09:29:38.434 INFO:teuthology.orchestra.run.smithi134.stdout: "image_id": "fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9", 2024-09-21T09:29:38.434 INFO:teuthology.orchestra.run.smithi134.stdout: "repo_digests": [ 2024-09-21T09:29:38.434 INFO:teuthology.orchestra.run.smithi134.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2" 2024-09-21T09:29:38.435 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:29:38.435 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:29:38.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo mkdir -p /etc/ceph 2024-09-21T09:29:38.483 DEBUG:teuthology.orchestra.run.smithi134:> sudo mkdir -p /etc/ceph 2024-09-21T09:29:38.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod 777 /etc/ceph 2024-09-21T09:29:38.549 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 777 /etc/ceph 2024-09-21T09:29:38.577 INFO:tasks.cephadm:Writing seed config... 2024-09-21T09:29:38.578 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-21T09:29:38.578 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-21T09:29:38.578 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-21T09:29:38.578 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-21T09:29:38.578 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-21T09:29:38.578 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-21T09:29:38.578 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-21T09:29:38.578 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-21T09:29:38.578 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-21T09:29:38.579 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-21T09:29:38.579 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-21T09:29:38.579 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:29:38.579 DEBUG:teuthology.orchestra.run.smithi017:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-21T09:29:38.606 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ee55f8cc-77fb-11ef-baf6-efdc52797490 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-21T09:29:38.606 DEBUG:teuthology.orchestra.run.smithi017:mon.a> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service 2024-09-21T09:29:38.648 DEBUG:teuthology.orchestra.run.smithi017:mgr.y> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y.service 2024-09-21T09:29:38.691 INFO:tasks.cephadm:Bootstrapping... 2024-09-21T09:29:38.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.17 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:29:38.897 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:29:38.897 INFO:teuthology.orchestra.run.smithi017.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'ee55f8cc-77fb-11ef-baf6-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.17', '--skip-admin-label'] 2024-09-21T09:29:38.922 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:29:38.922 INFO:teuthology.orchestra.run.smithi017.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-21T09:29:38.922 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying podman|docker is present... 2024-09-21T09:29:38.944 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:29:38.944 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying lvm2 is present... 2024-09-21T09:29:38.944 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying time synchronization is in place... 2024-09-21T09:29:38.953 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:29:38.953 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:29:38.959 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:29:38.960 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-21T09:29:38.967 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout enabled 2024-09-21T09:29:38.974 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout active 2024-09-21T09:29:38.974 INFO:teuthology.orchestra.run.smithi017.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:29:38.974 INFO:teuthology.orchestra.run.smithi017.stdout:Repeating the final host check... 2024-09-21T09:29:38.997 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:29:38.997 INFO:teuthology.orchestra.run.smithi017.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-21T09:29:38.997 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl is present 2024-09-21T09:29:38.997 INFO:teuthology.orchestra.run.smithi017.stdout:lvcreate is present 2024-09-21T09:29:39.004 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:29:39.004 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:29:39.011 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:29:39.011 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-21T09:29:39.018 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout enabled 2024-09-21T09:29:39.025 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout active 2024-09-21T09:29:39.025 INFO:teuthology.orchestra.run.smithi017.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:29:39.025 INFO:teuthology.orchestra.run.smithi017.stdout:Host looks OK 2024-09-21T09:29:39.025 INFO:teuthology.orchestra.run.smithi017.stdout:Cluster fsid: ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:39.025 INFO:teuthology.orchestra.run.smithi017.stdout:Acquiring lock 140676864623520 on /run/cephadm/ee55f8cc-77fb-11ef-baf6-efdc52797490.lock 2024-09-21T09:29:39.025 INFO:teuthology.orchestra.run.smithi017.stdout:Lock 140676864623520 acquired on /run/cephadm/ee55f8cc-77fb-11ef-baf6-efdc52797490.lock 2024-09-21T09:29:39.026 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying IP 172.21.15.17 port 3300 ... 2024-09-21T09:29:39.026 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying IP 172.21.15.17 port 6789 ... 2024-09-21T09:29:39.026 INFO:teuthology.orchestra.run.smithi017.stdout:Base mon IP(s) is [172.21.15.17:3300, 172.21.15.17:6789], mon addrv is [v2:172.21.15.17:3300,v1:172.21.15.17:6789] 2024-09-21T09:29:39.029 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.17 metric 100 2024-09-21T09:29:39.029 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.17 metric 100 2024-09-21T09:29:39.033 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-21T09:29:39.033 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-21T09:29:39.033 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1791sec hoplimit 64 pref medium 2024-09-21T09:29:39.035 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-21T09:29:39.035 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-21T09:29:39.035 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:29:39.035 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-21T09:29:39.035 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe59:d992/64 scope link 2024-09-21T09:29:39.036 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:29:39.036 INFO:teuthology.orchestra.run.smithi017.stdout:Mon IP `172.21.15.17` is in CIDR network `172.21.0.0/20` 2024-09-21T09:29:39.036 INFO:teuthology.orchestra.run.smithi017.stdout:Mon IP `172.21.15.17` is in CIDR network `172.21.0.0/20` 2024-09-21T09:29:39.036 INFO:teuthology.orchestra.run.smithi017.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-21T09:29:39.037 INFO:teuthology.orchestra.run.smithi017.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-21T09:29:39.037 INFO:teuthology.orchestra.run.smithi017.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:29:39.568 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 2024-09-21T09:29:39.568 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:29:39.568 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Getting image source signatures 2024-09-21T09:29:39.568 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Copying blob sha256:3ba00a516579693832719224d909a7cd5333511394d4f35f95e899e50f0bfaeb 2024-09-21T09:29:39.569 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-21T09:29:39.569 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Copying config sha256:fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 2024-09-21T09:29:39.569 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-21T09:29:40.455 INFO:teuthology.orchestra.run.smithi017.stdout:ceph: stdout ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable) 2024-09-21T09:29:40.455 INFO:teuthology.orchestra.run.smithi017.stdout:Ceph version: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable) 2024-09-21T09:29:40.455 INFO:teuthology.orchestra.run.smithi017.stdout:Extracting ceph user uid/gid from container image... 2024-09-21T09:29:41.502 INFO:teuthology.orchestra.run.smithi017.stdout:stat: stdout 167 167 2024-09-21T09:29:41.502 INFO:teuthology.orchestra.run.smithi017.stdout:Creating initial keys... 2024-09-21T09:29:42.518 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph-authtool: stdout AQAFku5mRzs3MhAARNgTU7sjS3HaRwvtWPf54g== 2024-09-21T09:29:43.481 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph-authtool: stdout AQAGku5m24ULNRAASO6pCIH0Gd8KHXeb/pMokw== 2024-09-21T09:29:44.477 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph-authtool: stdout AQAHku5m1dnXMhAAlsqhKCOxyi0snOSEqqpUzw== 2024-09-21T09:29:44.478 INFO:teuthology.orchestra.run.smithi017.stdout:Creating initial monmap... 2024-09-21T09:29:45.466 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:29:45.467 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-21T09:29:45.467 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:45.467 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:29:45.467 INFO:teuthology.orchestra.run.smithi017.stdout:monmaptool for a [v2:172.21.15.17:3300,v1:172.21.15.17:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:29:45.467 INFO:teuthology.orchestra.run.smithi017.stdout:setting min_mon_release = octopus 2024-09-21T09:29:45.467 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: set fsid to ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:45.467 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:29:45.467 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:29:45.467 INFO:teuthology.orchestra.run.smithi017.stdout:Creating mon... 2024-09-21T09:29:46.413 INFO:teuthology.orchestra.run.smithi017.stdout:create mon.a on 2024-09-21T09:29:46.621 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-21T09:29:46.771 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-21T09:29:46.929 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490.target. 2024-09-21T09:29:46.929 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490.target. 2024-09-21T09:29:47.105 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a 2024-09-21T09:29:47.105 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to reset failed state of unit ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service: Unit ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service not loaded. 2024-09-21T09:29:47.282 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490.target.wants/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service → /etc/systemd/system/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@.service. 2024-09-21T09:29:47.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:47 smithi017 podman[28872]: 2024-09-21 09:29:47.429324892 +0000 UTC m=+0.016176420 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:29:47.844 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:29:47.844 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2024-09-21T09:29:47.852 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:29:47.852 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-21T09:29:47.852 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2024-09-21T09:29:47.852 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:29:47.853 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mon to start... 2024-09-21T09:29:47.853 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mon... 2024-09-21T09:29:47.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:47 smithi017 podman[28872]: 2024-09-21 09:29:47.721250264 +0000 UTC m=+0.308101784 container init bc74ca5078a17e35750d3851cea353258d72cf70d2e080721a9f1888f855c04c (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T09:29:47.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:47 smithi017 podman[28872]: 2024-09-21 09:29:47.726470962 +0000 UTC m=+0.313322476 container start bc74ca5078a17e35750d3851cea353258d72cf70d2e080721a9f1888f855c04c (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD) 2024-09-21T09:29:47.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:47 smithi017 ceph-mon[28910]: mkfs ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:47.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:47 smithi017 ceph-mon[28910]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:29:47.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:47 smithi017 bash[28872]: bc74ca5078a17e35750d3851cea353258d72cf70d2e080721a9f1888f855c04c 2024-09-21T09:29:47.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:47 smithi017 systemd[1]: Started Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:29:48.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:48 smithi017 ceph-mon[28910]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:29:48.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:48 smithi017 ceph-mon[28910]: monmap e1: 1 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:29:48.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:48 smithi017 ceph-mon[28910]: fsmap 2024-09-21T09:29:48.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:48 smithi017 ceph-mon[28910]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:29:48.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:48 smithi017 ceph-mon[28910]: mgrmap e1: no daemons active 2024-09-21T09:29:48.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:48 smithi017 ceph-mon[28910]: from='client.? 172.21.15.17:0/3485356639' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T09:29:48.889 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout cluster: 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout id: ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout services: 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.603597s) 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout data: 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-21T09:29:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout pgs: 2024-09-21T09:29:48.892 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:48.892 INFO:teuthology.orchestra.run.smithi017.stdout:mon is available 2024-09-21T09:29:48.892 INFO:teuthology.orchestra.run.smithi017.stdout:Assimilating anything we can from ceph.conf... 2024-09-21T09:29:49.871 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:49.871 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout fsid = ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.17:3300,v1:172.21.15.17:6789] 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:29:49.872 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:29:49.873 INFO:teuthology.orchestra.run.smithi017.stdout:Generating new minimal ceph.conf... 2024-09-21T09:29:50.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:49 smithi017 ceph-mon[28910]: from='client.? 172.21.15.17:0/2637859629' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:29:50.072 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:49 smithi017 ceph-mon[28910]: from='client.? 172.21.15.17:0/2637859629' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-21T09:29:50.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:50 smithi017 ceph-mon[28910]: from='client.? 172.21.15.17:0/775986903' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:50.867 INFO:teuthology.orchestra.run.smithi017.stdout:Restarting the monitor... 2024-09-21T09:29:51.145 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:50 smithi017 systemd[1]: Stopping Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:29:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[28887]: 2024-09-21T09:29:51.143+0000 7fbce3f29640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:29:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[28887]: 2024-09-21T09:29:51.143+0000 7fbce3f29640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-21T09:29:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:51 smithi017 podman[29228]: 2024-09-21 09:29:51.287240327 +0000 UTC m=+0.243527979 container died bc74ca5078a17e35750d3851cea353258d72cf70d2e080721a9f1888f855c04c (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T09:29:51.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:51 smithi017 podman[29228]: 2024-09-21 09:29:51.64104989 +0000 UTC m=+0.597337542 container cleanup bc74ca5078a17e35750d3851cea353258d72cf70d2e080721a9f1888f855c04c (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T09:29:51.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:51 smithi017 bash[29228]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:29:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:51 smithi017 podman[29245]: 2024-09-21 09:29:51.908833204 +0000 UTC m=+0.616068864 container remove bc74ca5078a17e35750d3851cea353258d72cf70d2e080721a9f1888f855c04c (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:29:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:29:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 systemd[1]: Stopped Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:29:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 systemd[1]: Starting Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:29:52.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 podman[29343]: 2024-09-21 09:29:52.172714474 +0000 UTC m=+0.016011130 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:29:52.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 podman[29343]: 2024-09-21 09:29:52.284697092 +0000 UTC m=+0.127993738 container create 52c5c6bf49345abeb6b6dbf2e5b057bc5a7aa49d47bc9f160c510c122d9f15e9 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T09:29:52.524 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 podman[29343]: 2024-09-21 09:29:52.495140523 +0000 UTC m=+0.338437173 container init 52c5c6bf49345abeb6b6dbf2e5b057bc5a7aa49d47bc9f160c510c122d9f15e9 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a) 2024-09-21T09:29:52.524 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 podman[29343]: 2024-09-21 09:29:52.500069744 +0000 UTC m=+0.343366393 container start 52c5c6bf49345abeb6b6dbf2e5b057bc5a7aa49d47bc9f160c510c122d9f15e9 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T09:29:52.524 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:29:52.524 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable), process ceph-mon, pid 7 2024-09-21T09:29:52.524 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: pidfile_write: ignore empty --pid-file 2024-09-21T09:29:52.524 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: load: jerasure load: lrc 2024-09-21T09:29:52.583 INFO:teuthology.orchestra.run.smithi017.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-21T09:29:52.824 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: RocksDB version: 6.15.5 2024-09-21T09:29:52.824 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T09:29:52.824 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Compile date Sep 20 2024 2024-09-21T09:29:52.824 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: DB SUMMARY 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: DB Session ID: PGKVSL83EDD3IAUE8GEV 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83686 ; 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.env: 0x5587bf107440 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.fs: Posix File System 2024-09-21T09:29:52.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.info_log: 0x5587c1014ec0 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.statistics: (nil) 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.use_fsync: 0 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:29:52.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:29:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:29:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.db_log_dir: 2024-09-21T09:29:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-21T09:29:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:29:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:29:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:29:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:29:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:29:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:29:52.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:29:52.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:29:52.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.write_buffer_manager: 0x5587c10d27e0 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.unordered_write: 0 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:29:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.row_cache: None 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.wal_filter: None 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.preserve_deletes: 0 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:29:52.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:29:52.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_open_files: -1 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Compression algorithms supported: 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: kZSTD supported: 0 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:29:52.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.merge_operator: 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_filter: None 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:29:52.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5587c0fbba38) 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_type: 0 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_index_type: 0 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_shortening: 1 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: hash_index_allow_collision: 1 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: checksum: 1 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: no_block_cache: 0 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache: 0x5587c0ff1350 2024-09-21T09:29:52.834 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_options: 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: capacity : 536870912 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: num_shard_bits : 4 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: strict_capacity_limit : 0 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_compressed: (nil) 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: persistent_cache: (nil) 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size: 4096 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size_deviation: 10 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_restart_interval: 16 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_block_restart_interval: 1 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: metadata_block_size: 4096 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: partition_filters: 0 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: use_delta_encoding: 1 2024-09-21T09:29:52.835 INFO:journalctl@ceph.mon.a.smithi017.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout: whole_key_filtering: 1 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout: verify_compression: 0 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout: format_version: 4 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_index_compression: 1 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_align: 0 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compression: NoCompression 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.num_levels: 7 2024-09-21T09:29:52.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:29:52.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:29:52.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:29:52.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:29:52.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.ttl: 2592000 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:29:52.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910992524697, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910992525473, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80259, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78501, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72714, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726910992, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "1f7def57-f1d6-4669-a7c2-40690f0a21ac", "db_session_id": "PGKVSL83EDD3IAUE8GEV"}} 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910992525855, "job": 1, "event": "recovery_finished"} 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5587c1024700 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: DB pointer 0x5587c0ff4000 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T09:29:52.842 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** DB Stats ** 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.3 0.00 0.00 1 0.001 0 0 2024-09-21T09:29:52.843 INFO:journalctl@ceph.mon.a.smithi017.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.3 0.00 0.00 1 0.001 0 0 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.3 0.00 0.00 1 0.001 0 0 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 151.3 0.00 0.00 1 0.001 0 0 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative compaction: 0.00 GB write, 23.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:29:52.844 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval compaction: 0.00 GB write, 23.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.3 0.00 0.00 1 0.001 0 0 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.3 0.00 0.00 1 0.001 0 0 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:29:52.845 INFO:journalctl@ceph.mon.a.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 151.3 0.00 0.00 1 0.001 0 0 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative compaction: 0.00 GB write, 23.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: starting mon.a rank 0 at public addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] at bind addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:52.846 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mon.a@-1(???) e1 preinit fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mon.a@-1(???).mds e1 new map 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mon.a@-1(???).mds e1 print_map 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout: e1 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout: legacy client fscid: -1 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout: No filesystems configured 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:29:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:29:52.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:29:52.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-21T09:29:52.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 bash[29343]: 52c5c6bf49345abeb6b6dbf2e5b057bc5a7aa49d47bc9f160c510c122d9f15e9 2024-09-21T09:29:52.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 systemd[1]: Started Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:29:52.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:29:52.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: monmap e1: 1 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:29:52.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: fsmap 2024-09-21T09:29:52.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:29:52.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:52 smithi017 ceph-mon[29380]: mgrmap e1: no daemons active 2024-09-21T09:29:53.607 INFO:teuthology.orchestra.run.smithi017.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-21T09:29:53.607 INFO:teuthology.orchestra.run.smithi017.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:29:53.607 INFO:teuthology.orchestra.run.smithi017.stdout:Creating mgr... 2024-09-21T09:29:53.607 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-21T09:29:53.607 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-21T09:29:53.788 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y 2024-09-21T09:29:53.789 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to reset failed state of unit ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y.service: Unit ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y.service not loaded. 2024-09-21T09:29:53.940 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490.target.wants/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y.service → /etc/systemd/system/ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@.service. 2024-09-21T09:29:54.294 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:54 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2418909176' entity='client.admin' 2024-09-21T09:29:54.295 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 podman[29618]: 2024-09-21 09:29:54.193707258 +0000 UTC m=+0.138445684 container create cd2cad7817e108cbab465fb29c70427eb1d5bafcc35760c39bf13079b8e1673d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:29:54.519 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:29:54.519 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2024-09-21T09:29:54.527 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:29:54.527 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-21T09:29:54.527 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2024-09-21T09:29:54.527 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:29:54.536 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:29:54.536 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2024-09-21T09:29:54.544 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:29:54.544 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-21T09:29:54.545 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2024-09-21T09:29:54.545 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-21T09:29:54.545 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr to start... 2024-09-21T09:29:54.545 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr... 2024-09-21T09:29:54.647 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 podman[29618]: 2024-09-21 09:29:54.395917705 +0000 UTC m=+0.340656150 container init cd2cad7817e108cbab465fb29c70427eb1d5bafcc35760c39bf13079b8e1673d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T09:29:54.647 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 podman[29618]: 2024-09-21 09:29:54.401083013 +0000 UTC m=+0.345821442 container start cd2cad7817e108cbab465fb29c70427eb1d5bafcc35760c39bf13079b8e1673d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True) 2024-09-21T09:29:54.647 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 bash[29618]: cd2cad7817e108cbab465fb29c70427eb1d5bafcc35760c39bf13079b8e1673d 2024-09-21T09:29:54.647 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 systemd[1]: Started Ceph mgr.y for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:29:54.648 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:54.518+0000 7f2aae800ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:29:54.648 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:54.578+0000 7f2aae800ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:29:54.648 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:54.645+0000 7f2aae800ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:29:54.923 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:54.776+0000 7f2aae800ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:29:54.924 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:54.921+0000 7f2aae800ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:29:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:55 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3213654347' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:29:55.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:55 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:55.217+0000 7f2aae800ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:29:55.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:55 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:55.280+0000 7f2aae800ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:29:55.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:55 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:55.449+0000 7f2aae800ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:29:55.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:55.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:29:55.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsid": "ee55f8cc-77fb-11ef-baf6-efdc52797490", 2024-09-21T09:29:55.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:29:55.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:29:55.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:29:55.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:55.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:29:55.718 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:29:55.718 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:29:55.718 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:29:55.718 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:29:55.718 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:29:55.718 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:29:55.718 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:29:55.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:29:55.722 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:29:55.723 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:55.723 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:29:55.723 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:29:55.723 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:29:47.761019+0000", 2024-09-21T09:29:55.723 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:29:55.723 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:55.723 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:29:55.723 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:29:55.723 INFO:teuthology.orchestra.run.smithi017.stdout:mgr not available, waiting (1/15)... 2024-09-21T09:29:56.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:56.331+0000 7f2aae800ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:29:56.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:56.397+0000 7f2aae800ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:29:57.441 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.138+0000 7f2aae800ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:29:57.441 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.253+0000 7f2aae800ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:29:57.441 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.378+0000 7f2aae800ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:29:57.441 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.439+0000 7f2aae800ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:29:57.699 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.569+0000 7f2aae800ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:29:57.699 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.638+0000 7f2aae800ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:29:57.699 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.697+0000 7f2aae800ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:29:57.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.841+0000 7f2aae800ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:29:57.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.907+0000 7f2aae800ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:29:58.280 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:57.978+0000 7f2aae800ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:29:58.280 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:58 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:58.062+0000 7f2aae800ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:29:58.280 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:58 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:58.122+0000 7f2aae800ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:29:58.280 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:58 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:58.181+0000 7f2aae800ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:29:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:58 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/934965401' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:29:58.992 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:58.992 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsid": "ee55f8cc-77fb-11ef-baf6-efdc52797490", 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:29:58.993 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:29:58.994 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:29:58.995 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:29:58.996 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:29:47.761019+0000", 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:29:58.997 INFO:teuthology.orchestra.run.smithi017.stdout:mgr not available, waiting (2/15)... 2024-09-21T09:29:59.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:29:58 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:29:58.967+0000 7f2aae800ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:29:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: Activating manager daemon y 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: mgrmap e2: y(active, starting, since 0.00286763s) 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: Manager daemon y is now available 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' 2024-09-21T09:29:59.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:29:59 smithi017 ceph-mon[29380]: from='mgr.14100 172.21.15.17:0/3410412927' entity='mgr.y' 2024-09-21T09:30:01.230 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:00 smithi017 ceph-mon[29380]: mgrmap e3: y(active, since 1.00699s) 2024-09-21T09:30:02.292 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:01 smithi017 ceph-mon[29380]: mgrmap e4: y(active, since 2s) 2024-09-21T09:30:03.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:02 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/4146924067' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:30:08.957 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:08.962 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:08.962 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsid": "ee55f8cc-77fb-11ef-baf6-efdc52797490", 2024-09-21T09:30:08.962 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:30:08.962 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:08.963 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:30:08.964 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:30:08.965 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:29:47.761019+0000", 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:30:08.966 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.967 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:30:08.967 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:08.967 INFO:teuthology.orchestra.run.smithi017.stdout:mgr is available 2024-09-21T09:30:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:12 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2027476690' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout fsid = ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:12.716 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:30:12.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:30:12.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:12.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:30:12.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:30:12.717 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:30:12.717 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling cephadm module... 2024-09-21T09:30:13.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:13 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/915052283' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-21T09:30:13.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:13 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ignoring --setuser ceph since I am not root 2024-09-21T09:30:13.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:13 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ignoring --setgroup ceph since I am not root 2024-09-21T09:30:13.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:13 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:13.548+0000 7fe6bfea9ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:30:13.676 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:13 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:13.605+0000 7fe6bfea9ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:13.676 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:13 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:13.672+0000 7fe6bfea9ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:30:13.947 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:13 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:13.802+0000 7fe6bfea9ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:13.947 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:13 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:13.944+0000 7fe6bfea9ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:30:14.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:14 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/915052283' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-21T09:30:14.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:14 smithi017 ceph-mon[29380]: mgrmap e5: y(active, since 14s) 2024-09-21T09:30:14.454 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:14.221+0000 7fe6bfea9ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:30:14.454 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:14.283+0000 7fe6bfea9ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:30:14.749 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:14.458+0000 7fe6bfea9ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:30:15.415 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:15 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:15.357+0000 7fe6bfea9ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:30:15.418 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:15.418 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-21T09:30:15.419 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:30:15.419 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:30:15.419 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:30:15.419 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:15.419 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for the mgr to restart... 2024-09-21T09:30:15.419 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr epoch 5... 2024-09-21T09:30:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:15 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/829567183' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:30:15.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:15 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:15.426+0000 7fe6bfea9ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:30:16.291 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:16.172+0000 7fe6bfea9ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:30:16.291 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:16.288+0000 7fe6bfea9ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:30:16.613 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:16.417+0000 7fe6bfea9ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:16.613 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:16.479+0000 7fe6bfea9ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:30:16.613 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:16.610+0000 7fe6bfea9ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:30:16.883 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:16.680+0000 7fe6bfea9ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:30:16.883 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:16.740+0000 7fe6bfea9ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:30:16.883 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:16.880+0000 7fe6bfea9ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:30:17.161 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:16.947+0000 7fe6bfea9ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:30:17.162 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:17 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:17.014+0000 7fe6bfea9ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:30:17.162 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:17 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:17.099+0000 7fe6bfea9ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:17.162 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:17 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:17.158+0000 7fe6bfea9ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:30:17.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:17 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:17.218+0000 7fe6bfea9ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:17.999+0000 7fe6bfea9ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: Active manager daemon y restarted 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: Activating manager daemon y 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: osdmap e2: 0 total, 0 up, 0 in 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: mgrmap e6: y(active, starting, since 0.00317574s) 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:30:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:30:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: Manager daemon y is now available 2024-09-21T09:30:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:30:18.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:18 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:30:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:19 smithi017 ceph-mon[29380]: Found migration_current of "None". Setting to last migration. 2024-09-21T09:30:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:19 smithi017 ceph-mon[29380]: mgrmap e7: y(active, since 1.00513s) 2024-09-21T09:30:19.707 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:19.707 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-21T09:30:19.707 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:30:19.707 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:19.708 INFO:teuthology.orchestra.run.smithi017.stdout:mgr epoch 5 is available 2024-09-21T09:30:19.708 INFO:teuthology.orchestra.run.smithi017.stdout:Setting orchestrator backend to cephadm... 2024-09-21T09:30:20.090 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-mon[29380]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:30:20.091 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-mon[29380]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:30:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:20.773+0000 7fe672258640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:30:20] ENGINE Error in HTTPServer.serve 2024-09-21T09:30:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: Traceback (most recent call last): 2024-09-21T09:30:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:30:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self._connections.run(self.expiration_interval) 2024-09-21T09:30:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:30:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self._run(expiration_interval) 2024-09-21T09:30:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:30:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:30:20.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: s = self.context.wrap_socket( 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: return self.sslsocket_class._create( 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self.do_handshake() 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self._sslobj.do_handshake() 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:30:20.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:21.438 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:21 smithi017 ceph-mon[29380]: mgrmap e8: y(active, since 2s) 2024-09-21T09:30:21.438 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:21 smithi017 ceph-mon[29380]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:21.439 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:21 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:21.439 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:21 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:21.439 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:21 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:21.439 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:21 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:22 smithi017 ceph-mon[29380]: [21/Sep/2024:09:30:20] ENGINE Bus STARTING 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:22 smithi017 ceph-mon[29380]: [21/Sep/2024:09:30:20] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:22 smithi017 ceph-mon[29380]: [21/Sep/2024:09:30:20] ENGINE Bus STARTED 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:22 smithi017 ceph-mon[29380]: [21/Sep/2024:09:30:20] ENGINE Error in HTTPServer.serve 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:30:22.242 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:30:22.243 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:30:22.244 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-21T09:30:22.244 INFO:teuthology.orchestra.run.smithi017.stdout:Generating ssh key... 2024-09-21T09:30:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-mon[29380]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:23.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: Generating public/private rsa key pair. 2024-09-21T09:30:23.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: Your identification has been saved in /tmp/tmptz3uypwd/key 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: Your public key has been saved in /tmp/tmptz3uypwd/key.pub 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: The key fingerprint is: 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: SHA256:efn7JW84Nx3RKS99r3SeVCusxYRVVb3PYsMdatgeGHA ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: The key's randomart image is: 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: +---[RSA 3072]----+ 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: | .*| 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: | . E . .| 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: | o . +| 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: | . oo. =.| 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: | S o.=o=o=| 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: | . ++*=+B| 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: | +*=*B| 2024-09-21T09:30:23.451 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: | o+=B*| 2024-09-21T09:30:23.452 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: | ...o*+| 2024-09-21T09:30:23.452 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: +----[SHA256]-----+ 2024-09-21T09:30:24.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:24 smithi017 ceph-mon[29380]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:24.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:24 smithi017 ceph-mon[29380]: Generating ssh key... 2024-09-21T09:30:24.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:24 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:24.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:24 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:24.889 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3ZFf+f3LzmJoeDtXN6WejFhXrCQjDCwq/ESth+mX4a3f2Xd4grWHoDv13KN2BMhUDnckfX68QmtQYwGs9upSJeeliNxjliQx2HGV7i/o37tefWOBPg1KhNrHUFfEqb93QJL49qAPsBRZVBXyKoziruOKah2+6Je+ZtNwP6AtDPBPE5FguMZTWdw5vtQKfDKodyxl2UoKxm4LhiS5giXVOKa+hDF3Wv8UGs2r25tEumdkwaze8ngaXgq9591n65jOSJ+qyza2h5osqgzFF1JGilw0SCoBx2GUXj/MMzL3BWkokplNPdAWKKsL0K2R7D0Dd8TgsSY6rgVLBWYxSLHtMna588cFyb5CAkx5UbkySxnF/4PAAfohB4kiv8uNDU7jb6iQrXCEzcUUoN+iob5L1D6lMXPVhosDacAx+Fo+7f4ajnmME1fqzJUuvYsxjMhoz0IUtbF212PwdODnl39zKoR2JVvJFKOm61Ucd1z81857NV0GiDrYIZPJwFc9KY1s= ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:24.889 INFO:teuthology.orchestra.run.smithi017.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-21T09:30:24.890 INFO:teuthology.orchestra.run.smithi017.stdout:Adding key to root@localhost authorized_keys... 2024-09-21T09:30:24.895 INFO:teuthology.orchestra.run.smithi017.stdout:Adding host smithi017... 2024-09-21T09:30:25.140 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:25 smithi017 ceph-mon[29380]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:26.880 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:26 smithi017 ceph-mon[29380]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi017", "addr": "172.21.15.17", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:27.797 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:27 smithi017 ceph-mon[29380]: Deploying cephadm binary to smithi017 2024-09-21T09:30:29.463 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Added host 'smithi017' with addr '172.21.15.17' 2024-09-21T09:30:29.463 INFO:teuthology.orchestra.run.smithi017.stdout:Deploying unmanaged mon service... 2024-09-21T09:30:30.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:29 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:30.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:29 smithi017 ceph-mon[29380]: Added host smithi017 2024-09-21T09:30:30.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:29 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:31.006 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-21T09:30:31.006 INFO:teuthology.orchestra.run.smithi017.stdout:Deploying unmanaged mgr service... 2024-09-21T09:30:31.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:31 smithi017 ceph-mon[29380]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:31.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:31 smithi017 ceph-mon[29380]: Saving service mon spec with placement count:5 2024-09-21T09:30:31.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:31 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:33.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:32 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:33.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:32 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:33.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:32 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:33.015 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-21T09:30:33.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:33 smithi017 ceph-mon[29380]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:33.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:33 smithi017 ceph-mon[29380]: Saving service mgr spec with placement count:2 2024-09-21T09:30:35.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:34 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2228655500' entity='client.admin' 2024-09-21T09:30:35.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:34 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:35.823 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling the dashboard module... 2024-09-21T09:30:36.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:36 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/192799778' entity='client.admin' 2024-09-21T09:30:36.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:36 smithi017 ceph-mon[29380]: from='mgr.14118 172.21.15.17:0/504007007' entity='mgr.y' 2024-09-21T09:30:37.533 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:37 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2562673388' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-21T09:30:37.533 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ignoring --setuser ceph since I am not root 2024-09-21T09:30:37.533 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ignoring --setgroup ceph since I am not root 2024-09-21T09:30:37.533 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:37.400+0000 7fb54e494ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:30:37.533 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:37.457+0000 7fb54e494ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:37.533 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:37.530+0000 7fb54e494ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:30:37.806 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:37.660+0000 7fb54e494ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:37.807 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:37.804+0000 7fb54e494ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:30:38.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:38 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2562673388' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-21T09:30:38.326 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:38 smithi017 ceph-mon[29380]: mgrmap e9: y(active, since 19s) 2024-09-21T09:30:38.326 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:38.083+0000 7fb54e494ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:30:38.326 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:38.146+0000 7fb54e494ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:30:38.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:38.322+0000 7fb54e494ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:30:39.049 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:39.050 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-21T09:30:39.050 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:30:39.050 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:30:39.050 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:30:39.051 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:39.051 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for the mgr to restart... 2024-09-21T09:30:39.051 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr epoch 9... 2024-09-21T09:30:39.298 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:39 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/494254816' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:30:39.299 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:39 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:39.218+0000 7fb54e494ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:30:39.299 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:39 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:39.290+0000 7fb54e494ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:30:40.309 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.029+0000 7fb54e494ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:30:40.310 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.145+0000 7fb54e494ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:30:40.310 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.270+0000 7fb54e494ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:40.564 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.332+0000 7fb54e494ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:30:40.564 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.465+0000 7fb54e494ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:30:40.564 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.537+0000 7fb54e494ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:30:40.879 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.599+0000 7fb54e494ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:30:40.880 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.741+0000 7fb54e494ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:30:40.880 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.809+0000 7fb54e494ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:30:40.880 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.878+0000 7fb54e494ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:30:41.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:40.962+0000 7fb54e494ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:41.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:41.023+0000 7fb54e494ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:30:41.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:41.083+0000 7fb54e494ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:30:42.115 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:41.862+0000 7fb54e494ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:30:42.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: Active manager daemon y restarted 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: Activating manager daemon y 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: osdmap e3: 0 total, 0 up, 0 in 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: mgrmap e10: y(active, starting, since 0.003096s) 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: Manager daemon y is now available 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:30:42.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:30:43.759 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:43.759 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-21T09:30:43.759 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:30:43.759 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:43.759 INFO:teuthology.orchestra.run.smithi017.stdout:mgr epoch 9 is available 2024-09-21T09:30:43.759 INFO:teuthology.orchestra.run.smithi017.stdout:Generating a dashboard self-signed certificate... 2024-09-21T09:30:44.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-mon[29380]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:30:44.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-mon[29380]: mgrmap e11: y(active, since 1.00561s) 2024-09-21T09:30:44.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-mon[29380]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:30:44.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:44.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:44.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:44.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:43.850+0000 7fb4feff2640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:30:43] ENGINE Error in HTTPServer.serve 2024-09-21T09:30:44.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: Traceback (most recent call last): 2024-09-21T09:30:44.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self._connections.run(self.expiration_interval) 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self._run(expiration_interval) 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: s = self.context.wrap_socket( 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: return self.sslsocket_class._create( 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self.do_handshake() 2024-09-21T09:30:44.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:30:44.026 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self._sslobj.do_handshake() 2024-09-21T09:30:44.026 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:30:44.026 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:30:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:30:45.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:44 smithi017 ceph-mon[29380]: [21/Sep/2024:09:30:43] ENGINE Bus STARTING 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:44 smithi017 ceph-mon[29380]: [21/Sep/2024:09:30:43] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:44 smithi017 ceph-mon[29380]: [21/Sep/2024:09:30:43] ENGINE Bus STARTED 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:44 smithi017 ceph-mon[29380]: [21/Sep/2024:09:30:43] ENGINE Error in HTTPServer.serve 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:30:45.116 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:44 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:44 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:45.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:44 smithi017 ceph-mon[29380]: mgrmap e12: y(active, since 2s) 2024-09-21T09:30:45.836 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-21T09:30:45.837 INFO:teuthology.orchestra.run.smithi017.stdout:Creating initial admin user... 2024-09-21T09:30:46.090 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:45 smithi017 ceph-mon[29380]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:47.368 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$V3/8rbcMuh447zN/VexEDuHBr08FEgxfE9ppj6NiwPzhyxwVgiFz2", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726911046, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-21T09:30:47.369 INFO:teuthology.orchestra.run.smithi017.stdout:Fetching dashboard port number... 2024-09-21T09:30:47.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:47 smithi017 ceph-mon[29380]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:47.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:47 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:48.482 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 8443 2024-09-21T09:30:48.494 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:30:48.494 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2024-09-21T09:30:48.502 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:30:48.502 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2024-09-21T09:30:48.503 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2024-09-21T09:30:48.503 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-21T09:30:48.505 INFO:teuthology.orchestra.run.smithi017.stdout:Ceph Dashboard is now available at: 2024-09-21T09:30:48.505 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:48.505 INFO:teuthology.orchestra.run.smithi017.stdout: URL: https://smithi017.front.sepia.ceph.com:8443/ 2024-09-21T09:30:48.505 INFO:teuthology.orchestra.run.smithi017.stdout: User: admin 2024-09-21T09:30:48.505 INFO:teuthology.orchestra.run.smithi017.stdout: Password: lwpznf3r0d 2024-09-21T09:30:48.505 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:48.506 INFO:teuthology.orchestra.run.smithi017.stdout:Saving cluster configuration to /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config directory 2024-09-21T09:30:48.506 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling autotune for osd_memory_target 2024-09-21T09:30:49.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:48 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1010408092' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-21T09:30:50.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:49 smithi017 ceph-mon[29380]: mgrmap e13: y(active, since 6s) 2024-09-21T09:30:50.968 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-21T09:30:50.968 INFO:teuthology.orchestra.run.smithi017.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-21T09:30:50.968 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:50.968 INFO:teuthology.orchestra.run.smithi017.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:30:50.968 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:50.968 INFO:teuthology.orchestra.run.smithi017.stdout:Or, if you are only running a single cluster on this host: 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout: ceph telemetry on 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout:For more information see: 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:30:50.969 INFO:teuthology.orchestra.run.smithi017.stdout:Bootstrap complete. 2024-09-21T09:30:51.004 INFO:tasks.cephadm:Fetching config... 2024-09-21T09:30:51.004 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:30:51.004 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-21T09:30:51.020 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-21T09:30:51.021 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:30:51.021 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-21T09:30:51.077 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-21T09:30:51.077 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:30:51.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/keyring of=/dev/stdout 2024-09-21T09:30:51.143 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-21T09:30:51.143 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:30:51.143 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-21T09:30:51.199 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-21T09:30:51.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3ZFf+f3LzmJoeDtXN6WejFhXrCQjDCwq/ESth+mX4a3f2Xd4grWHoDv13KN2BMhUDnckfX68QmtQYwGs9upSJeeliNxjliQx2HGV7i/o37tefWOBPg1KhNrHUFfEqb93QJL49qAPsBRZVBXyKoziruOKah2+6Je+ZtNwP6AtDPBPE5FguMZTWdw5vtQKfDKodyxl2UoKxm4LhiS5giXVOKa+hDF3Wv8UGs2r25tEumdkwaze8ngaXgq9591n65jOSJ+qyza2h5osqgzFF1JGilw0SCoBx2GUXj/MMzL3BWkokplNPdAWKKsL0K2R7D0Dd8TgsSY6rgVLBWYxSLHtMna588cFyb5CAkx5UbkySxnF/4PAAfohB4kiv8uNDU7jb6iQrXCEzcUUoN+iob5L1D6lMXPVhosDacAx+Fo+7f4ajnmME1fqzJUuvYsxjMhoz0IUtbF212PwdODnl39zKoR2JVvJFKOm61Ucd1z81857NV0GiDrYIZPJwFc9KY1s= ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:30:51.275 INFO:teuthology.orchestra.run.smithi017.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3ZFf+f3LzmJoeDtXN6WejFhXrCQjDCwq/ESth+mX4a3f2Xd4grWHoDv13KN2BMhUDnckfX68QmtQYwGs9upSJeeliNxjliQx2HGV7i/o37tefWOBPg1KhNrHUFfEqb93QJL49qAPsBRZVBXyKoziruOKah2+6Je+ZtNwP6AtDPBPE5FguMZTWdw5vtQKfDKodyxl2UoKxm4LhiS5giXVOKa+hDF3Wv8UGs2r25tEumdkwaze8ngaXgq9591n65jOSJ+qyza2h5osqgzFF1JGilw0SCoBx2GUXj/MMzL3BWkokplNPdAWKKsL0K2R7D0Dd8TgsSY6rgVLBWYxSLHtMna588cFyb5CAkx5UbkySxnF/4PAAfohB4kiv8uNDU7jb6iQrXCEzcUUoN+iob5L1D6lMXPVhosDacAx+Fo+7f4ajnmME1fqzJUuvYsxjMhoz0IUtbF212PwdODnl39zKoR2JVvJFKOm61Ucd1z81857NV0GiDrYIZPJwFc9KY1s= ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:51.306 DEBUG:teuthology.orchestra.run.smithi134:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3ZFf+f3LzmJoeDtXN6WejFhXrCQjDCwq/ESth+mX4a3f2Xd4grWHoDv13KN2BMhUDnckfX68QmtQYwGs9upSJeeliNxjliQx2HGV7i/o37tefWOBPg1KhNrHUFfEqb93QJL49qAPsBRZVBXyKoziruOKah2+6Je+ZtNwP6AtDPBPE5FguMZTWdw5vtQKfDKodyxl2UoKxm4LhiS5giXVOKa+hDF3Wv8UGs2r25tEumdkwaze8ngaXgq9591n65jOSJ+qyza2h5osqgzFF1JGilw0SCoBx2GUXj/MMzL3BWkokplNPdAWKKsL0K2R7D0Dd8TgsSY6rgVLBWYxSLHtMna588cFyb5CAkx5UbkySxnF/4PAAfohB4kiv8uNDU7jb6iQrXCEzcUUoN+iob5L1D6lMXPVhosDacAx+Fo+7f4ajnmME1fqzJUuvYsxjMhoz0IUtbF212PwdODnl39zKoR2JVvJFKOm61Ucd1z81857NV0GiDrYIZPJwFc9KY1s= ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:30:51.358 INFO:teuthology.orchestra.run.smithi134.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3ZFf+f3LzmJoeDtXN6WejFhXrCQjDCwq/ESth+mX4a3f2Xd4grWHoDv13KN2BMhUDnckfX68QmtQYwGs9upSJeeliNxjliQx2HGV7i/o37tefWOBPg1KhNrHUFfEqb93QJL49qAPsBRZVBXyKoziruOKah2+6Je+ZtNwP6AtDPBPE5FguMZTWdw5vtQKfDKodyxl2UoKxm4LhiS5giXVOKa+hDF3Wv8UGs2r25tEumdkwaze8ngaXgq9591n65jOSJ+qyza2h5osqgzFF1JGilw0SCoBx2GUXj/MMzL3BWkokplNPdAWKKsL0K2R7D0Dd8TgsSY6rgVLBWYxSLHtMna588cFyb5CAkx5UbkySxnF/4PAAfohB4kiv8uNDU7jb6iQrXCEzcUUoN+iob5L1D6lMXPVhosDacAx+Fo+7f4ajnmME1fqzJUuvYsxjMhoz0IUtbF212PwdODnl39zKoR2JVvJFKOm61Ucd1z81857NV0GiDrYIZPJwFc9KY1s= ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:51.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-21T09:30:51.536 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:51 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3402030151' entity='client.admin' 2024-09-21T09:30:51.624 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:30:54.410 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-21T09:30:54.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-21T09:30:54.668 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:54 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/501102864' entity='client.admin' 2024-09-21T09:30:54.889 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:30:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi017", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:30:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:57.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:30:57.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:30:57.900 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi134 2024-09-21T09:30:57.900 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:30:57.900 DEBUG:teuthology.orchestra.run.smithi134:> dd of=/etc/ceph/ceph.conf 2024-09-21T09:30:57.919 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:30:57.919 DEBUG:teuthology.orchestra.run.smithi134:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:30:57.977 INFO:tasks.cephadm:Adding host smithi134 to orchestrator... 2024-09-21T09:30:57.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch host add smithi134 2024-09-21T09:30:58.229 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:30:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:58 smithi017 ceph-mon[29380]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:58 smithi017 ceph-mon[29380]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:30:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:58 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:58 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:58 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:30:59.311 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:59 smithi017 ceph-mon[29380]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:30:59.311 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:59 smithi017 ceph-mon[29380]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:30:59.312 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:30:59 smithi017 ceph-mon[29380]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:31:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:00 smithi017 ceph-mon[29380]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi134", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:01 smithi017 ceph-mon[29380]: Deploying cephadm binary to smithi134 2024-09-21T09:31:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:02 smithi017 ceph-mon[29380]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:03.348 INFO:teuthology.orchestra.run.smithi017.stdout:Added host 'smithi134' with addr '172.21.15.134' 2024-09-21T09:31:03.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch host ls --format=json 2024-09-21T09:31:04.182 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:04.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:04 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:04.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:04 smithi017 ceph-mon[29380]: Added host smithi134 2024-09-21T09:31:04.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:04 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:04.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:04 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:04.604 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:04 smithi017 ceph-mon[29380]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:04.604 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:04 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:05.936 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:31:05.936 INFO:teuthology.orchestra.run.smithi017.stdout:[{"addr": "172.21.15.17", "hostname": "smithi017", "labels": [], "status": ""}, {"addr": "172.21.15.134", "hostname": "smithi134", "labels": [], "status": ""}] 2024-09-21T09:31:05.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:06.477 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-21T09:31:06.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd crush tunables default 2024-09-21T09:31:06.722 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:06.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:06 smithi017 ceph-mon[29380]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:06.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:06 smithi017 ceph-mon[29380]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:31:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi134", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:31:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:08 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1296752129' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T09:31:09.372 INFO:teuthology.orchestra.run.smithi017.stderr:adjusted tunables profile to default 2024-09-21T09:31:10.014 INFO:tasks.cephadm:Adding mon.a on smithi017 2024-09-21T09:31:10.014 INFO:tasks.cephadm:Adding mon.c on smithi017 2024-09-21T09:31:10.014 INFO:tasks.cephadm:Adding mon.b on smithi134 2024-09-21T09:31:10.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch apply mon '3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi134:172.21.15.134=b' 2024-09-21T09:31:10.258 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:10 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1296752129' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:31:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:10 smithi017 ceph-mon[29380]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:10 smithi017 ceph-mon[29380]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:11.894 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled mon update... 2024-09-21T09:31:12.595 DEBUG:teuthology.orchestra.run.smithi017:mon.c> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.c.service 2024-09-21T09:31:12.598 DEBUG:teuthology.orchestra.run.smithi134:mon.b> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.b.service 2024-09-21T09:31:12.601 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:31:12.601 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:31:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi134:172.21.15.134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: Saving service mon spec with placement smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi134:172.21.15.134=b;count:3 2024-09-21T09:31:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:13.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:13.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:13.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:13.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:13.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:13.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:13.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:12 smithi017 ceph-mon[29380]: Deploying daemon mon.b on smithi134 2024-09-21T09:31:13.431 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:14 smithi017 ceph-mon[29380]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:16.804 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:31:16.804 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","modified":"2024-09-21T09:29:44.848123Z","created":"2024-09-21T09:29:44.848123Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3300","nonce":0},{"type":"v1","addr":"172.21.15.17:6789","nonce":0}]},"addr":"172.21.15.17:6789/0","public_addr":"172.21.15.17:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T09:31:16.804 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-09-21T09:31:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:16 smithi017 ceph-mon[29380]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:16 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/1758743214' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:17.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:17 smithi134 systemd[1]: Started Ceph mon.b for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:31:18.864 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:31:18.865 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:31:19.117 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:31:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: mon.a calling monitor election 2024-09-21T09:31:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: mon.b calling monitor election 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: monmap e2: 2 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: fsmap 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: mgrmap e13: y(active, since 40s) 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: overall HEALTH_OK 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:22.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:22.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:22.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:22 smithi017 ceph-mon[29380]: Deploying daemon mon.c on smithi017 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: mon.a calling monitor election 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: mon.b calling monitor election 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: monmap e2: 2 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: fsmap 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: mgrmap e13: y(active, since 40s) 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: overall HEALTH_OK 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:22.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:22 smithi134 ceph-mon[33541]: Deploying daemon mon.c on smithi017 2024-09-21T09:31:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:23 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:23.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:23 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:23.846 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:31:23.846 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":2,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","modified":"2024-09-21T09:31:17.255095Z","created":"2024-09-21T09:29:44.848123Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3300","nonce":0},{"type":"v1","addr":"172.21.15.17:6789","nonce":0}]},"addr":"172.21.15.17:6789/0","public_addr":"172.21.15.17:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:3300","nonce":0},{"type":"v1","addr":"172.21.15.134:6789","nonce":0}]},"addr":"172.21.15.134:6789/0","public_addr":"172.21.15.134:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-21T09:31:23.847 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 2 2024-09-21T09:31:24.417 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:24 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/3888474790' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:24.417 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:24 smithi134 ceph-mon[33541]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:24 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/3888474790' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:24 smithi017 ceph-mon[29380]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.441 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:31:25.442 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:31:25.556 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 podman[40699]: 2024-09-21 09:31:25.342275896 +0000 UTC m=+0.016253041 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T09:31:25.692 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:31:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 podman[40699]: 2024-09-21 09:31:25.647433456 +0000 UTC m=+0.321410601 container init 316af690367dd0f1d316a8d69361cc25a7800a5c81dbbce1a7b0d7692e902795 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True) 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 podman[40699]: 2024-09-21 09:31:25.652145334 +0000 UTC m=+0.326122479 container start 316af690367dd0f1d316a8d69361cc25a7800a5c81dbbce1a7b0d7692e902795 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2) 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable), process ceph-mon, pid 7 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pidfile_write: ignore empty --pid-file 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: load: jerasure load: lrc 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: RocksDB version: 6.15.5 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Compile date Sep 20 2024 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: DB SUMMARY 2024-09-21T09:31:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: DB Session ID: EE392OYTYO81MY1EV8P6 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.env: 0x557d36998440 2024-09-21T09:31:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.fs: Posix File System 2024-09-21T09:31:25.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.info_log: 0x557d37b26ec0 2024-09-21T09:31:25.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:31:25.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.statistics: (nil) 2024-09-21T09:31:25.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.use_fsync: 0 2024-09-21T09:31:25.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:31:25.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:31:25.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:31:25.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:31:25.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:31:25.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:31:25.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:31:25.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.db_log_dir: 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:31:25.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.write_buffer_manager: 0x557d37be47b0 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.unordered_write: 0 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:31:25.956 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.row_cache: None 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.wal_filter: None 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.preserve_deletes: 0 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:31:25.957 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:31:25.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_open_files: -1 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Compression algorithms supported: 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: kZSTD supported: 0 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:31:25.959 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.merge_operator: 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_filter: None 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557d37acda38) 2024-09-21T09:31:25.960 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_type: 0 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_index_type: 0 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_shortening: 1 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: hash_index_allow_collision: 1 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: checksum: 1 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: no_block_cache: 0 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache: 0x557d37b031f0 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_options: 2024-09-21T09:31:25.961 INFO:journalctl@ceph.mon.c.smithi017.stdout: capacity : 536870912 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: num_shard_bits : 4 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: strict_capacity_limit : 0 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_compressed: (nil) 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: persistent_cache: (nil) 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size: 4096 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size_deviation: 10 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_restart_interval: 16 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_block_restart_interval: 1 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: metadata_block_size: 4096 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: partition_filters: 0 2024-09-21T09:31:25.962 INFO:journalctl@ceph.mon.c.smithi017.stdout: use_delta_encoding: 1 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: whole_key_filtering: 1 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: verify_compression: 0 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: format_version: 4 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_index_compression: 1 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_align: 0 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compression: NoCompression 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:31:25.963 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.num_levels: 7 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:31:25.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:31:25.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:31:25.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:31:25.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:31:25.968 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.ttl: 2592000 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911085676897, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-21T09:31:25.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911085677208, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726911085, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "7a97661d-3375-4595-82c7-c73b69f29e2c", "db_session_id": "EE392OYTYO81MY1EV8P6"}} 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911085677549, "job": 1, "event": "recovery_finished"} 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x557d37b36700 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: DB pointer 0x557d37b06000 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** DB Stats ** 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:31:25.970 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:31:25.971 INFO:journalctl@ceph.mon.c.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:31:25.972 INFO:journalctl@ceph.mon.c.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:31:25.973 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: using public_addrv [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: starting mon.c rank -1 at public addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] at bind addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(???) e0 preinit fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).mds e1 new map 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: e1 2024-09-21T09:31:25.974 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout: legacy client fscid: -1 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout: No filesystems configured 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-21T09:31:25.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:31:25.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:31:25.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:31:25.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:31:25.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: Added host smithi134 2024-09-21T09:31:25.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:25.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi134", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1296752129' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1296752129' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi134:172.21.15.134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: Saving service mon spec with placement smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi134:172.21.15.134=b;count:3 2024-09-21T09:31:25.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: Deploying daemon mon.b on smithi134 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/1758743214' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.a calling monitor election 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.b calling monitor election 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: monmap e2: 2 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: fsmap 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mgrmap e13: y(active, since 40s) 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: overall HEALTH_OK 2024-09-21T09:31:25.980 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: Deploying daemon mon.c on smithi017 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/3888474790' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 ceph-mon[40737]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 bash[40699]: 316af690367dd0f1d316a8d69361cc25a7800a5c81dbbce1a7b0d7692e902795 2024-09-21T09:31:25.981 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:25 smithi017 systemd[1]: Started Ceph mon.c for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:31:31.069 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:31:31.069 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":3,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","modified":"2024-09-21T09:31:25.701063Z","created":"2024-09-21T09:29:44.848123Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3300","nonce":0},{"type":"v1","addr":"172.21.15.17:6789","nonce":0}]},"addr":"172.21.15.17:6789/0","public_addr":"172.21.15.17:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:3300","nonce":0},{"type":"v1","addr":"172.21.15.134:6789","nonce":0}]},"addr":"172.21.15.134:6789/0","public_addr":"172.21.15.134:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3301","nonce":0},{"type":"v1","addr":"172.21.15.17:6790","nonce":0}]},"addr":"172.21.15.17:6790/0","public_addr":"172.21.15.17:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-21T09:31:31.069 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 3 2024-09-21T09:31:31.069 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:31.069 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: mon.a calling monitor election 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: mon.b calling monitor election 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: mon.c calling monitor election 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:31:31.071 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:31.071 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: fsmap 2024-09-21T09:31:31.071 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:31.071 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: mgrmap e13: y(active, since 48s) 2024-09-21T09:31:31.071 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: overall HEALTH_OK 2024-09-21T09:31:31.071 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.071 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.071 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.071 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:30 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: mon.a calling monitor election 2024-09-21T09:31:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: mon.b calling monitor election 2024-09-21T09:31:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: mon.c calling monitor election 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: fsmap 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: mgrmap e13: y(active, since 48s) 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: overall HEALTH_OK 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: mon.a calling monitor election 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: mon.b calling monitor election 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: mon.c calling monitor election 2024-09-21T09:31:31.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: fsmap 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: mgrmap e13: y(active, since 48s) 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: overall HEALTH_OK 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:30 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:31.638 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-21T09:31:31.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph config generate-minimal-conf 2024-09-21T09:31:31.896 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:31.989 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:31 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:31.989 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:31 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/1182133851' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:31.990 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:31 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:32.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:31 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:32.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:31 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/1182133851' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:32.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:31 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:32.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:31 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:32.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:31 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/1182133851' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:32.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:31 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:33.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:33.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:33.771 INFO:teuthology.orchestra.run.smithi017.stdout:# minimal ceph.conf for ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:33.771 INFO:teuthology.orchestra.run.smithi017.stdout:[global] 2024-09-21T09:31:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: fsid = ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: mon_host = [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: Reconfiguring daemon mon.a on smithi017 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3212485805' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:31:34.077 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:31:34.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:34.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:34.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:31:34.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:34.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:34.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: Reconfiguring daemon mon.a on smithi017 2024-09-21T09:31:34.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:33 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3212485805' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:31:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:31:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:31:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: Reconfiguring daemon mon.a on smithi017 2024-09-21T09:31:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:33 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3212485805' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.827 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-21T09:31:34.827 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:31:34.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:31:34.859 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:31:34.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:31:34.928 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:31:34.928 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:31:34.960 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:31:34.961 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:31:35.029 INFO:tasks.cephadm:Adding mgr.y on smithi017 2024-09-21T09:31:35.029 INFO:tasks.cephadm:Adding mgr.x on smithi134 2024-09-21T09:31:35.029 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch apply mgr '2;smithi017=y;smithi134=x' 2024-09-21T09:31:35.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:34 smithi017 ceph-mon[29380]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:34 smithi017 ceph-mon[40737]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:35.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:34 smithi134 ceph-mon[33541]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:35.311 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:31:36.468 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:36.468 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:36.468 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:36 smithi134 ceph-mon[33541]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:31:36.468 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:36.468 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:36.468 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:36.468 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:36 smithi134 ceph-mon[33541]: Reconfiguring daemon mon.c on smithi017 2024-09-21T09:31:36.572 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:36.572 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:36.572 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[29380]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:31:36.572 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:36.572 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:36.572 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:36.572 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[29380]: Reconfiguring daemon mon.c on smithi017 2024-09-21T09:31:36.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:36.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:36.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[40737]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:31:36.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:36.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:36.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:36.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:36 smithi017 ceph-mon[40737]: Reconfiguring daemon mon.c on smithi017 2024-09-21T09:31:36.868 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled mgr update... 2024-09-21T09:31:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:37 smithi134 ceph-mon[33541]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:37 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:37 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:37 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:37 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:37 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:37 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[29380]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[40737]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:37 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:37.721 DEBUG:teuthology.orchestra.run.smithi134:mgr.x> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.x.service 2024-09-21T09:31:37.724 DEBUG:tasks.cephadm:set 0 configs 2024-09-21T09:31:37.724 INFO:tasks.cephadm:Deploying OSDs... 2024-09-21T09:31:37.724 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:31:37.724 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:31:37.745 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:31:37.746 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_1 2024-09-21T09:31:37.804 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:31:37.805 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:37.805 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 5h/5d Inode: 973 Links: 1 2024-09-21T09:31:37.805 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:37.805 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:37.805 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2024-09-21 09:31:25.165141510 +0000 2024-09-21T09:31:37.805 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:37.805 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:37.805 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:37.805 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:31:37.873 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2024-09-21T09:31:37.874 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2024-09-21T09:31:37.874 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.000127806 s, 4.0 MB/s 2024-09-21T09:31:37.875 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:31:37.935 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_2 2024-09-21T09:31:37.993 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:31:37.993 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:37.993 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 5h/5d Inode: 971 Links: 1 2024-09-21T09:31:37.993 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:37.994 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:37.994 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2024-09-21 09:31:25.166141499 +0000 2024-09-21T09:31:37.994 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:37.994 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:37.994 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:37.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:31:38.060 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2024-09-21T09:31:38.062 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2024-09-21T09:31:38.062 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.000117048 s, 4.4 MB/s 2024-09-21T09:31:38.063 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:31:38.121 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_3 2024-09-21T09:31:38.178 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:31:38.178 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:38.178 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 5h/5d Inode: 976 Links: 1 2024-09-21T09:31:38.178 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:38.179 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:38.179 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2024-09-21 09:31:25.166141499 +0000 2024-09-21T09:31:38.179 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:38.179 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:38.179 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:38.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:31:38.244 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2024-09-21T09:31:38.244 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2024-09-21T09:31:38.244 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.000116637 s, 4.4 MB/s 2024-09-21T09:31:38.245 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:31:38.303 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_4 2024-09-21T09:31:38.360 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:31:38.360 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:38.360 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 5h/5d Inode: 965 Links: 1 2024-09-21T09:31:38.360 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:38.360 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:38.360 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2024-09-21 09:31:25.166141499 +0000 2024-09-21T09:31:38.360 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:38.360 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:38.361 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: 2024-09-21 09:26:49.395051127 +0000 2024-09-21T09:31:38.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:31:38.425 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2024-09-21T09:31:38.425 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2024-09-21T09:31:38.425 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.00012232 s, 4.2 MB/s 2024-09-21T09:31:38.427 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:31:38.485 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:31:38.485 DEBUG:teuthology.orchestra.run.smithi134:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:31:38.503 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:31:38.503 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_1 2024-09-21T09:31:38.558 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:31:38.559 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:38.559 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-09-21T09:31:38.559 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:38.559 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:38.559 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-09-21 09:31:15.670704744 +0000 2024-09-21T09:31:38.559 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-09-21 09:26:49.269038650 +0000 2024-09-21T09:31:38.559 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-09-21 09:26:49.269038650 +0000 2024-09-21T09:31:38.559 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: 2024-09-21 09:26:49.269038650 +0000 2024-09-21T09:31:38.559 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:31:38.593 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:38 smithi134 ceph-mon[33541]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:31:38.594 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:38 smithi134 ceph-mon[33541]: Reconfiguring daemon mon.b on smithi134 2024-09-21T09:31:38.594 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:38 smithi134 ceph-mon[33541]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi017=y;smithi134=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:38.594 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:38 smithi134 ceph-mon[33541]: Saving service mgr spec with placement smithi017=y;smithi134=x;count:2 2024-09-21T09:31:38.620 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-09-21T09:31:38.620 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-09-21T09:31:38.620 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.00019099 s, 2.7 MB/s 2024-09-21T09:31:38.621 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:31:38.677 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_2 2024-09-21T09:31:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:38 smithi017 ceph-mon[29380]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:31:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:38 smithi017 ceph-mon[29380]: Reconfiguring daemon mon.b on smithi134 2024-09-21T09:31:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:38 smithi017 ceph-mon[29380]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi017=y;smithi134=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:38 smithi017 ceph-mon[29380]: Saving service mgr spec with placement smithi017=y;smithi134=x;count:2 2024-09-21T09:31:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:38 smithi017 ceph-mon[40737]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:31:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:38 smithi017 ceph-mon[40737]: Reconfiguring daemon mon.b on smithi134 2024-09-21T09:31:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:38 smithi017 ceph-mon[40737]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi017=y;smithi134=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:38 smithi017 ceph-mon[40737]: Saving service mgr spec with placement smithi017=y;smithi134=x;count:2 2024-09-21T09:31:38.733 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:31:38.733 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:38.733 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 966 Links: 1 2024-09-21T09:31:38.733 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:38.733 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:38.733 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-09-21 09:31:15.671704720 +0000 2024-09-21T09:31:38.734 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-09-21 09:26:49.268038674 +0000 2024-09-21T09:31:38.734 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-09-21 09:26:49.268038674 +0000 2024-09-21T09:31:38.734 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: 2024-09-21 09:26:49.268038674 +0000 2024-09-21T09:31:38.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:31:38.798 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-09-21T09:31:38.798 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-09-21T09:31:38.798 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000240707 s, 2.1 MB/s 2024-09-21T09:31:38.799 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:31:38.855 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_3 2024-09-21T09:31:38.911 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:31:38.911 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:38.911 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 978 Links: 1 2024-09-21T09:31:38.911 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:38.911 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:38.912 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-09-21 09:31:15.672704696 +0000 2024-09-21T09:31:38.912 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-09-21 09:26:49.269038650 +0000 2024-09-21T09:31:38.912 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-09-21 09:26:49.269038650 +0000 2024-09-21T09:31:38.912 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: 2024-09-21 09:26:49.269038650 +0000 2024-09-21T09:31:38.912 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:31:38.977 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-09-21T09:31:38.977 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-09-21T09:31:38.977 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.00019876 s, 2.6 MB/s 2024-09-21T09:31:38.978 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:31:39.034 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_4 2024-09-21T09:31:39.091 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:31:39.091 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:39.091 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 971 Links: 1 2024-09-21T09:31:39.091 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:39.091 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:39.091 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-09-21 09:31:15.672704696 +0000 2024-09-21T09:31:39.091 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-09-21 09:26:49.268038674 +0000 2024-09-21T09:31:39.091 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-09-21 09:26:49.268038674 +0000 2024-09-21T09:31:39.091 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: 2024-09-21 09:26:49.268038674 +0000 2024-09-21T09:31:39.091 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:31:39.153 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-09-21T09:31:39.154 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-09-21T09:31:39.154 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000262779 s, 1.9 MB/s 2024-09-21T09:31:39.155 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:31:39.211 INFO:tasks.cephadm:Deploying osd.0 on smithi017 with /dev/vg_nvme/lv_4... 2024-09-21T09:31:39.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:31:39.470 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:39.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:39.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:39.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:39.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:39.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:39.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:39.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:39.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:39.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:39.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:39.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:31:39.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:39.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:40 smithi134 ceph-mon[33541]: Deploying daemon mgr.x on smithi134 2024-09-21T09:31:40.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:40 smithi134 systemd[1]: Starting Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:31:40.586 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:40 smithi017 ceph-mon[29380]: Deploying daemon mgr.x on smithi134 2024-09-21T09:31:40.586 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:40 smithi017 ceph-mon[40737]: Deploying daemon mgr.x on smithi134 2024-09-21T09:31:40.929 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:40 smithi134 podman[35589]: 2024-09-21 09:31:40.510696558 +0000 UTC m=+0.015915689 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T09:31:40.930 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:40 smithi134 podman[35589]: 2024-09-21 09:31:40.617501593 +0000 UTC m=+0.122720765 container create c61de7d532224a9a89517a34a9e8405ebd8e1e677e83690ec54ba5de1a154449 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T09:31:40.930 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:40 smithi134 podman[35589]: 2024-09-21 09:31:40.828245776 +0000 UTC m=+0.333464910 container init c61de7d532224a9a89517a34a9e8405ebd8e1e677e83690ec54ba5de1a154449 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T09:31:40.930 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:40 smithi134 podman[35589]: 2024-09-21 09:31:40.833325268 +0000 UTC m=+0.338544396 container start c61de7d532224a9a89517a34a9e8405ebd8e1e677e83690ec54ba5de1a154449 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T09:31:40.930 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:40 smithi134 ceph-mgr[35626]: -- 172.21.15.134:0/691744371 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55835e39a340 con 0x55835e3a7000 2024-09-21T09:31:40.930 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:40 smithi134 bash[35589]: c61de7d532224a9a89517a34a9e8405ebd8e1e677e83690ec54ba5de1a154449 2024-09-21T09:31:41.208 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:40 smithi134 systemd[1]: Started Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:31:41.208 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:40.948+0000 7f2909c81ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:31:41.208 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:41.010+0000 7f2909c81ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:31:41.208 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:41.077+0000 7f2909c81ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:31:41.208 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:41.206+0000 7f2909c81ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:31:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[29380]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[40737]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:41 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:41.493 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:41.352+0000 7f2909c81ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:31:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-mon[33541]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:41.883 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:41.628+0000 7f2909c81ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:31:41.883 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:41.691+0000 7f2909c81ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:31:41.883 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:41.880+0000 7f2909c81ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:31:42.082 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:31:42.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi017:vg_nvme/lv_4 2024-09-21T09:31:42.363 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:43.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:43.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:43.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:43.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[29380]: Reconfiguring daemon mgr.y on smithi017 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:43.125 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:43.126 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.126 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:31:43.126 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:43.126 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:43.126 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:43.126 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:42 smithi017 ceph-mon[40737]: Reconfiguring daemon mgr.y on smithi017 2024-09-21T09:31:43.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:42.748+0000 7f2909c81ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:42.812+0000 7f2909c81ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:31:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:42 smithi134 ceph-mon[33541]: Reconfiguring daemon mgr.y on smithi017 2024-09-21T09:31:43.807 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:43 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:43.520+0000 7f2909c81ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:31:43.807 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:43 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:43.629+0000 7f2909c81ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:31:43.807 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:43 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:43.747+0000 7f2909c81ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:31:44.186 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:43 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:43.804+0000 7f2909c81ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:31:44.186 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:43 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:43.929+0000 7f2909c81ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:31:44.186 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:43 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:43.995+0000 7f2909c81ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:31:44.186 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:44.051+0000 7f2909c81ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:31:44.186 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:44.184+0000 7f2909c81ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:31:44.447 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:44.246+0000 7f2909c81ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:31:44.447 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:44.311+0000 7f2909c81ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:31:44.447 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:44.389+0000 7f2909c81ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:31:44.447 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:44.445+0000 7f2909c81ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:31:44.743 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:44.501+0000 7f2909c81ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:31:45.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:44 smithi017 ceph-mon[29380]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:45.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:44 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:31:45.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:44 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:31:45.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:44 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:45.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:44 smithi017 ceph-mon[40737]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:45.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:44 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:31:45.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:44 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:31:45.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:44 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:45.234 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-mon[33541]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:45.234 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:31:45.234 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:31:45.234 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:44 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:45.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:31:45.232+0000 7f2909c81ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:31:46.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:46.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: Standby manager daemon x started 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:46.107 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: Standby manager daemon x started 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:46.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:45 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: Standby manager daemon x started 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/3228068242' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:45 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:46 smithi017 ceph-mon[29380]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:46 smithi017 ceph-mon[29380]: mgrmap e14: y(active, since 64s), standbys: x 2024-09-21T09:31:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:46 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:31:47.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:46 smithi017 ceph-mon[40737]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:47.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:46 smithi017 ceph-mon[40737]: mgrmap e14: y(active, since 64s), standbys: x 2024-09-21T09:31:47.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:46 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:31:47.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:46 smithi134 ceph-mon[33541]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:46 smithi134 ceph-mon[33541]: mgrmap e14: y(active, since 64s), standbys: x 2024-09-21T09:31:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:46 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:31:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2606273353' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb258889-3128-4c2e-b5da-343281340eb4"}]: dispatch 2024-09-21T09:31:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb258889-3128-4c2e-b5da-343281340eb4"}]: dispatch 2024-09-21T09:31:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb258889-3128-4c2e-b5da-343281340eb4"}]': finished 2024-09-21T09:31:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[29380]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:31:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:31:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/857836401' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:31:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2606273353' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb258889-3128-4c2e-b5da-343281340eb4"}]: dispatch 2024-09-21T09:31:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb258889-3128-4c2e-b5da-343281340eb4"}]: dispatch 2024-09-21T09:31:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb258889-3128-4c2e-b5da-343281340eb4"}]': finished 2024-09-21T09:31:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[40737]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:31:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:31:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:47 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/857836401' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:31:48.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:47 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2606273353' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb258889-3128-4c2e-b5da-343281340eb4"}]: dispatch 2024-09-21T09:31:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:47 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb258889-3128-4c2e-b5da-343281340eb4"}]: dispatch 2024-09-21T09:31:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:47 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb258889-3128-4c2e-b5da-343281340eb4"}]': finished 2024-09-21T09:31:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:47 smithi134 ceph-mon[33541]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:31:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:47 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:31:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:47 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/857836401' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:31:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:49 smithi017 ceph-mon[29380]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:49 smithi017 ceph-mon[40737]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:49.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:49 smithi134 ceph-mon[33541]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:51.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:51 smithi017 ceph-mon[29380]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:51.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:51 smithi017 ceph-mon[40737]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:51.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:51 smithi134 ceph-mon[33541]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:53.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:53 smithi017 ceph-mon[29380]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:53.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:53 smithi017 ceph-mon[40737]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:53.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:53 smithi134 ceph-mon[33541]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:31:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:54 smithi017 ceph-mon[29380]: Deploying daemon osd.0 on smithi017 2024-09-21T09:31:54.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:31:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:54 smithi017 ceph-mon[40737]: Deploying daemon osd.0 on smithi017 2024-09-21T09:31:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:31:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:54 smithi134 ceph-mon[33541]: Deploying daemon osd.0 on smithi017 2024-09-21T09:31:55.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:55 smithi017 ceph-mon[29380]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:55.087 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:55 smithi017 ceph-mon[40737]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:55 smithi134 ceph-mon[33541]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:57.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:57 smithi017 ceph-mon[29380]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:57.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:57 smithi017 ceph-mon[40737]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:57.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:57 smithi134 ceph-mon[33541]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:58.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:58 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:58.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:58 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:58.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:58 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:58.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:58 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:58.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:58 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:58.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:58 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:58.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:58 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:58 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:58 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:31:59.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:31:59 smithi017 ceph-mon[29380]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:59.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:31:59 smithi017 ceph-mon[40737]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:59.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:31:59 smithi134 ceph-mon[33541]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:01.094 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 0 on host 'smithi017' 2024-09-21T09:32:01.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:01 smithi017 ceph-mon[29380]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:01.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:01 smithi017 ceph-mon[29380]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:32:01.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:01 smithi017 ceph-mon[40737]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:01.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:01 smithi017 ceph-mon[40737]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:32:01.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:01 smithi134 ceph-mon[33541]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:01 smithi134 ceph-mon[33541]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:32:01.928 DEBUG:teuthology.orchestra.run.smithi017:osd.0> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T09:32:01.930 INFO:tasks.cephadm:Deploying osd.1 on smithi017 with /dev/vg_nvme/lv_3... 2024-09-21T09:32:01.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:32:02.200 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[45076]: 2024-09-21T09:32:02.069+0000 7f4547388640 -1 osd.0 0 waiting for initial osdmap 2024-09-21T09:32:02.200 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[45076]: 2024-09-21T09:32:02.073+0000 7f45429c0640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:32:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:32:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:32:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:32:02.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:32:02.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:02 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: from='osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:32:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:02 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.689 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:03.329 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:03 smithi017 ceph-mon[29380]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:03.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:03 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:03.330 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:03 smithi017 ceph-mon[40737]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:03.330 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:03 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:03.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:03 smithi134 ceph-mon[33541]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:03.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:03 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:04.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[40737]: purged_snaps scrub starts 2024-09-21T09:32:04.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[40737]: purged_snaps scrub ok 2024-09-21T09:32:04.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[40737]: osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645] boot 2024-09-21T09:32:04.158 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[40737]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:32:04.158 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:04.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[29380]: purged_snaps scrub starts 2024-09-21T09:32:04.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[29380]: purged_snaps scrub ok 2024-09-21T09:32:04.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[29380]: osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645] boot 2024-09-21T09:32:04.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[29380]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:32:04.158 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:04 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:04.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:04 smithi134 ceph-mon[33541]: purged_snaps scrub starts 2024-09-21T09:32:04.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:04 smithi134 ceph-mon[33541]: purged_snaps scrub ok 2024-09-21T09:32:04.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:04 smithi134 ceph-mon[33541]: osd.0 [v2:172.21.15.17:6802/314719645,v1:172.21.15.17:6803/314719645] boot 2024-09-21T09:32:04.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:04 smithi134 ceph-mon[33541]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:32:04.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:04 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:05.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:05.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:32:05.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:05.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:05.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[29380]: Detected new or changed devices on smithi017 2024-09-21T09:32:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi017 to 16068M 2024-09-21T09:32:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:06.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[40737]: Detected new or changed devices on smithi017 2024-09-21T09:32:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi017 to 16068M 2024-09-21T09:32:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:06 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:06.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:06 smithi134 ceph-mon[33541]: Detected new or changed devices on smithi017 2024-09-21T09:32:06.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:06 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi017 to 16068M 2024-09-21T09:32:06.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:06 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:06.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:06 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:06.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:06 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:06.559 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:32:06.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi017:vg_nvme/lv_3 2024-09-21T09:32:06.828 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:07.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:07 smithi134 ceph-mon[33541]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:07 smithi017 ceph-mon[29380]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:07 smithi017 ceph-mon[40737]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:09.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:09 smithi017 ceph-mon[40737]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:09.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:09 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:09.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:09 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:09.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:09 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:09.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:09 smithi017 ceph-mon[29380]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:09.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:09 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:09.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:09 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:09.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:09 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:09.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:09 smithi134 ceph-mon[33541]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:09 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:09 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:09 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:10.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:10 smithi134 ceph-mon[33541]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:10.529 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:10 smithi017 ceph-mon[29380]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:10.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:10 smithi017 ceph-mon[40737]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[29380]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1366778424' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe0af53-c763-4d08-8051-7f781782ab0b"}]: dispatch 2024-09-21T09:32:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe0af53-c763-4d08-8051-7f781782ab0b"}]: dispatch 2024-09-21T09:32:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efe0af53-c763-4d08-8051-7f781782ab0b"}]': finished 2024-09-21T09:32:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[29380]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:32:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[40737]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1366778424' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe0af53-c763-4d08-8051-7f781782ab0b"}]: dispatch 2024-09-21T09:32:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe0af53-c763-4d08-8051-7f781782ab0b"}]: dispatch 2024-09-21T09:32:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efe0af53-c763-4d08-8051-7f781782ab0b"}]': finished 2024-09-21T09:32:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[40737]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:32:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:11 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:11.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:11 smithi134 ceph-mon[33541]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:11 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1366778424' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe0af53-c763-4d08-8051-7f781782ab0b"}]: dispatch 2024-09-21T09:32:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:11 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe0af53-c763-4d08-8051-7f781782ab0b"}]: dispatch 2024-09-21T09:32:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:11 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efe0af53-c763-4d08-8051-7f781782ab0b"}]': finished 2024-09-21T09:32:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:11 smithi134 ceph-mon[33541]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:32:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:11 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:12.658 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:12 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/457653130' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:12.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:12 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/457653130' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:12.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:12 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/457653130' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:13.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:13 smithi134 ceph-mon[33541]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:13.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:13 smithi017 ceph-mon[29380]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:13.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:13 smithi017 ceph-mon[40737]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:15 smithi017 ceph-mon[29380]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:15 smithi017 ceph-mon[40737]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:15.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:15 smithi134 ceph-mon[33541]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:17.254 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:17 smithi017 ceph-mon[29380]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:17.255 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:17 smithi017 ceph-mon[40737]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:17.255 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:17 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:32:17.255 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:17 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:17.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:17 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:32:17.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:17 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:17.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:17 smithi134 ceph-mon[33541]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:17 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:32:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:17 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:18 smithi017 ceph-mon[29380]: Deploying daemon osd.1 on smithi017 2024-09-21T09:32:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:18 smithi017 ceph-mon[40737]: Deploying daemon osd.1 on smithi017 2024-09-21T09:32:18.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:18 smithi134 ceph-mon[33541]: Deploying daemon osd.1 on smithi017 2024-09-21T09:32:19.516 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:19 smithi017 ceph-mon[29380]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:19.516 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:19 smithi017 ceph-mon[40737]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:19.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:19 smithi134 ceph-mon[33541]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:21.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:21 smithi017 ceph-mon[29380]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:21.355 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:21 smithi017 ceph-mon[40737]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:21.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:21 smithi134 ceph-mon[33541]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:22.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:22.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:22.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:22 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:22.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:22 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:22.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:22 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:22.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:22 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:22.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:22 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:23.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:23 smithi017 ceph-mon[29380]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:23.364 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:23 smithi017 ceph-mon[40737]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:23.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:23 smithi134 ceph-mon[33541]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:25.073 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 1 on host 'smithi017' 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[29380]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[29380]: from='osd.1 [v2:172.21.15.17:6810/1139898755,v1:172.21.15.17:6811/1139898755]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[29380]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[40737]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[40737]: from='osd.1 [v2:172.21.15.17:6810/1139898755,v1:172.21.15.17:6811/1139898755]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[40737]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:25.369 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:25 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:25.700 DEBUG:teuthology.orchestra.run.smithi017:osd.1> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.1.service 2024-09-21T09:32:25.702 INFO:tasks.cephadm:Deploying osd.2 on smithi017 with /dev/vg_nvme/lv_2... 2024-09-21T09:32:25.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:32:25.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:25 smithi134 ceph-mon[33541]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:25.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:25 smithi134 ceph-mon[33541]: from='osd.1 [v2:172.21.15.17:6810/1139898755,v1:172.21.15.17:6811/1139898755]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:32:25.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:25 smithi134 ceph-mon[33541]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:32:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:25 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:25 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:26.003 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:26.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[29380]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:32:26.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[29380]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:32:26.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:26.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[29380]: from='osd.1 [v2:172.21.15.17:6810/1139898755,v1:172.21.15.17:6811/1139898755]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:26.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[29380]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:26.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:26.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:26.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[40737]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:32:26.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[40737]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:32:26.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:26.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[40737]: from='osd.1 [v2:172.21.15.17:6810/1139898755,v1:172.21.15.17:6811/1139898755]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:26.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[40737]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:26.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:26.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:26.432 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[51310]: 2024-09-21T09:32:26.285+0000 7fd9b790c640 -1 osd.1 0 waiting for initial osdmap 2024-09-21T09:32:26.432 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:26 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[51310]: 2024-09-21T09:32:26.288+0000 7fd9b1f42640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:32:26.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:26 smithi134 ceph-mon[33541]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:32:26.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:26 smithi134 ceph-mon[33541]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:32:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:26 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:26 smithi134 ceph-mon[33541]: from='osd.1 [v2:172.21.15.17:6810/1139898755,v1:172.21.15.17:6811/1139898755]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:26 smithi134 ceph-mon[33541]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:26 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:26 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:27.541 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[29380]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[29380]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[29380]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[40737]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[40737]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[40737]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:27.542 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:27 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:27 smithi134 ceph-mon[33541]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:27 smithi134 ceph-mon[33541]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:32:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:27 smithi134 ceph-mon[33541]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:32:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:27 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:27 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:27 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:28.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[29380]: purged_snaps scrub starts 2024-09-21T09:32:28.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[29380]: purged_snaps scrub ok 2024-09-21T09:32:28.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[29380]: osd.1 [v2:172.21.15.17:6810/1139898755,v1:172.21.15.17:6811/1139898755] boot 2024-09-21T09:32:28.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[29380]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:32:28.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:28.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[40737]: purged_snaps scrub starts 2024-09-21T09:32:28.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[40737]: purged_snaps scrub ok 2024-09-21T09:32:28.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[40737]: osd.1 [v2:172.21.15.17:6810/1139898755,v1:172.21.15.17:6811/1139898755] boot 2024-09-21T09:32:28.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[40737]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:32:28.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:28 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:28.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:28 smithi134 ceph-mon[33541]: purged_snaps scrub starts 2024-09-21T09:32:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:28 smithi134 ceph-mon[33541]: purged_snaps scrub ok 2024-09-21T09:32:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:28 smithi134 ceph-mon[33541]: osd.1 [v2:172.21.15.17:6810/1139898755,v1:172.21.15.17:6811/1139898755] boot 2024-09-21T09:32:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:28 smithi134 ceph-mon[33541]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:32:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:28 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:29.480 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:32:29.498 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:29 smithi017 ceph-mon[29380]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:29.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:29 smithi017 ceph-mon[29380]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:32:29.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:29 smithi017 ceph-mon[40737]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:29.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:29 smithi017 ceph-mon[40737]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:32:29.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi017:vg_nvme/lv_2 2024-09-21T09:32:29.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:29 smithi134 ceph-mon[33541]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:29 smithi134 ceph-mon[33541]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:32:30.012 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:31.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:31 smithi017 ceph-mon[40737]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:31.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:31 smithi017 ceph-mon[29380]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:31.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:31 smithi134 ceph-mon[33541]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:32.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: Detected new or changed devices on smithi017 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi017 to 8034M 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:32.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:32.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:32.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:32.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:32.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:32.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:33.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: Detected new or changed devices on smithi017 2024-09-21T09:32:33.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:33.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:33.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi017 to 8034M 2024-09-21T09:32:33.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:33.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: Detected new or changed devices on smithi017 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi017 to 8034M 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:33.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:33.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:33.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:33.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:33.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:34.072 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:33 smithi017 ceph-mon[29380]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:34.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:33 smithi017 ceph-mon[40737]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:34.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:33 smithi134 ceph-mon[33541]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[40737]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/366639572' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59c1433f-c409-4616-a7b9-7a11d8fb28d8"}]: dispatch 2024-09-21T09:32:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/366639572' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59c1433f-c409-4616-a7b9-7a11d8fb28d8"}]': finished 2024-09-21T09:32:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[40737]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:32:35.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:35.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[29380]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:35.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/366639572' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59c1433f-c409-4616-a7b9-7a11d8fb28d8"}]: dispatch 2024-09-21T09:32:35.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/366639572' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59c1433f-c409-4616-a7b9-7a11d8fb28d8"}]': finished 2024-09-21T09:32:35.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[29380]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:32:35.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:34 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:35.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:34 smithi134 ceph-mon[33541]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:35.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:34 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/366639572' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59c1433f-c409-4616-a7b9-7a11d8fb28d8"}]: dispatch 2024-09-21T09:32:35.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:34 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/366639572' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59c1433f-c409-4616-a7b9-7a11d8fb28d8"}]': finished 2024-09-21T09:32:35.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:34 smithi134 ceph-mon[33541]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:32:35.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:34 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:35 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2143418134' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:35 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2143418134' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:36.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:35 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2143418134' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:36 smithi017 ceph-mon[29380]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:36 smithi017 ceph-mon[40737]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:37.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:36 smithi134 ceph-mon[33541]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:38 smithi017 ceph-mon[29380]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:38 smithi017 ceph-mon[40737]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:39.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:38 smithi134 ceph-mon[33541]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:40 smithi017 ceph-mon[29380]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:40 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:32:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:40 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:40 smithi017 ceph-mon[40737]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:40 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:32:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:40 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:41.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:40 smithi134 ceph-mon[33541]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:40 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:32:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:40 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:42.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:41 smithi017 ceph-mon[29380]: Deploying daemon osd.2 on smithi017 2024-09-21T09:32:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:41 smithi017 ceph-mon[40737]: Deploying daemon osd.2 on smithi017 2024-09-21T09:32:42.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:41 smithi134 ceph-mon[33541]: Deploying daemon osd.2 on smithi017 2024-09-21T09:32:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:42 smithi017 ceph-mon[29380]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:42 smithi017 ceph-mon[40737]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:43.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:42 smithi134 ceph-mon[33541]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:45.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:44 smithi134 ceph-mon[33541]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:45.301 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:44 smithi017 ceph-mon[29380]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:45.302 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:44 smithi017 ceph-mon[40737]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:46.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:45 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:46.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:45 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:46.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:45 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:46.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:45 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:46.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:45 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:46.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:45 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:46.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:45 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:46.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:45 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:46.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:45 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:47.211 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:46 smithi017 ceph-mon[29380]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:47.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:46 smithi017 ceph-mon[40737]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:47.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:46 smithi134 ceph-mon[33541]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:48.996 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:48 smithi017 ceph-mon[29380]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:48.996 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:48 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:48.996 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:48 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:48.996 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:48 smithi017 ceph-mon[40737]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:48.996 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:48 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:48.996 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:48 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:49.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:48 smithi134 ceph-mon[33541]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:49.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:48 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:49.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:48 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:49.420 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 2 on host 'smithi017' 2024-09-21T09:32:50.082 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:49 smithi017 ceph-mon[29380]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:32:50.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:49 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:50.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:49 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:50.083 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:49 smithi017 ceph-mon[40737]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:32:50.083 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:49 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:50.083 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:49 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:50.111 DEBUG:teuthology.orchestra.run.smithi017:osd.2> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.2.service 2024-09-21T09:32:50.113 INFO:tasks.cephadm:Deploying osd.3 on smithi017 with /dev/vg_nvme/lv_1... 2024-09-21T09:32:50.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:32:50.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:49 smithi134 ceph-mon[33541]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:32:50.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:49 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:50.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:49 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:50.928 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:51.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:50 smithi134 ceph-mon[33541]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:51.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:50 smithi134 ceph-mon[33541]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:32:51.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:50 smithi134 ceph-mon[33541]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:32:51.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:50 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:51.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:50 smithi134 ceph-mon[33541]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:51.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[29380]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:51.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[29380]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:32:51.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[29380]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:32:51.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:51.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[29380]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:51.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[40737]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:51.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[40737]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:32:51.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[40737]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:32:51.341 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:51.341 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-mon[40737]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:32:51.341 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[57700]: 2024-09-21T09:32:50.968+0000 7f0b839c3640 -1 osd.2 0 waiting for initial osdmap 2024-09-21T09:32:51.341 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:50 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[57700]: 2024-09-21T09:32:50.971+0000 7f0b7fffd640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:32:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:51 smithi017 ceph-mon[29380]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:32:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:51 smithi017 ceph-mon[29380]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:32:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:52.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:51 smithi017 ceph-mon[40737]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:32:52.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:51 smithi017 ceph-mon[40737]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:32:52.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:52.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:52.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:51 smithi134 ceph-mon[33541]: from='osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:32:52.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:51 smithi134 ceph-mon[33541]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:32:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:53.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:52 smithi134 ceph-mon[33541]: purged_snaps scrub starts 2024-09-21T09:32:53.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:52 smithi134 ceph-mon[33541]: purged_snaps scrub ok 2024-09-21T09:32:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:52 smithi134 ceph-mon[33541]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:52 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:52 smithi134 ceph-mon[33541]: osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048] boot 2024-09-21T09:32:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:52 smithi134 ceph-mon[33541]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:32:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:52 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:53.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[29380]: purged_snaps scrub starts 2024-09-21T09:32:53.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[29380]: purged_snaps scrub ok 2024-09-21T09:32:53.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[29380]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:53.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:53.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[29380]: osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048] boot 2024-09-21T09:32:53.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[29380]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:32:53.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:53.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[40737]: purged_snaps scrub starts 2024-09-21T09:32:53.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[40737]: purged_snaps scrub ok 2024-09-21T09:32:53.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[40737]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:32:53.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:53.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[40737]: osd.2 [v2:172.21.15.17:6818/3614951048,v1:172.21.15.17:6819/3614951048] boot 2024-09-21T09:32:53.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[40737]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:32:53.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:52 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:32:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T09:32:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:54 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T09:32:54.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:54.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:54.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:54 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T09:32:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:32:55.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi017:vg_nvme/lv_1 2024-09-21T09:32:55.351 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[29380]: Detected new or changed devices on smithi017 2024-09-21T09:32:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi017 to 5356M 2024-09-21T09:32:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[29380]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:32:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T09:32:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[29380]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:32:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:32:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[40737]: Detected new or changed devices on smithi017 2024-09-21T09:32:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi017 to 5356M 2024-09-21T09:32:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[40737]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:32:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T09:32:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[40737]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:32:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:32:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:55 smithi134 ceph-mon[33541]: Detected new or changed devices on smithi017 2024-09-21T09:32:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:55 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi017 to 5356M 2024-09-21T09:32:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:55 smithi134 ceph-mon[33541]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:32:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T09:32:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:55 smithi134 ceph-mon[33541]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:32:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:32:56.746 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:56 smithi017 sudo[61899]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:56.746 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:56 smithi017 sudo[61899]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:56.746 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:56 smithi017 sudo[61899]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:56.746 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:56 smithi017 sudo[61899]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:57.002 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:56 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:32:57.002 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:56 smithi017 ceph-mon[29380]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:32:57.002 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:56 smithi017 ceph-mon[29380]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:32:57.003 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:56 smithi017 sudo[61949]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:57.003 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:56 smithi017 sudo[61949]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:57.003 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:56 smithi017 sudo[61949]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:57.003 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:32:56 smithi017 sudo[61949]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:57.006 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:56 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:32:57.006 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:56 smithi017 ceph-mon[40737]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:32:57.006 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:56 smithi017 ceph-mon[40737]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:32:57.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:56 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:32:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:56 smithi134 ceph-mon[33541]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:32:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:56 smithi134 ceph-mon[33541]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:32:57.688 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:57 smithi017 sudo[62061]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:57.689 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:57 smithi017 sudo[62061]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:57.689 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:57 smithi017 sudo[62061]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:57.689 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:57 smithi017 sudo[62061]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:57.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[29380]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T09:32:57.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[29380]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[40737]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[40737]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:57.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:57 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:57.949 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:57 smithi017 sudo[62133]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:57.949 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:57 smithi017 sudo[62133]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:57.949 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:57 smithi017 sudo[62133]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:57.949 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:32:57 smithi017 sudo[62133]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:58.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:57 smithi134 ceph-mon[33541]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T09:32:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:57 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:32:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:57 smithi134 ceph-mon[33541]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:57 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:57 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:57 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:58.617 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:58 smithi017 sudo[62145]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:58.617 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:58 smithi017 sudo[62145]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:58.617 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:58 smithi017 sudo[62145]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:58.617 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:58 smithi017 sudo[62145]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:58.901 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:58 smithi017 sudo[62169]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:58.901 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:58 smithi017 sudo[62169]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:58.901 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:58 smithi017 sudo[62169]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:58.901 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:32:58 smithi017 sudo[62169]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:59.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 sudo[62209]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:59.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 sudo[62209]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:59.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 sudo[62209]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:59.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 sudo[62209]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:59.948 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:59 smithi134 ceph-mon[33541]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:32:59.948 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:59 smithi134 ceph-mon[33541]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:59.948 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:59 smithi134 sudo[36280]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:59.948 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:59 smithi134 sudo[36280]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:59.948 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:59 smithi134 sudo[36280]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:59.948 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:32:59 smithi134 sudo[36280]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:59 smithi017 ceph-mon[40737]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:32:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:32:59 smithi017 ceph-mon[40737]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 sudo[62242]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 sudo[62242]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 sudo[62242]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 sudo[62242]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 ceph-mon[29380]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:32:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:32:59 smithi017 ceph-mon[29380]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:33:00.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:00 smithi134 sudo[36283]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:33:00.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:00 smithi134 sudo[36283]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:33:00.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:00 smithi134 sudo[36283]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:33:00.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:00 smithi134 sudo[36283]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:33:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:00 smithi017 sudo[62278]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:33:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:00 smithi017 sudo[62278]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:33:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:00 smithi017 sudo[62278]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:33:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:00 smithi017 sudo[62278]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:33:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2889228776' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fb2cdbe-3662-4ee4-b09d-96dffa74d226"}]: dispatch 2024-09-21T09:33:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fb2cdbe-3662-4ee4-b09d-96dffa74d226"}]: dispatch 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:33:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fb2cdbe-3662-4ee4-b09d-96dffa74d226"}]': finished 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 sudo[62329]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 sudo[62329]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 sudo[62329]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 sudo[62329]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2889228776' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fb2cdbe-3662-4ee4-b09d-96dffa74d226"}]: dispatch 2024-09-21T09:33:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fb2cdbe-3662-4ee4-b09d-96dffa74d226"}]: dispatch 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fb2cdbe-3662-4ee4-b09d-96dffa74d226"}]': finished 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:33:01.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:33:01.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2889228776' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fb2cdbe-3662-4ee4-b09d-96dffa74d226"}]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fb2cdbe-3662-4ee4-b09d-96dffa74d226"}]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:33:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:33:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fb2cdbe-3662-4ee4-b09d-96dffa74d226"}]': finished 2024-09-21T09:33:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:33:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:33:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:33:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:33:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:02 smithi017 ceph-mon[29380]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:33:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:02 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1910666905' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:02 smithi017 ceph-mon[29380]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:33:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:02 smithi017 ceph-mon[40737]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:33:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:02 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1910666905' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:02 smithi017 ceph-mon[40737]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:33:02.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:02 smithi134 ceph-mon[33541]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:33:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:02 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1910666905' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:02 smithi134 ceph-mon[33541]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:33:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:03 smithi017 ceph-mon[40737]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:03 smithi017 ceph-mon[29380]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:03.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:03 smithi134 ceph-mon[33541]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:05.393 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:05 smithi017 ceph-mon[29380]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-21T09:33:05.393 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:05 smithi017 ceph-mon[40737]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-21T09:33:05.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:05 smithi134 ceph-mon[33541]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-21T09:33:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:07 smithi017 ceph-mon[29380]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-21T09:33:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:07 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:33:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:07 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:07 smithi017 ceph-mon[40737]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-21T09:33:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:07 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:33:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:07 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:07.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:07 smithi134 ceph-mon[33541]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-21T09:33:07.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:07 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:33:07.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:07 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:08.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:08 smithi017 ceph-mon[29380]: Deploying daemon osd.3 on smithi017 2024-09-21T09:33:08.354 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:08 smithi017 ceph-mon[40737]: Deploying daemon osd.3 on smithi017 2024-09-21T09:33:08.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:08 smithi134 ceph-mon[33541]: Deploying daemon osd.3 on smithi017 2024-09-21T09:33:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:09 smithi017 ceph-mon[29380]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T09:33:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:09 smithi017 ceph-mon[40737]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T09:33:09.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:09 smithi134 ceph-mon[33541]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T09:33:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:11 smithi017 ceph-mon[29380]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T09:33:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:11 smithi017 ceph-mon[40737]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T09:33:11.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:11 smithi134 ceph-mon[33541]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T09:33:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:12 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:12 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:12 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:12.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:12 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:12 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:12 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:13 smithi017 ceph-mon[40737]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 67 KiB/s wr, 4 op/s 2024-09-21T09:33:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:13 smithi017 ceph-mon[29380]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 67 KiB/s wr, 4 op/s 2024-09-21T09:33:13.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:13 smithi134 ceph-mon[33541]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 67 KiB/s wr, 4 op/s 2024-09-21T09:33:14.738 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 3 on host 'smithi017' 2024-09-21T09:33:15.331 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[29380]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:33:15.331 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[29380]: from='osd.3 [v2:172.21.15.17:6826/823707214,v1:172.21.15.17:6827/823707214]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:15.331 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[29380]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:15.331 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:15.331 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:15.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[40737]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:33:15.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[40737]: from='osd.3 [v2:172.21.15.17:6826/823707214,v1:172.21.15.17:6827/823707214]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:15.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[40737]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:15.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:15.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:15 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:15.359 DEBUG:teuthology.orchestra.run.smithi017:osd.3> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T09:33:15.362 INFO:tasks.cephadm:Deploying osd.4 on smithi134 with /dev/vg_nvme/lv_4... 2024-09-21T09:33:15.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:33:15.613 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:33:15.734 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:15 smithi134 ceph-mon[33541]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T09:33:15.734 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:15 smithi134 ceph-mon[33541]: from='osd.3 [v2:172.21.15.17:6826/823707214,v1:172.21.15.17:6827/823707214]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:15.734 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:15 smithi134 ceph-mon[33541]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:15.734 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:15 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:15.735 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:15 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[40737]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:33:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[40737]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:33:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[40737]: from='osd.3 [v2:172.21.15.17:6826/823707214,v1:172.21.15.17:6827/823707214]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:33:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[40737]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:33:16.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:16.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:16.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[29380]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:33:16.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[29380]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:33:16.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:16.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[29380]: from='osd.3 [v2:172.21.15.17:6826/823707214,v1:172.21.15.17:6827/823707214]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:33:16.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[29380]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:33:16.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:16.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:16.452 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[64024]: 2024-09-21T09:33:16.283+0000 7fb0baced640 -1 osd.3 0 waiting for initial osdmap 2024-09-21T09:33:16.452 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:33:16 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[64024]: 2024-09-21T09:33:16.287+0000 7fb0b3b20640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:33:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:16 smithi134 ceph-mon[33541]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:33:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:16 smithi134 ceph-mon[33541]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:33:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:16 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:16 smithi134 ceph-mon[33541]: from='osd.3 [v2:172.21.15.17:6826/823707214,v1:172.21.15.17:6827/823707214]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:33:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:16 smithi134 ceph-mon[33541]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2024-09-21T09:33:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:16 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:16 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[29380]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[29380]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:33:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[29380]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T09:33:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[40737]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[40737]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:33:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[40737]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T09:33:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:17 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:17.474 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:17 smithi134 ceph-mon[33541]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:17.475 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:17 smithi134 ceph-mon[33541]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2024-09-21T09:33:17.475 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:17 smithi134 ceph-mon[33541]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T09:33:17.475 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:17 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:17.475 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:17 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:17.475 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:17 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:33:18.172 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi134:vg_nvme/lv_4 2024-09-21T09:33:18.386 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[29380]: purged_snaps scrub starts 2024-09-21T09:33:18.386 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[29380]: purged_snaps scrub ok 2024-09-21T09:33:18.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[29380]: osd.3 [v2:172.21.15.17:6826/823707214,v1:172.21.15.17:6827/823707214] boot 2024-09-21T09:33:18.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[29380]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:33:18.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:18.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[40737]: purged_snaps scrub starts 2024-09-21T09:33:18.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[40737]: purged_snaps scrub ok 2024-09-21T09:33:18.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[40737]: osd.3 [v2:172.21.15.17:6826/823707214,v1:172.21.15.17:6827/823707214] boot 2024-09-21T09:33:18.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[40737]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:33:18.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:18 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:18.412 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:33:18.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:18 smithi134 ceph-mon[33541]: purged_snaps scrub starts 2024-09-21T09:33:18.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:18 smithi134 ceph-mon[33541]: purged_snaps scrub ok 2024-09-21T09:33:18.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:18 smithi134 ceph-mon[33541]: osd.3 [v2:172.21.15.17:6826/823707214,v1:172.21.15.17:6827/823707214] boot 2024-09-21T09:33:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:18 smithi134 ceph-mon[33541]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:33:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:18 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:19.479 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.480 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:19.480 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:19.480 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:19.480 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T09:33:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:19.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:19.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:19.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:19.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:20.622 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:20 smithi134 ceph-mon[33541]: Detected new or changed devices on smithi017 2024-09-21T09:33:20.622 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:20 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi017 to 4017M 2024-09-21T09:33:20.622 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:20 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:20.622 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:20 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:20.622 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:20 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[29380]: Detected new or changed devices on smithi017 2024-09-21T09:33:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi017 to 4017M 2024-09-21T09:33:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[40737]: Detected new or changed devices on smithi017 2024-09-21T09:33:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi017 to 4017M 2024-09-21T09:33:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:20 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:21 smithi134 ceph-mon[33541]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:21 smithi134 ceph-mon[33541]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:21 smithi017 ceph-mon[29380]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:21 smithi017 ceph-mon[29380]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:21 smithi017 ceph-mon[40737]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:21 smithi017 ceph-mon[40737]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[29380]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/1084980200' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6fde21-8847-4673-8b7f-523f387faf50"}]: dispatch 2024-09-21T09:33:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6fde21-8847-4673-8b7f-523f387faf50"}]: dispatch 2024-09-21T09:33:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d6fde21-8847-4673-8b7f-523f387faf50"}]': finished 2024-09-21T09:33:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[29380]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:33:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:23.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/1679320715' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[40737]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/1084980200' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6fde21-8847-4673-8b7f-523f387faf50"}]: dispatch 2024-09-21T09:33:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6fde21-8847-4673-8b7f-523f387faf50"}]: dispatch 2024-09-21T09:33:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d6fde21-8847-4673-8b7f-523f387faf50"}]': finished 2024-09-21T09:33:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[40737]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:33:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:23 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/1679320715' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:23.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:23 smithi134 ceph-mon[33541]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:23 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/1084980200' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6fde21-8847-4673-8b7f-523f387faf50"}]: dispatch 2024-09-21T09:33:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:23 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d6fde21-8847-4673-8b7f-523f387faf50"}]: dispatch 2024-09-21T09:33:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:23 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d6fde21-8847-4673-8b7f-523f387faf50"}]': finished 2024-09-21T09:33:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:23 smithi134 ceph-mon[33541]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:33:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:23 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:23 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/1679320715' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:25.487 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:25 smithi134 ceph-mon[33541]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:25 smithi017 ceph-mon[29380]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:25 smithi017 ceph-mon[40737]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:27.620 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:27 smithi134 ceph-mon[33541]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:27 smithi017 ceph-mon[29380]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:27 smithi017 ceph-mon[40737]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:29.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:29 smithi134 ceph-mon[33541]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:29.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:29 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:33:29.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:29 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:29.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:29 smithi134 ceph-mon[33541]: Deploying daemon osd.4 on smithi134 2024-09-21T09:33:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:29 smithi017 ceph-mon[29380]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:29 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:33:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:29 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:29 smithi017 ceph-mon[29380]: Deploying daemon osd.4 on smithi134 2024-09-21T09:33:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:29 smithi017 ceph-mon[40737]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:29 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:33:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:29 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:29 smithi017 ceph-mon[40737]: Deploying daemon osd.4 on smithi134 2024-09-21T09:33:31.585 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:31 smithi134 ceph-mon[33541]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:31 smithi017 ceph-mon[29380]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:31 smithi017 ceph-mon[40737]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:33.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:33 smithi134 ceph-mon[33541]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:33.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:33 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:33.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:33 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:33.577 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:33 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:33 smithi017 ceph-mon[29380]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:33 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:33 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:33 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:33 smithi017 ceph-mon[40737]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:33 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:33 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:33 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:35 smithi134 ceph-mon[33541]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:35 smithi017 ceph-mon[29380]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:35 smithi017 ceph-mon[40737]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:36.597 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 4 on host 'smithi134' 2024-09-21T09:33:37.165 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.165 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.165 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:37.166 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:37.166 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.166 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:37.166 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='osd.4 [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:37.166 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:37.166 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:37.166 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.166 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:36 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.192 DEBUG:teuthology.orchestra.run.smithi134:osd.4> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.4.service 2024-09-21T09:33:37.194 INFO:tasks.cephadm:Deploying osd.5 on smithi134 with /dev/vg_nvme/lv_3... 2024-09-21T09:33:37.194 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:33:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='osd.4 [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='osd.4 [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:36 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:37.866 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:33:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[29380]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:33:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[29380]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:33:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[29380]: from='osd.4 [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:33:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[29380]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:33:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[40737]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:33:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[40737]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:33:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[40737]: from='osd.4 [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:33:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:37 smithi017 ceph-mon[40737]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:33:38.223 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:37 smithi134 ceph-mon[33541]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:33:38.223 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:37 smithi134 ceph-mon[33541]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:33:38.224 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:37 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:38.224 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:37 smithi134 ceph-mon[33541]: from='osd.4 [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:33:38.224 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:37 smithi134 ceph-mon[33541]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:33:38.224 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 09:33:37 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[38931]: 2024-09-21T09:33:37.906+0000 7feec20fd640 -1 osd.4 0 waiting for initial osdmap 2024-09-21T09:33:38.224 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 09:33:37 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[38931]: 2024-09-21T09:33:37.909+0000 7feebd735640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:33:39.163 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:38 smithi134 ceph-mon[33541]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:39.163 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:38 smithi134 ceph-mon[33541]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:33:39.163 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:38 smithi134 ceph-mon[33541]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T09:33:39.163 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:38 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:39.163 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:38 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[29380]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[29380]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:33:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[29380]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T09:33:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[40737]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[40737]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:33:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[40737]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T09:33:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:38 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[40737]: purged_snaps scrub starts 2024-09-21T09:33:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[40737]: purged_snaps scrub ok 2024-09-21T09:33:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[40737]: osd.4 [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249] boot 2024-09-21T09:33:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[40737]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:33:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[40737]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:33:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[29380]: purged_snaps scrub starts 2024-09-21T09:33:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[29380]: purged_snaps scrub ok 2024-09-21T09:33:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[29380]: osd.4 [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249] boot 2024-09-21T09:33:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[29380]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:33:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:39 smithi017 ceph-mon[29380]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:33:40.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:39 smithi134 ceph-mon[33541]: purged_snaps scrub starts 2024-09-21T09:33:40.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:39 smithi134 ceph-mon[33541]: purged_snaps scrub ok 2024-09-21T09:33:40.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:39 smithi134 ceph-mon[33541]: osd.4 [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249] boot 2024-09-21T09:33:40.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:39 smithi134 ceph-mon[33541]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:33:40.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:39 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:40.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:39 smithi134 ceph-mon[33541]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:33:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[29380]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[29380]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[40737]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:40 smithi017 ceph-mon[40737]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T09:33:41.241 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:40 smithi134 ceph-mon[33541]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:33:41.241 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:40 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.241 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:40 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.241 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:40 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:41.241 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:40 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.241 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:40 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:41.241 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:40 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:41.241 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:40 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:41.241 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:40 smithi134 ceph-mon[33541]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T09:33:41.966 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:41 smithi134 ceph-mon[33541]: Detected new or changed devices on smithi134 2024-09-21T09:33:41.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:41 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi134 to 17088M 2024-09-21T09:33:41.973 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:33:41.993 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi134:vg_nvme/lv_3 2024-09-21T09:33:42.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:41 smithi017 ceph-mon[29380]: Detected new or changed devices on smithi134 2024-09-21T09:33:42.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:41 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi134 to 17088M 2024-09-21T09:33:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:41 smithi017 ceph-mon[40737]: Detected new or changed devices on smithi134 2024-09-21T09:33:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:41 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi134 to 17088M 2024-09-21T09:33:42.231 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:33:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:42 smithi017 ceph-mon[29380]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:42 smithi017 ceph-mon[40737]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:43.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:42 smithi134 ceph-mon[33541]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:45.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[29380]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:33:45.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[29380]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:45.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:45.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:45.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:45.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[40737]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:33:45.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[40737]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:45.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:45.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:45.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:44 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:45.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:44 smithi134 ceph-mon[33541]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:33:45.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:44 smithi134 ceph-mon[33541]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:45.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:44 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:44 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:44 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:46.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:46 smithi134 ceph-mon[33541]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-21T09:33:46.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:46 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/2303529443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2bae1d15-c9ee-4f67-82ed-31fb25c11749"}]: dispatch 2024-09-21T09:33:46.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:46 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2bae1d15-c9ee-4f67-82ed-31fb25c11749"}]: dispatch 2024-09-21T09:33:46.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:46 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2bae1d15-c9ee-4f67-82ed-31fb25c11749"}]': finished 2024-09-21T09:33:46.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:46 smithi134 ceph-mon[33541]: osdmap e34: 6 total, 5 up, 6 in 2024-09-21T09:33:46.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:46 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:33:46.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:46 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/3010352994' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[29380]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-21T09:33:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/2303529443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2bae1d15-c9ee-4f67-82ed-31fb25c11749"}]: dispatch 2024-09-21T09:33:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2bae1d15-c9ee-4f67-82ed-31fb25c11749"}]: dispatch 2024-09-21T09:33:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2bae1d15-c9ee-4f67-82ed-31fb25c11749"}]': finished 2024-09-21T09:33:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[29380]: osdmap e34: 6 total, 5 up, 6 in 2024-09-21T09:33:47.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:33:47.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/3010352994' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:47.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[40737]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-21T09:33:47.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/2303529443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2bae1d15-c9ee-4f67-82ed-31fb25c11749"}]: dispatch 2024-09-21T09:33:47.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2bae1d15-c9ee-4f67-82ed-31fb25c11749"}]: dispatch 2024-09-21T09:33:47.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2bae1d15-c9ee-4f67-82ed-31fb25c11749"}]': finished 2024-09-21T09:33:47.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[40737]: osdmap e34: 6 total, 5 up, 6 in 2024-09-21T09:33:47.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:33:47.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:46 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/3010352994' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:48 smithi017 ceph-mon[29380]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:33:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:48 smithi017 ceph-mon[40737]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:33:49.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:48 smithi134 ceph-mon[33541]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:33:50.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:50 smithi134 ceph-mon[33541]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:33:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:50 smithi017 ceph-mon[29380]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:33:51.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:50 smithi017 ceph-mon[40737]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:33:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:52 smithi017 ceph-mon[29380]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:33:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:52 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:33:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:52 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:52 smithi017 ceph-mon[29380]: Deploying daemon osd.5 on smithi134 2024-09-21T09:33:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:52 smithi017 ceph-mon[40737]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:33:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:52 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:33:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:52 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:53.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:52 smithi017 ceph-mon[40737]: Deploying daemon osd.5 on smithi134 2024-09-21T09:33:53.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:52 smithi134 ceph-mon[33541]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:33:53.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:52 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:33:53.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:52 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:52 smithi134 ceph-mon[33541]: Deploying daemon osd.5 on smithi134 2024-09-21T09:33:55.138 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:54 smithi134 ceph-mon[33541]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:54 smithi017 ceph-mon[29380]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:54 smithi017 ceph-mon[40737]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:56 smithi017 ceph-mon[29380]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:57.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:56 smithi017 ceph-mon[40737]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:57.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:56 smithi134 ceph-mon[33541]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:57 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:57 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:57 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:57 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:58.211 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:57 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:58.211 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:57 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:58.211 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:57 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:33:59.341 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:33:59 smithi134 ceph-mon[33541]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:33:59 smithi017 ceph-mon[29380]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:33:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:33:59 smithi017 ceph-mon[40737]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:34:00.777 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 5 on host 'smithi134' 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='osd.5 [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:00 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:34:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='osd.5 [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:34:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='osd.5 [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:34:01.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:01.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:00 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:01.384 DEBUG:teuthology.orchestra.run.smithi134:osd.5> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.5.service 2024-09-21T09:34:01.386 INFO:tasks.cephadm:Deploying osd.6 on smithi134 with /dev/vg_nvme/lv_2... 2024-09-21T09:34:01.387 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:34:02.196 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:34:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[29380]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:34:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[29380]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:34:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[29380]: from='osd.5 [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[29380]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[40737]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:34:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[40737]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:34:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[40737]: from='osd.5 [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:01 smithi017 ceph-mon[40737]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:01 smithi134 ceph-mon[33541]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:34:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:01 smithi134 ceph-mon[33541]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:34:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:01 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:01 smithi134 ceph-mon[33541]: from='osd.5 [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:01 smithi134 ceph-mon[33541]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:02.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 09:34:01 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T09:34:01.900+0000 7f4e96707640 -1 osd.5 0 waiting for initial osdmap 2024-09-21T09:34:02.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 09:34:01 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T09:34:01.905+0000 7f4e93542640 -1 osd.5 36 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:34:03.144 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:02 smithi134 ceph-mon[33541]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:34:03.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:02 smithi134 ceph-mon[33541]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T09:34:03.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:02 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:03.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:02 smithi134 ceph-mon[33541]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:34:03.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:02 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:03.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:02 smithi134 ceph-mon[33541]: osd.5 [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742] boot 2024-09-21T09:34:03.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:02 smithi134 ceph-mon[33541]: osdmap e37: 6 total, 6 up, 6 in 2024-09-21T09:34:03.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:02 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[29380]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:34:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[29380]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T09:34:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[29380]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:34:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[29380]: osd.5 [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742] boot 2024-09-21T09:34:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[29380]: osdmap e37: 6 total, 6 up, 6 in 2024-09-21T09:34:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[40737]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:34:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[40737]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T09:34:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[40737]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:34:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[40737]: osd.5 [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742] boot 2024-09-21T09:34:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[40737]: osdmap e37: 6 total, 6 up, 6 in 2024-09-21T09:34:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:02 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:03 smithi134 ceph-mon[33541]: purged_snaps scrub starts 2024-09-21T09:34:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:03 smithi134 ceph-mon[33541]: purged_snaps scrub ok 2024-09-21T09:34:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:03 smithi134 ceph-mon[33541]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:34:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:03 smithi017 ceph-mon[40737]: purged_snaps scrub starts 2024-09-21T09:34:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:03 smithi017 ceph-mon[40737]: purged_snaps scrub ok 2024-09-21T09:34:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:03 smithi017 ceph-mon[40737]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:34:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:03 smithi017 ceph-mon[29380]: purged_snaps scrub starts 2024-09-21T09:34:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:03 smithi017 ceph-mon[29380]: purged_snaps scrub ok 2024-09-21T09:34:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:03 smithi017 ceph-mon[29380]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:34:04.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:04 smithi134 ceph-mon[33541]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:04 smithi134 ceph-mon[33541]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T09:34:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:04 smithi017 ceph-mon[29380]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:04 smithi017 ceph-mon[29380]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T09:34:05.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:04 smithi017 ceph-mon[40737]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:05.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:04 smithi017 ceph-mon[40737]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T09:34:06.151 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:05 smithi134 ceph-mon[33541]: Detected new or changed devices on smithi134 2024-09-21T09:34:06.151 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.151 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:06.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:06.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:06.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:06.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:05 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[29380]: Detected new or changed devices on smithi134 2024-09-21T09:34:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[40737]: Detected new or changed devices on smithi134 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:06.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:06.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:05 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:06.850 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:34:06.870 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi134:vg_nvme/lv_2 2024-09-21T09:34:07.109 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:34:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:06 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi134 to 8544M 2024-09-21T09:34:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:06 smithi017 ceph-mon[29380]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:06 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi134 to 8544M 2024-09-21T09:34:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:06 smithi017 ceph-mon[40737]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:07.221 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:06 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi134 to 8544M 2024-09-21T09:34:07.221 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:06 smithi134 ceph-mon[33541]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:08 smithi017 ceph-mon[29380]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:08 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:08 smithi017 ceph-mon[40737]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:08 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:08 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:08 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:09.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:08 smithi134 ceph-mon[33541]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:09.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:08 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:08 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:08 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:09 smithi017 ceph-mon[29380]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:10.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:09 smithi017 ceph-mon[40737]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:10.235 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:09 smithi134 ceph-mon[33541]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:10 smithi017 ceph-mon[29380]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:10 smithi017 ceph-mon[40737]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:10 smithi134 ceph-mon[33541]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/834145320' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8f4f0e-887e-424a-bb0c-cb9cd88e6107"}]: dispatch 2024-09-21T09:34:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8f4f0e-887e-424a-bb0c-cb9cd88e6107"}]: dispatch 2024-09-21T09:34:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc8f4f0e-887e-424a-bb0c-cb9cd88e6107"}]': finished 2024-09-21T09:34:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[40737]: osdmap e40: 7 total, 6 up, 7 in 2024-09-21T09:34:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/932810682' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/834145320' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8f4f0e-887e-424a-bb0c-cb9cd88e6107"}]: dispatch 2024-09-21T09:34:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8f4f0e-887e-424a-bb0c-cb9cd88e6107"}]: dispatch 2024-09-21T09:34:12.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc8f4f0e-887e-424a-bb0c-cb9cd88e6107"}]': finished 2024-09-21T09:34:12.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[29380]: osdmap e40: 7 total, 6 up, 7 in 2024-09-21T09:34:12.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:12.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:11 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/932810682' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:12.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:11 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/834145320' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8f4f0e-887e-424a-bb0c-cb9cd88e6107"}]: dispatch 2024-09-21T09:34:12.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:11 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8f4f0e-887e-424a-bb0c-cb9cd88e6107"}]: dispatch 2024-09-21T09:34:12.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:11 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc8f4f0e-887e-424a-bb0c-cb9cd88e6107"}]': finished 2024-09-21T09:34:12.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:11 smithi134 ceph-mon[33541]: osdmap e40: 7 total, 6 up, 7 in 2024-09-21T09:34:12.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:11 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:12.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:11 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/932810682' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:13 smithi017 ceph-mon[40737]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:13 smithi017 ceph-mon[29380]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:13.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:13 smithi134 ceph-mon[33541]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:15.362 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:15 smithi134 ceph-mon[33541]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:15 smithi017 ceph-mon[29380]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:15 smithi017 ceph-mon[40737]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:17 smithi017 ceph-mon[29380]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:17 smithi017 ceph-mon[40737]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:17.464 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:17 smithi134 ceph-mon[33541]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:18 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:34:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:18 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:18 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:34:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:18 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:18.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:18 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:34:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:18 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:19.430 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:19 smithi134 ceph-mon[33541]: Deploying daemon osd.6 on smithi134 2024-09-21T09:34:19.430 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:19 smithi134 ceph-mon[33541]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:19 smithi017 ceph-mon[29380]: Deploying daemon osd.6 on smithi134 2024-09-21T09:34:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:19 smithi017 ceph-mon[29380]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:19 smithi017 ceph-mon[40737]: Deploying daemon osd.6 on smithi134 2024-09-21T09:34:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:19 smithi017 ceph-mon[40737]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:21 smithi017 ceph-mon[40737]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:21 smithi017 ceph-mon[29380]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:21 smithi134 ceph-mon[33541]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:23.333 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:23 smithi134 ceph-mon[33541]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:23.333 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:23 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:23.333 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:23 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:23.333 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:23 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:23 smithi017 ceph-mon[29380]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:23 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:23 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:23 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:23 smithi017 ceph-mon[40737]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:23 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:23 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:23 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:25 smithi017 ceph-mon[29380]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:25 smithi017 ceph-mon[40737]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:25 smithi134 ceph-mon[33541]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:25.847 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 6 on host 'smithi134' 2024-09-21T09:34:26.450 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:26 smithi134 ceph-mon[33541]: from='osd.6 [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:26.450 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:26 smithi134 ceph-mon[33541]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:26.450 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:26 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:26.450 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:26 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:26 smithi017 ceph-mon[29380]: from='osd.6 [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:26 smithi017 ceph-mon[29380]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:26 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:26 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:26 smithi017 ceph-mon[40737]: from='osd.6 [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:26 smithi017 ceph-mon[40737]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:26 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:26 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:26.477 DEBUG:teuthology.orchestra.run.smithi134:osd.6> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.6.service 2024-09-21T09:34:26.480 INFO:tasks.cephadm:Deploying osd.7 on smithi134 with /dev/vg_nvme/lv_1... 2024-09-21T09:34:26.480 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:34:26.785 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:34:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[29380]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[29380]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:34:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[29380]: osdmap e41: 7 total, 6 up, 7 in 2024-09-21T09:34:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[29380]: from='osd.6 [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[29380]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[40737]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[40737]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:34:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[40737]: osdmap e41: 7 total, 6 up, 7 in 2024-09-21T09:34:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[40737]: from='osd.6 [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[40737]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:27 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:27.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-mon[33541]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-mon[33541]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:34:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-mon[33541]: osdmap e41: 7 total, 6 up, 7 in 2024-09-21T09:34:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-mon[33541]: from='osd.6 [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-mon[33541]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:27.494 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T09:34:27.192+0000 7fd9f20ef640 -1 osd.6 0 waiting for initial osdmap 2024-09-21T09:34:27.494 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 09:34:27 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T09:34:27.198+0000 7fd9edf28640 -1 osd.6 42 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:34:28.209 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:28 smithi134 ceph-mon[33541]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:34:28.209 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:28 smithi134 ceph-mon[33541]: osdmap e42: 7 total, 6 up, 7 in 2024-09-21T09:34:28.209 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:28 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:28.209 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:28 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:28.209 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:28 smithi134 ceph-mon[33541]: osd.6 [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667] boot 2024-09-21T09:34:28.209 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:28 smithi134 ceph-mon[33541]: osdmap e43: 7 total, 7 up, 7 in 2024-09-21T09:34:28.210 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:28 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[29380]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:34:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[29380]: osdmap e42: 7 total, 6 up, 7 in 2024-09-21T09:34:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[29380]: osd.6 [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667] boot 2024-09-21T09:34:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[29380]: osdmap e43: 7 total, 7 up, 7 in 2024-09-21T09:34:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[40737]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[40737]: osdmap e42: 7 total, 6 up, 7 in 2024-09-21T09:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[40737]: osd.6 [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667] boot 2024-09-21T09:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[40737]: osdmap e43: 7 total, 7 up, 7 in 2024-09-21T09:34:28.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:28 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:29.332 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:29 smithi134 ceph-mon[33541]: purged_snaps scrub starts 2024-09-21T09:34:29.332 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:29 smithi134 ceph-mon[33541]: purged_snaps scrub ok 2024-09-21T09:34:29.332 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:29 smithi134 ceph-mon[33541]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:29.332 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:29 smithi134 ceph-mon[33541]: osdmap e44: 7 total, 7 up, 7 in 2024-09-21T09:34:29.338 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-09-21T09:34:29.358 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi134:vg_nvme/lv_1 2024-09-21T09:34:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:29 smithi017 ceph-mon[29380]: purged_snaps scrub starts 2024-09-21T09:34:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:29 smithi017 ceph-mon[29380]: purged_snaps scrub ok 2024-09-21T09:34:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:29 smithi017 ceph-mon[29380]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:29 smithi017 ceph-mon[29380]: osdmap e44: 7 total, 7 up, 7 in 2024-09-21T09:34:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:29 smithi017 ceph-mon[40737]: purged_snaps scrub starts 2024-09-21T09:34:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:29 smithi017 ceph-mon[40737]: purged_snaps scrub ok 2024-09-21T09:34:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:29 smithi017 ceph-mon[40737]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:34:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:29 smithi017 ceph-mon[40737]: osdmap e44: 7 total, 7 up, 7 in 2024-09-21T09:34:30.719 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:34:30.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:30 smithi134 ceph-mon[33541]: osdmap e45: 7 total, 7 up, 7 in 2024-09-21T09:34:30.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:30 smithi134 ceph-mon[33541]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:30 smithi017 ceph-mon[40737]: osdmap e45: 7 total, 7 up, 7 in 2024-09-21T09:34:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:30 smithi017 ceph-mon[40737]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:30 smithi017 ceph-mon[29380]: osdmap e45: 7 total, 7 up, 7 in 2024-09-21T09:34:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:30 smithi017 ceph-mon[29380]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:33.222 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:33.222 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.222 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.222 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:33.222 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:33.222 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:33.222 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.222 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:33.223 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:33.223 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.223 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:33.223 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:33.223 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:33.223 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:32 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:32 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:33.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:33 smithi134 ceph-mon[33541]: Detected new or changed devices on smithi134 2024-09-21T09:34:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:33 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi134 to 5696M 2024-09-21T09:34:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:33 smithi134 ceph-mon[33541]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:33 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:33 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:33 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:34.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[29380]: Detected new or changed devices on smithi134 2024-09-21T09:34:34.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi134 to 5696M 2024-09-21T09:34:34.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[29380]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:34.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:34.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:34.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:34.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[40737]: Detected new or changed devices on smithi134 2024-09-21T09:34:34.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi134 to 5696M 2024-09-21T09:34:34.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[40737]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:34.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:34.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:34.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:33 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:35.239 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:34 smithi134 ceph-mon[33541]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:34 smithi017 ceph-mon[29380]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:34 smithi017 ceph-mon[40737]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:36.098 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:35 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/3624284610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "edff61f5-4a21-484b-9924-e15db0d909a4"}]: dispatch 2024-09-21T09:34:36.099 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:35 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "edff61f5-4a21-484b-9924-e15db0d909a4"}]: dispatch 2024-09-21T09:34:36.099 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:35 smithi134 ceph-mon[33541]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "edff61f5-4a21-484b-9924-e15db0d909a4"}]': finished 2024-09-21T09:34:36.099 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:35 smithi134 ceph-mon[33541]: osdmap e46: 8 total, 7 up, 8 in 2024-09-21T09:34:36.099 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:35 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/3624284610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "edff61f5-4a21-484b-9924-e15db0d909a4"}]: dispatch 2024-09-21T09:34:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "edff61f5-4a21-484b-9924-e15db0d909a4"}]: dispatch 2024-09-21T09:34:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[29380]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "edff61f5-4a21-484b-9924-e15db0d909a4"}]': finished 2024-09-21T09:34:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[29380]: osdmap e46: 8 total, 7 up, 8 in 2024-09-21T09:34:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/3624284610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "edff61f5-4a21-484b-9924-e15db0d909a4"}]: dispatch 2024-09-21T09:34:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "edff61f5-4a21-484b-9924-e15db0d909a4"}]: dispatch 2024-09-21T09:34:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[40737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "edff61f5-4a21-484b-9924-e15db0d909a4"}]': finished 2024-09-21T09:34:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[40737]: osdmap e46: 8 total, 7 up, 8 in 2024-09-21T09:34:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:35 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:36 smithi134 ceph-mon[33541]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:36 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/3630797805' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:36 smithi017 ceph-mon[29380]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:36 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/3630797805' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:36 smithi017 ceph-mon[40737]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:36 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/3630797805' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:39.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:38 smithi134 ceph-mon[33541]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:39.251 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:38 smithi017 ceph-mon[29380]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:39.252 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:38 smithi017 ceph-mon[40737]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:41.106 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:40 smithi134 ceph-mon[33541]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:40 smithi017 ceph-mon[29380]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:40 smithi017 ceph-mon[40737]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:42.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:41 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:34:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:41 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:34:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:41 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:41 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:34:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:41 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:43.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:42 smithi134 ceph-mon[33541]: Deploying daemon osd.7 on smithi134 2024-09-21T09:34:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:42 smithi134 ceph-mon[33541]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:42 smithi017 ceph-mon[29380]: Deploying daemon osd.7 on smithi134 2024-09-21T09:34:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:42 smithi017 ceph-mon[29380]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:42 smithi017 ceph-mon[40737]: Deploying daemon osd.7 on smithi134 2024-09-21T09:34:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:42 smithi017 ceph-mon[40737]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:44 smithi134 ceph-mon[33541]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:44 smithi017 ceph-mon[29380]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:44 smithi017 ceph-mon[40737]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:47.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:46 smithi134 ceph-mon[33541]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:46 smithi017 ceph-mon[29380]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:46 smithi017 ceph-mon[40737]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:48.099 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:47 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:48.099 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:47 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:48.099 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:47 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:47 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:47 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:47 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:47 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:47 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:47 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:49.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:49 smithi134 ceph-mon[33541]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:49 smithi017 ceph-mon[29380]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:49 smithi017 ceph-mon[40737]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:50.836 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 7 on host 'smithi134' 2024-09-21T09:34:51.132 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:51.132 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.132 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.133 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:51.133 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:51.133 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.133 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='osd.7 [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:51.133 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:51.133 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:51.133 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.133 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:51 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:51.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:51.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='osd.7 [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:51.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:51.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:51.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:51.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:51.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:51.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='osd.7 [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:51.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:51.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:51.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:51 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:51.482 DEBUG:teuthology.orchestra.run.smithi134:osd.7> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.7.service 2024-09-21T09:34:51.484 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-21T09:34:51.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T09:34:51.750 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:34:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[29380]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:34:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[29380]: osdmap e47: 8 total, 7 up, 8 in 2024-09-21T09:34:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[29380]: from='osd.7 [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[29380]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[40737]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:34:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[40737]: osdmap e47: 8 total, 7 up, 8 in 2024-09-21T09:34:52.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:52.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[40737]: from='osd.7 [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:52.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:52 smithi017 ceph-mon[40737]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:52 smithi134 ceph-mon[33541]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:34:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:52 smithi134 ceph-mon[33541]: osdmap e47: 8 total, 7 up, 8 in 2024-09-21T09:34:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:52 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:52 smithi134 ceph-mon[33541]: from='osd.7 [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:52 smithi134 ceph-mon[33541]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-09-21T09:34:52.494 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 09:34:52 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T09:34:52.174+0000 7f6d43187640 -1 osd.7 0 waiting for initial osdmap 2024-09-21T09:34:52.494 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 09:34:52 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T09:34:52.180+0000 7f6d3dfbe640 -1 osd.7 48 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:34:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:53 smithi134 ceph-mon[33541]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:53 smithi134 ceph-mon[33541]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:34:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:53 smithi134 ceph-mon[33541]: osdmap e48: 8 total, 7 up, 8 in 2024-09-21T09:34:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:53 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:53 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:53 smithi134 ceph-mon[33541]: osd.7 [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708] boot 2024-09-21T09:34:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:53 smithi134 ceph-mon[33541]: osdmap e49: 8 total, 8 up, 8 in 2024-09-21T09:34:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:53 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[29380]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[29380]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:34:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[29380]: osdmap e48: 8 total, 7 up, 8 in 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[29380]: osd.7 [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708] boot 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[29380]: osdmap e49: 8 total, 8 up, 8 in 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[40737]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[40737]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[40737]: osdmap e48: 8 total, 7 up, 8 in 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[40737]: osd.7 [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708] boot 2024-09-21T09:34:53.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[40737]: osdmap e49: 8 total, 8 up, 8 in 2024-09-21T09:34:53.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:53 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:53.846 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:34:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:54 smithi017 ceph-mon[29380]: purged_snaps scrub starts 2024-09-21T09:34:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:54 smithi017 ceph-mon[29380]: purged_snaps scrub ok 2024-09-21T09:34:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:54 smithi017 ceph-mon[29380]: osdmap e50: 8 total, 8 up, 8 in 2024-09-21T09:34:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:54 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3386296322' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:34:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:54 smithi017 ceph-mon[40737]: purged_snaps scrub starts 2024-09-21T09:34:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:54 smithi017 ceph-mon[40737]: purged_snaps scrub ok 2024-09-21T09:34:54.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:54 smithi017 ceph-mon[40737]: osdmap e50: 8 total, 8 up, 8 in 2024-09-21T09:34:54.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:54 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3386296322' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:34:54.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:54 smithi134 ceph-mon[33541]: purged_snaps scrub starts 2024-09-21T09:34:54.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:54 smithi134 ceph-mon[33541]: purged_snaps scrub ok 2024-09-21T09:34:54.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:54 smithi134 ceph-mon[33541]: osdmap e50: 8 total, 8 up, 8 in 2024-09-21T09:34:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:54 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3386296322' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:34:54.573 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1726911292,"num_in_osds":8,"osd_in_since":1726911275,"num_remapped_pgs":0} 2024-09-21T09:34:54.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:34:54.827 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:34:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: Detected new or changed devices on smithi134 2024-09-21T09:34:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi134 to 4272M 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[29380]: osdmap e51: 8 total, 8 up, 8 in 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: Detected new or changed devices on smithi134 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi134 to 4272M 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:55.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:55.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:55 smithi017 ceph-mon[40737]: osdmap e51: 8 total, 8 up, 8 in 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: Detected new or changed devices on smithi134 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi134 to 4272M 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:55.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:34:55.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:55 smithi134 ceph-mon[33541]: osdmap e51: 8 total, 8 up, 8 in 2024-09-21T09:34:56.889 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:34:56.890 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":51,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T09:34:54.576537+0000","last_up_change":"2024-09-21T09:34:52.573445+0000","last_in_change":"2024-09-21T09:34:35.544589+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6803","nonce":314719645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6805","nonce":314719645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6809","nonce":314719645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6807","nonce":314719645}]},"public_addr":"172.21.15.17:6803/314719645","cluster_addr":"172.21.15.17:6805/314719645","heartbeat_back_addr":"172.21.15.17:6809/314719645","heartbeat_front_addr":"172.21.15.17:6807/314719645","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6811","nonce":1139898755}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6813","nonce":1139898755}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6817","nonce":1139898755}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6815","nonce":1139898755}]},"public_addr":"172.21.15.17:6811/1139898755","cluster_addr":"172.21.15.17:6813/1139898755","heartbeat_back_addr":"172.21.15.17:6817/1139898755","heartbeat_front_addr":"172.21.15.17:6815/1139898755","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6819","nonce":3614951048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6821","nonce":3614951048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6825","nonce":3614951048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6823","nonce":3614951048}]},"public_addr":"172.21.15.17:6819/3614951048","cluster_addr":"172.21.15.17:6821/3614951048","heartbeat_back_addr":"172.21.15.17:6825/3614951048","heartbeat_front_addr":"172.21.15.17:6823/3614951048","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6827","nonce":823707214}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6829","nonce":823707214}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6833","nonce":823707214}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6831","nonce":823707214}]},"public_addr":"172.21.15.17:6827/823707214","cluster_addr":"172.21.15.17:6829/823707214","heartbeat_back_addr":"172.21.15.17:6833/823707214","heartbeat_front_addr":"172.21.15.17:6831/823707214","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6801","nonce":2226434249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6803","nonce":2226434249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6807","nonce":2226434249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6805","nonce":2226434249}]},"public_addr":"172.21.15.134:6801/2226434249","cluster_addr":"172.21.15.134:6803/2226434249","heartbeat_back_addr":"172.21.15.134:6807/2226434249","heartbeat_front_addr":"172.21.15.134:6805/2226434249","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:34:57.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:57 smithi017 ceph-mon[29380]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:57.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:57 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/147177574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:34:57.555 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:57 smithi017 ceph-mon[40737]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:57.555 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:57 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/147177574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:34:57.584 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-21T09:32:54.362446+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-21T09:34:57.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-21T09:34:57.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:57 smithi134 ceph-mon[33541]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:57 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/147177574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:34:57.840 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:34:59.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:34:59 smithi017 ceph-mon[29380]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:59.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:34:59 smithi017 ceph-mon[40737]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:34:59.736 INFO:teuthology.orchestra.run.smithi017.stdout:pg_num: 1 2024-09-21T09:34:59.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:34:59 smithi134 ceph-mon[33541]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:00.364 INFO:tasks.cephadm:Adding prometheus.a on smithi134 2024-09-21T09:35:00.364 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch apply prometheus '1;smithi134=a' 2024-09-21T09:35:00.617 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:35:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:00 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2692102764' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:35:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:00 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2692102764' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:35:00.739 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:00 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2692102764' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:35:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:01 smithi017 ceph-mon[29380]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:01 smithi017 ceph-mon[40737]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:01.735 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:01 smithi134 ceph-mon[33541]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:02.398 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled prometheus update... 2024-09-21T09:35:03.018 DEBUG:teuthology.orchestra.run.smithi134:prometheus.a> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@prometheus.a.service 2024-09-21T09:35:03.020 INFO:tasks.cephadm:Adding node-exporter.a on smithi017 2024-09-21T09:35:03.021 INFO:tasks.cephadm:Adding node-exporter.b on smithi134 2024-09-21T09:35:03.021 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch apply node-exporter '2;smithi017=a;smithi134=b' 2024-09-21T09:35:03.350 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:03 smithi134 ceph-mon[33541]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:03 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:03 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:03 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:03 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:35:03.559 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:03 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[29380]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:35:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:35:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:35:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[40737]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:35:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:35:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:35:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:03 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:04 smithi017 ceph-mon[29380]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:04 smithi017 ceph-mon[29380]: Saving service prometheus spec with placement smithi134=a;count:1 2024-09-21T09:35:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:04 smithi017 ceph-mon[29380]: Deploying daemon prometheus.a on smithi134 2024-09-21T09:35:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:04 smithi017 ceph-mon[40737]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:04.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:04 smithi017 ceph-mon[40737]: Saving service prometheus spec with placement smithi134=a;count:1 2024-09-21T09:35:04.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:04 smithi017 ceph-mon[40737]: Deploying daemon prometheus.a on smithi134 2024-09-21T09:35:04.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:04 smithi134 ceph-mon[33541]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:04.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:04 smithi134 ceph-mon[33541]: Saving service prometheus spec with placement smithi134=a;count:1 2024-09-21T09:35:04.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:04 smithi134 ceph-mon[33541]: Deploying daemon prometheus.a on smithi134 2024-09-21T09:35:05.372 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:05 smithi134 ceph-mon[33541]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:05 smithi017 ceph-mon[29380]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:05 smithi017 ceph-mon[40737]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:06.008 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled node-exporter update... 2024-09-21T09:35:07.025 DEBUG:teuthology.orchestra.run.smithi017:node-exporter.a> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.a.service 2024-09-21T09:35:07.028 DEBUG:teuthology.orchestra.run.smithi134:node-exporter.b> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.b.service 2024-09-21T09:35:07.031 INFO:tasks.cephadm:Adding alertmanager.a on smithi017 2024-09-21T09:35:07.031 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch apply alertmanager '1;smithi017=a' 2024-09-21T09:35:07.308 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:35:07.428 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:07 smithi134 ceph-mon[33541]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:07.428 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:07 smithi134 ceph-mon[33541]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi017=a;smithi134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:07.428 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:07 smithi134 ceph-mon[33541]: Saving service node-exporter spec with placement smithi017=a;smithi134=b;count:2 2024-09-21T09:35:07.428 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:07 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:07 smithi017 ceph-mon[29380]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:07 smithi017 ceph-mon[29380]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi017=a;smithi134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:07 smithi017 ceph-mon[29380]: Saving service node-exporter spec with placement smithi017=a;smithi134=b;count:2 2024-09-21T09:35:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:07 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:07 smithi017 ceph-mon[40737]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:07 smithi017 ceph-mon[40737]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi017=a;smithi134=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:07 smithi017 ceph-mon[40737]: Saving service node-exporter spec with placement smithi017=a;smithi134=b;count:2 2024-09-21T09:35:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:07 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:09.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:09 smithi134 ceph-mon[33541]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:09 smithi017 ceph-mon[29380]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:09 smithi017 ceph-mon[40737]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:11.064 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:11 smithi134 ceph-mon[33541]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:11 smithi017 ceph-mon[29380]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:11 smithi017 ceph-mon[40737]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:11.494 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled alertmanager update... 2024-09-21T09:35:12.129 DEBUG:teuthology.orchestra.run.smithi017:alertmanager.a> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@alertmanager.a.service 2024-09-21T09:35:12.132 INFO:tasks.cephadm:Adding grafana.a on smithi134 2024-09-21T09:35:12.132 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph orch apply grafana '1;smithi134=a' 2024-09-21T09:35:12.717 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:12 smithi134 ceph-mon[33541]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi017=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:12.717 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:12 smithi134 ceph-mon[33541]: Saving service alertmanager spec with placement smithi017=a;count:1 2024-09-21T09:35:12.717 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:12 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:12.717 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:12 smithi134 ceph-mon[33541]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:12 smithi017 ceph-mon[29380]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi017=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:12 smithi017 ceph-mon[29380]: Saving service alertmanager spec with placement smithi017=a;count:1 2024-09-21T09:35:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:12 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:12 smithi017 ceph-mon[29380]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:12 smithi017 ceph-mon[40737]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi017=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:12 smithi017 ceph-mon[40737]: Saving service alertmanager spec with placement smithi017=a;count:1 2024-09-21T09:35:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:12 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:12 smithi017 ceph-mon[40737]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:14.338 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:35:15.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:14 smithi134 ceph-mon[33541]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:14 smithi017 ceph-mon[29380]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:15.470 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:14 smithi017 ceph-mon[40737]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:16.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:16 smithi134 systemd[1]: Starting Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:35:16.978 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled grafana update... 2024-09-21T09:35:17.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:16 smithi134 ceph-mon[33541]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:17.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:16 smithi134 podman[62231]: 2024-09-21 09:35:16.867821473 +0000 UTC m=+0.246233819 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:35:17.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:16 smithi134 podman[62231]: 2024-09-21 09:35:16.991704855 +0000 UTC m=+0.370117185 container create 32110c11daeb298a5c43bd5c2a8dd01b37b0bb40fc3492c355b9eda7b4ce966a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:16 smithi017 ceph-mon[29380]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:16 smithi017 ceph-mon[40737]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:17.605 DEBUG:teuthology.orchestra.run.smithi134:grafana.a> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@grafana.a.service 2024-09-21T09:35:17.607 INFO:tasks.cephadm:Setting up client nodes... 2024-09-21T09:35:17.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:35:17.872 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:18.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-mon[33541]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:18.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-mon[33541]: Saving service grafana spec with placement smithi134=a;count:1 2024-09-21T09:35:18.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:18.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 podman[62231]: 2024-09-21 09:35:17.856625502 +0000 UTC m=+1.235037841 container init 32110c11daeb298a5c43bd5c2a8dd01b37b0bb40fc3492c355b9eda7b4ce966a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:18.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 podman[62231]: 2024-09-21 09:35:17.861892898 +0000 UTC m=+1.240305228 container start 32110c11daeb298a5c43bd5c2a8dd01b37b0bb40fc3492c355b9eda7b4ce966a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:18.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.886Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:35:18.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.886Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:35:18.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.886Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi134 (none))" 2024-09-21T09:35:18.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.886Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.886Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.887Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.887Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.889Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.889Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.891Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.891Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.406µs 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.891Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.891Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.891Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.551µs wal_replay_duration=174.064µs wbl_replay_duration=485ns total_replay_duration=226.985µs 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.892Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.892Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.892Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:35:18.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.907Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.288725ms db_storage=723ns remote_storage=1.026µs web_handler=316ns query_engine=522ns scrape=375.579µs scrape_sd=15.848µs notify=639ns notify_sd=1.37µs rules=14.70436ms tracing=6.05µs 2024-09-21T09:35:18.246 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.907Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:35:18.246 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:35:17.907Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:35:18.246 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 bash[62231]: 32110c11daeb298a5c43bd5c2a8dd01b37b0bb40fc3492c355b9eda7b4ce966a 2024-09-21T09:35:18.246 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:35:17 smithi134 systemd[1]: Started Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:35:18.263 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:17 smithi017 ceph-mon[29380]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:18.264 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:17 smithi017 ceph-mon[29380]: Saving service grafana spec with placement smithi134=a;count:1 2024-09-21T09:35:18.264 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:17 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:18.264 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:17 smithi017 ceph-mon[40737]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi134=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:18.264 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:17 smithi017 ceph-mon[40737]: Saving service grafana spec with placement smithi134=a;count:1 2024-09-21T09:35:18.264 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:17 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[29380]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[40737]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:35:19.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ignoring --setuser ceph since I am not root 2024-09-21T09:35:19.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ignoring --setgroup ceph since I am not root 2024-09-21T09:35:19.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-mgr[29655]: -- 172.21.15.17:0/2276099537 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5647ea3fa340 con 0x5647ea406c00 2024-09-21T09:35:19.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:19.179+0000 7f0e23d2fec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:35:19.293 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-mon[33541]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:19.293 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.293 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.294 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' 2024-09-21T09:35:19.294 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:35:19.294 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ignoring --setuser ceph since I am not root 2024-09-21T09:35:19.294 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ignoring --setgroup ceph since I am not root 2024-09-21T09:35:19.294 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-mgr[35626]: -- 172.21.15.134:0/89625099 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56104721a340 con 0x561047226c00 2024-09-21T09:35:19.294 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:19.173+0000 7f9340fb5ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:35:19.294 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:19.226+0000 7f9340fb5ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:35:19.294 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:19.290+0000 7f9340fb5ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:35:19.551 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:19.413+0000 7f9340fb5ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:35:19.551 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:19.548+0000 7f9340fb5ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:35:19.586 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:19.236+0000 7f0e23d2fec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:35:19.587 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:19.307+0000 7f0e23d2fec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:35:19.587 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:19.438+0000 7f0e23d2fec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:35:19.587 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:19.584+0000 7f0e23d2fec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:35:19.730 INFO:teuthology.orchestra.run.smithi017.stdout:[client.0] 2024-09-21T09:35:19.731 INFO:teuthology.orchestra.run.smithi017.stdout: key = AQBXk+5mGVZCKxAANJCkIw1DzpOZaouHQE5ONg== 2024-09-21T09:35:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-mon[29380]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:35:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-mon[29380]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-21T09:35:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/302884068' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/302884068' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:20.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-mon[40737]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:35:20.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-mon[40737]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-21T09:35:20.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/302884068' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/302884068' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:20.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:19.856+0000 7f0e23d2fec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:35:20.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:19 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:19.918+0000 7f0e23d2fec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:35:20.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:20.090+0000 7f0e23d2fec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:35:20.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:19.801+0000 7f9340fb5ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:35:20.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:19.861+0000 7f9340fb5ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:35:20.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:20.021+0000 7f9340fb5ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:35:20.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:20 smithi134 ceph-mon[33541]: from='mgr.14150 172.21.15.17:0/2134984267' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:35:20.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:20 smithi134 ceph-mon[33541]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-21T09:35:20.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:20 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/302884068' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:20.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:20 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/302884068' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:20.519 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T09:35:20.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-21T09:35:20.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-21T09:35:20.560 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:35:20.815 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:35:21.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:20.889+0000 7f9340fb5ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:35:21.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:20.955+0000 7f9340fb5ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:35:21.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:20.980+0000 7f0e23d2fec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:35:21.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:21 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:21.050+0000 7f0e23d2fec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:35:21.927 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:21 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:21.808+0000 7f0e23d2fec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:35:21.927 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:21 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:21.924+0000 7f0e23d2fec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:35:21.949 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:21.656+0000 7f9340fb5ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:35:21.949 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:21.766+0000 7f9340fb5ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:35:21.950 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:21.888+0000 7f9340fb5ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:35:22.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.050+0000 7f0e23d2fec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:35:22.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.113+0000 7f0e23d2fec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:35:22.207 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:21.947+0000 7f9340fb5ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:35:22.207 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:22.074+0000 7f9340fb5ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:35:22.207 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:22.141+0000 7f9340fb5ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:35:22.207 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:22.205+0000 7f9340fb5ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:35:22.474 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:22.343+0000 7f9340fb5ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:35:22.475 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:22.408+0000 7f9340fb5ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:35:22.475 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:22.472+0000 7f9340fb5ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:35:22.517 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.244+0000 7f0e23d2fec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:35:22.517 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.314+0000 7f0e23d2fec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:35:22.517 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.374+0000 7f0e23d2fec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:35:22.517 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.514+0000 7f0e23d2fec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:35:22.743 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:22.553+0000 7f9340fb5ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:35:22.743 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:22.610+0000 7f9340fb5ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:35:22.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:22.666+0000 7f9340fb5ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:35:22.795 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.581+0000 7f0e23d2fec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:35:22.795 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.648+0000 7f0e23d2fec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:35:22.796 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.733+0000 7f0e23d2fec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:35:22.796 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.793+0000 7f0e23d2fec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:35:23.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:22 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:22.852+0000 7f0e23d2fec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-mon[33541]: Standby manager daemon x restarted 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-mon[33541]: Standby manager daemon x started 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:23.433+0000 7f9340fb5ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: [21/Sep/2024:09:35:23] ENGINE Bus STARTING 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: CherryPy Checker: 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: The Application mounted at '' has an empty config. 2024-09-21T09:35:23.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: 2024-09-21T09:35:23.745 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: [21/Sep/2024:09:35:23] ENGINE Serving on http://:::9283 2024-09-21T09:35:23.745 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: [21/Sep/2024:09:35:23] ENGINE Bus STARTED 2024-09-21T09:35:23.840 INFO:teuthology.orchestra.run.smithi134.stdout:[client.1] 2024-09-21T09:35:23.841 INFO:teuthology.orchestra.run.smithi134.stdout: key = AQBbk+5mFiGsMRAAbAAzmyH0lwjwR+Apq6nE3A== 2024-09-21T09:35:23.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[29380]: Standby manager daemon x restarted 2024-09-21T09:35:23.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[29380]: Standby manager daemon x started 2024-09-21T09:35:23.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:35:23.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:35:23.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:35:23.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:35:23.854 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:23.635+0000 7f0e23d2fec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:35:23.854 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:35:23] ENGINE Bus STARTING 2024-09-21T09:35:23.854 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: CherryPy Checker: 2024-09-21T09:35:23.854 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: The Application mounted at '' has an empty config. 2024-09-21T09:35:23.854 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:23.854 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:35:23] ENGINE Serving on http://:::9283 2024-09-21T09:35:23.854 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:35:23] ENGINE Bus STARTED 2024-09-21T09:35:23.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[40737]: Standby manager daemon x restarted 2024-09-21T09:35:23.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[40737]: Standby manager daemon x started 2024-09-21T09:35:23.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:35:23.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:35:23.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:35:23.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:23 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/586316343' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:35:24.524 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-09-21T09:35:24.524 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-21T09:35:24.524 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-21T09:35:24.561 INFO:tasks.cephadm:Creating RBD pool 2024-09-21T09:35:24.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create rbd 8 2024-09-21T09:35:24.763 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: Active manager daemon y restarted 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: Activating manager daemon y 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: osdmap e52: 8 total, 8 up, 8 in 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: mgrmap e18: y(active, starting, since 0.00643718s), standbys: x 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:35:24.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: Manager daemon y is now available 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:24.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:24.766 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:24.766 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:24.766 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='client.? 172.21.15.134:0/1254237687' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:24.766 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:24.766 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[29380]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:24.766 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-21T09:35:24.766 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: Active manager daemon y restarted 2024-09-21T09:35:24.766 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: Activating manager daemon y 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: osdmap e52: 8 total, 8 up, 8 in 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: mgrmap e18: y(active, starting, since 0.00643718s), standbys: x 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:35:24.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: Manager daemon y is now available 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:24.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:24.769 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='client.? 172.21.15.134:0/1254237687' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:24.769 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:24.769 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:24 smithi017 ceph-mon[40737]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-21T09:35:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: Active manager daemon y restarted 2024-09-21T09:35:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: Activating manager daemon y 2024-09-21T09:35:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: osdmap e52: 8 total, 8 up, 8 in 2024-09-21T09:35:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: mgrmap e18: y(active, starting, since 0.00643718s), standbys: x 2024-09-21T09:35:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:35:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:35:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:35:24.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: Manager daemon y is now available 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='client.? 172.21.15.134:0/1254237687' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:24.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:24 smithi134 ceph-mon[33541]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:25.164 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-mon[29380]: mgrmap e19: y(active, since 1.01432s), standbys: x 2024-09-21T09:35:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-mon[29380]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-mon[40737]: mgrmap e19: y(active, since 1.01432s), standbys: x 2024-09-21T09:35:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-mon[40737]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:25.549+0000 7f0dc6af2640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:35:25] ENGINE Error in HTTPServer.serve 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: Traceback (most recent call last): 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self._connections.run(self.expiration_interval) 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self._run(expiration_interval) 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:35:25.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: s = self.context.wrap_socket( 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: return self.sslsocket_class._create( 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self.do_handshake() 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: self._sslobj.do_handshake() 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:35:25.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:35:25.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:25 smithi134 ceph-mon[33541]: mgrmap e19: y(active, since 1.01432s), standbys: x 2024-09-21T09:35:25.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:25 smithi134 ceph-mon[33541]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:25 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:26.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[29380]: [21/Sep/2024:09:35:25] ENGINE Bus STARTING 2024-09-21T09:35:26.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[29380]: [21/Sep/2024:09:35:25] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:35:26.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[29380]: [21/Sep/2024:09:35:25] ENGINE Bus STARTED 2024-09-21T09:35:26.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[29380]: [21/Sep/2024:09:35:25] ENGINE Error in HTTPServer.serve 2024-09-21T09:35:26.779 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:35:26.779 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:35:26.779 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:35:26.779 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:35:26.779 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:35:26.780 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[29380]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[29380]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[40737]: [21/Sep/2024:09:35:25] ENGINE Bus STARTING 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[40737]: [21/Sep/2024:09:35:25] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[40737]: [21/Sep/2024:09:35:25] ENGINE Bus STARTED 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[40737]: [21/Sep/2024:09:35:25] ENGINE Error in HTTPServer.serve 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.c.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:35:26.781 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:35:26.782 INFO:journalctl@ceph.mon.c.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:35:26.783 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:35:26.783 INFO:journalctl@ceph.mon.c.smithi017.stdout: self.do_handshake() 2024-09-21T09:35:26.783 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:35:26.783 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:35:26.783 INFO:journalctl@ceph.mon.c.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:35:26.783 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[40737]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:26.783 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:26.783 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:26.783 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:26 smithi017 ceph-mon[40737]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-21T09:35:26.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:26 smithi134 ceph-mon[33541]: [21/Sep/2024:09:35:25] ENGINE Bus STARTING 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:26 smithi134 ceph-mon[33541]: [21/Sep/2024:09:35:25] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:26 smithi134 ceph-mon[33541]: [21/Sep/2024:09:35:25] ENGINE Bus STARTED 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:26 smithi134 ceph-mon[33541]: [21/Sep/2024:09:35:25] ENGINE Error in HTTPServer.serve 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: Traceback (most recent call last): 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._run(expiration_interval) 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: s = self.context.wrap_socket( 2024-09-21T09:35:26.969 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout: return self.sslsocket_class._create( 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout: self.do_handshake() 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._sslobj.do_handshake() 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:26 smithi134 ceph-mon[33541]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:26 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:26 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:26.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:26 smithi134 ceph-mon[33541]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-21T09:35:27.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:35:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:35:28.145 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'rbd' created 2024-09-21T09:35:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi134 to 4016M 2024-09-21T09:35:28.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2034167005' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:35:28.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:35:28.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:35:28.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi134 to 4016M 2024-09-21T09:35:28.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.455 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2034167005' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:35:28.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:35:28.456 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:28 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:35:28.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi134 to 4016M 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2034167005' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:35:28.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:28.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:35:28.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:28 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:35:28.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- rbd pool init rbd 2024-09-21T09:35:29.146 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:35:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:35:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-21T09:35:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2034167005' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:35:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: osdmap e53: 8 total, 8 up, 8 in 2024-09-21T09:35:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-21T09:35:29.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2034167005' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:35:29.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: osdmap e53: 8 total, 8 up, 8 in 2024-09-21T09:35:29.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:29 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2034167005' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: osdmap e53: 8 total, 8 up, 8 in 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:29.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:29 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[29380]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[29380]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[29380]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[29380]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[29380]: Deploying daemon alertmanager.a on smithi017 2024-09-21T09:35:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[29380]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:35:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[40737]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[40737]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[40737]: Deploying daemon alertmanager.a on smithi017 2024-09-21T09:35:30.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:30 smithi017 ceph-mon[40737]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:35:30.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:30 smithi134 ceph-mon[33541]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:30.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:30 smithi134 ceph-mon[33541]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:30 smithi134 ceph-mon[33541]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:30 smithi134 ceph-mon[33541]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:30 smithi134 ceph-mon[33541]: Deploying daemon alertmanager.a on smithi017 2024-09-21T09:35:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:30 smithi134 ceph-mon[33541]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:35:31.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:31 smithi017 ceph-mon[29380]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:31.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:31 smithi017 ceph-mon[29380]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:35:31.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:31 smithi017 ceph-mon[29380]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:35:31.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:31 smithi017 ceph-mon[40737]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:31.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:31 smithi017 ceph-mon[40737]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:35:31.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:31 smithi017 ceph-mon[40737]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:35:31.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:31 smithi134 ceph-mon[33541]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:31.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:31 smithi134 ceph-mon[33541]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:35:31.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:31 smithi134 ceph-mon[33541]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:35:32.415 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:32 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3647907716' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:35:32.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:32 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3647907716' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:35:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:32 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3647907716' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:35:33.353 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:33 smithi017 ceph-mon[29380]: pgmap v10: 9 pgs: 4 creating+activating, 5 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T09:35:33.353 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:33 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3647907716' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:35:33.353 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:33 smithi017 ceph-mon[29380]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:35:33.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:33 smithi017 ceph-mon[40737]: pgmap v10: 9 pgs: 4 creating+activating, 5 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T09:35:33.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:33 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3647907716' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:35:33.354 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:33 smithi017 ceph-mon[40737]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:35:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:33 smithi134 ceph-mon[33541]: pgmap v10: 9 pgs: 4 creating+activating, 5 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-21T09:35:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:33 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3647907716' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:35:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:33 smithi134 ceph-mon[33541]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:35:34.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:34 smithi017 ceph-mon[29380]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:35:34.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:34 smithi017 ceph-mon[40737]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:35:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:34 smithi134 ceph-mon[33541]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:35:35.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:35 smithi017 ceph-mon[29380]: pgmap v13: 9 pgs: 4 creating+activating, 5 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 15 op/s 2024-09-21T09:35:35.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:35 smithi017 ceph-mon[29380]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:35:35.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:35 smithi017 ceph-mon[29380]: Cluster is now healthy 2024-09-21T09:35:35.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:35 smithi017 ceph-mon[29380]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:35:35.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:35 smithi017 ceph-mon[40737]: pgmap v13: 9 pgs: 4 creating+activating, 5 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 15 op/s 2024-09-21T09:35:35.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:35 smithi017 ceph-mon[40737]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:35:35.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:35 smithi017 ceph-mon[40737]: Cluster is now healthy 2024-09-21T09:35:35.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:35 smithi017 ceph-mon[40737]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:35:35.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:35 smithi134 ceph-mon[33541]: pgmap v13: 9 pgs: 4 creating+activating, 5 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 15 op/s 2024-09-21T09:35:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:35 smithi134 ceph-mon[33541]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:35:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:35 smithi134 ceph-mon[33541]: Cluster is now healthy 2024-09-21T09:35:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:35 smithi134 ceph-mon[33541]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:35:36.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph config log 1 --format=json 2024-09-21T09:35:36.907 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:37.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:37 smithi134 ceph-mon[33541]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 372 B/s wr, 0 op/s 2024-09-21T09:35:37.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:35:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:35:37.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:37 smithi017 ceph-mon[29380]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 372 B/s wr, 0 op/s 2024-09-21T09:35:37.322 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:37 smithi017 ceph-mon[40737]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 372 B/s wr, 0 op/s 2024-09-21T09:35:38.034 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:37 smithi017 podman[71441]: 2024-09-21 09:35:37.855778838 +0000 UTC m=+0.211891910 container create b8bd452e6bf978dc8a6f549f283d1bfa333ef34dcbd6ab37bd3a9efbd9eff514 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 podman[71441]: 2024-09-21 09:35:38.158402756 +0000 UTC m=+0.514515823 container init b8bd452e6bf978dc8a6f549f283d1bfa333ef34dcbd6ab37bd3a9efbd9eff514 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 podman[71441]: 2024-09-21 09:35:38.16364292 +0000 UTC m=+0.519755986 container start b8bd452e6bf978dc8a6f549f283d1bfa333ef34dcbd6ab37bd3a9efbd9eff514 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:38.176Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:38.176Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:38.177Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.17 port=9094 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:38.178Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:38.201Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:38.201Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:38.204Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:38.204Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 bash[71441]: b8bd452e6bf978dc8a6f549f283d1bfa333ef34dcbd6ab37bd3a9efbd9eff514 2024-09-21T09:35:38.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:38 smithi017 systemd[1]: Started Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:35:38.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:35:38] ENGINE Bus STOPPING 2024-09-21T09:35:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:35:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 367 B/s wr, 0 op/s 2024-09-21T09:35:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:35:38.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 367 B/s wr, 0 op/s 2024-09-21T09:35:38.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:38.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 367 B/s wr, 0 op/s 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:38 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:39.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:35:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:35:39.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:35:38] ENGINE Bus STOPPED 2024-09-21T09:35:39.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:35:38] ENGINE Bus STARTING 2024-09-21T09:35:39.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:35:38] ENGINE Serving on http://:::9283 2024-09-21T09:35:39.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:35:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:35:38] ENGINE Bus STARTED 2024-09-21T09:35:39.454 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:35:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[29380]: Deploying daemon grafana.a on smithi134 2024-09-21T09:35:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3677492823' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:35:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[40737]: Deploying daemon grafana.a on smithi134 2024-09-21T09:35:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:39.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:39.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:39 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3677492823' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:35:39.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:39 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:39.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:39 smithi134 ceph-mon[33541]: Deploying daemon grafana.a on smithi134 2024-09-21T09:35:39.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:39 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:39 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:39 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3677492823' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:35:40.357 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:40.178Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00016159s 2024-09-21T09:35:40.429 INFO:teuthology.orchestra.run.smithi017.stdout:[{"version":19,"timestamp":"2024-09-21T09:35:38.433300+0000","name":"","changes":[{"name":"mgr/mgr/dashboard/GRAFANA_API_SSL_VERIFY","new_value":"false"}]}] 2024-09-21T09:35:40.429 INFO:tasks.ceph_manager:config epoch is 19 2024-09-21T09:35:40.429 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:35:40.429 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:35:40.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:35:40.682 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:40 smithi017 ceph-mon[29380]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 311 B/s wr, 0 op/s 2024-09-21T09:35:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:40 smithi017 ceph-mon[40737]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 311 B/s wr, 0 op/s 2024-09-21T09:35:40.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:40 smithi134 ceph-mon[33541]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 311 B/s wr, 0 op/s 2024-09-21T09:35:42.397 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:35:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:42 smithi017 ceph-mon[29380]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:35:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:42 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3943795624' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:35:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:42 smithi017 ceph-mon[40737]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:35:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:42 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3943795624' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:35:42.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:42 smithi134 ceph-mon[33541]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:35:42.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:42 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3943795624' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:35:43.012 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":21,"flags":0,"active_gid":24332,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6800","nonce":3625002403},{"type":"v1","addr":"172.21.15.17:6801","nonce":3625002403}]},"active_addr":"172.21.15.17:6801/3625002403","active_change":"2024-09-21T09:35:23.638854+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3512479442}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":877591523}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3671582122}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3086297034}]}]} 2024-09-21T09:35:43.017 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:35:43.017 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:35:43.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:35:43.277 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:44 smithi017 ceph-mon[40737]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 214 B/s wr, 0 op/s 2024-09-21T09:35:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:44 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:44 smithi017 ceph-mon[29380]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 214 B/s wr, 0 op/s 2024-09-21T09:35:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:44 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:44.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:44 smithi134 ceph-mon[33541]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 214 B/s wr, 0 op/s 2024-09-21T09:35:44.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:44 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:45.114 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:35:45.114 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":59,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T09:35:37.625880+0000","last_up_change":"2024-09-21T09:34:52.573445+0000","last_in_change":"2024-09-21T09:34:35.544589+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:27.624125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6803","nonce":314719645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6805","nonce":314719645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6809","nonce":314719645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6807","nonce":314719645}]},"public_addr":"172.21.15.17:6803/314719645","cluster_addr":"172.21.15.17:6805/314719645","heartbeat_back_addr":"172.21.15.17:6809/314719645","heartbeat_front_addr":"172.21.15.17:6807/314719645","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6811","nonce":1139898755}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6813","nonce":1139898755}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6817","nonce":1139898755}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6815","nonce":1139898755}]},"public_addr":"172.21.15.17:6811/1139898755","cluster_addr":"172.21.15.17:6813/1139898755","heartbeat_back_addr":"172.21.15.17:6817/1139898755","heartbeat_front_addr":"172.21.15.17:6815/1139898755","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6819","nonce":3614951048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6821","nonce":3614951048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6825","nonce":3614951048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6823","nonce":3614951048}]},"public_addr":"172.21.15.17:6819/3614951048","cluster_addr":"172.21.15.17:6821/3614951048","heartbeat_back_addr":"172.21.15.17:6825/3614951048","heartbeat_front_addr":"172.21.15.17:6823/3614951048","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6827","nonce":823707214}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6829","nonce":823707214}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6833","nonce":823707214}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6831","nonce":823707214}]},"public_addr":"172.21.15.17:6827/823707214","cluster_addr":"172.21.15.17:6829/823707214","heartbeat_back_addr":"172.21.15.17:6833/823707214","heartbeat_front_addr":"172.21.15.17:6831/823707214","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6801","nonce":2226434249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6803","nonce":2226434249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6807","nonce":2226434249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6805","nonce":2226434249}]},"public_addr":"172.21.15.134:6801/2226434249","cluster_addr":"172.21.15.134:6803/2226434249","heartbeat_back_addr":"172.21.15.134:6807/2226434249","heartbeat_front_addr":"172.21.15.134:6805/2226434249","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/2291861207":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/3206271349":"2024-09-22T09:35:23.638701+0000","172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/2266250644":"2024-09-22T09:35:23.638701+0000","172.21.15.17:6801/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/1454017192":"2024-09-22T09:35:23.638701+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:35:45.712 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:35:45.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:35:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:45 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1247566484' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:45 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1247566484' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:45.965 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:45.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:45 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1247566484' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:46.843 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:46 smithi134 ceph-mon[33541]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:47.099 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:35:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:35:47.131 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:46 smithi017 ceph-mon[29380]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:47.131 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:46 smithi017 ceph-mon[40737]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:47.802 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:35:47.802 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":59,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T09:35:37.625880+0000","last_up_change":"2024-09-21T09:34:52.573445+0000","last_in_change":"2024-09-21T09:34:35.544589+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:27.624125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6803","nonce":314719645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6805","nonce":314719645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6809","nonce":314719645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6807","nonce":314719645}]},"public_addr":"172.21.15.17:6803/314719645","cluster_addr":"172.21.15.17:6805/314719645","heartbeat_back_addr":"172.21.15.17:6809/314719645","heartbeat_front_addr":"172.21.15.17:6807/314719645","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6811","nonce":1139898755}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6813","nonce":1139898755}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6817","nonce":1139898755}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6815","nonce":1139898755}]},"public_addr":"172.21.15.17:6811/1139898755","cluster_addr":"172.21.15.17:6813/1139898755","heartbeat_back_addr":"172.21.15.17:6817/1139898755","heartbeat_front_addr":"172.21.15.17:6815/1139898755","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6819","nonce":3614951048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6821","nonce":3614951048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6825","nonce":3614951048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6823","nonce":3614951048}]},"public_addr":"172.21.15.17:6819/3614951048","cluster_addr":"172.21.15.17:6821/3614951048","heartbeat_back_addr":"172.21.15.17:6825/3614951048","heartbeat_front_addr":"172.21.15.17:6823/3614951048","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6827","nonce":823707214}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6829","nonce":823707214}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6833","nonce":823707214}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6831","nonce":823707214}]},"public_addr":"172.21.15.17:6827/823707214","cluster_addr":"172.21.15.17:6829/823707214","heartbeat_back_addr":"172.21.15.17:6833/823707214","heartbeat_front_addr":"172.21.15.17:6831/823707214","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6801","nonce":2226434249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6803","nonce":2226434249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6807","nonce":2226434249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6805","nonce":2226434249}]},"public_addr":"172.21.15.134:6801/2226434249","cluster_addr":"172.21.15.134:6803/2226434249","heartbeat_back_addr":"172.21.15.134:6807/2226434249","heartbeat_front_addr":"172.21.15.134:6805/2226434249","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/2291861207":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/3206271349":"2024-09-22T09:35:23.638701+0000","172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/2266250644":"2024-09-22T09:35:23.638701+0000","172.21.15.17:6801/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/1454017192":"2024-09-22T09:35:23.638701+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:35:48.450 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:35:48 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:35:48.181Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003304081s 2024-09-21T09:35:48.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:35:48.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:35:48.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:35:48.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:35:48.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:35:48.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:35:48.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:35:48.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:35:48.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:48 smithi017 ceph-mon[29380]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:48.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:48 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3270681522' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:48.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:48 smithi017 ceph-mon[40737]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:48.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:48 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3270681522' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:48.983 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:48 smithi134 ceph-mon[33541]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:48.983 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:48 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3270681522' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:48.983 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:48 smithi134 podman[64393]: 2024-09-21 09:35:48.842842185 +0000 UTC m=+0.120697708 container create f7936515cf4ed7a70e4d4d3fa60ecf6632e8b7e3abf494992adb14ae1060f9f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, version=8.5, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., distribution-scope=public, io.openshift.tags=base rhel8, io.openshift.expose-services=, com.redhat.component=ubi8-container, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container) 2024-09-21T09:35:49.126 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:49.127 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:49.128 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:49.128 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:49.129 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:49.131 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:49.133 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:49.161 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:49.330 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 podman[64393]: 2024-09-21 09:35:49.078312266 +0000 UTC m=+0.356167758 container init f7936515cf4ed7a70e4d4d3fa60ecf6632e8b7e3abf494992adb14ae1060f9f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, io.buildah.version=1.32.0, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.openshift.expose-services=, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-09-21T09:35:49.330 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 podman[64393]: 2024-09-21 09:35:49.083228195 +0000 UTC m=+0.361083676 container start f7936515cf4ed7a70e4d4d3fa60ecf6632e8b7e3abf494992adb14ae1060f9f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, architecture=x86_64, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.openshift.expose-services=, io.buildah.version=1.32.0, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-09-21T09:35:49.330 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 bash[64393]: f7936515cf4ed7a70e4d4d3fa60ecf6632e8b7e3abf494992adb14ae1060f9f6 2024-09-21T09:35:49.330 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 systemd[1]: Started Ceph grafana.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:35:49.330 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236604832Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236857465Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236891259Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.23690408Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236914581Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236925168Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236934698Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236945082Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236955734Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236965812Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236984197Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.236994338Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.237004388Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:35:49.331 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=settings t=2024-09-21T09:35:49.237015127Z level=info msg="App mode production" 2024-09-21T09:35:49.332 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=sqlstore t=2024-09-21T09:35:49.237080477Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:35:49.332 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=sqlstore t=2024-09-21T09:35:49.237104288Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:35:49.332 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.23838728Z level=info msg="Starting DB migrations" 2024-09-21T09:35:49.332 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.238505216Z level=info msg="Executing migration" id="create migration_log table" 2024-09-21T09:35:49.332 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.239035704Z level=info msg="Executing migration" id="create user table" 2024-09-21T09:35:49.332 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.23946558Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-21T09:35:49.332 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.242519105Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-21T09:35:49.332 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.244973206Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-21T09:35:49.332 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.247579541Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.250345147Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.253259949Z level=info msg="Executing migration" id="create user table v2" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.25365764Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.256412928Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.256849239Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.25963177Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.262374113Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.265076383Z level=info msg="Executing migration" id="Update user table charset" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.267811852Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-21T09:35:49.333 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.270630987Z level=info msg="Executing migration" id="Add missing user data" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.273538095Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.276249079Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.278935492Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.281624172Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.284271621Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.287009231Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.289634617Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.292378172Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.29497559Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.297590649Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.300348402Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.300760786Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-21T09:35:49.334 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.303461131Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.306133018Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.308803782Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.309469651Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.31219687Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.312619809Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.314909222Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.317627308Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.318097687Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.318955856Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.319828938Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.320535339Z level=info msg="Executing migration" id="create star table" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.321389053Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-21T09:35:49.335 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.321954392Z level=info msg="Executing migration" id="create org table v1" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.322370914Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.323153105Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.324098829Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.324739168Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.325553496Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.326504188Z level=info msg="Executing migration" id="Update org table charset" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.327156864Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.327977918Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.328787994Z level=info msg="Executing migration" id="create dashboard table" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.329698603Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-21T09:35:49.336 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.330515993Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-21T09:35:49.580 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.331308412Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-21T09:35:49.580 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.332162507Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-21T09:35:49.580 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.333003771Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-21T09:35:49.580 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.333863111Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-21T09:35:49.581 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.334718235Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-21T09:35:49.581 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.3355534Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-21T09:35:49.581 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.336394638Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-21T09:35:49.581 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.337274482Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-21T09:35:49.581 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.338093964Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-21T09:35:49.581 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.33892981Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-21T09:35:49.582 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.339749718Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-21T09:35:49.582 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.340631085Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-21T09:35:49.582 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.341637036Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-21T09:35:49.582 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.342312242Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-21T09:35:49.582 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.343249436Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-21T09:35:49.582 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.343997973Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-21T09:35:49.582 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.344966387Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-21T09:35:49.582 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.345715756Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-21T09:35:49.582 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.348474278Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-21T09:35:49.584 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.348860823Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-21T09:35:49.584 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.351421588Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-21T09:35:49.584 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.354044664Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-21T09:35:49.584 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.35692025Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-21T09:35:49.584 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.359731614Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-21T09:35:49.584 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.362565852Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-21T09:35:49.584 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.365448435Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-21T09:35:49.584 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.36796229Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.370774314Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.373505553Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.376212617Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.378977068Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.381480278Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.381921659Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.382846905Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.383507288Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.384272798Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.384789232Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.385200349Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.386080589Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-21T09:35:49.585 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.386972661Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.387864928Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.388584893Z level=info msg="Executing migration" id="create data_source table" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.389519674Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.390150626Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.390554538Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.39137753Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.391773912Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.392792757Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.393769335Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.394511008Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.395488272Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.396274682Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.39704618Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-21T09:35:49.586 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.398012699Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.398713525Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.399664538Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.400424217Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.401373866Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.402195654Z level=info msg="Executing migration" id="Add uid column" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.402948244Z level=info msg="Executing migration" id="Update uid value" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.403927405Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.404589313Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.405578111Z level=info msg="Executing migration" id="create api_key table" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.408275459Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.411120539Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.413588335Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.416456782Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-21T09:35:49.587 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.419092739Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.421794975Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.422246152Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.425016369Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.427706878Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.430304495Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.430875889Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.433446459Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.436306979Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.436681411Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.436946109Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.437496599Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.439684907Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-21T09:35:49.588 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.442491817Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.445067379Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.445717446Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.448420386Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.451138973Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.453840641Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.454330773Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.456913638Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.459575898Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.462026056Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.462530824Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.465258805Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.467943571Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.470645764Z level=info msg="Executing migration" id="create quota table v1" 2024-09-21T09:35:49.589 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.471171017Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.473969855Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.476434891Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.47687814Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.477317529Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.479950039Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.480277256Z level=info msg="Executing migration" id="create session table" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.480749132Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.481091705Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.481399057Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.481803772Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.48256393Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.482857466Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-21T09:35:49.590 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.483196238Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.483484097Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.483772847Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.484199474Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.484980617Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.485624492Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.485914806Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.486549254Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.48715139Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.487898101Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.488322585Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.489137644Z level=info msg="Executing migration" id="create alert table v1" 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.489597292Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-21T09:35:49.591 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.490379079Z level=info msg="Executing migration" id="add index alert state" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.49115896Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.491602998Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.491970611Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.492381727Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.492764162Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.494177382Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.49454904Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.494980421Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.495296932Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.495634336Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.496019244Z level=info msg="Executing migration" id="Add column is_default" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.496632166Z level=info msg="Executing migration" id="Add column frequency" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.497299179Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.497938646Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-21T09:35:49.592 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.498589486Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.499008949Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.499293502Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.499557191Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.499970087Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.500392338Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.50074254Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.501134318Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.501541343Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.502185295Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.502812916Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.503132565Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.503540811Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-21T09:35:49.593 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.503914329Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.504588805Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.504853791Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.505278995Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.505681557Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.505970773Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.506397341Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.507148175Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.507575093Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.50799964Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.508468315Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.509260016Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.510042563Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.510747068Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-21T09:35:49.594 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.511233451Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.511988837Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.512777018Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.513837428Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.514384436Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.516028004Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.516913975Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.517972885Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.518356285Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.51915237Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.519500999Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.520225478Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.521188646Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-21T09:35:49.595 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.522313316Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.524984672Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.527798833Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.530461948Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.530925186Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.533513923Z level=info msg="Executing migration" id="Move region to single row" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.536134668Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.538984848Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.539460677Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.542068686Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.542540186Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.544736317Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.545260415Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.547614912Z level=info msg="Executing migration" id="create test_data table" 2024-09-21T09:35:49.596 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.548282379Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-21T09:35:49.597 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.550774697Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-21T09:35:49.597 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.55356702Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-21T09:35:49.597 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.556080222Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-21T09:35:49.597 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.55640147Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-21T09:35:49.597 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.55720443Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-21T09:35:49.597 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.557536407Z level=info msg="Executing migration" id="create team table" 2024-09-21T09:35:49.597 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.558068595Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-21T09:35:49.597 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.558848329Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-21T09:35:49.597 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.55932223Z level=info msg="Executing migration" id="create team member table" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.560166387Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.561011241Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.561787392Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.562277509Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.56303988Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.563853704Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.564643635Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.565070786Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.56596522Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.566655531Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.56761121Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.568305238Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.569132669Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-21T09:35:49.598 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.570063505Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.570806069Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.57173022Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.572460362Z level=info msg="Executing migration" id="create tag table" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.573326444Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.574204437Z level=info msg="Executing migration" id="create login attempt table" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.575144476Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.575849443Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.576805959Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.578960452Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.579782524Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.580607354Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.581425734Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.582276867Z level=info msg="Executing migration" id="create user auth table" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.583120059Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-21T09:35:49.599 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.583955137Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-21T09:35:49.831 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.586724685Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-21T09:35:49.831 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.589502674Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-21T09:35:49.831 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.592304641Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-21T09:35:49.832 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.595098658Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-21T09:35:49.832 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.597803326Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-21T09:35:49.832 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.60067924Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-21T09:35:49.832 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.60340439Z level=info msg="Executing migration" id="create server_lock table" 2024-09-21T09:35:49.832 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.606176932Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-21T09:35:49.832 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.608948337Z level=info msg="Executing migration" id="create user auth token table" 2024-09-21T09:35:49.833 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.611548581Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-21T09:35:49.833 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.614295396Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-21T09:35:49.833 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.617150468Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-21T09:35:49.833 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.619660483Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-21T09:35:49.833 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.622352064Z level=info msg="Executing migration" id="create cache_data table" 2024-09-21T09:35:49.833 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.625074004Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-21T09:35:49.833 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.627925027Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-21T09:35:49.833 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.630589251Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-21T09:35:49.834 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.633300083Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-21T09:35:49.834 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.63592228Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-21T09:35:49.834 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.638781339Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-21T09:35:49.834 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.641394603Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-21T09:35:49.834 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.644113485Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-21T09:35:49.834 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.64671549Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-21T09:35:49.835 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.649520906Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-21T09:35:49.835 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.652044451Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-21T09:35:49.835 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.654725463Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-21T09:35:49.835 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.65729086Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-21T09:35:49.835 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.660013271Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-21T09:35:49.835 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.662690841Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-21T09:35:49.835 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.665504584Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-21T09:35:49.836 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.667785348Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-21T09:35:49.836 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.670549537Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-21T09:35:49.836 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.673082493Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-21T09:35:49.836 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.675746687Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-21T09:35:49.836 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.678459582Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-21T09:35:49.836 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.679126789Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-21T09:35:49.837 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.680094722Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-21T09:35:49.837 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.680762613Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-21T09:35:49.837 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.68161608Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-21T09:35:49.837 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.682443694Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-21T09:35:49.837 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.683287854Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-21T09:35:49.837 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.684176875Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-21T09:35:49.837 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.687623321Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-21T09:35:49.837 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.691062572Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-21T09:35:49.838 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.691865601Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-21T09:35:49.838 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.692624688Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-21T09:35:49.838 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.693522143Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-21T09:35:49.838 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.694320936Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-21T09:35:49.838 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.695152557Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-21T09:35:49.838 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.69612194Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-21T09:35:49.839 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.696792758Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-21T09:35:49.839 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.697710121Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-21T09:35:49.839 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.698605168Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-21T09:35:49.839 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.699537136Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-21T09:35:49.840 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.700406856Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-21T09:35:49.840 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.701280003Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-21T09:35:49.841 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.702046972Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-21T09:35:49.841 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.70292289Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-21T09:35:49.841 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.703746755Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-21T09:35:49.841 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.704625553Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-21T09:35:49.841 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.705484675Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-21T09:35:49.841 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.706353892Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-21T09:35:49.841 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.707256466Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-21T09:35:49.842 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.7080757Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-21T09:35:49.842 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.709026293Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-21T09:35:49.842 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.709752064Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-21T09:35:49.842 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.710682588Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-21T09:35:49.842 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.711665042Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-21T09:35:49.842 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.71253922Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-21T09:35:49.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.713410398Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-21T09:35:49.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.716267913Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-21T09:35:49.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.719134707Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-21T09:35:49.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.72172578Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-21T09:35:49.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.724506415Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-21T09:35:49.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.7271086Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-21T09:35:49.843 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.72988414Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-21T09:35:49.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.732694728Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-21T09:35:49.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.735443461Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-21T09:35:49.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.736496136Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-21T09:35:49.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.737305867Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-21T09:35:49.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.738286237Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-21T09:35:49.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.740965372Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-21T09:35:49.844 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.743752437Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-21T09:35:49.845 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.746445773Z level=info msg="Executing migration" id="create alert_image table" 2024-09-21T09:35:49.845 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.748903382Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-21T09:35:49.845 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.751649872Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-21T09:35:49.845 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.75437111Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-21T09:35:49.845 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.754841231Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-21T09:35:49.845 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.757694475Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:35:49.845 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.757829851Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:35:49.846 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.760216267Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-21T09:35:49.846 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.762833258Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-21T09:35:49.846 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.765697754Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-21T09:35:49.846 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.765845181Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-21T09:35:49.846 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.765966346Z level=info msg="alerts found to migrate" alerts=0 2024-09-21T09:35:49.846 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.76843542Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-21T09:35:49.847 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.771148471Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-21T09:35:49.847 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.773841017Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-21T09:35:49.847 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.776385962Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-21T09:35:49.847 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.779213457Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-21T09:35:49.847 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.782101694Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-21T09:35:49.847 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.784688979Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-21T09:35:49.847 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.787366184Z level=info msg="Executing migration" id="create data_keys table" 2024-09-21T09:35:49.848 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.789982162Z level=info msg="Executing migration" id="create secrets table" 2024-09-21T09:35:49.848 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.792639149Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-21T09:35:49.848 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.796976089Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-21T09:35:49.848 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.799726282Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-21T09:35:49.848 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.8024742Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-21T09:35:49.848 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.806811665Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-21T09:35:49.848 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.811068077Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-21T09:35:49.848 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.813720297Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-21T09:35:49.849 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.816425162Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-21T09:35:49.849 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.818933054Z level=info msg="Executing migration" id="create permission table" 2024-09-21T09:35:49.849 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.821674626Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-21T09:35:49.849 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.824099072Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-21T09:35:49.849 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.826456763Z level=info msg="Executing migration" id="create role table" 2024-09-21T09:35:49.849 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.829264208Z level=info msg="Executing migration" id="add column display_name" 2024-09-21T09:35:49.849 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.831903826Z level=info msg="Executing migration" id="add column group_name" 2024-09-21T09:35:50.239 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.834419544Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-21T09:35:50.239 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.837032499Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-21T09:35:50.239 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.837845447Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-21T09:35:50.239 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.838663545Z level=info msg="Executing migration" id="create team role table" 2024-09-21T09:35:50.239 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.839447346Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-21T09:35:50.240 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.84023974Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-21T09:35:50.240 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.841020895Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-21T09:35:50.240 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.841834894Z level=info msg="Executing migration" id="create user role table" 2024-09-21T09:35:50.240 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.842651183Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-21T09:35:50.240 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.843442201Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-21T09:35:50.240 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.844226844Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-21T09:35:50.241 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.845083153Z level=info msg="Executing migration" id="create builtin role table" 2024-09-21T09:35:50.241 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.845868781Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-21T09:35:50.241 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.846722556Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-21T09:35:50.241 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.847508586Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-21T09:35:50.241 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.848538893Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-21T09:35:50.242 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.849054964Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-21T09:35:50.242 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.849845161Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-21T09:35:50.242 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.850284754Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-21T09:35:50.242 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.850678108Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-21T09:35:50.242 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.851066302Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-21T09:35:50.242 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.851489988Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-21T09:35:50.243 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.852530856Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-21T09:35:50.243 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.852930573Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-21T09:35:50.243 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.85336074Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-21T09:35:50.243 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.853667716Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-21T09:35:50.243 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.85407269Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-21T09:35:50.243 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.854528987Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-21T09:35:50.243 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.854859412Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-21T09:35:50.244 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.855171492Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-21T09:35:50.244 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.855485502Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-21T09:35:50.244 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.855865366Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-21T09:35:50.244 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.856304958Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-21T09:35:50.244 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.857292293Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-21T09:35:50.244 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.857583382Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-21T09:35:50.244 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.857986582Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-21T09:35:50.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.858405399Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-21T09:35:50.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.858820747Z level=info msg="Executing migration" id="add correlation config column" 2024-09-21T09:35:50.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.859814079Z level=info msg="Executing migration" id="create entity_events table" 2024-09-21T09:35:50.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.860457811Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-21T09:35:50.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.861184497Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:35:50.245 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.861291403Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:35:50.246 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.861604619Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:35:50.246 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.861698493Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:35:50.246 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.861983366Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-21T09:35:50.246 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.862438318Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-21T09:35:50.246 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.862826278Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:35:50.246 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.863312431Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:35:50.247 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.86375195Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:35:50.247 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.864190861Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:35:50.247 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.864628931Z level=info msg="Executing migration" id="Drop public config table" 2024-09-21T09:35:50.247 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.865060362Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-21T09:35:50.247 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.865494253Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:35:50.247 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.865921792Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:35:50.248 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.866443026Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-21T09:35:50.248 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.866935658Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-21T09:35:50.249 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.870070919Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-21T09:35:50.249 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.8711334Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-21T09:35:50.249 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.872210647Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-21T09:35:50.249 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.872533222Z level=info msg="Executing migration" id="add share column" 2024-09-21T09:35:50.249 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.873585828Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-21T09:35:50.250 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.874101097Z level=info msg="Executing migration" id="create file table" 2024-09-21T09:35:50.250 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.874579792Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-21T09:35:50.250 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.87505726Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-21T09:35:50.250 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.875581179Z level=info msg="Executing migration" id="create file_meta table" 2024-09-21T09:35:50.250 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.876463136Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-21T09:35:50.250 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.877318176Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-21T09:35:50.251 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.878288317Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-21T09:35:50.251 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.879040816Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-21T09:35:50.251 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.879824658Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-21T09:35:50.251 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.880627826Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-21T09:35:50.251 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.8816725Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-21T09:35:50.251 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.882644539Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-21T09:35:50.252 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.883393032Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-21T09:35:50.252 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.884335881Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-21T09:35:50.252 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.885074636Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-21T09:35:50.252 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.885953987Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-21T09:35:50.252 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.887145309Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-21T09:35:50.252 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.888189267Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-21T09:35:50.252 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.88897013Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-21T09:35:50.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.899220329Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-21T09:35:50.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.901997312Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-21T09:35:50.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.904671994Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-21T09:35:50.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.90838236Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-21T09:35:50.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.911049669Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-21T09:35:50.253 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.913967623Z level=info msg="Executing migration" id="create folder table" 2024-09-21T09:35:50.254 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.916755972Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-21T09:35:50.254 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.919350482Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-21T09:35:50.254 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.922082348Z level=info msg="Executing migration" id="Update folder title length" 2024-09-21T09:35:50.254 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.924825359Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-21T09:35:50.254 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=migrator t=2024-09-21T09:35:49.927548528Z level=info msg="migrations completed" performed=485 skipped=0 duration=689.078291ms 2024-09-21T09:35:50.254 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=sqlstore t=2024-09-21T09:35:49.928001411Z level=info msg="Created default organization" 2024-09-21T09:35:50.255 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:49 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=plugin.loader t=2024-09-21T09:35:49.984717295Z level=info msg="Plugin registered" pluginID=input 2024-09-21T09:35:50.255 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=plugin.loader t=2024-09-21T09:35:50.011702329Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-21T09:35:50.255 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=plugin.loader t=2024-09-21T09:35:50.011719543Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-21T09:35:50.255 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=secrets t=2024-09-21T09:35:50.011770853Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:35:50.255 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=query_data t=2024-09-21T09:35:50.013407507Z level=info msg="Query Service initialization" 2024-09-21T09:35:50.256 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=live.push_http t=2024-09-21T09:35:50.015725126Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:35:50.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[29380]: Deploying daemon node-exporter.a on smithi017 2024-09-21T09:35:50.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[29380]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:50.422 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.422 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.422 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.422 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.422 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[40737]: Deploying daemon node-exporter.a on smithi017 2024-09-21T09:35:50.422 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:50 smithi017 ceph-mon[40737]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:50.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-mon[33541]: Deploying daemon node-exporter.a on smithi017 2024-09-21T09:35:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-mon[33541]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:50.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=infra.usagestats.collector t=2024-09-21T09:35:50.722640936Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:35:50.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=provisioning.datasources t=2024-09-21T09:35:50.722943596Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-21T09:35:50.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=provisioning.datasources t=2024-09-21T09:35:50.756608693Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:35:50.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=provisioning.datasources t=2024-09-21T09:35:50.788282783Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-21T09:35:50.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=provisioning.alerting t=2024-09-21T09:35:50.809568848Z level=info msg="starting to provision alerting" 2024-09-21T09:35:50.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=provisioning.alerting t=2024-09-21T09:35:50.809593835Z level=info msg="finished to provision alerting" 2024-09-21T09:35:50.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=grafanaStorageLogger t=2024-09-21T09:35:50.809733566Z level=info msg="storage starting" 2024-09-21T09:35:50.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=ngalert.state.manager t=2024-09-21T09:35:50.809762414Z level=info msg="Warming state cache for startup" 2024-09-21T09:35:50.995 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=http.server t=2024-09-21T09:35:50.811937432Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:35:50.995 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=sqlstore.transactions t=2024-09-21T09:35:50.820063924Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:35:50.995 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=ngalert.state.manager t=2024-09-21T09:35:50.820174152Z level=info msg="State cache has been initialized" states=0 duration=10.412062ms 2024-09-21T09:35:50.995 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:35:50.820213727Z level=info msg="starting MultiOrg Alertmanager" 2024-09-21T09:35:50.995 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=ticker t=2024-09-21T09:35:50.820233451Z level=info msg=starting first_tick=2024-09-21T09:36:00Z 2024-09-21T09:35:50.995 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=sqlstore.transactions t=2024-09-21T09:35:50.853156892Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:35:50.995 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=sqlstore.transactions t=2024-09-21T09:35:50.894671175Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:35:50.995 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:35:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[64408]: logger=sqlstore.transactions t=2024-09-21T09:35:50.909056782Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-21T09:35:52.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:52 smithi134 ceph-mon[33541]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:52 smithi017 ceph-mon[29380]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:52 smithi017 ceph-mon[40737]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:53.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:53 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:53.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:53 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:53.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:53 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:53.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:53 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:53 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:53 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:35:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:54 smithi017 ceph-mon[29380]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:54 smithi017 ceph-mon[40737]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:54.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:54 smithi134 ceph-mon[33541]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:56.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:56 smithi134 ceph-mon[33541]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:56.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:35:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:35:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:35:57.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:56 smithi017 ceph-mon[29380]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:57.015 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:56 smithi017 ceph-mon[40737]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:57.044 INFO:teuthology.orchestra.run.smithi017.stdout:55834574892 2024-09-21T09:35:57.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:35:58.561 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:35:58 smithi017 systemd[1]: Starting Ceph node-exporter.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:35:58.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:35:58 smithi017 ceph-mon[29380]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:58.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:35:58 smithi017 ceph-mon[40737]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:35:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:35:58 smithi134 ceph-mon[33541]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:00.140 INFO:teuthology.orchestra.run.smithi017.stdout:133143986204 2024-09-21T09:36:00.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:36:00.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:00 smithi017 ceph-mon[29380]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:00.782 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:00 smithi017 ceph-mon[40737]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:00.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:00 smithi134 ceph-mon[33541]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:02.428 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:02 smithi017 bash[73918]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T09:36:02.451 INFO:teuthology.orchestra.run.smithi017.stdout:184683593749 2024-09-21T09:36:02.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:36:02.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:02 smithi017 bash[73918]: Getting image source signatures 2024-09-21T09:36:02.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:02 smithi017 bash[73918]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T09:36:02.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:02 smithi017 bash[73918]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T09:36:02.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:02 smithi017 bash[73918]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T09:36:02.984 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:02 smithi017 ceph-mon[29380]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:02.985 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:02 smithi017 ceph-mon[40737]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:02.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:02 smithi134 ceph-mon[33541]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:04.061 INFO:teuthology.orchestra.run.smithi017.stdout:34359738417 2024-09-21T09:36:04.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:36:04.626 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:04 smithi017 bash[73918]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T09:36:04.692 INFO:teuthology.orchestra.run.smithi017.stdout:158913789977 2024-09-21T09:36:04.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:36:04.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:04 smithi017 ceph-mon[29380]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:04.896 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:04 smithi017 ceph-mon[40737]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:04 smithi134 ceph-mon[33541]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:05.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:04 smithi017 bash[73918]: Writing manifest to image destination 2024-09-21T09:36:05.450 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:05 smithi017 podman[73918]: 2024-09-21 09:36:05.243456163 +0000 UTC m=+3.452109999 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:36:06.009 INFO:teuthology.orchestra.run.smithi017.stdout:210453397520 2024-09-21T09:36:06.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:36:06.331 INFO:teuthology.orchestra.run.smithi017.stdout:77309411368 2024-09-21T09:36:06.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:36:06.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:06 smithi134 ceph-mon[33541]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:06.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:36:06 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:36:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:36:07.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:06 smithi017 ceph-mon[29380]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:07.168 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:06 smithi017 ceph-mon[40737]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:07.168 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:06 smithi017 podman[73918]: 2024-09-21 09:36:06.716537165 +0000 UTC m=+4.925190999 container create c1d4600d9989ee43391f76460eaa0db194787373f329a876cf128112192592b1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:07.168 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 podman[73918]: 2024-09-21 09:36:07.166934545 +0000 UTC m=+5.375588394 container init c1d4600d9989ee43391f76460eaa0db194787373f329a876cf128112192592b1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:07.395 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:07.409 INFO:teuthology.orchestra.run.smithi017.stdout:111669149733 2024-09-21T09:36:07.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:36:07.433 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 podman[73918]: 2024-09-21 09:36:07.172129848 +0000 UTC m=+5.380783682 container start c1d4600d9989ee43391f76460eaa0db194787373f329a876cf128112192592b1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:07.433 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.176Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:36:07.433 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.176Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:36:07.433 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.176Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:36:07.433 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:36:07.433 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:36:07.433 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:36:07.433 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:36:07.433 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:36:07.434 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:36:07.434 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:36:07.434 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:36:07.434 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:36:07.434 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:36:07.434 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:36:07.434 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:36:07.434 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:36:07.434 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.177Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:36:07.435 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:36:07.436 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[74466]: ts=2024-09-21T09:36:07.178Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 bash[73918]: c1d4600d9989ee43391f76460eaa0db194787373f329a876cf128112192592b1 2024-09-21T09:36:07.437 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:36:07 smithi017 systemd[1]: Started Ceph node-exporter.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:08.171 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:08.180 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:08.183 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:08.185 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:08.340 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:08 smithi134 bash[64685]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[29380]: Deploying daemon node-exporter.b on smithi134 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[29380]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[40737]: Deploying daemon node-exporter.b on smithi134 2024-09-21T09:36:08.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:08 smithi017 ceph-mon[40737]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:08.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:08 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:08 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:08 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:08 smithi134 ceph-mon[33541]: Deploying daemon node-exporter.b on smithi134 2024-09-21T09:36:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:08 smithi134 ceph-mon[33541]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:08.744 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:08 smithi134 bash[64685]: Getting image source signatures 2024-09-21T09:36:08.744 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:08 smithi134 bash[64685]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T09:36:08.744 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:08 smithi134 bash[64685]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T09:36:08.744 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:08 smithi134 bash[64685]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T09:36:09.405 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:09 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:09 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:09.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:09 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:10.353 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 bash[64685]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T09:36:10.353 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 bash[64685]: Writing manifest to image destination 2024-09-21T09:36:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:10 smithi017 ceph-mon[29380]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:10.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:10 smithi017 ceph-mon[40737]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:10.608 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-mon[33541]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:10.608 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 podman[64685]: 2024-09-21 09:36:10.498596473 +0000 UTC m=+2.367113118 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:36:10.608 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 podman[64685]: 2024-09-21 09:36:10.58875114 +0000 UTC m=+2.457267780 container create a405017b5317f8f31d672a2169654fc775b2c5e64868cc26525c3dfac44f785c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:36:10.901 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 podman[64685]: 2024-09-21 09:36:10.807960506 +0000 UTC m=+2.676477165 container init a405017b5317f8f31d672a2169654fc775b2c5e64868cc26525c3dfac44f785c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:36:10.901 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 podman[64685]: 2024-09-21 09:36:10.81334223 +0000 UTC m=+2.681858874 container start a405017b5317f8f31d672a2169654fc775b2c5e64868cc26525c3dfac44f785c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:36:10.901 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.817Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:36:10.901 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.817Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:36:10.901 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.817Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:36:10.901 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:36:10.901 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:36:10.901 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:36:10.901 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:36:10.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:36:10.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:36:10.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:36:10.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:36:10.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:36:10.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:36:10.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:36:10.902 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:36:10.903 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:36:10.904 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.818Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.819Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.819Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.819Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[64761]: ts=2024-09-21T09:36:10.819Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:36:10.905 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 bash[64685]: a405017b5317f8f31d672a2169654fc775b2c5e64868cc26525c3dfac44f785c 2024-09-21T09:36:11.243 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:36:10 smithi134 systemd[1]: Started Ceph node-exporter.b for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:11 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:12.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:11 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:11 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:11 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:11 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:11 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:11 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:12.535 INFO:teuthology.orchestra.run.smithi017.stdout:34359738419 2024-09-21T09:36:13.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:12 smithi017 ceph-mon[29380]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:13.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:12 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/584410206' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:36:13.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:12 smithi017 ceph-mon[40737]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:13.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:12 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/584410206' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:36:13.121 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738417 got 34359738419 for osd.0 2024-09-21T09:36:13.122 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:13.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:12 smithi134 ceph-mon[33541]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:13.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:12 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/584410206' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:36:13.721 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:13.918 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:14.732 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:14 smithi017 ceph-mon[29380]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:14.732 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:14 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:14.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:14 smithi017 ceph-mon[40737]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:14.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:14 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:14.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:14 smithi134 ceph-mon[33541]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:14.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:14 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:15 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:15 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:15 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2258041129' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:36:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:15 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:15 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:15 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2258041129' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:36:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:15 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:15 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:15 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2258041129' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:36:16.600 INFO:teuthology.orchestra.run.smithi017.stdout:184683593752 2024-09-21T09:36:16.942 INFO:teuthology.orchestra.run.smithi017.stdout:55834574895 2024-09-21T09:36:16.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:16 smithi017 ceph-mon[29380]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:16.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:16 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2892180370' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:36:16.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:16 smithi017 ceph-mon[40737]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:16 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2892180370' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:36:16.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:16 smithi134 ceph-mon[33541]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:16.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:16 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2892180370' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:36:17.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:36:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:36:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:36:18.648 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574895 for osd.1 2024-09-21T09:36:18.648 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:18.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:18 smithi017 ceph-mon[40737]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:18 smithi017 ceph-mon[29380]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:18 smithi134 ceph-mon[33541]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:20.016 INFO:teuthology.orchestra.run.smithi017.stdout:158913789981 2024-09-21T09:36:20.099 INFO:teuthology.orchestra.run.smithi017.stdout:77309411372 2024-09-21T09:36:20.933 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:20 smithi017 ceph-mon[29380]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:20.933 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:20 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2269234119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:36:20.933 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:20 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1058001827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:36:20.933 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:20 smithi017 ceph-mon[40737]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:20.933 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:20 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2269234119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:36:20.933 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:20 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1058001827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:36:20.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:20 smithi134 ceph-mon[33541]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:20 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2269234119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:36:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:20 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1058001827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:36:21.454 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411372 for osd.2 2024-09-21T09:36:21.454 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:21.856 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593749 got 184683593752 for osd.6 2024-09-21T09:36:21.856 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:22.414 INFO:teuthology.orchestra.run.smithi017.stdout:111669149736 2024-09-21T09:36:22.690 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789977 got 158913789981 for osd.5 2024-09-21T09:36:22.690 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:22 smithi017 ceph-mon[29380]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:22 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/801800713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:36:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:22 smithi017 ceph-mon[40737]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:22 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/801800713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:36:22.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:22 smithi134 ceph-mon[33541]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:22 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/801800713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:36:23.134 INFO:teuthology.orchestra.run.smithi017.stdout:210453397524 2024-09-21T09:36:23.797 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397520 got 210453397524 for osd.7 2024-09-21T09:36:23.797 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:23.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:23 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2569841653' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:36:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:23 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:24.118 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:23 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2569841653' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:36:24.118 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:23 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:24.118 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:23 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2569841653' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:36:24.118 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:23 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:24.505 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149733 got 111669149736 for osd.3 2024-09-21T09:36:24.506 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:24.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:24 smithi134 ceph-mon[33541]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:25.004 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:24 smithi017 ceph-mon[29380]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:25.004 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:24 smithi017 ceph-mon[40737]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:25.044 INFO:teuthology.orchestra.run.smithi017.stdout:133143986209 2024-09-21T09:36:25.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:25 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3320038997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:36:25.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:25 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3320038997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:36:25.973 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986204 got 133143986209 for osd.4 2024-09-21T09:36:25.973 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:25.973 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:36:25.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:36:25.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:25 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3320038997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:36:26.851 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:26.988 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:26 smithi017 ceph-mon[29380]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:26.989 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:26 smithi017 ceph-mon[40737]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:26.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:26 smithi134 ceph-mon[33541]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:26.993 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:36:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:36:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:36:28.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:28 smithi134 ceph-mon[33541]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:28 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:29.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:28 smithi017 ceph-mon[29380]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:29.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:29.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:28 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:29.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:28 smithi017 ceph-mon[40737]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:29.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:29.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:28 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:29.026 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:36:29.026 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:36:29.706 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-09-21T09:36:27.660403+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385164,"kb_used_data":6316,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576052,"statfs":{"total":767960285184,"available":765517877248,"internally_reserved":0,"allocated":6467584,"data_stored":3738991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001775"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.266516+0000","last_change":"2024-09-21T09:35:34.097095+0000","last_active":"2024-09-21T09:35:38.266516+0000","last_peered":"2024-09-21T09:35:38.266516+0000","last_clean":"2024-09-21T09:35:38.266516+0000","last_became_active":"2024-09-21T09:35:29.148371+0000","last_became_peered":"2024-09-21T09:35:29.148371+0000","last_unstale":"2024-09-21T09:35:38.266516+0000","last_undegraded":"2024-09-21T09:35:38.266516+0000","last_fullsized":"2024-09-21T09:35:38.266516+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:04:07.399927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:37.661468+0000","last_change":"2024-09-21T09:35:34.097223+0000","last_active":"2024-09-21T09:35:37.661468+0000","last_peered":"2024-09-21T09:35:37.661468+0000","last_clean":"2024-09-21T09:35:37.661468+0000","last_became_active":"2024-09-21T09:35:29.378933+0000","last_became_peered":"2024-09-21T09:35:29.378933+0000","last_unstale":"2024-09-21T09:35:37.661468+0000","last_undegraded":"2024-09-21T09:35:37.661468+0000","last_fullsized":"2024-09-21T09:35:37.661468+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:04.627265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033167899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.018954+0000","last_change":"2024-09-21T09:35:34.101535+0000","last_active":"2024-09-21T09:35:38.018954+0000","last_peered":"2024-09-21T09:35:38.018954+0000","last_clean":"2024-09-21T09:35:38.018954+0000","last_became_active":"2024-09-21T09:35:29.753057+0000","last_became_peered":"2024-09-21T09:35:29.753057+0000","last_unstale":"2024-09-21T09:35:38.018954+0000","last_undegraded":"2024-09-21T09:35:38.018954+0000","last_fullsized":"2024-09-21T09:35:38.018954+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:31:49.246270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:37.661475+0000","last_change":"2024-09-21T09:35:34.097383+0000","last_active":"2024-09-21T09:35:37.661475+0000","last_peered":"2024-09-21T09:35:37.661475+0000","last_clean":"2024-09-21T09:35:37.661475+0000","last_became_active":"2024-09-21T09:35:29.752863+0000","last_became_peered":"2024-09-21T09:35:29.752863+0000","last_unstale":"2024-09-21T09:35:37.661475+0000","last_undegraded":"2024-09-21T09:35:37.661475+0000","last_fullsized":"2024-09-21T09:35:37.661475+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:00.231892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.266610+0000","last_change":"2024-09-21T09:35:34.096316+0000","last_active":"2024-09-21T09:35:38.266610+0000","last_peered":"2024-09-21T09:35:38.266610+0000","last_clean":"2024-09-21T09:35:38.266610+0000","last_became_active":"2024-09-21T09:35:29.148648+0000","last_became_peered":"2024-09-21T09:35:29.148648+0000","last_unstale":"2024-09-21T09:35:38.266610+0000","last_undegraded":"2024-09-21T09:35:38.266610+0000","last_fullsized":"2024-09-21T09:35:38.266610+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:29.243621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:37.628705+0000","last_change":"2024-09-21T09:35:34.090997+0000","last_active":"2024-09-21T09:35:37.628705+0000","last_peered":"2024-09-21T09:35:37.628705+0000","last_clean":"2024-09-21T09:35:37.628705+0000","last_became_active":"2024-09-21T09:35:29.752848+0000","last_became_peered":"2024-09-21T09:35:29.752848+0000","last_unstale":"2024-09-21T09:35:37.628705+0000","last_undegraded":"2024-09-21T09:35:37.628705+0000","last_fullsized":"2024-09-21T09:35:37.628705+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:57:11.329055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.018953+0000","last_change":"2024-09-21T09:35:34.101543+0000","last_active":"2024-09-21T09:35:38.018953+0000","last_peered":"2024-09-21T09:35:38.018953+0000","last_clean":"2024-09-21T09:35:38.018953+0000","last_became_active":"2024-09-21T09:35:29.753066+0000","last_became_peered":"2024-09-21T09:35:29.753066+0000","last_unstale":"2024-09-21T09:35:38.018953+0000","last_undegraded":"2024-09-21T09:35:38.018953+0000","last_fullsized":"2024-09-21T09:35:38.018953+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:49.271159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.266528+0000","last_change":"2024-09-21T09:35:34.095862+0000","last_active":"2024-09-21T09:35:38.266528+0000","last_peered":"2024-09-21T09:35:38.266528+0000","last_clean":"2024-09-21T09:35:38.266528+0000","last_became_active":"2024-09-21T09:35:29.148713+0000","last_became_peered":"2024-09-21T09:35:29.148713+0000","last_unstale":"2024-09-21T09:35:38.266528+0000","last_undegraded":"2024-09-21T09:35:38.266528+0000","last_fullsized":"2024-09-21T09:35:38.266528+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:18.348007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":49,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:35:34.101247+0000","last_change":"2024-09-21T09:34:54.592924+0000","last_active":"2024-09-21T09:35:34.101247+0000","last_peered":"2024-09-21T09:35:34.101247+0000","last_clean":"2024-09-21T09:35:34.101247+0000","last_became_active":"2024-09-21T09:34:54.588063+0000","last_became_peered":"2024-09-21T09:34:54.588063+0000","last_unstale":"2024-09-21T09:35:34.101247+0000","last_undegraded":"2024-09-21T09:35:34.101247+0000","last_fullsized":"2024-09-21T09:35:34.101247+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:02.981289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397525,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.59899999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.54200000000000004}]},{"osd":2,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.55300000000000005}]},{"osd":4,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.38},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.35799999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.502},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.48999999999999999}]}]},{"osd":6,"up_from":43,"seq":184683593754,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":835916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.58699999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.62},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.55300000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.501},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.435}]},{"osd":5,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.48599999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:35:57 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.53600000000000003}]}]},{"osd":5,"up_from":37,"seq":158913789983,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":245792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67100000000000004,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.442},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.53500000000000003,"15min":0.54700000000000004},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.58899999999999997,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.48899999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.39600000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67800000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.54500000000000004,"15min":0.55400000000000005},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63900000000000001,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.54600000000000004,"15min":0.55600000000000005},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.69999999999999996,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72999999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.45300000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.56899999999999995,"15min":0.57899999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.74099999999999999,"5min":0.88,"15min":0.88},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.54800000000000004,"15min":0.55500000000000005},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69899999999999995,"5min":0.754,"15min":0.754},"last":0.58599999999999997}]},{"osd":4,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39200000000000002,"15min":0.39100000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.439,"5min":0.40100000000000002,"15min":0.39500000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:35:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.55100000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.53100000000000003,"15min":0.53900000000000003},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.63500000000000001,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.76700000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.52900000000000003}]},{"osd":1,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.52700000000000002,"15min":0.53500000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.60799999999999998,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.56499999999999995,"15min":0.56999999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.67300000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.54200000000000004}]},{"osd":2,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.39100000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.748,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.36799999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.438}]},{"osd":3,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39800000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.74099999999999999,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.501},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.67800000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.57999999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.38800000000000001,"15min":0.38200000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56299999999999994,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.31,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.623,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.34899999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:35:28 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.37},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:35:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.40899999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:21 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.40500000000000003,"15min":0.40400000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.31},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.42499999999999999,"15min":0.42599999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.53900000000000003,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.25700000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:35:21 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.61799999999999999,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.442,"15min":0.44},"min":{"1min":0.33700000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:35:21 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.38},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.61099999999999999,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.26800000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.626,"15min":0.63100000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.84899999999999998,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.73599999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:36:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59199999999999997,"15min":0.59399999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.80800000000000005,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.89400000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.85799999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.61099999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.35199999999999998,"15min":0.34000000000000002},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.501,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.372,"15min":0.35599999999999998},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.57099999999999995,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.34200000000000003}]},{"osd":1,"last update":"Sat Sep 21 09:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.371},"min":{"1min":0.28899999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.377,"15min":0.35599999999999998},"min":{"1min":0.26500000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.27900000000000003}]},{"osd":3,"last update":"Sat Sep 21 09:36:19 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.45900000000000002,"15min":0.45700000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.54400000000000004,"5min":0.627,"15min":0.627},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.44500000000000001,"15min":0.439},"min":{"1min":0.30199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56899999999999995,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Sep 21 09:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.57499999999999996,"15min":0.57099999999999995},"min":{"1min":0.45600000000000002,"5min":0.437,"15min":0.437},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.79900000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:36:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.623,"15min":0.61299999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.962,"5min":1.962,"15min":1.962},"last":0.752}]},{"osd":6,"last update":"Sat Sep 21 09:35:27 2024","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.58599999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":246107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.34899999999999998,"15min":0.33300000000000002},"min":{"1min":0.23699999999999999,"5min":0.156,"15min":0.156},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.36599999999999999,"15min":0.34599999999999997},"min":{"1min":0.246,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.52000000000000002,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.28299999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39400000000000002,"15min":0.38300000000000001},"min":{"1min":0.25800000000000001,"5min":0.19,"15min":0.19},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.39700000000000002,"15min":0.39400000000000002},"min":{"1min":0.252,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.63500000000000001,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.308}]},{"osd":3,"last update":"Sat Sep 21 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.41599999999999998,"15min":0.41499999999999998},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.45500000000000002,"15min":0.45400000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.35999999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57999999999999996,"15min":0.57799999999999996},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.40400000000000003,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.57999999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58399999999999996,"15min":0.58999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73099999999999998,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.71299999999999997,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.59899999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:35:28 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.75800000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.60899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:35:34 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.34599999999999997,"15min":0.33600000000000002},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":0.57099999999999995,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.376,"5min":0.36199999999999999,"15min":0.35099999999999998},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53100000000000003,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.26500000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.41299999999999998,"15min":0.41999999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.51800000000000002,"5min":0.626,"15min":0.626},"last":0.372},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.40400000000000003,"15min":0.41199999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.52600000000000002,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.308}]},{"osd":3,"last update":"Sat Sep 21 09:35:24 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.48399999999999999,"15min":0.48599999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.69499999999999995,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.46600000000000003,"15min":0.47299999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.57799999999999996,"5min":0.753,"15min":0.753},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 21 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.53300000000000003,"15min":0.53700000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71699999999999997,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56100000000000005,"15min":0.56000000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.74099999999999999,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:35:28 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.57399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:36:29.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:36:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:29 smithi017 ceph-mon[40737]: from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:29 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:29 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:29 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:29.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:29 smithi017 ceph-mon[29380]: from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:29 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:29 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:29 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:29 smithi134 ceph-mon[33541]: from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:29 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:29 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:29 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:31.188 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:31.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:31 smithi017 ceph-mon[29380]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:36:31.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:31 smithi017 ceph-mon[29380]: Reconfiguring daemon alertmanager.a on smithi017 2024-09-21T09:36:31.331 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:31 smithi017 ceph-mon[29380]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:31.331 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:31 smithi017 ceph-mon[40737]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:36:31.331 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:31 smithi017 ceph-mon[40737]: Reconfiguring daemon alertmanager.a on smithi017 2024-09-21T09:36:31.331 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:31 smithi017 ceph-mon[40737]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:31 smithi134 ceph-mon[33541]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:36:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:31 smithi134 ceph-mon[33541]: Reconfiguring daemon alertmanager.a on smithi017 2024-09-21T09:36:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:31 smithi134 ceph-mon[33541]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:33 smithi017 ceph-mon[29380]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:33 smithi017 ceph-mon[40737]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:33.450 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:33 smithi017 systemd[1]: Stopping Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:36:33.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:33 smithi134 ceph-mon[33541]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:34.514 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:36:34.514 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:36:34.950 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[71455]: ts=2024-09-21T09:36:34.527Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:36:34.950 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:34 smithi017 podman[77067]: 2024-09-21 09:36:34.748007115 +0000 UTC m=+0.361310627 container died b8bd452e6bf978dc8a6f549f283d1bfa333ef34dcbd6ab37bd3a9efbd9eff514 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:35.236 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:35 smithi017 ceph-mon[29380]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:35.236 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:35 smithi017 ceph-mon[40737]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:35.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:35 smithi134 ceph-mon[33541]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:35.664 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:35 smithi017 podman[77067]: 2024-09-21 09:36:35.23533661 +0000 UTC m=+0.848640161 container cleanup b8bd452e6bf978dc8a6f549f283d1bfa333ef34dcbd6ab37bd3a9efbd9eff514 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:35.664 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:35 smithi017 bash[77067]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:36:35.690 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2024-09-21T09:36:33.661466+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385164,"kb_used_data":6316,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576052,"statfs":{"total":767960285184,"available":765517877248,"internally_reserved":0,"allocated":6467584,"data_stored":3738991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001958"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.266516+0000","last_change":"2024-09-21T09:35:34.097095+0000","last_active":"2024-09-21T09:35:38.266516+0000","last_peered":"2024-09-21T09:35:38.266516+0000","last_clean":"2024-09-21T09:35:38.266516+0000","last_became_active":"2024-09-21T09:35:29.148371+0000","last_became_peered":"2024-09-21T09:35:29.148371+0000","last_unstale":"2024-09-21T09:35:38.266516+0000","last_undegraded":"2024-09-21T09:35:38.266516+0000","last_fullsized":"2024-09-21T09:35:38.266516+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:04:07.399927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:37.661468+0000","last_change":"2024-09-21T09:35:34.097223+0000","last_active":"2024-09-21T09:35:37.661468+0000","last_peered":"2024-09-21T09:35:37.661468+0000","last_clean":"2024-09-21T09:35:37.661468+0000","last_became_active":"2024-09-21T09:35:29.378933+0000","last_became_peered":"2024-09-21T09:35:29.378933+0000","last_unstale":"2024-09-21T09:35:37.661468+0000","last_undegraded":"2024-09-21T09:35:37.661468+0000","last_fullsized":"2024-09-21T09:35:37.661468+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:04.627265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033167899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.018954+0000","last_change":"2024-09-21T09:35:34.101535+0000","last_active":"2024-09-21T09:35:38.018954+0000","last_peered":"2024-09-21T09:35:38.018954+0000","last_clean":"2024-09-21T09:35:38.018954+0000","last_became_active":"2024-09-21T09:35:29.753057+0000","last_became_peered":"2024-09-21T09:35:29.753057+0000","last_unstale":"2024-09-21T09:35:38.018954+0000","last_undegraded":"2024-09-21T09:35:38.018954+0000","last_fullsized":"2024-09-21T09:35:38.018954+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:31:49.246270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:37.661475+0000","last_change":"2024-09-21T09:35:34.097383+0000","last_active":"2024-09-21T09:35:37.661475+0000","last_peered":"2024-09-21T09:35:37.661475+0000","last_clean":"2024-09-21T09:35:37.661475+0000","last_became_active":"2024-09-21T09:35:29.752863+0000","last_became_peered":"2024-09-21T09:35:29.752863+0000","last_unstale":"2024-09-21T09:35:37.661475+0000","last_undegraded":"2024-09-21T09:35:37.661475+0000","last_fullsized":"2024-09-21T09:35:37.661475+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:00.231892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":21,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.266610+0000","last_change":"2024-09-21T09:35:34.096316+0000","last_active":"2024-09-21T09:35:38.266610+0000","last_peered":"2024-09-21T09:35:38.266610+0000","last_clean":"2024-09-21T09:35:38.266610+0000","last_became_active":"2024-09-21T09:35:29.148648+0000","last_became_peered":"2024-09-21T09:35:29.148648+0000","last_unstale":"2024-09-21T09:35:38.266610+0000","last_undegraded":"2024-09-21T09:35:38.266610+0000","last_fullsized":"2024-09-21T09:35:38.266610+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:29.243621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:37.628705+0000","last_change":"2024-09-21T09:35:34.090997+0000","last_active":"2024-09-21T09:35:37.628705+0000","last_peered":"2024-09-21T09:35:37.628705+0000","last_clean":"2024-09-21T09:35:37.628705+0000","last_became_active":"2024-09-21T09:35:29.752848+0000","last_became_peered":"2024-09-21T09:35:29.752848+0000","last_unstale":"2024-09-21T09:35:37.628705+0000","last_undegraded":"2024-09-21T09:35:37.628705+0000","last_fullsized":"2024-09-21T09:35:37.628705+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:57:11.329055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.018953+0000","last_change":"2024-09-21T09:35:34.101543+0000","last_active":"2024-09-21T09:35:38.018953+0000","last_peered":"2024-09-21T09:35:38.018953+0000","last_clean":"2024-09-21T09:35:38.018953+0000","last_became_active":"2024-09-21T09:35:29.753066+0000","last_became_peered":"2024-09-21T09:35:29.753066+0000","last_unstale":"2024-09-21T09:35:38.018953+0000","last_undegraded":"2024-09-21T09:35:38.018953+0000","last_fullsized":"2024-09-21T09:35:38.018953+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:49.271159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.266528+0000","last_change":"2024-09-21T09:35:34.095862+0000","last_active":"2024-09-21T09:35:38.266528+0000","last_peered":"2024-09-21T09:35:38.266528+0000","last_clean":"2024-09-21T09:35:38.266528+0000","last_became_active":"2024-09-21T09:35:29.148713+0000","last_became_peered":"2024-09-21T09:35:29.148713+0000","last_unstale":"2024-09-21T09:35:38.266528+0000","last_undegraded":"2024-09-21T09:35:38.266528+0000","last_fullsized":"2024-09-21T09:35:38.266528+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:18.348007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":49,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:35:34.101247+0000","last_change":"2024-09-21T09:34:54.592924+0000","last_active":"2024-09-21T09:35:34.101247+0000","last_peered":"2024-09-21T09:35:34.101247+0000","last_clean":"2024-09-21T09:35:34.101247+0000","last_became_active":"2024-09-21T09:34:54.588063+0000","last_became_peered":"2024-09-21T09:34:54.588063+0000","last_unstale":"2024-09-21T09:35:34.101247+0000","last_undegraded":"2024-09-21T09:35:34.101247+0000","last_fullsized":"2024-09-21T09:35:34.101247+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:02.981289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397526,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.68500000000000005}]},{"osd":1,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.64500000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.34000000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.438}]},{"osd":6,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.47399999999999998}]}]},{"osd":6,"up_from":43,"seq":184683593755,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":835916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.51200000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.58499999999999996}]},{"osd":2,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.55900000000000005}]},{"osd":4,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:35:57 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.31},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.40100000000000002}]}]},{"osd":5,"up_from":37,"seq":158913789984,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":245792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67100000000000004,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.53500000000000003,"15min":0.54700000000000004},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.58899999999999997,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.47699999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.39600000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67800000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.54500000000000004,"15min":0.55400000000000005},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63900000000000001,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.53800000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.54600000000000004,"15min":0.55600000000000005},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.69999999999999996,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72999999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.61699999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.56899999999999995,"15min":0.57899999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.74099999999999999,"5min":0.88,"15min":0.88},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.54800000000000004,"15min":0.55500000000000005},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69899999999999995,"5min":0.754,"15min":0.754},"last":0.503}]},{"osd":4,"last update":"Sat Sep 21 09:36:09 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39200000000000002,"15min":0.39100000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.439,"5min":0.40100000000000002,"15min":0.39500000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.68000000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:35:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.43099999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.53100000000000003,"15min":0.53900000000000003},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.63500000000000001,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.76700000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.52700000000000002,"15min":0.53500000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.60799999999999998,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.56499999999999995,"15min":0.56999999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.67300000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.38900000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.39100000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.748,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.36799999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39800000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.74099999999999999,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.501},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.67800000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.56899999999999995}]},{"osd":5,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.38800000000000001,"15min":0.38200000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56299999999999994,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.31,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.623,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.65700000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:36:29 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.40999999999999998,"15min":0.40000000000000002},"min":{"1min":0.33000000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.46999999999999997,"15min":0.46200000000000002},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67300000000000004,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Sep 21 09:35:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.63}]}]},{"osd":3,"up_from":26,"seq":111669149738,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40500000000000003,"15min":0.40400000000000003},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.61099999999999999,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.42099999999999999,"15min":0.42499999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.45100000000000001,"15min":0.45500000000000002},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.315},{"interface":"front","average":{"1min":0.439,"5min":0.442,"15min":0.44},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.47499999999999998,"15min":0.47699999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.443,"5min":0.46800000000000003,"15min":0.47199999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.33100000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.626,"15min":0.63100000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.84899999999999998,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.72599999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:36:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59199999999999997,"15min":0.59399999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.80800000000000005,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.877}]},{"osd":7,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.751}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297928,"kb_used_data":572,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447224,"statfs":{"total":95995035648,"available":95689957376,"internally_reserved":0,"allocated":585728,"data_stored":246088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.35199999999999998,"15min":0.34000000000000002},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.501,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.372,"15min":0.35599999999999998},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.57099999999999995,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.27200000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.371},"min":{"1min":0.28899999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.377,"15min":0.35599999999999998},"min":{"1min":0.26500000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:36:19 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.45900000000000002,"15min":0.45700000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.54400000000000004,"5min":0.627,"15min":0.627},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.438,"5min":0.44500000000000001,"15min":0.439},"min":{"1min":0.30199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56899999999999995,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.57499999999999996,"15min":0.57099999999999995},"min":{"1min":0.45600000000000002,"5min":0.437,"15min":0.437},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:36:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.623,"15min":0.61299999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.962,"5min":1.962,"15min":1.962},"last":0.58699999999999997}]},{"osd":6,"last update":"Sat Sep 21 09:35:27 2024","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.502},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.56899999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.61099999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":246107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.34899999999999998,"15min":0.33300000000000002},"min":{"1min":0.23699999999999999,"5min":0.156,"15min":0.156},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.371,"5min":0.36599999999999999,"15min":0.34599999999999997},"min":{"1min":0.246,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.52000000000000002,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.26300000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39400000000000002,"15min":0.38300000000000001},"min":{"1min":0.25800000000000001,"5min":0.19,"15min":0.19},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.39700000000000002,"15min":0.39400000000000002},"min":{"1min":0.252,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.63500000000000001,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.28000000000000003}]},{"osd":3,"last update":"Sat Sep 21 09:36:27 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.40799999999999997,"15min":0.41199999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.48599999999999999,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.44800000000000001,"15min":0.45200000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65400000000000003,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57999999999999996,"15min":0.57799999999999996},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56599999999999995,"15min":0.56699999999999995},"min":{"1min":0.40400000000000003,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.66500000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58399999999999996,"15min":0.58999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73099999999999998,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.71299999999999997,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.54400000000000004}]},{"osd":6,"last update":"Sat Sep 21 09:36:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.59599999999999997,"15min":0.59999999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.63300000000000001,"15min":0.63600000000000001},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.67700000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.64100000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":836456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:35:34 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.34599999999999997,"15min":0.33600000000000002},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":0.57099999999999995,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.376,"5min":0.36199999999999999,"15min":0.35099999999999998},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53100000000000003,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.27100000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.41299999999999998,"15min":0.41999999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.51800000000000002,"5min":0.626,"15min":0.626},"last":0.317},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.40400000000000003,"15min":0.41199999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.52600000000000002,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.30399999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.46600000000000003,"15min":0.47999999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.54900000000000004,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.437},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.45000000000000001,"15min":0.46800000000000003},"min":{"1min":0.29699999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.51500000000000001,"5min":0.753,"15min":0.753},"last":0.33600000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.53300000000000003,"15min":0.53700000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71699999999999997,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.57099999999999995}]},{"osd":5,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56100000000000005,"15min":0.56000000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.74099999999999999,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:36:29 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58499999999999996,"15min":0.58599999999999997},"min":{"1min":0.42499999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57699999999999996,"15min":0.57599999999999996},"min":{"1min":0.437,"5min":0.379,"15min":0.379},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.621},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.51000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:36:35.691 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:36:35.691 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:36:35.692 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:36:35.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:36:36.133 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:36 smithi017 ceph-mon[29380]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:36 smithi017 ceph-mon[40737]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:36.201 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:35 smithi017 podman[77097]: 2024-09-21 09:36:35.94751888 +0000 UTC m=+1.371953169 container remove b8bd452e6bf978dc8a6f549f283d1bfa333ef34dcbd6ab37bd3a9efbd9eff514 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:36.201 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:36 smithi017 podman[77097]: 2024-09-21 09:36:36.048241807 +0000 UTC m=+1.472676050 volume remove cbf756f0a4e69fd00a771d63abeb288cba56cbb16b7d0fb3fee1e9cb00d88a94 2024-09-21T09:36:36.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:36 smithi134 ceph-mon[33541]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:37.042 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:36 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:36:37.042 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:36 smithi017 systemd[1]: Stopped Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:37.042 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:36 smithi017 systemd[1]: Starting Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:36:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:37 smithi134 ceph-mon[33541]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:37.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:36:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:36:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:36:37.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:37 smithi017 ceph-mon[29380]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:37.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:37 smithi017 ceph-mon[40737]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:38.157 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:36:38.157 INFO:teuthology.orchestra.run.smithi017.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:36:38.450 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 podman[77387]: 2024-09-21 09:36:38.098834027 +0000 UTC m=+0.017521581 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-21T09:36:38.450 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 podman[77387]: 2024-09-21 09:36:38.200863679 +0000 UTC m=+0.119551288 volume create 751396c7b39e6ee1080396c9ad6fe24fd418729ac2bd208b1a39990d931afbae 2024-09-21T09:36:38.450 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 podman[77387]: 2024-09-21 09:36:38.309441714 +0000 UTC m=+0.228129258 container create b40d1bacbd5033c71dc3ad74e2060dea5980b1cab98a0ce482f308564473cb83 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:39.050 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 podman[77387]: 2024-09-21 09:36:38.787825722 +0000 UTC m=+0.706513274 container init b40d1bacbd5033c71dc3ad74e2060dea5980b1cab98a0ce482f308564473cb83 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 podman[77387]: 2024-09-21 09:36:38.793201861 +0000 UTC m=+0.711889409 container start b40d1bacbd5033c71dc3ad74e2060dea5980b1cab98a0ce482f308564473cb83 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:38.807Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:38.807Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:38.808Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.17 port=9094 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:38.809Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:38.832Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:38.833Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:38.835Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:38.836Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 bash[77387]: b40d1bacbd5033c71dc3ad74e2060dea5980b1cab98a0ce482f308564473cb83 2024-09-21T09:36:39.051 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:38 smithi017 systemd[1]: Started Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[29380]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1131978147' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:36:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[40737]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1131978147' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:36:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:39 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:39.476 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:36:39.476 INFO:tasks.cephadm:Setup complete, yielding 2024-09-21T09:36:39.476 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T09:36:39.490 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi017.front.sepia.ceph.com 2024-09-21T09:36:39.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- bash -c 'ceph fs volume create foo' 2024-09-21T09:36:39.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:39 smithi134 ceph-mon[33541]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:39 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1131978147' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:36:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:39 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:39 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:39 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:39.747 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:40.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:40 smithi017 ceph-mon[29380]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:36:40.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:40 smithi017 ceph-mon[29380]: Reconfiguring daemon prometheus.a on smithi134 2024-09-21T09:36:40.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:40 smithi017 ceph-mon[40737]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:36:40.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:40 smithi017 ceph-mon[40737]: Reconfiguring daemon prometheus.a on smithi134 2024-09-21T09:36:40.205 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:40 smithi134 ceph-mon[33541]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:36:40.205 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:40 smithi134 ceph-mon[33541]: Reconfiguring daemon prometheus.a on smithi134 2024-09-21T09:36:41.092 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:40.809Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000193631s 2024-09-21T09:36:41.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:41 smithi134 ceph-mon[33541]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:41.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:41 smithi017 ceph-mon[29380]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:41.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:41 smithi017 ceph-mon[40737]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:42.419 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:42.420 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:42 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:42 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:42 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:42 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:42.743 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 systemd[1]: Stopping Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:36:43.113 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:36:43.113 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:36:43.113 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:36:43.113 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:36:43.113 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:36:43.113 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:36:43.113 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:36:43.114 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:36:43.114 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:36:43.114 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:36:43.114 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:36:43.114 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[62310]: ts=2024-09-21T09:36:42.841Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:36:43.114 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:42 smithi134 podman[65639]: 2024-09-21 09:36:42.851624984 +0000 UTC m=+0.118431234 container died 32110c11daeb298a5c43bd5c2a8dd01b37b0bb40fc3492c355b9eda7b4ce966a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[40737]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[40737]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-21T09:36:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[40737]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:36:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[29380]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[29380]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-21T09:36:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[29380]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:36:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[29357]: 2024-09-21T09:36:43.121+0000 7f604d136640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:36:43.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:43 smithi134 podman[65639]: 2024-09-21 09:36:43.222173872 +0000 UTC m=+0.488980122 container cleanup 32110c11daeb298a5c43bd5c2a8dd01b37b0bb40fc3492c355b9eda7b4ce966a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:43.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:43 smithi134 bash[65639]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:36:43.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:43 smithi134 ceph-mon[33541]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:43 smithi134 ceph-mon[33541]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:43 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-21T09:36:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:43 smithi134 ceph-mon[33541]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:36:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:43 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:43 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:43.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-21T09:36:43.892 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:43 smithi134 podman[65656]: 2024-09-21 09:36:43.548579696 +0000 UTC m=+0.699002698 container remove 32110c11daeb298a5c43bd5c2a8dd01b37b0bb40fc3492c355b9eda7b4ce966a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:43.892 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:43 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:36:43.892 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:43 smithi134 systemd[1]: Stopped Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:43.892 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:43 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.086s CPU time. 2024-09-21T09:36:43.892 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:43 smithi134 systemd[1]: Starting Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:36:44.138 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:44.152 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:43 smithi134 podman[65751]: 2024-09-21 09:36:43.79466573 +0000 UTC m=+0.016587426 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:36:44.152 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:43 smithi134 podman[65751]: 2024-09-21 09:36:43.932928891 +0000 UTC m=+0.154850583 container create b50e22f97e5d14e90270738c523336c7e4c04085efd2cc6d39437673a2a224ab (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:44.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: fsmap foo:0 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: Saving service mds.foo spec with placement count:2 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:44.153 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-mon[33541]: pgmap v53: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:44.386 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:44] ENGINE Bus STOPPING 2024-09-21T09:36:44.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-21T09:36:44.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:36:44.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:44.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:44.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:36:44.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T09:36:44.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:36:44.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-21T09:36:44.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: fsmap foo:0 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: Saving service mds.foo spec with placement count:2 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[29380]: pgmap v53: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:36:44.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-21T09:36:44.389 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: fsmap foo:0 2024-09-21T09:36:44.389 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: Saving service mds.foo spec with placement count:2 2024-09-21T09:36:44.389 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:44.389 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-mon[40737]: pgmap v53: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:44.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 podman[65751]: 2024-09-21 09:36:44.151503897 +0000 UTC m=+0.373425587 container init b50e22f97e5d14e90270738c523336c7e4c04085efd2cc6d39437673a2a224ab (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:44.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 podman[65751]: 2024-09-21 09:36:44.156595388 +0000 UTC m=+0.378517073 container start b50e22f97e5d14e90270738c523336c7e4c04085efd2cc6d39437673a2a224ab (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:44.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.180Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:36:44.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.180Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:36:44.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.180Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi134 (none))" 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.180Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.180Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.181Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.182Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.183Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.183Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.185Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.185Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.644µs 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.185Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.186Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.186Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.186Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.803µs wal_replay_duration=440.435µs wbl_replay_duration=113ns total_replay_duration=480.266µs 2024-09-21T09:36:44.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.186Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:36:44.496 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.186Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:36:44.496 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.186Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:36:44.496 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.201Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.990326ms db_storage=700ns remote_storage=1.035µs web_handler=552ns query_engine=501ns scrape=368.955µs scrape_sd=25.627µs notify=24.131µs notify_sd=7.341µs rules=14.348475ms tracing=5.668µs 2024-09-21T09:36:44.496 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.202Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:36:44.496 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:36:44.202Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:36:44.496 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 bash[65751]: b50e22f97e5d14e90270738c523336c7e4c04085efd2cc6d39437673a2a224ab 2024-09-21T09:36:44.496 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:36:44 smithi134 systemd[1]: Started Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:44.674 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:36:44.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:44] ENGINE Bus STOPPED 2024-09-21T09:36:44.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:44] ENGINE Bus STARTING 2024-09-21T09:36:44.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:44] ENGINE Serving on http://:::9283 2024-09-21T09:36:44.675 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:44] ENGINE Bus STARTED 2024-09-21T09:36:44.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:44] ENGINE Bus STOPPING 2024-09-21T09:36:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:36:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:45 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:45.283 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:36:45.283 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE Bus STOPPED 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE Bus STARTING 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE Serving on http://:::9283 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE Bus STARTED 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:45.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:45.285 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi134.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:45.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:45.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi134.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:45.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:45.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:45.641 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE Bus STOPPING 2024-09-21T09:36:45.891 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:36:45.892 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE Bus STOPPED 2024-09-21T09:36:45.892 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE Bus STARTING 2024-09-21T09:36:45.892 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE Serving on http://:::9283 2024-09-21T09:36:45.892 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:36:45] ENGINE Bus STARTED 2024-09-21T09:36:46.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:46 smithi017 ceph-mon[29380]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:46.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:46 smithi017 ceph-mon[40737]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:46.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:46 smithi134 ceph-mon[33541]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:46.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-21T09:36:47.343 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:48.950 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:36:48 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:36:48.811Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002736326s 2024-09-21T09:36:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:48 smithi017 ceph-mon[29380]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:48 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:48 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:48.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:48 smithi017 ceph-mon[40737]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:48.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:48 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:48.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:48 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:49.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:48 smithi134 ceph-mon[33541]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:49.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:48 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:49.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:48 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:50.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:36:50 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:36:50] "GET /metrics HTTP/1.1" 200 203786 "" "Prometheus/2.43.0" 2024-09-21T09:36:50.565 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T09:36:50.580 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:36:50.581 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:36:50.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:36:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:50 smithi017 ceph-mon[29380]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:51.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:50 smithi017 ceph-mon[40737]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:51.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:50 smithi134 ceph-mon[33541]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:51.277 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:52.403 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:52.403 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: Deploying daemon mds.foo.smithi017.coodep on smithi017 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[29380]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:52.404 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:52.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:52.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:52.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:52.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:52.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:52.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: Deploying daemon mds.foo.smithi017.coodep on smithi017 2024-09-21T09:36:52.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:52 smithi017 ceph-mon[40737]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: Deploying daemon mds.foo.smithi017.coodep on smithi017 2024-09-21T09:36:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:52 smithi134 ceph-mon[33541]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:54.820 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:54 smithi017 ceph-mon[29380]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:54.820 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:54 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:54.820 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:54 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:54.820 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:54 smithi017 ceph-mon[40737]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:54.820 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:54 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:54.820 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:54 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:36:54.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:54 smithi134 ceph-mon[33541]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:54.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:54 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:54.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:54 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:55.528 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":21,"flags":0,"active_gid":24332,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6800","nonce":3625002403},{"type":"v1","addr":"172.21.15.17:6801","nonce":3625002403}]},"active_addr":"172.21.15.17:6801/3625002403","active_change":"2024-09-21T09:35:23.638854+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":52,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3512479442}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":877591523}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3671582122}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3086297034}]}]} 2024-09-21T09:36:55.532 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:36:55.532 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:36:55.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:36:55.799 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:55.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: mds.? [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] up:boot 2024-09-21T09:36:55.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: daemon mds.foo.smithi017.coodep assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-21T09:36:55.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:36:55.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T09:36:55.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: Cluster is now healthy 2024-09-21T09:36:55.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: fsmap foo:0 1 up:standby 2024-09-21T09:36:55.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.coodep"}]: dispatch 2024-09-21T09:36:55.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: fsmap foo:1 {0=foo.smithi017.coodep=up:creating} 2024-09-21T09:36:55.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: daemon mds.foo.smithi017.coodep is now active in filesystem foo as rank 0 2024-09-21T09:36:55.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:55.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:55.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:55.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:55.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:55.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:55.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:55.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: Deploying daemon mds.foo.smithi134.bwovvb on smithi134 2024-09-21T09:36:55.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3604380951' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:36:55.923 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: mds.? [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] up:boot 2024-09-21T09:36:55.923 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: daemon mds.foo.smithi017.coodep assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-21T09:36:55.923 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:36:55.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T09:36:55.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: Cluster is now healthy 2024-09-21T09:36:55.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: fsmap foo:0 1 up:standby 2024-09-21T09:36:55.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.coodep"}]: dispatch 2024-09-21T09:36:55.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: fsmap foo:1 {0=foo.smithi017.coodep=up:creating} 2024-09-21T09:36:55.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: daemon mds.foo.smithi017.coodep is now active in filesystem foo as rank 0 2024-09-21T09:36:55.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:55.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:55.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:55.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:55.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:55.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:55.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:55.929 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: Deploying daemon mds.foo.smithi134.bwovvb on smithi134 2024-09-21T09:36:55.929 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:55 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3604380951' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:36:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: mds.? [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] up:boot 2024-09-21T09:36:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: daemon mds.foo.smithi017.coodep assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-21T09:36:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:36:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T09:36:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: Cluster is now healthy 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: fsmap foo:0 1 up:standby 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.coodep"}]: dispatch 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: fsmap foo:1 {0=foo.smithi017.coodep=up:creating} 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: daemon mds.foo.smithi017.coodep is now active in filesystem foo as rank 0 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: Deploying daemon mds.foo.smithi134.bwovvb on smithi134 2024-09-21T09:36:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:55 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3604380951' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:36:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:56 smithi017 ceph-mon[29380]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:56 smithi017 ceph-mon[29380]: mds.? [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] up:active 2024-09-21T09:36:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:56 smithi017 ceph-mon[29380]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 2024-09-21T09:36:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:56 smithi017 ceph-mon[40737]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:56 smithi017 ceph-mon[40737]: mds.? [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] up:active 2024-09-21T09:36:56.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:56 smithi017 ceph-mon[40737]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 2024-09-21T09:36:56.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:56 smithi134 ceph-mon[33541]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:56.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:56 smithi134 ceph-mon[33541]: mds.? [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] up:active 2024-09-21T09:36:56.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:56 smithi134 ceph-mon[33541]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 2024-09-21T09:36:57.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:36:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:36:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:36:57.734 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:36:57.734 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":63,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T09:36:44.125907+0000","last_up_change":"2024-09-21T09:34:52.573445+0000","last_in_change":"2024-09-21T09:34:35.544589+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:27.624125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:41.967182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:42.538769+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6803","nonce":314719645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6805","nonce":314719645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6809","nonce":314719645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6807","nonce":314719645}]},"public_addr":"172.21.15.17:6803/314719645","cluster_addr":"172.21.15.17:6805/314719645","heartbeat_back_addr":"172.21.15.17:6809/314719645","heartbeat_front_addr":"172.21.15.17:6807/314719645","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6811","nonce":1139898755}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6813","nonce":1139898755}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6817","nonce":1139898755}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6815","nonce":1139898755}]},"public_addr":"172.21.15.17:6811/1139898755","cluster_addr":"172.21.15.17:6813/1139898755","heartbeat_back_addr":"172.21.15.17:6817/1139898755","heartbeat_front_addr":"172.21.15.17:6815/1139898755","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6819","nonce":3614951048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6821","nonce":3614951048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6825","nonce":3614951048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6823","nonce":3614951048}]},"public_addr":"172.21.15.17:6819/3614951048","cluster_addr":"172.21.15.17:6821/3614951048","heartbeat_back_addr":"172.21.15.17:6825/3614951048","heartbeat_front_addr":"172.21.15.17:6823/3614951048","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6827","nonce":823707214}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6829","nonce":823707214}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6833","nonce":823707214}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6831","nonce":823707214}]},"public_addr":"172.21.15.17:6827/823707214","cluster_addr":"172.21.15.17:6829/823707214","heartbeat_back_addr":"172.21.15.17:6833/823707214","heartbeat_front_addr":"172.21.15.17:6831/823707214","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6801","nonce":2226434249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6803","nonce":2226434249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6807","nonce":2226434249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6805","nonce":2226434249}]},"public_addr":"172.21.15.134:6801/2226434249","cluster_addr":"172.21.15.134:6803/2226434249","heartbeat_back_addr":"172.21.15.134:6807/2226434249","heartbeat_front_addr":"172.21.15.134:6805/2226434249","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/2291861207":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/3206271349":"2024-09-22T09:35:23.638701+0000","172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/2266250644":"2024-09-22T09:35:23.638701+0000","172.21.15.17:6801/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/1454017192":"2024-09-22T09:35:23.638701+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:36:58.478 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:36:58.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:36:58.504 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.504 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.504 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.504 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.504 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.504 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.504 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: mds.? [v2:172.21.15.134:6832/3208820776,v1:172.21.15.134:6833/3208820776] up:boot 2024-09-21T09:36:58.504 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"}]: dispatch 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3585728064' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: mds.? [v2:172.21.15.134:6832/3208820776,v1:172.21.15.134:6833/3208820776] up:boot 2024-09-21T09:36:58.505 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:36:58.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"}]: dispatch 2024-09-21T09:36:58.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:58.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:58 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3585728064' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: mds.? [v2:172.21.15.134:6832/3208820776,v1:172.21.15.134:6833/3208820776] up:boot 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"}]: dispatch 2024-09-21T09:36:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:36:58.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:58 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3585728064' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:36:59.171 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:59 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:59 smithi017 ceph-mon[29380]: mds.? [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] up:active 2024-09-21T09:36:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:36:59 smithi017 ceph-mon[29380]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:36:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:59 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:59 smithi017 ceph-mon[40737]: mds.? [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] up:active 2024-09-21T09:36:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:36:59 smithi017 ceph-mon[40737]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:36:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:59 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:36:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:59 smithi134 ceph-mon[33541]: mds.? [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] up:active 2024-09-21T09:36:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:36:59 smithi134 ceph-mon[33541]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:37:00.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:00 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:00] "GET /metrics HTTP/1.1" 200 208747 "" "Prometheus/2.43.0" 2024-09-21T09:37:00.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:00 smithi134 ceph-mon[33541]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:01.103 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:37:01.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:00 smithi017 ceph-mon[29380]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:01.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:00 smithi017 ceph-mon[40737]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:01.104 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":63,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T09:36:44.125907+0000","last_up_change":"2024-09-21T09:34:52.573445+0000","last_in_change":"2024-09-21T09:34:35.544589+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:27.624125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:41.967182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:42.538769+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6803","nonce":314719645}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6805","nonce":314719645}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6809","nonce":314719645}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":314719645},{"type":"v1","addr":"172.21.15.17:6807","nonce":314719645}]},"public_addr":"172.21.15.17:6803/314719645","cluster_addr":"172.21.15.17:6805/314719645","heartbeat_back_addr":"172.21.15.17:6809/314719645","heartbeat_front_addr":"172.21.15.17:6807/314719645","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6811","nonce":1139898755}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6813","nonce":1139898755}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6817","nonce":1139898755}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1139898755},{"type":"v1","addr":"172.21.15.17:6815","nonce":1139898755}]},"public_addr":"172.21.15.17:6811/1139898755","cluster_addr":"172.21.15.17:6813/1139898755","heartbeat_back_addr":"172.21.15.17:6817/1139898755","heartbeat_front_addr":"172.21.15.17:6815/1139898755","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6819","nonce":3614951048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6821","nonce":3614951048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6825","nonce":3614951048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":3614951048},{"type":"v1","addr":"172.21.15.17:6823","nonce":3614951048}]},"public_addr":"172.21.15.17:6819/3614951048","cluster_addr":"172.21.15.17:6821/3614951048","heartbeat_back_addr":"172.21.15.17:6825/3614951048","heartbeat_front_addr":"172.21.15.17:6823/3614951048","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6827","nonce":823707214}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6829","nonce":823707214}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6833","nonce":823707214}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":823707214},{"type":"v1","addr":"172.21.15.17:6831","nonce":823707214}]},"public_addr":"172.21.15.17:6827/823707214","cluster_addr":"172.21.15.17:6829/823707214","heartbeat_back_addr":"172.21.15.17:6833/823707214","heartbeat_front_addr":"172.21.15.17:6831/823707214","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6801","nonce":2226434249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6803","nonce":2226434249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6807","nonce":2226434249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6805","nonce":2226434249}]},"public_addr":"172.21.15.134:6801/2226434249","cluster_addr":"172.21.15.134:6803/2226434249","heartbeat_back_addr":"172.21.15.134:6807/2226434249","heartbeat_front_addr":"172.21.15.134:6805/2226434249","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/2291861207":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/3206271349":"2024-09-22T09:35:23.638701+0000","172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/2266250644":"2024-09-22T09:35:23.638701+0000","172.21.15.17:6801/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/1454017192":"2024-09-22T09:35:23.638701+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:37:02.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:37:02.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:37:02.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:37:02.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:37:02.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:37:02.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:37:02.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:37:02.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:37:02.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:02 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:02.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:02 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:02.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:02 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/752062562' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:37:02.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:02 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:02.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:02 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:02.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:02 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/752062562' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:37:02.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:02 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:02 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:02 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/752062562' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:37:02.975 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:02.975 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:02.976 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:02.980 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:02.981 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:02.981 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:03 smithi017 ceph-mon[29380]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:03 smithi017 ceph-mon[29380]: mds.? [v2:172.21.15.134:6832/3208820776,v1:172.21.15.134:6833/3208820776] up:standby 2024-09-21T09:37:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:03 smithi017 ceph-mon[29380]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:37:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:03 smithi017 ceph-mon[40737]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:03 smithi017 ceph-mon[40737]: mds.? [v2:172.21.15.134:6832/3208820776,v1:172.21.15.134:6833/3208820776] up:standby 2024-09-21T09:37:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:03 smithi017 ceph-mon[40737]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:37:03.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:03 smithi134 ceph-mon[33541]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:03.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:03 smithi134 ceph-mon[33541]: mds.? [v2:172.21.15.134:6832/3208820776,v1:172.21.15.134:6833/3208820776] up:standby 2024-09-21T09:37:03.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:03 smithi134 ceph-mon[33541]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:37:04.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:04.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:04.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:04.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:04.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:04.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:37:04.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[29380]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:04.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:04.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:04.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:04.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:04.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:04.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:37:04.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:04 smithi017 ceph-mon[40737]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:04.696 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:04 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:04.696 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:04 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:04.696 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:04 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:04.696 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:04 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:04.696 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:04 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:04.697 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:04 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:37:04.697 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:04 smithi134 ceph-mon[33541]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:06 smithi017 ceph-mon[29380]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:06.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:06 smithi017 ceph-mon[40737]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:06.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:06 smithi134 ceph-mon[33541]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:07.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:37:06 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:37:07.894 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:08.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:08 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:08.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:08 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:08.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:08 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:08.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:08 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:08.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:08 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:08 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:09.300 INFO:teuthology.orchestra.run.smithi017.stdout:55834574907 2024-09-21T09:37:09.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:37:09.328 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:09 smithi017 ceph-mon[29380]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:09.328 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:09 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:09.328 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:09 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:09.328 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:09 smithi017 ceph-mon[40737]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:09.328 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:09 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:09.328 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:09 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:09 smithi134 ceph-mon[33541]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:09 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:09 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:10.328 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:10 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:10] "GET /metrics HTTP/1.1" 200 239747 "" "Prometheus/2.43.0" 2024-09-21T09:37:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:11 smithi017 ceph-mon[29380]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:11 smithi017 ceph-mon[40737]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:11.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:11 smithi134 ceph-mon[33541]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:37:13.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:13 smithi017 ceph-mon[29380]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:13.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:13 smithi017 ceph-mon[40737]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:13.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:13 smithi134 ceph-mon[33541]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:13.841 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:15 smithi017 ceph-mon[29380]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:15 smithi017 ceph-mon[40737]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:15.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:15 smithi134 ceph-mon[33541]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:15.588 INFO:teuthology.orchestra.run.smithi017.stdout:77309411383 2024-09-21T09:37:15.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:37:16.029 INFO:teuthology.orchestra.run.smithi017.stdout:133143986219 2024-09-21T09:37:16.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:37:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:17 smithi017 ceph-mon[29380]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:17 smithi017 ceph-mon[40737]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:17.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:17 smithi134 ceph-mon[33541]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:17.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:37:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:37:17.560 INFO:teuthology.orchestra.run.smithi017.stdout:111669149747 2024-09-21T09:37:17.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:37:18.188 INFO:teuthology.orchestra.run.smithi017.stdout:158913789994 2024-09-21T09:37:18.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:37:19.251 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:19 smithi017 ceph-mon[29380]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:19.252 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:19 smithi017 ceph-mon[40737]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:19.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:19 smithi134 ceph-mon[33541]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:20.157 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:20] "GET /metrics HTTP/1.1" 200 239747 "" "Prometheus/2.43.0" 2024-09-21T09:37:20.188 INFO:teuthology.orchestra.run.smithi017.stdout:210453397536 2024-09-21T09:37:20.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:37:20.918 INFO:teuthology.orchestra.run.smithi017.stdout:34359738433 2024-09-21T09:37:20.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:37:21.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:21 smithi017 ceph-mon[29380]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:21 smithi017 ceph-mon[40737]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:21.459 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:21.460 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:21.461 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:21.463 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:21.474 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:21.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:21 smithi134 ceph-mon[33541]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:23.121 INFO:teuthology.orchestra.run.smithi017.stdout:184683593766 2024-09-21T09:37:23.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:37:23.430 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:23 smithi017 ceph-mon[29380]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:23 smithi017 ceph-mon[40737]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:23.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:23 smithi134 ceph-mon[33541]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:24.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:24.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:24 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:24 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:24 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:24.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:24.675 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:25.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[29380]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:25.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T09:37:25.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-09-21T09:37:25.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T09:37:25.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[29380]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:37:25.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[40737]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:25.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T09:37:25.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-09-21T09:37:25.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T09:37:25.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:25 smithi017 ceph-mon[40737]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:37:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:25 smithi134 ceph-mon[33541]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:25 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T09:37:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:25 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-09-21T09:37:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:25 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T09:37:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:25 smithi134 ceph-mon[33541]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:37:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:26 smithi017 ceph-mon[29380]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:37:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:26 smithi017 ceph-mon[40737]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:37:26.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:26 smithi134 ceph-mon[33541]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:37:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:27 smithi134 ceph-mon[33541]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:27.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:37:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:37:27.388 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:27 smithi017 ceph-mon[29380]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:27.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:27 smithi017 ceph-mon[40737]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:28.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:28 smithi017 ceph-mon[29380]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:28.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:28 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2045474106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:37:28.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:28 smithi017 ceph-mon[40737]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:28.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:28 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2045474106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:37:28.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:28 smithi134 ceph-mon[33541]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:28 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2045474106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:37:28.833 INFO:teuthology.orchestra.run.smithi017.stdout:133143986223 2024-09-21T09:37:29.567 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986219 got 133143986223 for osd.4 2024-09-21T09:37:29.567 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:30.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:30] "GET /metrics HTTP/1.1" 200 239728 "" "Prometheus/2.43.0" 2024-09-21T09:37:30.773 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:30 smithi017 ceph-mon[29380]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:30.773 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:30 smithi017 ceph-mon[40737]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:30.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:30 smithi134 ceph-mon[33541]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:31.037 INFO:teuthology.orchestra.run.smithi017.stdout:55834574912 2024-09-21T09:37:31.686 INFO:teuthology.orchestra.run.smithi017.stdout:34359738436 2024-09-21T09:37:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:31 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/696171563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:37:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:31 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/529328289' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:37:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:31 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/696171563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:37:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:31 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/529328289' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:37:31.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:31 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/696171563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:37:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:31 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/529328289' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:37:32.615 INFO:teuthology.orchestra.run.smithi017.stdout:158913789997 2024-09-21T09:37:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:32 smithi017 ceph-mon[29380]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:32 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1527028194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:37:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:32 smithi017 ceph-mon[40737]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:32 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1527028194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:37:32.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:32 smithi134 ceph-mon[33541]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:32.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:32 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1527028194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:37:33.446 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738433 got 34359738436 for osd.0 2024-09-21T09:37:33.446 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:34.340 INFO:teuthology.orchestra.run.smithi017.stdout:210453397539 2024-09-21T09:37:34.553 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574907 got 55834574912 for osd.1 2024-09-21T09:37:34.553 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:34 smithi134 ceph-mon[33541]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:34 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3280049292' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:37:35.038 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:34 smithi017 ceph-mon[29380]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:35.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:34 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3280049292' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:37:35.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:34 smithi017 ceph-mon[40737]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:35.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:34 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3280049292' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:37:35.623 INFO:teuthology.orchestra.run.smithi017.stdout:111669149751 2024-09-21T09:37:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:35 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3508241444' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:37:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:35 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3508241444' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:37:35.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:35 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3508241444' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:37:36.420 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:36.570 INFO:teuthology.orchestra.run.smithi017.stdout:77309411388 2024-09-21T09:37:36.792 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:36 smithi017 ceph-mon[29380]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:36.793 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:36 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3777577081' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:37:36.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:36 smithi017 ceph-mon[40737]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:36.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:36 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3777577081' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:37:36.819 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789994 got 158913789997 for osd.5 2024-09-21T09:37:36.819 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:36 smithi134 ceph-mon[33541]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:36 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3777577081' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:37:36.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:37:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:37:37.494 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149747 got 111669149751 for osd.3 2024-09-21T09:37:37.494 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:38.232 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397536 got 210453397539 for osd.7 2024-09-21T09:37:38.232 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:38.933 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:38 smithi017 ceph-mon[29380]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:38.933 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:38 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:38.934 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:38 smithi017 ceph-mon[40737]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:38.934 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:38 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:38.959 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411388 for osd.2 2024-09-21T09:37:38.959 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:39.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:38 smithi134 ceph-mon[33541]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:38 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:40.298 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:40] "GET /metrics HTTP/1.1" 200 239776 "" "Prometheus/2.43.0" 2024-09-21T09:37:40.375 INFO:teuthology.orchestra.run.smithi017.stdout:184683593770 2024-09-21T09:37:40.936 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:40 smithi017 ceph-mon[29380]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:40.936 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:40 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/4135892478' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:37:40.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:40 smithi017 ceph-mon[40737]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:40.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:40 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/4135892478' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:37:41.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:40 smithi134 ceph-mon[33541]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:40 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/4135892478' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:37:41.329 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593766 got 184683593770 for osd.6 2024-09-21T09:37:41.329 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:41.329 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:37:41.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:37:41.609 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[40737]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:42.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:42.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:42.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[29380]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:42.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:42.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:42.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:42 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:42 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:42 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:42 smithi134 ceph-mon[33541]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:42 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:42 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:42 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:43.471 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:37:43.472 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:37:44.180 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":85,"stamp":"2024-09-21T09:37:41.675806+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121,"ondisk_log_size":121,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385860,"kb_used_data":6988,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575356,"statfs":{"total":767960285184,"available":765517164544,"internally_reserved":0,"allocated":7155712,"data_stored":4094125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002620"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816224+0000","last_change":"2024-09-21T09:36:43.389780+0000","last_active":"2024-09-21T09:37:24.816224+0000","last_peered":"2024-09-21T09:37:24.816224+0000","last_clean":"2024-09-21T09:37:24.816224+0000","last_became_active":"2024-09-21T09:36:43.389430+0000","last_became_peered":"2024-09-21T09:36:43.389430+0000","last_unstale":"2024-09-21T09:37:24.816224+0000","last_undegraded":"2024-09-21T09:37:24.816224+0000","last_fullsized":"2024-09-21T09:37:24.816224+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:16.104966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104167+0000","last_change":"2024-09-21T09:36:43.112938+0000","last_active":"2024-09-21T09:37:24.104167+0000","last_peered":"2024-09-21T09:37:24.104167+0000","last_clean":"2024-09-21T09:37:24.104167+0000","last_became_active":"2024-09-21T09:36:43.112638+0000","last_became_peered":"2024-09-21T09:36:43.112638+0000","last_unstale":"2024-09-21T09:37:24.104167+0000","last_undegraded":"2024-09-21T09:37:24.104167+0000","last_fullsized":"2024-09-21T09:37:24.104167+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:49.176718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102902+0000","last_change":"2024-09-21T09:36:43.388929+0000","last_active":"2024-09-21T09:37:24.102902+0000","last_peered":"2024-09-21T09:37:24.102902+0000","last_clean":"2024-09-21T09:37:24.102902+0000","last_became_active":"2024-09-21T09:36:43.388613+0000","last_became_peered":"2024-09-21T09:36:43.388613+0000","last_unstale":"2024-09-21T09:37:24.102902+0000","last_undegraded":"2024-09-21T09:37:24.102902+0000","last_fullsized":"2024-09-21T09:37:24.102902+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:51.496717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816121+0000","last_change":"2024-09-21T09:36:43.114686+0000","last_active":"2024-09-21T09:37:24.816121+0000","last_peered":"2024-09-21T09:37:24.816121+0000","last_clean":"2024-09-21T09:37:24.816121+0000","last_became_active":"2024-09-21T09:36:43.114341+0000","last_became_peered":"2024-09-21T09:36:43.114341+0000","last_unstale":"2024-09-21T09:37:24.816121+0000","last_undegraded":"2024-09-21T09:37:24.816121+0000","last_fullsized":"2024-09-21T09:37:24.816121+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:29.444341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816549+0000","last_change":"2024-09-21T09:36:43.388944+0000","last_active":"2024-09-21T09:37:24.816549+0000","last_peered":"2024-09-21T09:37:24.816549+0000","last_clean":"2024-09-21T09:37:24.816549+0000","last_became_active":"2024-09-21T09:36:43.388602+0000","last_became_peered":"2024-09-21T09:36:43.388602+0000","last_unstale":"2024-09-21T09:37:24.816549+0000","last_undegraded":"2024-09-21T09:37:24.816549+0000","last_fullsized":"2024-09-21T09:37:24.816549+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:54.714424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104090+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:37:24.104090+0000","last_peered":"2024-09-21T09:37:24.104090+0000","last_clean":"2024-09-21T09:37:24.104090+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:37:24.104090+0000","last_undegraded":"2024-09-21T09:37:24.104090+0000","last_fullsized":"2024-09-21T09:37:24.104090+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816440+0000","last_change":"2024-09-21T09:36:43.389236+0000","last_active":"2024-09-21T09:37:24.816440+0000","last_peered":"2024-09-21T09:37:24.816440+0000","last_clean":"2024-09-21T09:37:24.816440+0000","last_became_active":"2024-09-21T09:36:43.388483+0000","last_became_peered":"2024-09-21T09:36:43.388483+0000","last_unstale":"2024-09-21T09:37:24.816440+0000","last_undegraded":"2024-09-21T09:37:24.816440+0000","last_fullsized":"2024-09-21T09:37:24.816440+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:11.754320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":21,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.101034+0000","last_change":"2024-09-21T09:36:43.112754+0000","last_active":"2024-09-21T09:37:24.101034+0000","last_peered":"2024-09-21T09:37:24.101034+0000","last_clean":"2024-09-21T09:37:24.101034+0000","last_became_active":"2024-09-21T09:36:43.112516+0000","last_became_peered":"2024-09-21T09:36:43.112516+0000","last_unstale":"2024-09-21T09:37:24.101034+0000","last_undegraded":"2024-09-21T09:37:24.101034+0000","last_fullsized":"2024-09-21T09:37:24.101034+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:03.284919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104326+0000","last_change":"2024-09-21T09:36:43.117848+0000","last_active":"2024-09-21T09:37:24.104326+0000","last_peered":"2024-09-21T09:37:24.104326+0000","last_clean":"2024-09-21T09:37:24.104326+0000","last_became_active":"2024-09-21T09:36:43.117530+0000","last_became_peered":"2024-09-21T09:36:43.117530+0000","last_unstale":"2024-09-21T09:37:24.104326+0000","last_undegraded":"2024-09-21T09:37:24.104326+0000","last_fullsized":"2024-09-21T09:37:24.104326+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:24.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102739+0000","last_change":"2024-09-21T09:36:43.389124+0000","last_active":"2024-09-21T09:37:24.102739+0000","last_peered":"2024-09-21T09:37:24.102739+0000","last_clean":"2024-09-21T09:37:24.102739+0000","last_became_active":"2024-09-21T09:36:43.388588+0000","last_became_peered":"2024-09-21T09:36:43.388588+0000","last_unstale":"2024-09-21T09:37:24.102739+0000","last_undegraded":"2024-09-21T09:37:24.102739+0000","last_fullsized":"2024-09-21T09:37:24.102739+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:57.562156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104448+0000","last_change":"2024-09-21T09:36:43.118880+0000","last_active":"2024-09-21T09:37:24.104448+0000","last_peered":"2024-09-21T09:37:24.104448+0000","last_clean":"2024-09-21T09:37:24.104448+0000","last_became_active":"2024-09-21T09:36:43.118607+0000","last_became_peered":"2024-09-21T09:36:43.118607+0000","last_unstale":"2024-09-21T09:37:24.104448+0000","last_undegraded":"2024-09-21T09:37:24.104448+0000","last_fullsized":"2024-09-21T09:37:24.104448+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:03.173398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099832+0000","last_change":"2024-09-21T09:36:43.144659+0000","last_active":"2024-09-21T09:37:24.099832+0000","last_peered":"2024-09-21T09:37:24.099832+0000","last_clean":"2024-09-21T09:37:24.099832+0000","last_became_active":"2024-09-21T09:36:43.144018+0000","last_became_peered":"2024-09-21T09:36:43.144018+0000","last_unstale":"2024-09-21T09:37:24.099832+0000","last_undegraded":"2024-09-21T09:37:24.099832+0000","last_fullsized":"2024-09-21T09:37:24.099832+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:54.660256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104317+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:37:24.104317+0000","last_peered":"2024-09-21T09:37:24.104317+0000","last_clean":"2024-09-21T09:37:24.104317+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:37:24.104317+0000","last_undegraded":"2024-09-21T09:37:24.104317+0000","last_fullsized":"2024-09-21T09:37:24.104317+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:37:25.318285+0000","last_change":"2024-09-21T09:37:25.318285+0000","last_active":"2024-09-21T09:37:25.318285+0000","last_peered":"2024-09-21T09:37:25.318285+0000","last_clean":"2024-09-21T09:37:25.318285+0000","last_became_active":"2024-09-21T09:37:25.317840+0000","last_became_peered":"2024-09-21T09:37:25.317840+0000","last_unstale":"2024-09-21T09:37:25.318285+0000","last_undegraded":"2024-09-21T09:37:25.318285+0000","last_fullsized":"2024-09-21T09:37:25.318285+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:43.798072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103104+0000","last_change":"2024-09-21T09:36:43.123804+0000","last_active":"2024-09-21T09:37:24.103104+0000","last_peered":"2024-09-21T09:37:24.103104+0000","last_clean":"2024-09-21T09:37:24.103104+0000","last_became_active":"2024-09-21T09:36:43.123391+0000","last_became_peered":"2024-09-21T09:36:43.123391+0000","last_unstale":"2024-09-21T09:37:24.103104+0000","last_undegraded":"2024-09-21T09:37:24.103104+0000","last_fullsized":"2024-09-21T09:37:24.103104+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:28:09.612297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816271+0000","last_change":"2024-09-21T09:36:43.141849+0000","last_active":"2024-09-21T09:37:24.816271+0000","last_peered":"2024-09-21T09:37:24.816271+0000","last_clean":"2024-09-21T09:37:24.816271+0000","last_became_active":"2024-09-21T09:36:43.141555+0000","last_became_peered":"2024-09-21T09:36:43.141555+0000","last_unstale":"2024-09-21T09:37:24.816271+0000","last_undegraded":"2024-09-21T09:37:24.816271+0000","last_fullsized":"2024-09-21T09:37:24.816271+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:13:06.417256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816151+0000","last_change":"2024-09-21T09:36:43.115249+0000","last_active":"2024-09-21T09:37:24.816151+0000","last_peered":"2024-09-21T09:37:24.816151+0000","last_clean":"2024-09-21T09:37:24.816151+0000","last_became_active":"2024-09-21T09:36:43.113061+0000","last_became_peered":"2024-09-21T09:36:43.113061+0000","last_unstale":"2024-09-21T09:37:24.816151+0000","last_undegraded":"2024-09-21T09:37:24.816151+0000","last_fullsized":"2024-09-21T09:37:24.816151+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:41.227755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100138+0000","last_change":"2024-09-21T09:36:43.144546+0000","last_active":"2024-09-21T09:37:24.100138+0000","last_peered":"2024-09-21T09:37:24.100138+0000","last_clean":"2024-09-21T09:37:24.100138+0000","last_became_active":"2024-09-21T09:36:43.144158+0000","last_became_peered":"2024-09-21T09:36:43.144158+0000","last_unstale":"2024-09-21T09:37:24.100138+0000","last_undegraded":"2024-09-21T09:37:24.100138+0000","last_fullsized":"2024-09-21T09:37:24.100138+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:29:35.319596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102050+0000","last_change":"2024-09-21T09:36:43.118430+0000","last_active":"2024-09-21T09:37:24.102050+0000","last_peered":"2024-09-21T09:37:24.102050+0000","last_clean":"2024-09-21T09:37:24.102050+0000","last_became_active":"2024-09-21T09:36:43.118182+0000","last_became_peered":"2024-09-21T09:36:43.118182+0000","last_unstale":"2024-09-21T09:37:24.102050+0000","last_undegraded":"2024-09-21T09:37:24.102050+0000","last_fullsized":"2024-09-21T09:37:24.102050+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.215983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100570+0000","last_change":"2024-09-21T09:36:43.139430+0000","last_active":"2024-09-21T09:37:24.100570+0000","last_peered":"2024-09-21T09:37:24.100570+0000","last_clean":"2024-09-21T09:37:24.100570+0000","last_became_active":"2024-09-21T09:36:43.139180+0000","last_became_peered":"2024-09-21T09:36:43.139180+0000","last_unstale":"2024-09-21T09:37:24.100570+0000","last_undegraded":"2024-09-21T09:37:24.100570+0000","last_fullsized":"2024-09-21T09:37:24.100570+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:25:34.229203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099829+0000","last_change":"2024-09-21T09:36:43.115450+0000","last_active":"2024-09-21T09:37:24.099829+0000","last_peered":"2024-09-21T09:37:24.099829+0000","last_clean":"2024-09-21T09:37:24.099829+0000","last_became_active":"2024-09-21T09:36:43.111483+0000","last_became_peered":"2024-09-21T09:36:43.111483+0000","last_unstale":"2024-09-21T09:37:24.099829+0000","last_undegraded":"2024-09-21T09:37:24.099829+0000","last_fullsized":"2024-09-21T09:37:24.099829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:48.980330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100512+0000","last_change":"2024-09-21T09:36:43.140046+0000","last_active":"2024-09-21T09:37:24.100512+0000","last_peered":"2024-09-21T09:37:24.100512+0000","last_clean":"2024-09-21T09:37:24.100512+0000","last_became_active":"2024-09-21T09:36:43.139680+0000","last_became_peered":"2024-09-21T09:36:43.139680+0000","last_unstale":"2024-09-21T09:37:24.100512+0000","last_undegraded":"2024-09-21T09:37:24.100512+0000","last_fullsized":"2024-09-21T09:37:24.100512+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:05:31.198975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815906+0000","last_change":"2024-09-21T09:36:43.120067+0000","last_active":"2024-09-21T09:37:24.815906+0000","last_peered":"2024-09-21T09:37:24.815906+0000","last_clean":"2024-09-21T09:37:24.815906+0000","last_became_active":"2024-09-21T09:36:43.119588+0000","last_became_peered":"2024-09-21T09:36:43.119588+0000","last_unstale":"2024-09-21T09:37:24.815906+0000","last_undegraded":"2024-09-21T09:37:24.815906+0000","last_fullsized":"2024-09-21T09:37:24.815906+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:53.994599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100335+0000","last_change":"2024-09-21T09:36:43.146324+0000","last_active":"2024-09-21T09:37:24.100335+0000","last_peered":"2024-09-21T09:37:24.100335+0000","last_clean":"2024-09-21T09:37:24.100335+0000","last_became_active":"2024-09-21T09:36:43.146030+0000","last_became_peered":"2024-09-21T09:36:43.146030+0000","last_unstale":"2024-09-21T09:37:24.100335+0000","last_undegraded":"2024-09-21T09:37:24.100335+0000","last_fullsized":"2024-09-21T09:37:24.100335+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:28.017328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":55,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103983+0000","last_change":"2024-09-21T09:34:54.592924+0000","last_active":"2024-09-21T09:37:24.103983+0000","last_peered":"2024-09-21T09:37:24.103983+0000","last_clean":"2024-09-21T09:37:24.103983+0000","last_became_active":"2024-09-21T09:34:54.588063+0000","last_became_peered":"2024-09-21T09:34:54.588063+0000","last_unstale":"2024-09-21T09:37:24.103983+0000","last_undegraded":"2024-09-21T09:37:24.103983+0000","last_fullsized":"2024-09-21T09:37:24.103983+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:02.981289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102813+0000","last_change":"2024-09-21T09:35:34.095862+0000","last_active":"2024-09-21T09:37:24.102813+0000","last_peered":"2024-09-21T09:37:24.102813+0000","last_clean":"2024-09-21T09:37:24.102813+0000","last_became_active":"2024-09-21T09:35:29.148713+0000","last_became_peered":"2024-09-21T09:35:29.148713+0000","last_unstale":"2024-09-21T09:37:24.102813+0000","last_undegraded":"2024-09-21T09:37:24.102813+0000","last_fullsized":"2024-09-21T09:37:24.102813+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:18.348007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816074+0000","last_change":"2024-09-21T09:36:43.115501+0000","last_active":"2024-09-21T09:37:24.816074+0000","last_peered":"2024-09-21T09:37:24.816074+0000","last_clean":"2024-09-21T09:37:24.816074+0000","last_became_active":"2024-09-21T09:36:43.115158+0000","last_became_peered":"2024-09-21T09:36:43.115158+0000","last_unstale":"2024-09-21T09:37:24.816074+0000","last_undegraded":"2024-09-21T09:37:24.816074+0000","last_fullsized":"2024-09-21T09:37:24.816074+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:35:27.744791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.101345+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:37:24.101345+0000","last_peered":"2024-09-21T09:37:24.101345+0000","last_clean":"2024-09-21T09:37:24.101345+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:37:24.101345+0000","last_undegraded":"2024-09-21T09:37:24.101345+0000","last_fullsized":"2024-09-21T09:37:24.101345+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104048+0000","last_change":"2024-09-21T09:35:34.101543+0000","last_active":"2024-09-21T09:37:24.104048+0000","last_peered":"2024-09-21T09:37:24.104048+0000","last_clean":"2024-09-21T09:37:24.104048+0000","last_became_active":"2024-09-21T09:35:29.753066+0000","last_became_peered":"2024-09-21T09:35:29.753066+0000","last_unstale":"2024-09-21T09:37:24.104048+0000","last_undegraded":"2024-09-21T09:37:24.104048+0000","last_fullsized":"2024-09-21T09:37:24.104048+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:49.271159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815829+0000","last_change":"2024-09-21T09:36:43.113941+0000","last_active":"2024-09-21T09:37:24.815829+0000","last_peered":"2024-09-21T09:37:24.815829+0000","last_clean":"2024-09-21T09:37:24.815829+0000","last_became_active":"2024-09-21T09:36:43.113641+0000","last_became_peered":"2024-09-21T09:36:43.113641+0000","last_unstale":"2024-09-21T09:37:24.815829+0000","last_undegraded":"2024-09-21T09:37:24.815829+0000","last_fullsized":"2024-09-21T09:37:24.815829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:49.055900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816479+0000","last_change":"2024-09-21T09:36:43.142144+0000","last_active":"2024-09-21T09:37:24.816479+0000","last_peered":"2024-09-21T09:37:24.816479+0000","last_clean":"2024-09-21T09:37:24.816479+0000","last_became_active":"2024-09-21T09:36:43.141540+0000","last_became_peered":"2024-09-21T09:36:43.141540+0000","last_unstale":"2024-09-21T09:37:24.816479+0000","last_undegraded":"2024-09-21T09:37:24.816479+0000","last_fullsized":"2024-09-21T09:37:24.816479+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:30.370892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":24,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098456+0000","last_change":"2024-09-21T09:35:34.090997+0000","last_active":"2024-09-21T09:37:24.098456+0000","last_peered":"2024-09-21T09:37:24.098456+0000","last_clean":"2024-09-21T09:37:24.098456+0000","last_became_active":"2024-09-21T09:35:29.752848+0000","last_became_peered":"2024-09-21T09:35:29.752848+0000","last_unstale":"2024-09-21T09:37:24.098456+0000","last_undegraded":"2024-09-21T09:37:24.098456+0000","last_fullsized":"2024-09-21T09:37:24.098456+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:57:11.329055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098555+0000","last_change":"2024-09-21T09:36:43.120577+0000","last_active":"2024-09-21T09:37:24.098555+0000","last_peered":"2024-09-21T09:37:24.098555+0000","last_clean":"2024-09-21T09:37:24.098555+0000","last_became_active":"2024-09-21T09:36:43.120247+0000","last_became_peered":"2024-09-21T09:36:43.120247+0000","last_unstale":"2024-09-21T09:37:24.098555+0000","last_undegraded":"2024-09-21T09:37:24.098555+0000","last_fullsized":"2024-09-21T09:37:24.098555+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:30:10.958363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100418+0000","last_change":"2024-09-21T09:36:43.137959+0000","last_active":"2024-09-21T09:37:24.100418+0000","last_peered":"2024-09-21T09:37:24.100418+0000","last_clean":"2024-09-21T09:37:24.100418+0000","last_became_active":"2024-09-21T09:36:43.137656+0000","last_became_peered":"2024-09-21T09:36:43.137656+0000","last_unstale":"2024-09-21T09:37:24.100418+0000","last_undegraded":"2024-09-21T09:37:24.100418+0000","last_fullsized":"2024-09-21T09:37:24.100418+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:41.758559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":27,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103560+0000","last_change":"2024-09-21T09:35:34.096316+0000","last_active":"2024-09-21T09:37:24.103560+0000","last_peered":"2024-09-21T09:37:24.103560+0000","last_clean":"2024-09-21T09:37:24.103560+0000","last_became_active":"2024-09-21T09:35:29.148648+0000","last_became_peered":"2024-09-21T09:35:29.148648+0000","last_unstale":"2024-09-21T09:37:24.103560+0000","last_undegraded":"2024-09-21T09:37:24.103560+0000","last_fullsized":"2024-09-21T09:37:24.103560+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:29.243621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100278+0000","last_change":"2024-09-21T09:36:43.119551+0000","last_active":"2024-09-21T09:37:24.100278+0000","last_peered":"2024-09-21T09:37:24.100278+0000","last_clean":"2024-09-21T09:37:24.100278+0000","last_became_active":"2024-09-21T09:36:43.119105+0000","last_became_peered":"2024-09-21T09:36:43.119105+0000","last_unstale":"2024-09-21T09:37:24.100278+0000","last_undegraded":"2024-09-21T09:37:24.100278+0000","last_fullsized":"2024-09-21T09:37:24.100278+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:01.710977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098912+0000","last_change":"2024-09-21T09:36:43.145633+0000","last_active":"2024-09-21T09:37:24.098912+0000","last_peered":"2024-09-21T09:37:24.098912+0000","last_clean":"2024-09-21T09:37:24.098912+0000","last_became_active":"2024-09-21T09:36:43.145311+0000","last_became_peered":"2024-09-21T09:36:43.145311+0000","last_unstale":"2024-09-21T09:37:24.098912+0000","last_undegraded":"2024-09-21T09:37:24.098912+0000","last_fullsized":"2024-09-21T09:37:24.098912+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:00.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103050+0000","last_change":"2024-09-21T09:36:43.123593+0000","last_active":"2024-09-21T09:37:24.103050+0000","last_peered":"2024-09-21T09:37:24.103050+0000","last_clean":"2024-09-21T09:37:24.103050+0000","last_became_active":"2024-09-21T09:36:43.123321+0000","last_became_peered":"2024-09-21T09:36:43.123321+0000","last_unstale":"2024-09-21T09:37:24.103050+0000","last_undegraded":"2024-09-21T09:37:24.103050+0000","last_fullsized":"2024-09-21T09:37:24.103050+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:01.340388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099092+0000","last_change":"2024-09-21T09:35:34.097383+0000","last_active":"2024-09-21T09:37:24.099092+0000","last_peered":"2024-09-21T09:37:24.099092+0000","last_clean":"2024-09-21T09:37:24.099092+0000","last_became_active":"2024-09-21T09:35:29.752863+0000","last_became_peered":"2024-09-21T09:35:29.752863+0000","last_unstale":"2024-09-21T09:37:24.099092+0000","last_undegraded":"2024-09-21T09:37:24.099092+0000","last_fullsized":"2024-09-21T09:37:24.099092+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:00.231892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:37:25.317643+0000","last_change":"2024-09-21T09:37:25.317643+0000","last_active":"2024-09-21T09:37:25.317643+0000","last_peered":"2024-09-21T09:37:25.317643+0000","last_clean":"2024-09-21T09:37:25.317643+0000","last_became_active":"2024-09-21T09:37:25.317397+0000","last_became_peered":"2024-09-21T09:37:25.317397+0000","last_unstale":"2024-09-21T09:37:25.317643+0000","last_undegraded":"2024-09-21T09:37:25.317643+0000","last_fullsized":"2024-09-21T09:37:25.317643+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:45.758571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099014+0000","last_change":"2024-09-21T09:36:43.120667+0000","last_active":"2024-09-21T09:37:24.099014+0000","last_peered":"2024-09-21T09:37:24.099014+0000","last_clean":"2024-09-21T09:37:24.099014+0000","last_became_active":"2024-09-21T09:36:43.120292+0000","last_became_peered":"2024-09-21T09:36:43.120292+0000","last_unstale":"2024-09-21T09:37:24.099014+0000","last_undegraded":"2024-09-21T09:37:24.099014+0000","last_fullsized":"2024-09-21T09:37:24.099014+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:36:33.835796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104342+0000","last_change":"2024-09-21T09:35:34.101535+0000","last_active":"2024-09-21T09:37:24.104342+0000","last_peered":"2024-09-21T09:37:24.104342+0000","last_clean":"2024-09-21T09:37:24.104342+0000","last_became_active":"2024-09-21T09:35:29.753057+0000","last_became_peered":"2024-09-21T09:35:29.753057+0000","last_unstale":"2024-09-21T09:37:24.104342+0000","last_undegraded":"2024-09-21T09:37:24.104342+0000","last_fullsized":"2024-09-21T09:37:24.104342+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:31:49.246270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816290+0000","last_change":"2024-09-21T09:36:43.389413+0000","last_active":"2024-09-21T09:37:24.816290+0000","last_peered":"2024-09-21T09:37:24.816290+0000","last_clean":"2024-09-21T09:37:24.816290+0000","last_became_active":"2024-09-21T09:36:43.389065+0000","last_became_peered":"2024-09-21T09:36:43.389065+0000","last_unstale":"2024-09-21T09:37:24.816290+0000","last_undegraded":"2024-09-21T09:37:24.816290+0000","last_fullsized":"2024-09-21T09:37:24.816290+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:44.539113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816494+0000","last_change":"2024-09-21T09:36:43.112562+0000","last_active":"2024-09-21T09:37:24.816494+0000","last_peered":"2024-09-21T09:37:24.816494+0000","last_clean":"2024-09-21T09:37:24.816494+0000","last_became_active":"2024-09-21T09:36:43.112231+0000","last_became_peered":"2024-09-21T09:36:43.112231+0000","last_unstale":"2024-09-21T09:37:24.816494+0000","last_undegraded":"2024-09-21T09:37:24.816494+0000","last_fullsized":"2024-09-21T09:37:24.816494+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:15.209506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099457+0000","last_change":"2024-09-21T09:35:34.097223+0000","last_active":"2024-09-21T09:37:24.099457+0000","last_peered":"2024-09-21T09:37:24.099457+0000","last_clean":"2024-09-21T09:37:24.099457+0000","last_became_active":"2024-09-21T09:35:29.378933+0000","last_became_peered":"2024-09-21T09:35:29.378933+0000","last_unstale":"2024-09-21T09:37:24.099457+0000","last_undegraded":"2024-09-21T09:37:24.099457+0000","last_fullsized":"2024-09-21T09:37:24.099457+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:04.627265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033167899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815818+0000","last_change":"2024-09-21T09:36:43.390345+0000","last_active":"2024-09-21T09:37:24.815818+0000","last_peered":"2024-09-21T09:37:24.815818+0000","last_clean":"2024-09-21T09:37:24.815818+0000","last_became_active":"2024-09-21T09:36:43.389359+0000","last_became_peered":"2024-09-21T09:36:43.389359+0000","last_unstale":"2024-09-21T09:37:24.815818+0000","last_undegraded":"2024-09-21T09:37:24.815818+0000","last_fullsized":"2024-09-21T09:37:24.815818+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:20.223990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816599+0000","last_change":"2024-09-21T09:36:43.121629+0000","last_active":"2024-09-21T09:37:24.816599+0000","last_peered":"2024-09-21T09:37:24.816599+0000","last_clean":"2024-09-21T09:37:24.816599+0000","last_became_active":"2024-09-21T09:36:43.121342+0000","last_became_peered":"2024-09-21T09:36:43.121342+0000","last_unstale":"2024-09-21T09:37:24.816599+0000","last_undegraded":"2024-09-21T09:37:24.816599+0000","last_fullsized":"2024-09-21T09:37:24.816599+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:23.037498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102222+0000","last_change":"2024-09-21T09:35:34.097095+0000","last_active":"2024-09-21T09:37:24.102222+0000","last_peered":"2024-09-21T09:37:24.102222+0000","last_clean":"2024-09-21T09:37:24.102222+0000","last_became_active":"2024-09-21T09:35:29.148371+0000","last_became_peered":"2024-09-21T09:35:29.148371+0000","last_unstale":"2024-09-21T09:37:24.102222+0000","last_undegraded":"2024-09-21T09:37:24.102222+0000","last_fullsized":"2024-09-21T09:37:24.102222+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:04:07.399927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099679+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:37:24.099679+0000","last_peered":"2024-09-21T09:37:24.099679+0000","last_clean":"2024-09-21T09:37:24.099679+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:37:24.099679+0000","last_undegraded":"2024-09-21T09:37:24.099679+0000","last_fullsized":"2024-09-21T09:37:24.099679+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100102+0000","last_change":"2024-09-21T09:36:43.139540+0000","last_active":"2024-09-21T09:37:24.100102+0000","last_peered":"2024-09-21T09:37:24.100102+0000","last_clean":"2024-09-21T09:37:24.100102+0000","last_became_active":"2024-09-21T09:36:43.139266+0000","last_became_peered":"2024-09-21T09:36:43.139266+0000","last_unstale":"2024-09-21T09:37:24.100102+0000","last_undegraded":"2024-09-21T09:37:24.100102+0000","last_fullsized":"2024-09-21T09:37:24.100102+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:50.717738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"63'6","reported_seq":19,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100175+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:37:24.100175+0000","last_peered":"2024-09-21T09:37:24.100175+0000","last_clean":"2024-09-21T09:37:24.100175+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:37:24.100175+0000","last_undegraded":"2024-09-21T09:37:24.100175+0000","last_fullsized":"2024-09-21T09:37:24.100175+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099582+0000","last_change":"2024-09-21T09:36:43.140342+0000","last_active":"2024-09-21T09:37:24.099582+0000","last_peered":"2024-09-21T09:37:24.099582+0000","last_clean":"2024-09-21T09:37:24.099582+0000","last_became_active":"2024-09-21T09:36:43.140092+0000","last_became_peered":"2024-09-21T09:36:43.140092+0000","last_unstale":"2024-09-21T09:37:24.099582+0000","last_undegraded":"2024-09-21T09:37:24.099582+0000","last_fullsized":"2024-09-21T09:37:24.099582+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:02.983516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104137+0000","last_change":"2024-09-21T09:36:43.119387+0000","last_active":"2024-09-21T09:37:24.104137+0000","last_peered":"2024-09-21T09:37:24.104137+0000","last_clean":"2024-09-21T09:37:24.104137+0000","last_became_active":"2024-09-21T09:36:43.118703+0000","last_became_peered":"2024-09-21T09:36:43.118703+0000","last_unstale":"2024-09-21T09:37:24.104137+0000","last_undegraded":"2024-09-21T09:37:24.104137+0000","last_fullsized":"2024-09-21T09:37:24.104137+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:58.053141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102915+0000","last_change":"2024-09-21T09:36:43.119455+0000","last_active":"2024-09-21T09:37:24.102915+0000","last_peered":"2024-09-21T09:37:24.102915+0000","last_clean":"2024-09-21T09:37:24.102915+0000","last_became_active":"2024-09-21T09:36:43.119034+0000","last_became_peered":"2024-09-21T09:36:43.119034+0000","last_unstale":"2024-09-21T09:37:24.102915+0000","last_undegraded":"2024-09-21T09:37:24.102915+0000","last_fullsized":"2024-09-21T09:37:24.102915+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:25.444791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815906+0000","last_change":"2024-09-21T09:36:43.389046+0000","last_active":"2024-09-21T09:37:24.815906+0000","last_peered":"2024-09-21T09:37:24.815906+0000","last_clean":"2024-09-21T09:37:24.815906+0000","last_became_active":"2024-09-21T09:36:43.388637+0000","last_became_peered":"2024-09-21T09:36:43.388637+0000","last_unstale":"2024-09-21T09:37:24.815906+0000","last_undegraded":"2024-09-21T09:37:24.815906+0000","last_fullsized":"2024-09-21T09:37:24.815906+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:12.885309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816567+0000","last_change":"2024-09-21T09:36:43.120236+0000","last_active":"2024-09-21T09:37:24.816567+0000","last_peered":"2024-09-21T09:37:24.816567+0000","last_clean":"2024-09-21T09:37:24.816567+0000","last_became_active":"2024-09-21T09:36:43.116719+0000","last_became_peered":"2024-09-21T09:36:43.116719+0000","last_unstale":"2024-09-21T09:37:24.816567+0000","last_undegraded":"2024-09-21T09:37:24.816567+0000","last_fullsized":"2024-09-21T09:37:24.816567+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:46.780407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815781+0000","last_change":"2024-09-21T09:36:43.143466+0000","last_active":"2024-09-21T09:37:24.815781+0000","last_peered":"2024-09-21T09:37:24.815781+0000","last_clean":"2024-09-21T09:37:24.815781+0000","last_became_active":"2024-09-21T09:36:43.143059+0000","last_became_peered":"2024-09-21T09:36:43.143059+0000","last_unstale":"2024-09-21T09:37:24.815781+0000","last_undegraded":"2024-09-21T09:37:24.815781+0000","last_fullsized":"2024-09-21T09:37:24.815781+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:32.604473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:37:25.318535+0000","last_change":"2024-09-21T09:37:25.318535+0000","last_active":"2024-09-21T09:37:25.318535+0000","last_peered":"2024-09-21T09:37:25.318535+0000","last_clean":"2024-09-21T09:37:25.318535+0000","last_became_active":"2024-09-21T09:37:25.317932+0000","last_became_peered":"2024-09-21T09:37:25.317932+0000","last_unstale":"2024-09-21T09:37:25.318535+0000","last_undegraded":"2024-09-21T09:37:25.318535+0000","last_fullsized":"2024-09-21T09:37:25.318535+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:52.502093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815868+0000","last_change":"2024-09-21T09:36:43.121951+0000","last_active":"2024-09-21T09:37:24.815868+0000","last_peered":"2024-09-21T09:37:24.815868+0000","last_clean":"2024-09-21T09:37:24.815868+0000","last_became_active":"2024-09-21T09:36:43.121656+0000","last_became_peered":"2024-09-21T09:36:43.121656+0000","last_unstale":"2024-09-21T09:37:24.815868+0000","last_undegraded":"2024-09-21T09:37:24.815868+0000","last_fullsized":"2024-09-21T09:37:24.815868+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:43.045642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816407+0000","last_change":"2024-09-21T09:36:43.388937+0000","last_active":"2024-09-21T09:37:24.816407+0000","last_peered":"2024-09-21T09:37:24.816407+0000","last_clean":"2024-09-21T09:37:24.816407+0000","last_became_active":"2024-09-21T09:36:43.388558+0000","last_became_peered":"2024-09-21T09:36:43.388558+0000","last_unstale":"2024-09-21T09:37:24.816407+0000","last_undegraded":"2024-09-21T09:37:24.816407+0000","last_fullsized":"2024-09-21T09:37:24.816407+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:29:58.442452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098426+0000","last_change":"2024-09-21T09:36:43.118044+0000","last_active":"2024-09-21T09:37:24.098426+0000","last_peered":"2024-09-21T09:37:24.098426+0000","last_clean":"2024-09-21T09:37:24.098426+0000","last_became_active":"2024-09-21T09:36:43.117752+0000","last_became_peered":"2024-09-21T09:36:43.117752+0000","last_unstale":"2024-09-21T09:37:24.098426+0000","last_undegraded":"2024-09-21T09:37:24.098426+0000","last_fullsized":"2024-09-21T09:37:24.098426+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:31:47.718963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816379+0000","last_change":"2024-09-21T09:36:43.142724+0000","last_active":"2024-09-21T09:37:24.816379+0000","last_peered":"2024-09-21T09:37:24.816379+0000","last_clean":"2024-09-21T09:37:24.816379+0000","last_became_active":"2024-09-21T09:36:43.140767+0000","last_became_peered":"2024-09-21T09:36:43.140767+0000","last_unstale":"2024-09-21T09:37:24.816379+0000","last_undegraded":"2024-09-21T09:37:24.816379+0000","last_fullsized":"2024-09-21T09:37:24.816379+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:03.041550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100014+0000","last_change":"2024-09-21T09:36:43.115705+0000","last_active":"2024-09-21T09:37:24.100014+0000","last_peered":"2024-09-21T09:37:24.100014+0000","last_clean":"2024-09-21T09:37:24.100014+0000","last_became_active":"2024-09-21T09:36:43.115439+0000","last_became_peered":"2024-09-21T09:36:43.115439+0000","last_unstale":"2024-09-21T09:37:24.100014+0000","last_undegraded":"2024-09-21T09:37:24.100014+0000","last_fullsized":"2024-09-21T09:37:24.100014+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:06:12.185663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098645+0000","last_change":"2024-09-21T09:36:43.138262+0000","last_active":"2024-09-21T09:37:24.098645+0000","last_peered":"2024-09-21T09:37:24.098645+0000","last_clean":"2024-09-21T09:37:24.098645+0000","last_became_active":"2024-09-21T09:36:43.137957+0000","last_became_peered":"2024-09-21T09:36:43.137957+0000","last_unstale":"2024-09-21T09:37:24.098645+0000","last_undegraded":"2024-09-21T09:37:24.098645+0000","last_fullsized":"2024-09-21T09:37:24.098645+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:01.178073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816357+0000","last_change":"2024-09-21T09:36:43.116111+0000","last_active":"2024-09-21T09:37:24.816357+0000","last_peered":"2024-09-21T09:37:24.816357+0000","last_clean":"2024-09-21T09:37:24.816357+0000","last_became_active":"2024-09-21T09:36:43.115022+0000","last_became_peered":"2024-09-21T09:36:43.115022+0000","last_unstale":"2024-09-21T09:37:24.816357+0000","last_undegraded":"2024-09-21T09:37:24.816357+0000","last_fullsized":"2024-09-21T09:37:24.816357+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:48.706807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100070+0000","last_change":"2024-09-21T09:36:43.148280+0000","last_active":"2024-09-21T09:37:24.100070+0000","last_peered":"2024-09-21T09:37:24.100070+0000","last_clean":"2024-09-21T09:37:24.100070+0000","last_became_active":"2024-09-21T09:36:43.148010+0000","last_became_peered":"2024-09-21T09:36:43.148010+0000","last_unstale":"2024-09-21T09:37:24.100070+0000","last_undegraded":"2024-09-21T09:37:24.100070+0000","last_fullsized":"2024-09-21T09:37:24.100070+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:26:00.852481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102981+0000","last_change":"2024-09-21T09:36:43.117958+0000","last_active":"2024-09-21T09:37:24.102981+0000","last_peered":"2024-09-21T09:37:24.102981+0000","last_clean":"2024-09-21T09:37:24.102981+0000","last_became_active":"2024-09-21T09:36:43.117652+0000","last_became_peered":"2024-09-21T09:36:43.117652+0000","last_unstale":"2024-09-21T09:37:24.102981+0000","last_undegraded":"2024-09-21T09:37:24.102981+0000","last_fullsized":"2024-09-21T09:37:24.102981+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:58.389534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099888+0000","last_change":"2024-09-21T09:36:43.144525+0000","last_active":"2024-09-21T09:37:24.099888+0000","last_peered":"2024-09-21T09:37:24.099888+0000","last_clean":"2024-09-21T09:37:24.099888+0000","last_became_active":"2024-09-21T09:36:43.144223+0000","last_became_peered":"2024-09-21T09:36:43.144223+0000","last_unstale":"2024-09-21T09:37:24.099888+0000","last_undegraded":"2024-09-21T09:37:24.099888+0000","last_fullsized":"2024-09-21T09:37:24.099888+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:57:40.702512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103042+0000","last_change":"2024-09-21T09:36:43.115791+0000","last_active":"2024-09-21T09:37:24.103042+0000","last_peered":"2024-09-21T09:37:24.103042+0000","last_clean":"2024-09-21T09:37:24.103042+0000","last_became_active":"2024-09-21T09:36:43.115525+0000","last_became_peered":"2024-09-21T09:36:43.115525+0000","last_unstale":"2024-09-21T09:37:24.103042+0000","last_undegraded":"2024-09-21T09:37:24.103042+0000","last_fullsized":"2024-09-21T09:37:24.103042+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:28.396722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815984+0000","last_change":"2024-09-21T09:36:43.140639+0000","last_active":"2024-09-21T09:37:24.815984+0000","last_peered":"2024-09-21T09:37:24.815984+0000","last_clean":"2024-09-21T09:37:24.815984+0000","last_became_active":"2024-09-21T09:36:43.139865+0000","last_became_peered":"2024-09-21T09:36:43.139865+0000","last_unstale":"2024-09-21T09:37:24.815984+0000","last_undegraded":"2024-09-21T09:37:24.815984+0000","last_fullsized":"2024-09-21T09:37:24.815984+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:01:15.881225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816282+0000","last_change":"2024-09-21T09:36:43.117111+0000","last_active":"2024-09-21T09:37:24.816282+0000","last_peered":"2024-09-21T09:37:24.816282+0000","last_clean":"2024-09-21T09:37:24.816282+0000","last_became_active":"2024-09-21T09:36:43.114575+0000","last_became_peered":"2024-09-21T09:36:43.114575+0000","last_unstale":"2024-09-21T09:37:24.816282+0000","last_undegraded":"2024-09-21T09:37:24.816282+0000","last_fullsized":"2024-09-21T09:37:24.816282+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:13.218433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099576+0000","last_change":"2024-09-21T09:36:43.140043+0000","last_active":"2024-09-21T09:37:24.099576+0000","last_peered":"2024-09-21T09:37:24.099576+0000","last_clean":"2024-09-21T09:37:24.099576+0000","last_became_active":"2024-09-21T09:36:43.139784+0000","last_became_peered":"2024-09-21T09:36:43.139784+0000","last_unstale":"2024-09-21T09:37:24.099576+0000","last_undegraded":"2024-09-21T09:37:24.099576+0000","last_fullsized":"2024-09-21T09:37:24.099576+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:05.226038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816574+0000","last_change":"2024-09-21T09:36:43.119924+0000","last_active":"2024-09-21T09:37:24.816574+0000","last_peered":"2024-09-21T09:37:24.816574+0000","last_clean":"2024-09-21T09:37:24.816574+0000","last_became_active":"2024-09-21T09:36:43.116283+0000","last_became_peered":"2024-09-21T09:36:43.116283+0000","last_unstale":"2024-09-21T09:37:24.816574+0000","last_undegraded":"2024-09-21T09:37:24.816574+0000","last_fullsized":"2024-09-21T09:37:24.816574+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:16:31.191718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397541,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1252,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1282048,"data_stored":881739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.502,"15min":0.502},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67600000000000005,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.623},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.52100000000000002,"15min":0.51700000000000002},"min":{"1min":0.39900000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.69499999999999995,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.63600000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.502,"15min":0.499},"min":{"1min":0.40600000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.63700000000000001,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.39800000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.71799999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39700000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.70799999999999996,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.52600000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52000000000000002,"15min":0.51900000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.70599999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.42299999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.65500000000000003,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41299999999999998,"15min":0.41199999999999998},"min":{"1min":0.28999999999999998,"5min":0.248,"15min":0.248},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.47499999999999998,"15min":0.48499999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.59099999999999997,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.57899999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.47199999999999998,"15min":0.47699999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.50900000000000001,"15min":0.51300000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.67000000000000004,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.52100000000000002,"15min":0.52700000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.64800000000000002}]}]},{"osd":6,"up_from":43,"seq":184683593770,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298596,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446556,"statfs":{"total":95995035648,"available":95689273344,"internally_reserved":0,"allocated":1265664,"data_stored":881053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.54800000000000004,"15min":0.53900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53800000000000003,"15min":0.52800000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71299999999999997,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.53400000000000003}]},{"osd":1,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.38600000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.53300000000000003,"15min":0.50900000000000001},"min":{"1min":0.45300000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.56599999999999995,"15min":0.55800000000000005},"min":{"1min":0.48399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.53800000000000003,"15min":0.52500000000000002},"min":{"1min":0.44900000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.58299999999999996}]},{"osd":3,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54600000000000004,"15min":0.53800000000000003},"min":{"1min":0.42699999999999999,"5min":0.375,"15min":0.375},"max":{"1min":0.72499999999999998,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.56200000000000006,"15min":0.54200000000000004},"min":{"1min":0.42699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.626}]},{"osd":4,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.45300000000000001,"15min":0.44400000000000001},"min":{"1min":0.32400000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.61899999999999999,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.46600000000000003,"15min":0.45400000000000001},"min":{"1min":0.33500000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.51300000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.46700000000000003,"15min":0.45000000000000001},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.502},{"interface":"front","average":{"1min":0.5,"5min":0.47399999999999998,"15min":0.46000000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.73699999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.68300000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:36:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.51700000000000002,"15min":0.51300000000000001},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.55200000000000005,"15min":0.54200000000000004},"min":{"1min":0.39500000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.56699999999999995}]}]},{"osd":5,"up_from":37,"seq":158913789999,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":289947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53000000000000003,"15min":0.54800000000000004},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.628,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.52000000000000002,"15min":0.54200000000000004},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.61599999999999999,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.54800000000000004,"15min":0.56299999999999994},"min":{"1min":0.35499999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.65300000000000002,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.497,"5min":0.53300000000000003,"15min":0.55000000000000004},"min":{"1min":0.33100000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66700000000000004,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.60499999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.53500000000000003,"15min":0.55300000000000005},"min":{"1min":0.317,"5min":0.314,"15min":0.314},"max":{"1min":0.67800000000000005,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53300000000000003,"15min":0.53900000000000003},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.63900000000000001,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.61899999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.55700000000000005,"15min":0.57499999999999996},"min":{"1min":0.40400000000000003,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70099999999999996,"5min":0.88,"15min":0.88},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54400000000000004,"15min":0.55400000000000005},"min":{"1min":0.41699999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.65700000000000003,"5min":0.754,"15min":0.754},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.39700000000000002,"15min":0.39300000000000002},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.61199999999999999,"5min":0.624,"15min":0.624},"last":0.432},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40400000000000003,"15min":0.39600000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.60299999999999998,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.53800000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:37:35 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.45500000000000002,"15min":0.442},"min":{"1min":0.311,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.47399999999999998,"15min":0.46999999999999997},"min":{"1min":0.32800000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67700000000000005,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.47599999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.47999999999999998,"15min":0.47799999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.69499999999999995,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.32100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.66200000000000003,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.55600000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986226,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298024,"kb_used_data":668,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447128,"statfs":{"total":95995035648,"available":95689859072,"internally_reserved":0,"allocated":684032,"data_stored":291301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.53600000000000003,"15min":0.54100000000000004},"min":{"1min":0.379,"5min":0.318,"15min":0.318},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56000000000000005,"15min":0.56599999999999995},"min":{"1min":0.38900000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.74199999999999999,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.64400000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:36:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52600000000000002,"15min":0.53400000000000003},"min":{"1min":0.35799999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.749,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.38900000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71899999999999997,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.59399999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:36:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54500000000000004,"15min":0.54400000000000004},"min":{"1min":0.379,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.753,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38600000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.55600000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:36:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.76600000000000001,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.36299999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.39900000000000002,"15min":0.38600000000000001},"min":{"1min":0.29199999999999998,"5min":0.25,"15min":0.25},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.441},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.40999999999999998,"15min":0.40100000000000002},"min":{"1min":0.30299999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.65700000000000003,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.36699999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.42799999999999999,"15min":0.40600000000000003},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.48199999999999998,"15min":0.46600000000000003},"min":{"1min":0.34000000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68200000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.498}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.45900000000000002,"15min":0.45500000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67300000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.48199999999999998,"15min":0.47899999999999998},"min":{"1min":0.28999999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.69799999999999995,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.51400000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":663552,"data_stored":289693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.39400000000000002,"15min":0.40000000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.51700000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.40899999999999997,"15min":0.42099999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.495,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.437,"15min":0.45100000000000001},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.627,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.316},{"interface":"front","average":{"1min":0.378,"5min":0.42899999999999999,"15min":0.436},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56100000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.34200000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.46100000000000002,"15min":0.47199999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.51400000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.45200000000000001,"15min":0.46700000000000003},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.54600000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.29399999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:36:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.59599999999999997,"15min":0.60499999999999998},"min":{"1min":0.40799999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80900000000000005,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.626,"15min":0.63100000000000001},"min":{"1min":0.48099999999999998,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.77600000000000002,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.52900000000000003}]},{"osd":5,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.59299999999999997,"15min":0.59499999999999997},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.78400000000000003,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.433,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.66700000000000004}]},{"osd":6,"last update":"Sat Sep 21 09:37:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.61699999999999999,"15min":0.621},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.748,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.626,"15min":0.63},"min":{"1min":0.42399999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.72699999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.56000000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:36:56 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.75900000000000001,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.751}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":667648,"data_stored":290207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:00 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.371,"15min":0.34699999999999998},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":1.782,"5min":1.782,"15min":1.782},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.38300000000000001,"15min":0.35899999999999999},"min":{"1min":0.249,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.35399999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:37:00 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.40400000000000003,"15min":0.377},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.0659999999999998,"5min":2.0659999999999998,"15min":2.0659999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.38,"15min":0.35699999999999998},"min":{"1min":0.26200000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.58499999999999996,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:37:19 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.44700000000000001,"15min":0.45300000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.56100000000000005,"5min":0.627,"15min":0.627},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.373,"5min":0.432,"15min":0.434},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.502,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.43099999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:36:41 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57799999999999996,"15min":0.57199999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.75,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57499999999999996,"15min":0.57199999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79900000000000004,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.61399999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:37:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.443,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.81899999999999995,"5min":0.88,"15min":0.88},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.61899999999999999,"15min":0.61099999999999999},"min":{"1min":0.432,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84699999999999998,"5min":1.962,"15min":1.962},"last":0.76900000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:37:32 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.65200000000000002,"15min":0.66300000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80600000000000005,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.60799999999999998,"15min":0.624},"min":{"1min":0.40500000000000003,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73699999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.70299999999999996}]},{"osd":7,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.63900000000000001,"15min":0.64200000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.66200000000000003,"15min":0.65800000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.9930000000000001,"5min":1.9930000000000001,"15min":1.9930000000000001},"last":0.748}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":663552,"data_stored":289622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:33 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.35399999999999998,"15min":0.33400000000000002},"min":{"1min":0.19900000000000001,"5min":0.156,"15min":0.156},"max":{"1min":0.54300000000000004,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.376,"15min":0.34999999999999998},"min":{"1min":0.25900000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.48799999999999999,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.375}]},{"osd":2,"last update":"Sat Sep 21 09:36:57 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.39200000000000002,"15min":0.38200000000000001},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.59899999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.32400000000000001,"5min":0.38300000000000001,"15min":0.38900000000000001},"min":{"1min":0.23799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.52100000000000002,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.31}]},{"osd":3,"last update":"Sat Sep 21 09:37:27 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.39700000000000002,"15min":0.40899999999999997},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.436,"15min":0.44800000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56100000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.44400000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:36:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.755,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.78200000000000003,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 21 09:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.41699999999999998,"5min":0.37,"15min":0.37},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:37:33 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.59699999999999998,"15min":0.60099999999999998},"min":{"1min":0.432,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.62},{"interface":"front","average":{"1min":0.625,"5min":0.63100000000000001,"15min":0.63500000000000001},"min":{"1min":0.495,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:36:57 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.56799999999999995,"15min":0.55800000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.73999999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.64100000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298592,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446560,"statfs":{"total":95995035648,"available":95689277440,"internally_reserved":0,"allocated":1261568,"data_stored":880563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:37:39 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.35299999999999998,"15min":0.33900000000000002},"min":{"1min":0.245,"5min":0.184,"15min":0.184},"max":{"1min":0.58199999999999996,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.38,"15min":0.35699999999999998},"min":{"1min":0.28699999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.59499999999999997,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:37:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.40600000000000003,"15min":0.41799999999999998},"min":{"1min":0.23300000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.371,"5min":0.39500000000000002,"15min":0.40899999999999997},"min":{"1min":0.23699999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.45100000000000001,"15min":0.47499999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.627,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.439,"15min":0.46400000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64500000000000002,"5min":0.753,"15min":0.753},"last":0.443}]},{"osd":4,"last update":"Sat Sep 21 09:36:48 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53200000000000003,"15min":0.53700000000000003},"min":{"1min":0.40100000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71799999999999997,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.38200000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79100000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.78600000000000003}]},{"osd":5,"last update":"Sat Sep 21 09:37:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56299999999999994,"15min":0.56100000000000005},"min":{"1min":0.41599999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73099999999999998,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42699999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.64500000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:37:29 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.59199999999999997,"15min":0.58799999999999997},"min":{"1min":0.46000000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81000000000000005,"5min":0.878,"15min":0.878},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.58099999999999996,"15min":0.57699999999999996},"min":{"1min":0.501,"5min":0.379,"15min":0.379},"max":{"1min":0.76200000000000001,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.72999999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:36:57 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59999999999999998,"15min":0.60099999999999998},"min":{"1min":0.46800000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59599999999999997,"15min":0.59699999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77300000000000002,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.69699999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:37:44.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:37:44.438 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:44.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:44 smithi017 ceph-mon[29380]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:44.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:44 smithi017 ceph-mon[29380]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:44.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:44 smithi017 ceph-mon[40737]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:44.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:44 smithi017 ceph-mon[40737]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:44.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:44 smithi134 ceph-mon[33541]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:44.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:44 smithi134 ceph-mon[33541]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:37:46.422 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:37:46.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:46 smithi134 ceph-mon[33541]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:46.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:37:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:37:47.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:46 smithi017 ceph-mon[29380]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:47.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:46 smithi017 ceph-mon[40737]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:47.146 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":87,"stamp":"2024-09-21T09:37:45.676584+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121,"ondisk_log_size":121,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385860,"kb_used_data":6988,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575356,"statfs":{"total":767960285184,"available":765517164544,"internally_reserved":0,"allocated":7155712,"data_stored":4094125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002445"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816224+0000","last_change":"2024-09-21T09:36:43.389780+0000","last_active":"2024-09-21T09:37:24.816224+0000","last_peered":"2024-09-21T09:37:24.816224+0000","last_clean":"2024-09-21T09:37:24.816224+0000","last_became_active":"2024-09-21T09:36:43.389430+0000","last_became_peered":"2024-09-21T09:36:43.389430+0000","last_unstale":"2024-09-21T09:37:24.816224+0000","last_undegraded":"2024-09-21T09:37:24.816224+0000","last_fullsized":"2024-09-21T09:37:24.816224+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:16.104966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104167+0000","last_change":"2024-09-21T09:36:43.112938+0000","last_active":"2024-09-21T09:37:24.104167+0000","last_peered":"2024-09-21T09:37:24.104167+0000","last_clean":"2024-09-21T09:37:24.104167+0000","last_became_active":"2024-09-21T09:36:43.112638+0000","last_became_peered":"2024-09-21T09:36:43.112638+0000","last_unstale":"2024-09-21T09:37:24.104167+0000","last_undegraded":"2024-09-21T09:37:24.104167+0000","last_fullsized":"2024-09-21T09:37:24.104167+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:49.176718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102902+0000","last_change":"2024-09-21T09:36:43.388929+0000","last_active":"2024-09-21T09:37:24.102902+0000","last_peered":"2024-09-21T09:37:24.102902+0000","last_clean":"2024-09-21T09:37:24.102902+0000","last_became_active":"2024-09-21T09:36:43.388613+0000","last_became_peered":"2024-09-21T09:36:43.388613+0000","last_unstale":"2024-09-21T09:37:24.102902+0000","last_undegraded":"2024-09-21T09:37:24.102902+0000","last_fullsized":"2024-09-21T09:37:24.102902+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:51.496717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816121+0000","last_change":"2024-09-21T09:36:43.114686+0000","last_active":"2024-09-21T09:37:24.816121+0000","last_peered":"2024-09-21T09:37:24.816121+0000","last_clean":"2024-09-21T09:37:24.816121+0000","last_became_active":"2024-09-21T09:36:43.114341+0000","last_became_peered":"2024-09-21T09:36:43.114341+0000","last_unstale":"2024-09-21T09:37:24.816121+0000","last_undegraded":"2024-09-21T09:37:24.816121+0000","last_fullsized":"2024-09-21T09:37:24.816121+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:29.444341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816549+0000","last_change":"2024-09-21T09:36:43.388944+0000","last_active":"2024-09-21T09:37:24.816549+0000","last_peered":"2024-09-21T09:37:24.816549+0000","last_clean":"2024-09-21T09:37:24.816549+0000","last_became_active":"2024-09-21T09:36:43.388602+0000","last_became_peered":"2024-09-21T09:36:43.388602+0000","last_unstale":"2024-09-21T09:37:24.816549+0000","last_undegraded":"2024-09-21T09:37:24.816549+0000","last_fullsized":"2024-09-21T09:37:24.816549+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:54.714424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104090+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:37:24.104090+0000","last_peered":"2024-09-21T09:37:24.104090+0000","last_clean":"2024-09-21T09:37:24.104090+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:37:24.104090+0000","last_undegraded":"2024-09-21T09:37:24.104090+0000","last_fullsized":"2024-09-21T09:37:24.104090+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816440+0000","last_change":"2024-09-21T09:36:43.389236+0000","last_active":"2024-09-21T09:37:24.816440+0000","last_peered":"2024-09-21T09:37:24.816440+0000","last_clean":"2024-09-21T09:37:24.816440+0000","last_became_active":"2024-09-21T09:36:43.388483+0000","last_became_peered":"2024-09-21T09:36:43.388483+0000","last_unstale":"2024-09-21T09:37:24.816440+0000","last_undegraded":"2024-09-21T09:37:24.816440+0000","last_fullsized":"2024-09-21T09:37:24.816440+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:11.754320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":21,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.101034+0000","last_change":"2024-09-21T09:36:43.112754+0000","last_active":"2024-09-21T09:37:24.101034+0000","last_peered":"2024-09-21T09:37:24.101034+0000","last_clean":"2024-09-21T09:37:24.101034+0000","last_became_active":"2024-09-21T09:36:43.112516+0000","last_became_peered":"2024-09-21T09:36:43.112516+0000","last_unstale":"2024-09-21T09:37:24.101034+0000","last_undegraded":"2024-09-21T09:37:24.101034+0000","last_fullsized":"2024-09-21T09:37:24.101034+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:03.284919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104326+0000","last_change":"2024-09-21T09:36:43.117848+0000","last_active":"2024-09-21T09:37:24.104326+0000","last_peered":"2024-09-21T09:37:24.104326+0000","last_clean":"2024-09-21T09:37:24.104326+0000","last_became_active":"2024-09-21T09:36:43.117530+0000","last_became_peered":"2024-09-21T09:36:43.117530+0000","last_unstale":"2024-09-21T09:37:24.104326+0000","last_undegraded":"2024-09-21T09:37:24.104326+0000","last_fullsized":"2024-09-21T09:37:24.104326+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:24.503053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102739+0000","last_change":"2024-09-21T09:36:43.389124+0000","last_active":"2024-09-21T09:37:24.102739+0000","last_peered":"2024-09-21T09:37:24.102739+0000","last_clean":"2024-09-21T09:37:24.102739+0000","last_became_active":"2024-09-21T09:36:43.388588+0000","last_became_peered":"2024-09-21T09:36:43.388588+0000","last_unstale":"2024-09-21T09:37:24.102739+0000","last_undegraded":"2024-09-21T09:37:24.102739+0000","last_fullsized":"2024-09-21T09:37:24.102739+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:57.562156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104448+0000","last_change":"2024-09-21T09:36:43.118880+0000","last_active":"2024-09-21T09:37:24.104448+0000","last_peered":"2024-09-21T09:37:24.104448+0000","last_clean":"2024-09-21T09:37:24.104448+0000","last_became_active":"2024-09-21T09:36:43.118607+0000","last_became_peered":"2024-09-21T09:36:43.118607+0000","last_unstale":"2024-09-21T09:37:24.104448+0000","last_undegraded":"2024-09-21T09:37:24.104448+0000","last_fullsized":"2024-09-21T09:37:24.104448+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:03.173398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099832+0000","last_change":"2024-09-21T09:36:43.144659+0000","last_active":"2024-09-21T09:37:24.099832+0000","last_peered":"2024-09-21T09:37:24.099832+0000","last_clean":"2024-09-21T09:37:24.099832+0000","last_became_active":"2024-09-21T09:36:43.144018+0000","last_became_peered":"2024-09-21T09:36:43.144018+0000","last_unstale":"2024-09-21T09:37:24.099832+0000","last_undegraded":"2024-09-21T09:37:24.099832+0000","last_fullsized":"2024-09-21T09:37:24.099832+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:54.660256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104317+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:37:24.104317+0000","last_peered":"2024-09-21T09:37:24.104317+0000","last_clean":"2024-09-21T09:37:24.104317+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:37:24.104317+0000","last_undegraded":"2024-09-21T09:37:24.104317+0000","last_fullsized":"2024-09-21T09:37:24.104317+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:37:25.318285+0000","last_change":"2024-09-21T09:37:25.318285+0000","last_active":"2024-09-21T09:37:25.318285+0000","last_peered":"2024-09-21T09:37:25.318285+0000","last_clean":"2024-09-21T09:37:25.318285+0000","last_became_active":"2024-09-21T09:37:25.317840+0000","last_became_peered":"2024-09-21T09:37:25.317840+0000","last_unstale":"2024-09-21T09:37:25.318285+0000","last_undegraded":"2024-09-21T09:37:25.318285+0000","last_fullsized":"2024-09-21T09:37:25.318285+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:43.798072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103104+0000","last_change":"2024-09-21T09:36:43.123804+0000","last_active":"2024-09-21T09:37:24.103104+0000","last_peered":"2024-09-21T09:37:24.103104+0000","last_clean":"2024-09-21T09:37:24.103104+0000","last_became_active":"2024-09-21T09:36:43.123391+0000","last_became_peered":"2024-09-21T09:36:43.123391+0000","last_unstale":"2024-09-21T09:37:24.103104+0000","last_undegraded":"2024-09-21T09:37:24.103104+0000","last_fullsized":"2024-09-21T09:37:24.103104+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:28:09.612297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816271+0000","last_change":"2024-09-21T09:36:43.141849+0000","last_active":"2024-09-21T09:37:24.816271+0000","last_peered":"2024-09-21T09:37:24.816271+0000","last_clean":"2024-09-21T09:37:24.816271+0000","last_became_active":"2024-09-21T09:36:43.141555+0000","last_became_peered":"2024-09-21T09:36:43.141555+0000","last_unstale":"2024-09-21T09:37:24.816271+0000","last_undegraded":"2024-09-21T09:37:24.816271+0000","last_fullsized":"2024-09-21T09:37:24.816271+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:13:06.417256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816151+0000","last_change":"2024-09-21T09:36:43.115249+0000","last_active":"2024-09-21T09:37:24.816151+0000","last_peered":"2024-09-21T09:37:24.816151+0000","last_clean":"2024-09-21T09:37:24.816151+0000","last_became_active":"2024-09-21T09:36:43.113061+0000","last_became_peered":"2024-09-21T09:36:43.113061+0000","last_unstale":"2024-09-21T09:37:24.816151+0000","last_undegraded":"2024-09-21T09:37:24.816151+0000","last_fullsized":"2024-09-21T09:37:24.816151+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:41.227755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100138+0000","last_change":"2024-09-21T09:36:43.144546+0000","last_active":"2024-09-21T09:37:24.100138+0000","last_peered":"2024-09-21T09:37:24.100138+0000","last_clean":"2024-09-21T09:37:24.100138+0000","last_became_active":"2024-09-21T09:36:43.144158+0000","last_became_peered":"2024-09-21T09:36:43.144158+0000","last_unstale":"2024-09-21T09:37:24.100138+0000","last_undegraded":"2024-09-21T09:37:24.100138+0000","last_fullsized":"2024-09-21T09:37:24.100138+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:29:35.319596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102050+0000","last_change":"2024-09-21T09:36:43.118430+0000","last_active":"2024-09-21T09:37:24.102050+0000","last_peered":"2024-09-21T09:37:24.102050+0000","last_clean":"2024-09-21T09:37:24.102050+0000","last_became_active":"2024-09-21T09:36:43.118182+0000","last_became_peered":"2024-09-21T09:36:43.118182+0000","last_unstale":"2024-09-21T09:37:24.102050+0000","last_undegraded":"2024-09-21T09:37:24.102050+0000","last_fullsized":"2024-09-21T09:37:24.102050+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:30.215983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100570+0000","last_change":"2024-09-21T09:36:43.139430+0000","last_active":"2024-09-21T09:37:24.100570+0000","last_peered":"2024-09-21T09:37:24.100570+0000","last_clean":"2024-09-21T09:37:24.100570+0000","last_became_active":"2024-09-21T09:36:43.139180+0000","last_became_peered":"2024-09-21T09:36:43.139180+0000","last_unstale":"2024-09-21T09:37:24.100570+0000","last_undegraded":"2024-09-21T09:37:24.100570+0000","last_fullsized":"2024-09-21T09:37:24.100570+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:25:34.229203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099829+0000","last_change":"2024-09-21T09:36:43.115450+0000","last_active":"2024-09-21T09:37:24.099829+0000","last_peered":"2024-09-21T09:37:24.099829+0000","last_clean":"2024-09-21T09:37:24.099829+0000","last_became_active":"2024-09-21T09:36:43.111483+0000","last_became_peered":"2024-09-21T09:36:43.111483+0000","last_unstale":"2024-09-21T09:37:24.099829+0000","last_undegraded":"2024-09-21T09:37:24.099829+0000","last_fullsized":"2024-09-21T09:37:24.099829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:48.980330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100512+0000","last_change":"2024-09-21T09:36:43.140046+0000","last_active":"2024-09-21T09:37:24.100512+0000","last_peered":"2024-09-21T09:37:24.100512+0000","last_clean":"2024-09-21T09:37:24.100512+0000","last_became_active":"2024-09-21T09:36:43.139680+0000","last_became_peered":"2024-09-21T09:36:43.139680+0000","last_unstale":"2024-09-21T09:37:24.100512+0000","last_undegraded":"2024-09-21T09:37:24.100512+0000","last_fullsized":"2024-09-21T09:37:24.100512+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:05:31.198975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815906+0000","last_change":"2024-09-21T09:36:43.120067+0000","last_active":"2024-09-21T09:37:24.815906+0000","last_peered":"2024-09-21T09:37:24.815906+0000","last_clean":"2024-09-21T09:37:24.815906+0000","last_became_active":"2024-09-21T09:36:43.119588+0000","last_became_peered":"2024-09-21T09:36:43.119588+0000","last_unstale":"2024-09-21T09:37:24.815906+0000","last_undegraded":"2024-09-21T09:37:24.815906+0000","last_fullsized":"2024-09-21T09:37:24.815906+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:53.994599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100335+0000","last_change":"2024-09-21T09:36:43.146324+0000","last_active":"2024-09-21T09:37:24.100335+0000","last_peered":"2024-09-21T09:37:24.100335+0000","last_clean":"2024-09-21T09:37:24.100335+0000","last_became_active":"2024-09-21T09:36:43.146030+0000","last_became_peered":"2024-09-21T09:36:43.146030+0000","last_unstale":"2024-09-21T09:37:24.100335+0000","last_undegraded":"2024-09-21T09:37:24.100335+0000","last_fullsized":"2024-09-21T09:37:24.100335+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:28.017328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"52'79","reported_seq":55,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103983+0000","last_change":"2024-09-21T09:34:54.592924+0000","last_active":"2024-09-21T09:37:24.103983+0000","last_peered":"2024-09-21T09:37:24.103983+0000","last_clean":"2024-09-21T09:37:24.103983+0000","last_became_active":"2024-09-21T09:34:54.588063+0000","last_became_peered":"2024-09-21T09:34:54.588063+0000","last_unstale":"2024-09-21T09:37:24.103983+0000","last_undegraded":"2024-09-21T09:37:24.103983+0000","last_fullsized":"2024-09-21T09:37:24.103983+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:02.981289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102813+0000","last_change":"2024-09-21T09:35:34.095862+0000","last_active":"2024-09-21T09:37:24.102813+0000","last_peered":"2024-09-21T09:37:24.102813+0000","last_clean":"2024-09-21T09:37:24.102813+0000","last_became_active":"2024-09-21T09:35:29.148713+0000","last_became_peered":"2024-09-21T09:35:29.148713+0000","last_unstale":"2024-09-21T09:37:24.102813+0000","last_undegraded":"2024-09-21T09:37:24.102813+0000","last_fullsized":"2024-09-21T09:37:24.102813+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:18.348007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816074+0000","last_change":"2024-09-21T09:36:43.115501+0000","last_active":"2024-09-21T09:37:24.816074+0000","last_peered":"2024-09-21T09:37:24.816074+0000","last_clean":"2024-09-21T09:37:24.816074+0000","last_became_active":"2024-09-21T09:36:43.115158+0000","last_became_peered":"2024-09-21T09:36:43.115158+0000","last_unstale":"2024-09-21T09:37:24.816074+0000","last_undegraded":"2024-09-21T09:37:24.816074+0000","last_fullsized":"2024-09-21T09:37:24.816074+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:35:27.744791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.101345+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:37:24.101345+0000","last_peered":"2024-09-21T09:37:24.101345+0000","last_clean":"2024-09-21T09:37:24.101345+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:37:24.101345+0000","last_undegraded":"2024-09-21T09:37:24.101345+0000","last_fullsized":"2024-09-21T09:37:24.101345+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104048+0000","last_change":"2024-09-21T09:35:34.101543+0000","last_active":"2024-09-21T09:37:24.104048+0000","last_peered":"2024-09-21T09:37:24.104048+0000","last_clean":"2024-09-21T09:37:24.104048+0000","last_became_active":"2024-09-21T09:35:29.753066+0000","last_became_peered":"2024-09-21T09:35:29.753066+0000","last_unstale":"2024-09-21T09:37:24.104048+0000","last_undegraded":"2024-09-21T09:37:24.104048+0000","last_fullsized":"2024-09-21T09:37:24.104048+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:49.271159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815829+0000","last_change":"2024-09-21T09:36:43.113941+0000","last_active":"2024-09-21T09:37:24.815829+0000","last_peered":"2024-09-21T09:37:24.815829+0000","last_clean":"2024-09-21T09:37:24.815829+0000","last_became_active":"2024-09-21T09:36:43.113641+0000","last_became_peered":"2024-09-21T09:36:43.113641+0000","last_unstale":"2024-09-21T09:37:24.815829+0000","last_undegraded":"2024-09-21T09:37:24.815829+0000","last_fullsized":"2024-09-21T09:37:24.815829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:49.055900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816479+0000","last_change":"2024-09-21T09:36:43.142144+0000","last_active":"2024-09-21T09:37:24.816479+0000","last_peered":"2024-09-21T09:37:24.816479+0000","last_clean":"2024-09-21T09:37:24.816479+0000","last_became_active":"2024-09-21T09:36:43.141540+0000","last_became_peered":"2024-09-21T09:36:43.141540+0000","last_unstale":"2024-09-21T09:37:24.816479+0000","last_undegraded":"2024-09-21T09:37:24.816479+0000","last_fullsized":"2024-09-21T09:37:24.816479+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:30.370892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":24,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098456+0000","last_change":"2024-09-21T09:35:34.090997+0000","last_active":"2024-09-21T09:37:24.098456+0000","last_peered":"2024-09-21T09:37:24.098456+0000","last_clean":"2024-09-21T09:37:24.098456+0000","last_became_active":"2024-09-21T09:35:29.752848+0000","last_became_peered":"2024-09-21T09:35:29.752848+0000","last_unstale":"2024-09-21T09:37:24.098456+0000","last_undegraded":"2024-09-21T09:37:24.098456+0000","last_fullsized":"2024-09-21T09:37:24.098456+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:57:11.329055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098555+0000","last_change":"2024-09-21T09:36:43.120577+0000","last_active":"2024-09-21T09:37:24.098555+0000","last_peered":"2024-09-21T09:37:24.098555+0000","last_clean":"2024-09-21T09:37:24.098555+0000","last_became_active":"2024-09-21T09:36:43.120247+0000","last_became_peered":"2024-09-21T09:36:43.120247+0000","last_unstale":"2024-09-21T09:37:24.098555+0000","last_undegraded":"2024-09-21T09:37:24.098555+0000","last_fullsized":"2024-09-21T09:37:24.098555+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:30:10.958363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100418+0000","last_change":"2024-09-21T09:36:43.137959+0000","last_active":"2024-09-21T09:37:24.100418+0000","last_peered":"2024-09-21T09:37:24.100418+0000","last_clean":"2024-09-21T09:37:24.100418+0000","last_became_active":"2024-09-21T09:36:43.137656+0000","last_became_peered":"2024-09-21T09:36:43.137656+0000","last_unstale":"2024-09-21T09:37:24.100418+0000","last_undegraded":"2024-09-21T09:37:24.100418+0000","last_fullsized":"2024-09-21T09:37:24.100418+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:41.758559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":27,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103560+0000","last_change":"2024-09-21T09:35:34.096316+0000","last_active":"2024-09-21T09:37:24.103560+0000","last_peered":"2024-09-21T09:37:24.103560+0000","last_clean":"2024-09-21T09:37:24.103560+0000","last_became_active":"2024-09-21T09:35:29.148648+0000","last_became_peered":"2024-09-21T09:35:29.148648+0000","last_unstale":"2024-09-21T09:37:24.103560+0000","last_undegraded":"2024-09-21T09:37:24.103560+0000","last_fullsized":"2024-09-21T09:37:24.103560+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:22:29.243621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100278+0000","last_change":"2024-09-21T09:36:43.119551+0000","last_active":"2024-09-21T09:37:24.100278+0000","last_peered":"2024-09-21T09:37:24.100278+0000","last_clean":"2024-09-21T09:37:24.100278+0000","last_became_active":"2024-09-21T09:36:43.119105+0000","last_became_peered":"2024-09-21T09:36:43.119105+0000","last_unstale":"2024-09-21T09:37:24.100278+0000","last_undegraded":"2024-09-21T09:37:24.100278+0000","last_fullsized":"2024-09-21T09:37:24.100278+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:01.710977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098912+0000","last_change":"2024-09-21T09:36:43.145633+0000","last_active":"2024-09-21T09:37:24.098912+0000","last_peered":"2024-09-21T09:37:24.098912+0000","last_clean":"2024-09-21T09:37:24.098912+0000","last_became_active":"2024-09-21T09:36:43.145311+0000","last_became_peered":"2024-09-21T09:36:43.145311+0000","last_unstale":"2024-09-21T09:37:24.098912+0000","last_undegraded":"2024-09-21T09:37:24.098912+0000","last_fullsized":"2024-09-21T09:37:24.098912+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:00.422792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103050+0000","last_change":"2024-09-21T09:36:43.123593+0000","last_active":"2024-09-21T09:37:24.103050+0000","last_peered":"2024-09-21T09:37:24.103050+0000","last_clean":"2024-09-21T09:37:24.103050+0000","last_became_active":"2024-09-21T09:36:43.123321+0000","last_became_peered":"2024-09-21T09:36:43.123321+0000","last_unstale":"2024-09-21T09:37:24.103050+0000","last_undegraded":"2024-09-21T09:37:24.103050+0000","last_fullsized":"2024-09-21T09:37:24.103050+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:01.340388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099092+0000","last_change":"2024-09-21T09:35:34.097383+0000","last_active":"2024-09-21T09:37:24.099092+0000","last_peered":"2024-09-21T09:37:24.099092+0000","last_clean":"2024-09-21T09:37:24.099092+0000","last_became_active":"2024-09-21T09:35:29.752863+0000","last_became_peered":"2024-09-21T09:35:29.752863+0000","last_unstale":"2024-09-21T09:37:24.099092+0000","last_undegraded":"2024-09-21T09:37:24.099092+0000","last_fullsized":"2024-09-21T09:37:24.099092+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:00.231892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:37:25.317643+0000","last_change":"2024-09-21T09:37:25.317643+0000","last_active":"2024-09-21T09:37:25.317643+0000","last_peered":"2024-09-21T09:37:25.317643+0000","last_clean":"2024-09-21T09:37:25.317643+0000","last_became_active":"2024-09-21T09:37:25.317397+0000","last_became_peered":"2024-09-21T09:37:25.317397+0000","last_unstale":"2024-09-21T09:37:25.317643+0000","last_undegraded":"2024-09-21T09:37:25.317643+0000","last_fullsized":"2024-09-21T09:37:25.317643+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:45.758571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099014+0000","last_change":"2024-09-21T09:36:43.120667+0000","last_active":"2024-09-21T09:37:24.099014+0000","last_peered":"2024-09-21T09:37:24.099014+0000","last_clean":"2024-09-21T09:37:24.099014+0000","last_became_active":"2024-09-21T09:36:43.120292+0000","last_became_peered":"2024-09-21T09:36:43.120292+0000","last_unstale":"2024-09-21T09:37:24.099014+0000","last_undegraded":"2024-09-21T09:37:24.099014+0000","last_fullsized":"2024-09-21T09:37:24.099014+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:36:33.835796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104342+0000","last_change":"2024-09-21T09:35:34.101535+0000","last_active":"2024-09-21T09:37:24.104342+0000","last_peered":"2024-09-21T09:37:24.104342+0000","last_clean":"2024-09-21T09:37:24.104342+0000","last_became_active":"2024-09-21T09:35:29.753057+0000","last_became_peered":"2024-09-21T09:35:29.753057+0000","last_unstale":"2024-09-21T09:37:24.104342+0000","last_undegraded":"2024-09-21T09:37:24.104342+0000","last_fullsized":"2024-09-21T09:37:24.104342+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:31:49.246270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816290+0000","last_change":"2024-09-21T09:36:43.389413+0000","last_active":"2024-09-21T09:37:24.816290+0000","last_peered":"2024-09-21T09:37:24.816290+0000","last_clean":"2024-09-21T09:37:24.816290+0000","last_became_active":"2024-09-21T09:36:43.389065+0000","last_became_peered":"2024-09-21T09:36:43.389065+0000","last_unstale":"2024-09-21T09:37:24.816290+0000","last_undegraded":"2024-09-21T09:37:24.816290+0000","last_fullsized":"2024-09-21T09:37:24.816290+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:53:44.539113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816494+0000","last_change":"2024-09-21T09:36:43.112562+0000","last_active":"2024-09-21T09:37:24.816494+0000","last_peered":"2024-09-21T09:37:24.816494+0000","last_clean":"2024-09-21T09:37:24.816494+0000","last_became_active":"2024-09-21T09:36:43.112231+0000","last_became_peered":"2024-09-21T09:36:43.112231+0000","last_unstale":"2024-09-21T09:37:24.816494+0000","last_undegraded":"2024-09-21T09:37:24.816494+0000","last_fullsized":"2024-09-21T09:37:24.816494+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:15.209506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099457+0000","last_change":"2024-09-21T09:35:34.097223+0000","last_active":"2024-09-21T09:37:24.099457+0000","last_peered":"2024-09-21T09:37:24.099457+0000","last_clean":"2024-09-21T09:37:24.099457+0000","last_became_active":"2024-09-21T09:35:29.378933+0000","last_became_peered":"2024-09-21T09:35:29.378933+0000","last_unstale":"2024-09-21T09:37:24.099457+0000","last_undegraded":"2024-09-21T09:37:24.099457+0000","last_fullsized":"2024-09-21T09:37:24.099457+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:04.627265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033167899999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815818+0000","last_change":"2024-09-21T09:36:43.390345+0000","last_active":"2024-09-21T09:37:24.815818+0000","last_peered":"2024-09-21T09:37:24.815818+0000","last_clean":"2024-09-21T09:37:24.815818+0000","last_became_active":"2024-09-21T09:36:43.389359+0000","last_became_peered":"2024-09-21T09:36:43.389359+0000","last_unstale":"2024-09-21T09:37:24.815818+0000","last_undegraded":"2024-09-21T09:37:24.815818+0000","last_fullsized":"2024-09-21T09:37:24.815818+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:20.223990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816599+0000","last_change":"2024-09-21T09:36:43.121629+0000","last_active":"2024-09-21T09:37:24.816599+0000","last_peered":"2024-09-21T09:37:24.816599+0000","last_clean":"2024-09-21T09:37:24.816599+0000","last_became_active":"2024-09-21T09:36:43.121342+0000","last_became_peered":"2024-09-21T09:36:43.121342+0000","last_unstale":"2024-09-21T09:37:24.816599+0000","last_undegraded":"2024-09-21T09:37:24.816599+0000","last_fullsized":"2024-09-21T09:37:24.816599+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:23.037498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102222+0000","last_change":"2024-09-21T09:35:34.097095+0000","last_active":"2024-09-21T09:37:24.102222+0000","last_peered":"2024-09-21T09:37:24.102222+0000","last_clean":"2024-09-21T09:37:24.102222+0000","last_became_active":"2024-09-21T09:35:29.148371+0000","last_became_peered":"2024-09-21T09:35:29.148371+0000","last_unstale":"2024-09-21T09:37:24.102222+0000","last_undegraded":"2024-09-21T09:37:24.102222+0000","last_fullsized":"2024-09-21T09:37:24.102222+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:04:07.399927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099679+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:37:24.099679+0000","last_peered":"2024-09-21T09:37:24.099679+0000","last_clean":"2024-09-21T09:37:24.099679+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:37:24.099679+0000","last_undegraded":"2024-09-21T09:37:24.099679+0000","last_fullsized":"2024-09-21T09:37:24.099679+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100102+0000","last_change":"2024-09-21T09:36:43.139540+0000","last_active":"2024-09-21T09:37:24.100102+0000","last_peered":"2024-09-21T09:37:24.100102+0000","last_clean":"2024-09-21T09:37:24.100102+0000","last_became_active":"2024-09-21T09:36:43.139266+0000","last_became_peered":"2024-09-21T09:36:43.139266+0000","last_unstale":"2024-09-21T09:37:24.100102+0000","last_undegraded":"2024-09-21T09:37:24.100102+0000","last_fullsized":"2024-09-21T09:37:24.100102+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:50.717738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"63'6","reported_seq":19,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100175+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:37:24.100175+0000","last_peered":"2024-09-21T09:37:24.100175+0000","last_clean":"2024-09-21T09:37:24.100175+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:37:24.100175+0000","last_undegraded":"2024-09-21T09:37:24.100175+0000","last_fullsized":"2024-09-21T09:37:24.100175+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099582+0000","last_change":"2024-09-21T09:36:43.140342+0000","last_active":"2024-09-21T09:37:24.099582+0000","last_peered":"2024-09-21T09:37:24.099582+0000","last_clean":"2024-09-21T09:37:24.099582+0000","last_became_active":"2024-09-21T09:36:43.140092+0000","last_became_peered":"2024-09-21T09:36:43.140092+0000","last_unstale":"2024-09-21T09:37:24.099582+0000","last_undegraded":"2024-09-21T09:37:24.099582+0000","last_fullsized":"2024-09-21T09:37:24.099582+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:31:02.983516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.104137+0000","last_change":"2024-09-21T09:36:43.119387+0000","last_active":"2024-09-21T09:37:24.104137+0000","last_peered":"2024-09-21T09:37:24.104137+0000","last_clean":"2024-09-21T09:37:24.104137+0000","last_became_active":"2024-09-21T09:36:43.118703+0000","last_became_peered":"2024-09-21T09:36:43.118703+0000","last_unstale":"2024-09-21T09:37:24.104137+0000","last_undegraded":"2024-09-21T09:37:24.104137+0000","last_fullsized":"2024-09-21T09:37:24.104137+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:58.053141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102915+0000","last_change":"2024-09-21T09:36:43.119455+0000","last_active":"2024-09-21T09:37:24.102915+0000","last_peered":"2024-09-21T09:37:24.102915+0000","last_clean":"2024-09-21T09:37:24.102915+0000","last_became_active":"2024-09-21T09:36:43.119034+0000","last_became_peered":"2024-09-21T09:36:43.119034+0000","last_unstale":"2024-09-21T09:37:24.102915+0000","last_undegraded":"2024-09-21T09:37:24.102915+0000","last_fullsized":"2024-09-21T09:37:24.102915+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:25.444791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815906+0000","last_change":"2024-09-21T09:36:43.389046+0000","last_active":"2024-09-21T09:37:24.815906+0000","last_peered":"2024-09-21T09:37:24.815906+0000","last_clean":"2024-09-21T09:37:24.815906+0000","last_became_active":"2024-09-21T09:36:43.388637+0000","last_became_peered":"2024-09-21T09:36:43.388637+0000","last_unstale":"2024-09-21T09:37:24.815906+0000","last_undegraded":"2024-09-21T09:37:24.815906+0000","last_fullsized":"2024-09-21T09:37:24.815906+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:12.885309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816567+0000","last_change":"2024-09-21T09:36:43.120236+0000","last_active":"2024-09-21T09:37:24.816567+0000","last_peered":"2024-09-21T09:37:24.816567+0000","last_clean":"2024-09-21T09:37:24.816567+0000","last_became_active":"2024-09-21T09:36:43.116719+0000","last_became_peered":"2024-09-21T09:36:43.116719+0000","last_unstale":"2024-09-21T09:37:24.816567+0000","last_undegraded":"2024-09-21T09:37:24.816567+0000","last_fullsized":"2024-09-21T09:37:24.816567+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:46.780407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815781+0000","last_change":"2024-09-21T09:36:43.143466+0000","last_active":"2024-09-21T09:37:24.815781+0000","last_peered":"2024-09-21T09:37:24.815781+0000","last_clean":"2024-09-21T09:37:24.815781+0000","last_became_active":"2024-09-21T09:36:43.143059+0000","last_became_peered":"2024-09-21T09:36:43.143059+0000","last_unstale":"2024-09-21T09:37:24.815781+0000","last_undegraded":"2024-09-21T09:37:24.815781+0000","last_fullsized":"2024-09-21T09:37:24.815781+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:32.604473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:37:25.318535+0000","last_change":"2024-09-21T09:37:25.318535+0000","last_active":"2024-09-21T09:37:25.318535+0000","last_peered":"2024-09-21T09:37:25.318535+0000","last_clean":"2024-09-21T09:37:25.318535+0000","last_became_active":"2024-09-21T09:37:25.317932+0000","last_became_peered":"2024-09-21T09:37:25.317932+0000","last_unstale":"2024-09-21T09:37:25.318535+0000","last_undegraded":"2024-09-21T09:37:25.318535+0000","last_fullsized":"2024-09-21T09:37:25.318535+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:52.502093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815868+0000","last_change":"2024-09-21T09:36:43.121951+0000","last_active":"2024-09-21T09:37:24.815868+0000","last_peered":"2024-09-21T09:37:24.815868+0000","last_clean":"2024-09-21T09:37:24.815868+0000","last_became_active":"2024-09-21T09:36:43.121656+0000","last_became_peered":"2024-09-21T09:36:43.121656+0000","last_unstale":"2024-09-21T09:37:24.815868+0000","last_undegraded":"2024-09-21T09:37:24.815868+0000","last_fullsized":"2024-09-21T09:37:24.815868+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:43.045642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816407+0000","last_change":"2024-09-21T09:36:43.388937+0000","last_active":"2024-09-21T09:37:24.816407+0000","last_peered":"2024-09-21T09:37:24.816407+0000","last_clean":"2024-09-21T09:37:24.816407+0000","last_became_active":"2024-09-21T09:36:43.388558+0000","last_became_peered":"2024-09-21T09:36:43.388558+0000","last_unstale":"2024-09-21T09:37:24.816407+0000","last_undegraded":"2024-09-21T09:37:24.816407+0000","last_fullsized":"2024-09-21T09:37:24.816407+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:29:58.442452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098426+0000","last_change":"2024-09-21T09:36:43.118044+0000","last_active":"2024-09-21T09:37:24.098426+0000","last_peered":"2024-09-21T09:37:24.098426+0000","last_clean":"2024-09-21T09:37:24.098426+0000","last_became_active":"2024-09-21T09:36:43.117752+0000","last_became_peered":"2024-09-21T09:36:43.117752+0000","last_unstale":"2024-09-21T09:37:24.098426+0000","last_undegraded":"2024-09-21T09:37:24.098426+0000","last_fullsized":"2024-09-21T09:37:24.098426+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:31:47.718963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816379+0000","last_change":"2024-09-21T09:36:43.142724+0000","last_active":"2024-09-21T09:37:24.816379+0000","last_peered":"2024-09-21T09:37:24.816379+0000","last_clean":"2024-09-21T09:37:24.816379+0000","last_became_active":"2024-09-21T09:36:43.140767+0000","last_became_peered":"2024-09-21T09:36:43.140767+0000","last_unstale":"2024-09-21T09:37:24.816379+0000","last_undegraded":"2024-09-21T09:37:24.816379+0000","last_fullsized":"2024-09-21T09:37:24.816379+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:03.041550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100014+0000","last_change":"2024-09-21T09:36:43.115705+0000","last_active":"2024-09-21T09:37:24.100014+0000","last_peered":"2024-09-21T09:37:24.100014+0000","last_clean":"2024-09-21T09:37:24.100014+0000","last_became_active":"2024-09-21T09:36:43.115439+0000","last_became_peered":"2024-09-21T09:36:43.115439+0000","last_unstale":"2024-09-21T09:37:24.100014+0000","last_undegraded":"2024-09-21T09:37:24.100014+0000","last_fullsized":"2024-09-21T09:37:24.100014+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:06:12.185663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.098645+0000","last_change":"2024-09-21T09:36:43.138262+0000","last_active":"2024-09-21T09:37:24.098645+0000","last_peered":"2024-09-21T09:37:24.098645+0000","last_clean":"2024-09-21T09:37:24.098645+0000","last_became_active":"2024-09-21T09:36:43.137957+0000","last_became_peered":"2024-09-21T09:36:43.137957+0000","last_unstale":"2024-09-21T09:37:24.098645+0000","last_undegraded":"2024-09-21T09:37:24.098645+0000","last_fullsized":"2024-09-21T09:37:24.098645+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:01.178073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816357+0000","last_change":"2024-09-21T09:36:43.116111+0000","last_active":"2024-09-21T09:37:24.816357+0000","last_peered":"2024-09-21T09:37:24.816357+0000","last_clean":"2024-09-21T09:37:24.816357+0000","last_became_active":"2024-09-21T09:36:43.115022+0000","last_became_peered":"2024-09-21T09:36:43.115022+0000","last_unstale":"2024-09-21T09:37:24.816357+0000","last_undegraded":"2024-09-21T09:37:24.816357+0000","last_fullsized":"2024-09-21T09:37:24.816357+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:48.706807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.100070+0000","last_change":"2024-09-21T09:36:43.148280+0000","last_active":"2024-09-21T09:37:24.100070+0000","last_peered":"2024-09-21T09:37:24.100070+0000","last_clean":"2024-09-21T09:37:24.100070+0000","last_became_active":"2024-09-21T09:36:43.148010+0000","last_became_peered":"2024-09-21T09:36:43.148010+0000","last_unstale":"2024-09-21T09:37:24.100070+0000","last_undegraded":"2024-09-21T09:37:24.100070+0000","last_fullsized":"2024-09-21T09:37:24.100070+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:26:00.852481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.102981+0000","last_change":"2024-09-21T09:36:43.117958+0000","last_active":"2024-09-21T09:37:24.102981+0000","last_peered":"2024-09-21T09:37:24.102981+0000","last_clean":"2024-09-21T09:37:24.102981+0000","last_became_active":"2024-09-21T09:36:43.117652+0000","last_became_peered":"2024-09-21T09:36:43.117652+0000","last_unstale":"2024-09-21T09:37:24.102981+0000","last_undegraded":"2024-09-21T09:37:24.102981+0000","last_fullsized":"2024-09-21T09:37:24.102981+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:58.389534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099888+0000","last_change":"2024-09-21T09:36:43.144525+0000","last_active":"2024-09-21T09:37:24.099888+0000","last_peered":"2024-09-21T09:37:24.099888+0000","last_clean":"2024-09-21T09:37:24.099888+0000","last_became_active":"2024-09-21T09:36:43.144223+0000","last_became_peered":"2024-09-21T09:36:43.144223+0000","last_unstale":"2024-09-21T09:37:24.099888+0000","last_undegraded":"2024-09-21T09:37:24.099888+0000","last_fullsized":"2024-09-21T09:37:24.099888+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:57:40.702512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.103042+0000","last_change":"2024-09-21T09:36:43.115791+0000","last_active":"2024-09-21T09:37:24.103042+0000","last_peered":"2024-09-21T09:37:24.103042+0000","last_clean":"2024-09-21T09:37:24.103042+0000","last_became_active":"2024-09-21T09:36:43.115525+0000","last_became_peered":"2024-09-21T09:36:43.115525+0000","last_unstale":"2024-09-21T09:37:24.103042+0000","last_undegraded":"2024-09-21T09:37:24.103042+0000","last_fullsized":"2024-09-21T09:37:24.103042+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:28.396722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.815984+0000","last_change":"2024-09-21T09:36:43.140639+0000","last_active":"2024-09-21T09:37:24.815984+0000","last_peered":"2024-09-21T09:37:24.815984+0000","last_clean":"2024-09-21T09:37:24.815984+0000","last_became_active":"2024-09-21T09:36:43.139865+0000","last_became_peered":"2024-09-21T09:36:43.139865+0000","last_unstale":"2024-09-21T09:37:24.815984+0000","last_undegraded":"2024-09-21T09:37:24.815984+0000","last_fullsized":"2024-09-21T09:37:24.815984+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:01:15.881225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816282+0000","last_change":"2024-09-21T09:36:43.117111+0000","last_active":"2024-09-21T09:37:24.816282+0000","last_peered":"2024-09-21T09:37:24.816282+0000","last_clean":"2024-09-21T09:37:24.816282+0000","last_became_active":"2024-09-21T09:36:43.114575+0000","last_became_peered":"2024-09-21T09:36:43.114575+0000","last_unstale":"2024-09-21T09:37:24.816282+0000","last_undegraded":"2024-09-21T09:37:24.816282+0000","last_fullsized":"2024-09-21T09:37:24.816282+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:13.218433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.099576+0000","last_change":"2024-09-21T09:36:43.140043+0000","last_active":"2024-09-21T09:37:24.099576+0000","last_peered":"2024-09-21T09:37:24.099576+0000","last_clean":"2024-09-21T09:37:24.099576+0000","last_became_active":"2024-09-21T09:36:43.139784+0000","last_became_peered":"2024-09-21T09:36:43.139784+0000","last_unstale":"2024-09-21T09:37:24.099576+0000","last_undegraded":"2024-09-21T09:37:24.099576+0000","last_fullsized":"2024-09-21T09:37:24.099576+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:05.226038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-21T09:37:24.816574+0000","last_change":"2024-09-21T09:36:43.119924+0000","last_active":"2024-09-21T09:37:24.816574+0000","last_peered":"2024-09-21T09:37:24.816574+0000","last_clean":"2024-09-21T09:37:24.816574+0000","last_became_active":"2024-09-21T09:36:43.116283+0000","last_became_peered":"2024-09-21T09:36:43.116283+0000","last_unstale":"2024-09-21T09:37:24.816574+0000","last_undegraded":"2024-09-21T09:37:24.816574+0000","last_fullsized":"2024-09-21T09:37:24.816574+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:16:31.191718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397542,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1252,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1282048,"data_stored":881739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.502,"15min":0.502},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67600000000000005,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.52100000000000002,"15min":0.51700000000000002},"min":{"1min":0.39900000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.69499999999999995,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.52400000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.502,"15min":0.499},"min":{"1min":0.40600000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.63700000000000001,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.627},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.39800000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.71799999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.53500000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39700000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.70799999999999996,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.48299999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52000000000000002,"15min":0.51900000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.70599999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.44},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.42299999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.65500000000000003,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.56599999999999995}]},{"osd":4,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41299999999999998,"15min":0.41199999999999998},"min":{"1min":0.28999999999999998,"5min":0.248,"15min":0.248},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.47499999999999998,"15min":0.48499999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.59099999999999997,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.47199999999999998,"15min":0.47699999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.50900000000000001,"15min":0.51300000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.67000000000000004,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.52100000000000002,"15min":0.52700000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.59399999999999997}]}]},{"osd":6,"up_from":43,"seq":184683593771,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298596,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446556,"statfs":{"total":95995035648,"available":95689273344,"internally_reserved":0,"allocated":1265664,"data_stored":881053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.54800000000000004,"15min":0.53900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.44},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53800000000000003,"15min":0.52800000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71299999999999997,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.58099999999999996}]},{"osd":1,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.38600000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.53300000000000003,"15min":0.50900000000000001},"min":{"1min":0.45300000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.39500000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.56599999999999995,"15min":0.55800000000000005},"min":{"1min":0.48399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.53800000000000003,"15min":0.52500000000000002},"min":{"1min":0.44900000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.51900000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54600000000000004,"15min":0.53800000000000003},"min":{"1min":0.42699999999999999,"5min":0.375,"15min":0.375},"max":{"1min":0.72499999999999998,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.56200000000000006,"15min":0.54200000000000004},"min":{"1min":0.42699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.499}]},{"osd":4,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.45300000000000001,"15min":0.44400000000000001},"min":{"1min":0.32400000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.61899999999999999,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.46600000000000003,"15min":0.45400000000000001},"min":{"1min":0.33500000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.46700000000000003,"15min":0.45000000000000001},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.5,"5min":0.47399999999999998,"15min":0.46000000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.73699999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:36:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.51700000000000002,"15min":0.51300000000000001},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.55200000000000005,"15min":0.54200000000000004},"min":{"1min":0.39500000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.59899999999999998}]}]},{"osd":5,"up_from":37,"seq":158913790000,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":289947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53000000000000003,"15min":0.54800000000000004},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.628,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.52000000000000002,"15min":0.54200000000000004},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.61599999999999999,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.432}]},{"osd":1,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.54800000000000004,"15min":0.56299999999999994},"min":{"1min":0.35499999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.65300000000000002,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.497,"5min":0.53300000000000003,"15min":0.55000000000000004},"min":{"1min":0.33100000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66700000000000004,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.53500000000000003,"15min":0.55300000000000005},"min":{"1min":0.317,"5min":0.314,"15min":0.314},"max":{"1min":0.67800000000000005,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53300000000000003,"15min":0.53900000000000003},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.63900000000000001,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.55700000000000005,"15min":0.57499999999999996},"min":{"1min":0.40400000000000003,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70099999999999996,"5min":0.88,"15min":0.88},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54400000000000004,"15min":0.55400000000000005},"min":{"1min":0.41699999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.65700000000000003,"5min":0.754,"15min":0.754},"last":0.59499999999999997}]},{"osd":4,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.39700000000000002,"15min":0.39300000000000002},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.61199999999999999,"5min":0.624,"15min":0.624},"last":0.371},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40400000000000003,"15min":0.39600000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.60299999999999998,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 21 09:37:35 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.45500000000000002,"15min":0.442},"min":{"1min":0.311,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.47399999999999998,"15min":0.46999999999999997},"min":{"1min":0.32800000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67700000000000005,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.624}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.47999999999999998,"15min":0.47799999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.69499999999999995,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.32100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.66200000000000003,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.56599999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298024,"kb_used_data":668,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447128,"statfs":{"total":95995035648,"available":95689859072,"internally_reserved":0,"allocated":684032,"data_stored":291301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.53600000000000003,"15min":0.54100000000000004},"min":{"1min":0.379,"5min":0.318,"15min":0.318},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56000000000000005,"15min":0.56599999999999995},"min":{"1min":0.38900000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.74199999999999999,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.62}]},{"osd":1,"last update":"Sat Sep 21 09:36:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52600000000000002,"15min":0.53400000000000003},"min":{"1min":0.35799999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.749,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.38900000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71899999999999997,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.60799999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:36:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54500000000000004,"15min":0.54400000000000004},"min":{"1min":0.379,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.753,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38600000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:36:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.76600000000000001,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.36299999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59799999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.39900000000000002,"15min":0.38600000000000001},"min":{"1min":0.29199999999999998,"5min":0.25,"15min":0.25},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.432},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.40999999999999998,"15min":0.40100000000000002},"min":{"1min":0.30299999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.65700000000000003,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.502}]},{"osd":6,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.42799999999999999,"15min":0.40600000000000003},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.48199999999999998,"15min":0.46600000000000003},"min":{"1min":0.34000000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68200000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.56299999999999994}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.45900000000000002,"15min":0.45500000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67300000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.501,"5min":0.48199999999999998,"15min":0.47899999999999998},"min":{"1min":0.28999999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.69799999999999995,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.58299999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149754,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":663552,"data_stored":289693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.39400000000000002,"15min":0.40000000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.51700000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.40899999999999997,"15min":0.42099999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.495,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.47499999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.437,"15min":0.45100000000000001},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.627,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.378,"5min":0.42899999999999999,"15min":0.436},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56100000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.40699999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.46100000000000002,"15min":0.47199999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.51400000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.438},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.45200000000000001,"15min":0.46700000000000003},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.54600000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.373}]},{"osd":4,"last update":"Sat Sep 21 09:36:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.59599999999999997,"15min":0.60499999999999998},"min":{"1min":0.40799999999999997,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80900000000000005,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.62},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.626,"15min":0.63100000000000001},"min":{"1min":0.48099999999999998,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.77600000000000002,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.59299999999999997,"15min":0.59499999999999997},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.78400000000000003,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.433,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.57599999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:37:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.61699999999999999,"15min":0.621},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.748,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.626,"15min":0.63},"min":{"1min":0.42399999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.72699999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:36:56 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.629,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.75900000000000001,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.68100000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":667648,"data_stored":290207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:00 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.371,"15min":0.34699999999999998},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":1.782,"5min":1.782,"15min":1.782},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.38300000000000001,"15min":0.35899999999999999},"min":{"1min":0.249,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:37:00 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.40400000000000003,"15min":0.377},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":2.0659999999999998,"5min":2.0659999999999998,"15min":2.0659999999999998},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.38,"15min":0.35699999999999998},"min":{"1min":0.26200000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.58499999999999996,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.27600000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:37:19 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.44700000000000001,"15min":0.45300000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.56100000000000005,"5min":0.627,"15min":0.627},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.373,"5min":0.432,"15min":0.434},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.502,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.29699999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:37:42 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.57799999999999996,"15min":0.57199999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57499999999999996,"15min":0.57199999999999995},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.86899999999999999,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.55400000000000005}]},{"osd":5,"last update":"Sat Sep 21 09:37:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.443,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.81899999999999995,"5min":0.88,"15min":0.88},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.61899999999999999,"15min":0.61099999999999999},"min":{"1min":0.432,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84699999999999998,"5min":1.962,"15min":1.962},"last":0.54400000000000004}]},{"osd":6,"last update":"Sat Sep 21 09:37:32 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.65200000000000002,"15min":0.66300000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80600000000000005,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.60799999999999998,"15min":0.624},"min":{"1min":0.40500000000000003,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73699999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.626}]},{"osd":7,"last update":"Sat Sep 21 09:36:55 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.63900000000000001,"15min":0.64200000000000002},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.66200000000000003,"15min":0.65800000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.9930000000000001,"5min":1.9930000000000001,"15min":1.9930000000000001},"last":0.64100000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":663552,"data_stored":289622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:33 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.35399999999999998,"15min":0.33400000000000002},"min":{"1min":0.19900000000000001,"5min":0.156,"15min":0.156},"max":{"1min":0.54300000000000004,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.376,"15min":0.34999999999999998},"min":{"1min":0.25900000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.48799999999999999,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.47299999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:36:57 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.39200000000000002,"15min":0.38200000000000001},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.59899999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.32400000000000001,"5min":0.38300000000000001,"15min":0.38900000000000001},"min":{"1min":0.23799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.52100000000000002,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.38400000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:37:27 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.39700000000000002,"15min":0.40899999999999997},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.436,"15min":0.44800000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56100000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.49399999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:36:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.755,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.78200000000000003,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.41699999999999998,"5min":0.37,"15min":0.37},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.79000000000000004}]},{"osd":6,"last update":"Sat Sep 21 09:37:33 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.59699999999999998,"15min":0.60099999999999998},"min":{"1min":0.432,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.625,"5min":0.63100000000000001,"15min":0.63500000000000001},"min":{"1min":0.495,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.80300000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:36:57 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.56799999999999995,"15min":0.55800000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.73999999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.67700000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298592,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446560,"statfs":{"total":95995035648,"available":95689277440,"internally_reserved":0,"allocated":1261568,"data_stored":880563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:37:39 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.35299999999999998,"15min":0.33900000000000002},"min":{"1min":0.245,"5min":0.184,"15min":0.184},"max":{"1min":0.58199999999999996,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.438,"5min":0.38,"15min":0.35699999999999998},"min":{"1min":0.28699999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.59499999999999997,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.45300000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:37:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.40600000000000003,"15min":0.41799999999999998},"min":{"1min":0.23300000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.39500000000000002,"15min":0.40899999999999997},"min":{"1min":0.23699999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.46600000000000003}]},{"osd":3,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.45100000000000001,"15min":0.47499999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.627,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.439,"15min":0.46400000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64500000000000002,"5min":0.753,"15min":0.753},"last":0.437}]},{"osd":4,"last update":"Sat Sep 21 09:36:48 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53200000000000003,"15min":0.53700000000000003},"min":{"1min":0.40100000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71799999999999997,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.38200000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79100000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.72899999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:37:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.56299999999999994,"15min":0.56100000000000005},"min":{"1min":0.41599999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73099999999999998,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42699999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.71099999999999997}]},{"osd":6,"last update":"Sat Sep 21 09:37:29 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.59199999999999997,"15min":0.58799999999999997},"min":{"1min":0.46000000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81000000000000005,"5min":0.878,"15min":0.878},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.58099999999999996,"15min":0.57699999999999996},"min":{"1min":0.501,"5min":0.379,"15min":0.379},"max":{"1min":0.76200000000000001,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.749}]},{"osd":7,"last update":"Sat Sep 21 09:36:57 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59999999999999998,"15min":0.60099999999999998},"min":{"1min":0.46800000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59599999999999997,"15min":0.59699999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77300000000000002,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.68500000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:37:47.150 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:37:47.150 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:37:47.150 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:37:47.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:37:47.409 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:47.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:47 smithi017 ceph-mon[29380]: from='client.24671 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:47.830 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:47 smithi017 ceph-mon[40737]: from='client.24671 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:47.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:47 smithi134 ceph-mon[33541]: from='client.24671 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:48.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:48 smithi017 ceph-mon[29380]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:48.891 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:48 smithi017 ceph-mon[40737]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:48.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:48 smithi134 ceph-mon[33541]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:49.331 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:37:49.331 INFO:teuthology.orchestra.run.smithi017.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:37:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:49 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2554144592' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:37:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:49 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2554144592' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:37:50.103 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:37:50.103 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:37:50.114 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-21T09:37:50.114 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T09:37:50.122 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T09:37:50.122 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:37:50.123 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-21T09:37:50.123 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-09-21T09:37:50.125 INFO:tasks.rados:Beginning rados... 2024-09-21T09:37:50.125 INFO:tasks.rados:config is {'clients': ['client.0'], 'objects': 50, 'op_weights': {'delete': 50, 'read': 100, 'rollback': 50, 'snap_create': 50, 'snap_remove': 50, 'write': 100}, 'ops': 4000, 'write_append_excl': False} 2024-09-21T09:37:50.126 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:37:50.126 INFO:tasks.rados:config is {'clients': ['client.0'], 'objects': 50, 'op_weights': {'delete': 50, 'read': 100, 'rollback': 50, 'snap_create': 50, 'snap_remove': 50, 'write': 100}, 'ops': 4000, 'write_append_excl': False} 2024-09-21T09:37:50.126 INFO:tasks.rados:joining rados 2024-09-21T09:37:50.126 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:37:50.126 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T09:37:50.126 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi017.front.sepia.ceph.com 2024-09-21T09:37:50.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-21T09:37:50.127 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-09-21T09:37:50.127 INFO:tasks.rados:starting run 0 out of 1 2024-09-21T09:37:50.127 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-21T09:37:50.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_0 16 2024-09-21T09:37:50.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:49 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2554144592' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:37:50.402 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:50.406 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:50.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:50 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:50] "GET /metrics HTTP/1.1" 200 239776 "" "Prometheus/2.43.0" 2024-09-21T09:37:50.794 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:50 smithi017 ceph-mon[29380]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:50.795 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:50 smithi017 ceph-mon[40737]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:51.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:50 smithi134 ceph-mon[33541]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:52.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:52 smithi017 ceph-mon[29380]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:52.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:52 smithi017 ceph-mon[40737]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:53.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:52 smithi134 ceph-mon[33541]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:53.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:53 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:37:53] ENGINE Bus STOPPING 2024-09-21T09:37:53.855 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:53 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:37:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:37:53.855 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:53 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:37:53] ENGINE Bus STOPPED 2024-09-21T09:37:53.855 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:53 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:37:53] ENGINE Bus STARTING 2024-09-21T09:37:53.855 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:53 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:37:53] ENGINE Serving on http://:::9283 2024-09-21T09:37:53.855 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:37:53 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:37:53] ENGINE Bus STARTED 2024-09-21T09:37:54.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-21T09:37:54.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/668061756' entity='client.admin' 2024-09-21T09:37:54.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:37:54.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:54.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:54.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:54.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[40737]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:54.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:54.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/668061756' entity='client.admin' 2024-09-21T09:37:54.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:37:54.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:54.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:54.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:54.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[29380]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:54.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:54 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:54 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/668061756' entity='client.admin' 2024-09-21T09:37:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:54 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:37:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:54 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:54 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:54 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:37:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:54 smithi134 ceph-mon[33541]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:54 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:54.688 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:55.571 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_0' created 2024-09-21T09:37:55.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:55 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/568661802' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:55.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:55 smithi017 ceph-mon[29380]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:55.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:55 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/568661802' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:55.933 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:55 smithi017 ceph-mon[40737]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:55.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:55 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/568661802' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:55 smithi134 ceph-mon[33541]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:56.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:56 smithi017 ceph-mon[29380]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-21T09:37:56.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:56 smithi017 ceph-mon[29380]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:37:56.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:56 smithi017 ceph-mon[29380]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:56.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:56 smithi017 ceph-mon[40737]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-21T09:37:56.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:56 smithi017 ceph-mon[40737]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:37:56.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:56 smithi017 ceph-mon[40737]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:56.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-21T09:37:56.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:56 smithi134 ceph-mon[33541]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-21T09:37:56.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:56 smithi134 ceph-mon[33541]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:37:56.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:56 smithi134 ceph-mon[33541]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:57.171 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:57.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:37:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:37:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:37:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:57 smithi017 ceph-mon[40737]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:37:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:57 smithi017 ceph-mon[40737]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:37:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:57 smithi017 ceph-mon[29380]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:37:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:57 smithi017 ceph-mon[29380]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:37:57.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:57 smithi134 ceph-mon[33541]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:37:57.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:57 smithi134 ceph-mon[33541]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:37:58.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-21T09:37:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:58 smithi017 ceph-mon[29380]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:37:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:58 smithi017 ceph-mon[29380]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:58 smithi017 ceph-mon[40737]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:37:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:58 smithi017 ceph-mon[40737]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:58.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:58 smithi134 ceph-mon[33541]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:37:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:58 smithi134 ceph-mon[33541]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:37:59.582 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:59.760 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-21T09:38:00.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:59 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/83480712' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:38:00.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:37:59 smithi017 ceph-mon[29380]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:38:00.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:59 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/83480712' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:38:00.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:37:59 smithi017 ceph-mon[40737]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:38:00.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:59 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/83480712' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:38:00.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:37:59 smithi134 ceph-mon[33541]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:38:00.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:38:00 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:00] "GET /metrics HTTP/1.1" 200 239759 "" "Prometheus/2.43.0" 2024-09-21T09:38:01.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:00 smithi017 ceph-mon[29380]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:01.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:00 smithi017 ceph-mon[29380]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:38:01.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:00 smithi017 ceph-mon[29380]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:38:01.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:00 smithi017 ceph-mon[40737]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:01.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:00 smithi017 ceph-mon[40737]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:38:01.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:00 smithi017 ceph-mon[40737]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:38:01.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:00 smithi134 ceph-mon[33541]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:01.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:00 smithi134 ceph-mon[33541]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:38:01.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:00 smithi134 ceph-mon[33541]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:38:01.471 DEBUG:teuthology.orchestra.run.smithi017:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 100 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --pool unique_pool_0 2024-09-21T09:38:01.518 INFO:tasks.rados.rados.0.smithi017.stdout:adding op weight read -> 100 2024-09-21T09:38:01.519 INFO:tasks.rados.rados.0.smithi017.stdout:adding op weight write -> 100 2024-09-21T09:38:01.519 INFO:tasks.rados.rados.0.smithi017.stdout:adding op weight delete -> 50 2024-09-21T09:38:01.519 INFO:tasks.rados.rados.0.smithi017.stdout:adding op weight snap_create -> 50 2024-09-21T09:38:01.519 INFO:tasks.rados.rados.0.smithi017.stdout:adding op weight snap_remove -> 50 2024-09-21T09:38:01.519 INFO:tasks.rados.rados.0.smithi017.stdout:adding op weight rollback -> 50 2024-09-21T09:38:01.519 INFO:tasks.rados.rados.0.smithi017.stdout:ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable) 2024-09-21T09:38:01.520 INFO:tasks.rados.rados.0.smithi017.stdout:Configuration: 2024-09-21T09:38:01.520 INFO:tasks.rados.rados.0.smithi017.stdout: Number of operations: 4000 2024-09-21T09:38:01.520 INFO:tasks.rados.rados.0.smithi017.stdout: Number of objects: 50 2024-09-21T09:38:01.520 INFO:tasks.rados.rados.0.smithi017.stdout: Max in flight operations: 16 2024-09-21T09:38:01.520 INFO:tasks.rados.rados.0.smithi017.stdout: Object size (in bytes): 4000000 2024-09-21T09:38:01.520 INFO:tasks.rados.rados.0.smithi017.stdout: Write stride min: 400000 2024-09-21T09:38:01.520 INFO:tasks.rados.rados.0.smithi017.stdout: Write stride max: 800000 2024-09-21T09:38:01.773 INFO:tasks.rados.rados.0.smithi017.stdout:1: write initial oid 1 2024-09-21T09:38:01.773 INFO:tasks.rados.rados.0.smithi017.stdout:1: seq_num 0 ranges {416807=475249,1542129=474678} 2024-09-21T09:38:01.778 INFO:tasks.rados.rados.0.smithi017.stdout:1: writing smithi01785587-1 from 416807 to 892056 tid 1 2024-09-21T09:38:01.785 INFO:tasks.rados.rados.0.smithi017.stdout:1: writing smithi01785587-1 from 1542129 to 2016807 tid 2 2024-09-21T09:38:01.785 INFO:tasks.rados.rados.0.smithi017.stdout:2: write initial oid 2 2024-09-21T09:38:01.785 INFO:tasks.rados.rados.0.smithi017.stdout:2: seq_num 1 ranges {416807=475249,1542129=474678} 2024-09-21T09:38:01.789 INFO:tasks.rados.rados.0.smithi017.stdout:2: writing smithi01785587-2 from 416807 to 892056 tid 1 2024-09-21T09:38:01.796 INFO:tasks.rados.rados.0.smithi017.stdout:2: writing smithi01785587-2 from 1542129 to 2016807 tid 2 2024-09-21T09:38:01.796 INFO:tasks.rados.rados.0.smithi017.stdout:3: write initial oid 3 2024-09-21T09:38:01.796 INFO:tasks.rados.rados.0.smithi017.stdout:3: seq_num 2 ranges {433614=550498,1600611=433003} 2024-09-21T09:38:01.802 INFO:tasks.rados.rados.0.smithi017.stdout:3: writing smithi01785587-3 from 433614 to 984112 tid 1 2024-09-21T09:38:01.807 INFO:tasks.rados.rados.0.smithi017.stdout:3: writing smithi01785587-3 from 1600611 to 2033614 tid 2 2024-09-21T09:38:01.808 INFO:tasks.rados.rados.0.smithi017.stdout:4: write initial oid 4 2024-09-21T09:38:01.808 INFO:tasks.rados.rados.0.smithi017.stdout:4: seq_num 3 ranges {450421=625747,1659093=391328} 2024-09-21T09:38:01.813 INFO:tasks.rados.rados.0.smithi017.stdout:4: writing smithi01785587-4 from 450421 to 1076168 tid 1 2024-09-21T09:38:01.819 INFO:tasks.rados.rados.0.smithi017.stdout:4: writing smithi01785587-4 from 1659093 to 2050421 tid 2 2024-09-21T09:38:01.819 INFO:tasks.rados.rados.0.smithi017.stdout:5: write initial oid 5 2024-09-21T09:38:01.819 INFO:tasks.rados.rados.0.smithi017.stdout:5: seq_num 4 ranges {467228=700996,1717575=349653} 2024-09-21T09:38:01.825 INFO:tasks.rados.rados.0.smithi017.stdout:5: writing smithi01785587-5 from 467228 to 1168224 tid 1 2024-09-21T09:38:01.830 INFO:tasks.rados.rados.0.smithi017.stdout:5: writing smithi01785587-5 from 1717575 to 2067228 tid 2 2024-09-21T09:38:01.830 INFO:tasks.rados.rados.0.smithi017.stdout:6: write initial oid 6 2024-09-21T09:38:01.831 INFO:tasks.rados.rados.0.smithi017.stdout:6: seq_num 5 ranges {484035=776245,2059704=24331} 2024-09-21T09:38:01.837 INFO:tasks.rados.rados.0.smithi017.stdout:6: writing smithi01785587-6 from 484035 to 1260280 tid 1 2024-09-21T09:38:01.842 INFO:tasks.rados.rados.0.smithi017.stdout:6: writing smithi01785587-6 from 2059704 to 2084035 tid 2 2024-09-21T09:38:01.842 INFO:tasks.rados.rados.0.smithi017.stdout:7: write initial oid 7 2024-09-21T09:38:01.842 INFO:tasks.rados.rados.0.smithi017.stdout:7: seq_num 6 ranges {500842=451494,1718186=382656} 2024-09-21T09:38:01.847 INFO:tasks.rados.rados.0.smithi017.stdout:7: writing smithi01785587-7 from 500842 to 952336 tid 1 2024-09-21T09:38:01.854 INFO:tasks.rados.rados.0.smithi017.stdout:7: writing smithi01785587-7 from 1718186 to 2100842 tid 2 2024-09-21T09:38:01.854 INFO:tasks.rados.rados.0.smithi017.stdout:8: write initial oid 8 2024-09-21T09:38:01.854 INFO:tasks.rados.rados.0.smithi017.stdout:8: seq_num 7 ranges {517649=526743,1776668=340981} 2024-09-21T09:38:01.859 INFO:tasks.rados.rados.0.smithi017.stdout:8: writing smithi01785587-8 from 517649 to 1044392 tid 1 2024-09-21T09:38:01.865 INFO:tasks.rados.rados.0.smithi017.stdout:8: writing smithi01785587-8 from 1776668 to 2117649 tid 2 2024-09-21T09:38:01.865 INFO:tasks.rados.rados.0.smithi017.stdout:9: write initial oid 9 2024-09-21T09:38:01.865 INFO:tasks.rados.rados.0.smithi017.stdout:9: seq_num 8 ranges {534456=718345,1951503=182953} 2024-09-21T09:38:01.872 INFO:tasks.rados.rados.0.smithi017.stdout:9: writing smithi01785587-9 from 534456 to 1252801 tid 1 2024-09-21T09:38:01.877 INFO:tasks.rados.rados.0.smithi017.stdout:9: writing smithi01785587-9 from 1951503 to 2134456 tid 2 2024-09-21T09:38:01.877 INFO:tasks.rados.rados.0.smithi017.stdout:10: write initial oid 10 2024-09-21T09:38:01.877 INFO:tasks.rados.rados.0.smithi017.stdout:10: seq_num 9 ranges {551263=793594,1893632=257631} 2024-09-21T09:38:01.884 INFO:tasks.rados.rados.0.smithi017.stdout:10: writing smithi01785587-10 from 551263 to 1344857 tid 1 2024-09-21T09:38:01.889 INFO:tasks.rados.rados.0.smithi017.stdout:10: writing smithi01785587-10 from 1893632 to 2151263 tid 2 2024-09-21T09:38:01.889 INFO:tasks.rados.rados.0.smithi017.stdout:11: write initial oid 11 2024-09-21T09:38:01.889 INFO:tasks.rados.rados.0.smithi017.stdout:11: seq_num 10 ranges {568070=468843,1552114=615956} 2024-09-21T09:38:01.895 INFO:tasks.rados.rados.0.smithi017.stdout:11: writing smithi01785587-11 from 568070 to 1036913 tid 1 2024-09-21T09:38:01.901 INFO:tasks.rados.rados.0.smithi017.stdout:11: writing smithi01785587-11 from 1552114 to 2168070 tid 2 2024-09-21T09:38:01.901 INFO:tasks.rados.rados.0.smithi017.stdout:12: write initial oid 12 2024-09-21T09:38:01.901 INFO:tasks.rados.rados.0.smithi017.stdout:12: seq_num 11 ranges {584877=544092,1610596=562003} 2024-09-21T09:38:01.907 INFO:tasks.rados.rados.0.smithi017.stdout:12: writing smithi01785587-12 from 584877 to 1128969 tid 1 2024-09-21T09:38:01.913 INFO:tasks.rados.rados.0.smithi017.stdout:12: writing smithi01785587-12 from 1610596 to 2172599 tid 2 2024-09-21T09:38:01.913 INFO:tasks.rados.rados.0.smithi017.stdout:13: write initial oid 13 2024-09-21T09:38:01.913 INFO:tasks.rados.rados.0.smithi017.stdout:13: seq_num 12 ranges {601684=619341,1669078=532606} 2024-09-21T09:38:01.920 INFO:tasks.rados.rados.0.smithi017.stdout:13: writing smithi01785587-13 from 601684 to 1221025 tid 1 2024-09-21T09:38:01.925 INFO:tasks.rados.rados.0.smithi017.stdout:13: writing smithi01785587-13 from 1669078 to 2201684 tid 2 2024-09-21T09:38:01.925 INFO:tasks.rados.rados.0.smithi017.stdout:14: write initial oid 14 2024-09-21T09:38:01.926 INFO:tasks.rados.rados.0.smithi017.stdout:14: seq_num 13 ranges {618491=694590,2011207=207284} 2024-09-21T09:38:01.932 INFO:tasks.rados.rados.0.smithi017.stdout:14: writing smithi01785587-14 from 618491 to 1313081 tid 1 2024-09-21T09:38:01.938 INFO:tasks.rados.rados.0.smithi017.stdout:14: writing smithi01785587-14 from 2011207 to 2218491 tid 2 2024-09-21T09:38:01.938 INFO:tasks.rados.rados.0.smithi017.stdout:15: write initial oid 15 2024-09-21T09:38:01.938 INFO:tasks.rados.rados.0.smithi017.stdout:15: seq_num 14 ranges {635298=769839,2069689=165609} 2024-09-21T09:38:01.945 INFO:tasks.rados.rados.0.smithi017.stdout:15: writing smithi01785587-15 from 635298 to 1405137 tid 1 2024-09-21T09:38:01.950 INFO:tasks.rados.rados.0.smithi017.stdout:15: writing smithi01785587-15 from 2069689 to 2235298 tid 2 2024-09-21T09:38:01.950 INFO:tasks.rados.rados.0.smithi017.stdout:16: write initial oid 16 2024-09-21T09:38:01.950 INFO:tasks.rados.rados.0.smithi017.stdout:16: seq_num 15 ranges {652105=445088,1728171=452988} 2024-09-21T09:38:01.957 INFO:tasks.rados.rados.0.smithi017.stdout:16: writing smithi01785587-16 from 652105 to 1097193 tid 1 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:16: writing smithi01785587-16 from 1728171 to 2181159 tid 2 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:1: finishing write tid 1 to smithi01785587-1 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:1: finishing write tid 2 to smithi01785587-1 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:1: finishing write tid 3 to smithi01785587-1 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:1: finishing write tid 4 to smithi01785587-1 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 3 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:2: finishing write tid 1 to smithi01785587-2 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:1: done (15 left) 2024-09-21T09:38:01.963 INFO:tasks.rados.rados.0.smithi017.stdout:17: write initial oid 17 2024-09-21T09:38:01.964 INFO:tasks.rados.rados.0.smithi017.stdout:17: seq_num 16 ranges {668912=636690,1903006=365906} 2024-09-21T09:38:01.969 INFO:tasks.rados.rados.0.smithi017.stdout:17: writing smithi01785587-17 from 668912 to 1305602 tid 1 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:17: writing smithi01785587-17 from 1903006 to 2268912 tid 2 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:2: finishing write tid 2 to smithi01785587-2 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:2: finishing write tid 3 to smithi01785587-2 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:2: finishing write tid 4 to smithi01785587-2 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 3 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:3: finishing write tid 1 to smithi01785587-3 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:2: done (15 left) 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:18: write initial oid 18 2024-09-21T09:38:01.976 INFO:tasks.rados.rados.0.smithi017.stdout:18: seq_num 17 ranges {685719=711939,1845135=440584} 2024-09-21T09:38:01.983 INFO:tasks.rados.rados.0.smithi017.stdout:18: writing smithi01785587-18 from 685719 to 1397658 tid 1 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout:18: writing smithi01785587-18 from 1845135 to 2285719 tid 2 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout:3: finishing write tid 2 to smithi01785587-3 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout:3: finishing write tid 3 to smithi01785587-3 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout:3: finishing write tid 4 to smithi01785587-3 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 3 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout:4: finishing write tid 1 to smithi01785587-4 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout:3: done (15 left) 2024-09-21T09:38:01.988 INFO:tasks.rados.rados.0.smithi017.stdout:19: write initial oid 19 2024-09-21T09:38:01.989 INFO:tasks.rados.rados.0.smithi017.stdout:19: seq_num 18 ranges {702526=787188,1903617=398909} 2024-09-21T09:38:01.996 INFO:tasks.rados.rados.0.smithi017.stdout:19: writing smithi01785587-19 from 702526 to 1489714 tid 1 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:19: writing smithi01785587-19 from 1903617 to 2302526 tid 2 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:4: finishing write tid 2 to smithi01785587-4 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:4: finishing write tid 3 to smithi01785587-4 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:4: finishing write tid 4 to smithi01785587-4 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 3 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:5: finishing write tid 1 to smithi01785587-5 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:5: finishing write tid 2 to smithi01785587-5 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:5: finishing write tid 3 to smithi01785587-5 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:5: finishing write tid 4 to smithi01785587-5 2024-09-21T09:38:02.001 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 6 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:6: finishing write tid 1 to smithi01785587-6 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:6: finishing write tid 2 to smithi01785587-6 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:6: finishing write tid 3 to smithi01785587-6 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:6: finishing write tid 4 to smithi01785587-6 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 6 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:7: finishing write tid 1 to smithi01785587-7 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:7: finishing write tid 2 to smithi01785587-7 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:7: finishing write tid 3 to smithi01785587-7 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:4: done (15 left) 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:5: done (14 left) 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:6: done (13 left) 2024-09-21T09:38:02.002 INFO:tasks.rados.rados.0.smithi017.stdout:20: write initial oid 20 2024-09-21T09:38:02.003 INFO:tasks.rados.rados.0.smithi017.stdout:20: seq_num 19 ranges {719333=462437,1962099=357234} 2024-09-21T09:38:02.007 INFO:tasks.rados.rados.0.smithi017.stdout:20: writing smithi01785587-20 from 719333 to 1181770 tid 1 2024-09-21T09:38:02.013 INFO:tasks.rados.rados.0.smithi017.stdout:20: writing smithi01785587-20 from 1962099 to 2319333 tid 2 2024-09-21T09:38:02.013 INFO:tasks.rados.rados.0.smithi017.stdout:21: write initial oid 21 2024-09-21T09:38:02.013 INFO:tasks.rados.rados.0.smithi017.stdout:21: seq_num 20 ranges {736140=537686,2020581=315559} 2024-09-21T09:38:02.020 INFO:tasks.rados.rados.0.smithi017.stdout:21: writing smithi01785587-21 from 736140 to 1273826 tid 1 2024-09-21T09:38:02.026 INFO:tasks.rados.rados.0.smithi017.stdout:21: writing smithi01785587-21 from 2020581 to 2336140 tid 2 2024-09-21T09:38:02.026 INFO:tasks.rados.rados.0.smithi017.stdout:22: write initial oid 22 2024-09-21T09:38:02.026 INFO:tasks.rados.rados.0.smithi017.stdout:22: seq_num 21 ranges {752947=612935,1962710=390237} 2024-09-21T09:38:02.033 INFO:tasks.rados.rados.0.smithi017.stdout:22: writing smithi01785587-22 from 752947 to 1365882 tid 1 2024-09-21T09:38:02.039 INFO:tasks.rados.rados.0.smithi017.stdout:22: writing smithi01785587-22 from 1962710 to 2352947 tid 2 2024-09-21T09:38:02.039 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.039 INFO:tasks.rados.rados.0.smithi017.stdout:7: finishing write tid 4 to smithi01785587-7 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 3 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:8: finishing write tid 1 to smithi01785587-8 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:8: finishing write tid 2 to smithi01785587-8 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:8: finishing write tid 3 to smithi01785587-8 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:8: finishing write tid 4 to smithi01785587-8 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 3 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:9: finishing write tid 1 to smithi01785587-9 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:9: finishing write tid 2 to smithi01785587-9 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:9: finishing write tid 3 to smithi01785587-9 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:9: finishing write tid 4 to smithi01785587-9 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 6 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:38:02.040 INFO:tasks.rados.rados.0.smithi017.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:10: finishing write tid 1 to smithi01785587-10 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:10: finishing write tid 2 to smithi01785587-10 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:10: finishing write tid 3 to smithi01785587-10 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:10: finishing write tid 4 to smithi01785587-10 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 3 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:11: finishing write tid 1 to smithi01785587-11 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:11: finishing write tid 2 to smithi01785587-11 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:11: finishing write tid 3 to smithi01785587-11 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:11: finishing write tid 4 to smithi01785587-11 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 6 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:7: done (15 left) 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:8: done (14 left) 2024-09-21T09:38:02.041 INFO:tasks.rados.rados.0.smithi017.stdout:9: done (13 left) 2024-09-21T09:38:02.042 INFO:tasks.rados.rados.0.smithi017.stdout:10: done (12 left) 2024-09-21T09:38:02.042 INFO:tasks.rados.rados.0.smithi017.stdout:11: done (11 left) 2024-09-21T09:38:02.042 INFO:tasks.rados.rados.0.smithi017.stdout:23: write initial oid 23 2024-09-21T09:38:02.042 INFO:tasks.rados.rados.0.smithi017.stdout:23: seq_num 22 ranges {769754=688184,2021192=348562} 2024-09-21T09:38:02.047 INFO:tasks.rados.rados.0.smithi017.stdout:23: writing smithi01785587-23 from 769754 to 1457938 tid 1 2024-09-21T09:38:02.052 INFO:tasks.rados.rados.0.smithi017.stdout:23: writing smithi01785587-23 from 2021192 to 2369754 tid 2 2024-09-21T09:38:02.052 INFO:tasks.rados.rados.0.smithi017.stdout:24: write initial oid 24 2024-09-21T09:38:02.052 INFO:tasks.rados.rados.0.smithi017.stdout:24: seq_num 23 ranges {786561=479786,1796027=467664} 2024-09-21T09:38:02.059 INFO:tasks.rados.rados.0.smithi017.stdout:24: writing smithi01785587-24 from 786561 to 1266347 tid 1 2024-09-21T09:38:02.064 INFO:tasks.rados.rados.0.smithi017.stdout:24: writing smithi01785587-24 from 1796027 to 2263691 tid 2 2024-09-21T09:38:02.065 INFO:tasks.rados.rados.0.smithi017.stdout:25: write initial oid 25 2024-09-21T09:38:02.065 INFO:tasks.rados.rados.0.smithi017.stdout:25: seq_num 24 ranges {403368=555035,1454509=727675} 2024-09-21T09:38:02.070 INFO:tasks.rados.rados.0.smithi017.stdout:25: writing smithi01785587-25 from 403368 to 958403 tid 1 2024-09-21T09:38:02.076 INFO:tasks.rados.rados.0.smithi017.stdout:25: writing smithi01785587-25 from 1454509 to 2182184 tid 2 2024-09-21T09:38:02.076 INFO:tasks.rados.rados.0.smithi017.stdout:26: write initial oid 26 2024-09-21T09:38:02.077 INFO:tasks.rados.rados.0.smithi017.stdout:26: seq_num 25 ranges {420175=630284,1796638=471333} 2024-09-21T09:38:02.082 INFO:tasks.rados.rados.0.smithi017.stdout:26: writing smithi01785587-26 from 420175 to 1050459 tid 1 2024-09-21T09:38:02.089 INFO:tasks.rados.rados.0.smithi017.stdout:26: writing smithi01785587-26 from 1796638 to 2267971 tid 2 2024-09-21T09:38:02.089 INFO:tasks.rados.rados.0.smithi017.stdout:27: write initial oid 27 2024-09-21T09:38:02.089 INFO:tasks.rados.rados.0.smithi017.stdout:27: seq_num 26 ranges {436982=705533,1855120=581862} 2024-09-21T09:38:02.095 INFO:tasks.rados.rados.0.smithi017.stdout:27: writing smithi01785587-27 from 436982 to 1142515 tid 1 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:27: writing smithi01785587-27 from 1855120 to 2436982 tid 2 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:12: finishing write tid 1 to smithi01785587-12 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:12: finishing write tid 2 to smithi01785587-12 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:12: finishing write tid 3 to smithi01785587-12 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:12: finishing write tid 4 to smithi01785587-12 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 3 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:13: finishing write tid 1 to smithi01785587-13 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:12: done (15 left) 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:28: write initial oid 28 2024-09-21T09:38:02.103 INFO:tasks.rados.rados.0.smithi017.stdout:28: seq_num 27 ranges {453789=780782,1913602=475002} 2024-09-21T09:38:02.109 INFO:tasks.rados.rados.0.smithi017.stdout:28: writing smithi01785587-28 from 453789 to 1234571 tid 1 2024-09-21T09:38:02.116 INFO:tasks.rados.rados.0.smithi017.stdout:28: writing smithi01785587-28 from 1913602 to 2388604 tid 2 2024-09-21T09:38:02.116 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.116 INFO:tasks.rados.rados.0.smithi017.stdout:13: finishing write tid 2 to smithi01785587-13 2024-09-21T09:38:02.116 INFO:tasks.rados.rados.0.smithi017.stdout:13: finishing write tid 3 to smithi01785587-13 2024-09-21T09:38:02.116 INFO:tasks.rados.rados.0.smithi017.stdout:13: finishing write tid 4 to smithi01785587-13 2024-09-21T09:38:02.116 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 3 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-21T09:38:02.116 INFO:tasks.rados.rados.0.smithi017.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-09-21T09:38:02.117 INFO:tasks.rados.rados.0.smithi017.stdout:14: finishing write tid 1 to smithi01785587-14 2024-09-21T09:38:02.117 INFO:tasks.rados.rados.0.smithi017.stdout:13: done (15 left) 2024-09-21T09:38:02.117 INFO:tasks.rados.rados.0.smithi017.stdout:29: write initial oid 29 2024-09-21T09:38:02.117 INFO:tasks.rados.rados.0.smithi017.stdout:29: seq_num 28 ranges {470596=456031,1572084=618660} 2024-09-21T09:38:02.121 INFO:tasks.rados.rados.0.smithi017.stdout:29: writing smithi01785587-29 from 470596 to 926627 tid 1 2024-09-21T09:38:02.128 INFO:tasks.rados.rados.0.smithi017.stdout:29: writing smithi01785587-29 from 1572084 to 2190744 tid 2 2024-09-21T09:38:02.128 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.128 INFO:tasks.rados.rados.0.smithi017.stdout:14: finishing write tid 2 to smithi01785587-14 2024-09-21T09:38:02.128 INFO:tasks.rados.rados.0.smithi017.stdout:14: finishing write tid 3 to smithi01785587-14 2024-09-21T09:38:02.128 INFO:tasks.rados.rados.0.smithi017.stdout:14: finishing write tid 4 to smithi01785587-14 2024-09-21T09:38:02.128 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 9 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:38:02.128 INFO:tasks.rados.rados.0.smithi017.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:38:02.128 INFO:tasks.rados.rados.0.smithi017.stdout:15: finishing write tid 1 to smithi01785587-15 2024-09-21T09:38:02.128 INFO:tasks.rados.rados.0.smithi017.stdout:14: done (15 left) 2024-09-21T09:38:02.129 INFO:tasks.rados.rados.0.smithi017.stdout:30: write initial oid 30 2024-09-21T09:38:02.129 INFO:tasks.rados.rados.0.smithi017.stdout:30: seq_num 29 ranges {487403=531280,1514213=478671} 2024-09-21T09:38:02.133 INFO:tasks.rados.rados.0.smithi017.stdout:30: writing smithi01785587-30 from 487403 to 1018683 tid 1 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:30: writing smithi01785587-30 from 1514213 to 1992884 tid 2 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:15: finishing write tid 2 to smithi01785587-15 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:15: finishing write tid 3 to smithi01785587-15 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:15: finishing write tid 4 to smithi01785587-15 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 3 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:16: finishing write tid 1 to smithi01785587-16 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:15: done (15 left) 2024-09-21T09:38:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:31: write initial oid 31 2024-09-21T09:38:02.140 INFO:tasks.rados.rados.0.smithi017.stdout:31: seq_num 30 ranges {504210=606529,1572695=622329} 2024-09-21T09:38:02.145 INFO:tasks.rados.rados.0.smithi017.stdout:31: writing smithi01785587-31 from 504210 to 1110739 tid 1 2024-09-21T09:38:02.151 INFO:tasks.rados.rados.0.smithi017.stdout:31: writing smithi01785587-31 from 1572695 to 2195024 tid 2 2024-09-21T09:38:02.151 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.151 INFO:tasks.rados.rados.0.smithi017.stdout:16: finishing write tid 2 to smithi01785587-16 2024-09-21T09:38:02.151 INFO:tasks.rados.rados.0.smithi017.stdout:16: finishing write tid 3 to smithi01785587-16 2024-09-21T09:38:02.151 INFO:tasks.rados.rados.0.smithi017.stdout:16: finishing write tid 4 to smithi01785587-16 2024-09-21T09:38:02.152 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 6 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-21T09:38:02.152 INFO:tasks.rados.rados.0.smithi017.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-09-21T09:38:02.152 INFO:tasks.rados.rados.0.smithi017.stdout:17: finishing write tid 1 to smithi01785587-17 2024-09-21T09:38:02.152 INFO:tasks.rados.rados.0.smithi017.stdout:16: done (15 left) 2024-09-21T09:38:02.152 INFO:tasks.rados.rados.0.smithi017.stdout:32: write initial oid 32 2024-09-21T09:38:02.152 INFO:tasks.rados.rados.0.smithi017.stdout:32: seq_num 31 ranges {521017=798131,1747530=482340} 2024-09-21T09:38:02.159 INFO:tasks.rados.rados.0.smithi017.stdout:32: writing smithi01785587-32 from 521017 to 1319148 tid 1 2024-09-21T09:38:02.159 INFO:teuthology.orchestra.run.smithi017.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:02.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:01 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/674371772' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:38:02.160 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:01 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/674371772' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:32: writing smithi01785587-32 from 1747530 to 2229870 tid 2 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:17: finishing write tid 2 to smithi01785587-17 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:17: finishing write tid 3 to smithi01785587-17 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:17: finishing write tid 4 to smithi01785587-17 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 3 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:18: finishing write tid 1 to smithi01785587-18 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:18: finishing write tid 2 to smithi01785587-18 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:17: done (15 left) 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:33: write initial oid 33 2024-09-21T09:38:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:33: seq_num 32 ranges {537824=473380,1806012=625998} 2024-09-21T09:38:02.170 INFO:tasks.rados.rados.0.smithi017.stdout:33: writing smithi01785587-33 from 537824 to 1011204 tid 1 2024-09-21T09:38:02.178 INFO:tasks.rados.rados.0.smithi017.stdout:33: writing smithi01785587-33 from 1806012 to 2432010 tid 2 2024-09-21T09:38:02.178 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:18: finishing write tid 3 to smithi01785587-18 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:18: finishing write tid 4 to smithi01785587-18 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 3 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:19: finishing write tid 1 to smithi01785587-19 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:19: finishing write tid 2 to smithi01785587-19 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:19: finishing write tid 3 to smithi01785587-19 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:18: done (15 left) 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:34: write initial oid 34 2024-09-21T09:38:02.179 INFO:tasks.rados.rados.0.smithi017.stdout:34: seq_num 33 ranges {554631=548629,1748141=486009} 2024-09-21T09:38:02.184 INFO:tasks.rados.rados.0.smithi017.stdout:34: writing smithi01785587-34 from 554631 to 1103260 tid 1 2024-09-21T09:38:02.191 INFO:tasks.rados.rados.0.smithi017.stdout:34: writing smithi01785587-34 from 1748141 to 2234150 tid 2 2024-09-21T09:38:02.191 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.191 INFO:tasks.rados.rados.0.smithi017.stdout:19: finishing write tid 4 to smithi01785587-19 2024-09-21T09:38:02.191 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 6 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-21T09:38:02.191 INFO:tasks.rados.rados.0.smithi017.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-09-21T09:38:02.191 INFO:tasks.rados.rados.0.smithi017.stdout:20: finishing write tid 1 to smithi01785587-20 2024-09-21T09:38:02.191 INFO:tasks.rados.rados.0.smithi017.stdout:19: done (15 left) 2024-09-21T09:38:02.191 INFO:tasks.rados.rados.0.smithi017.stdout:35: write initial oid 35 2024-09-21T09:38:02.191 INFO:tasks.rados.rados.0.smithi017.stdout:35: seq_num 34 ranges {571438=623878,1806623=629667} 2024-09-21T09:38:02.197 INFO:tasks.rados.rados.0.smithi017.stdout:35: writing smithi01785587-35 from 571438 to 1195316 tid 1 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout:35: writing smithi01785587-35 from 1806623 to 2436290 tid 2 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout:20: finishing write tid 2 to smithi01785587-20 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout:20: finishing write tid 3 to smithi01785587-20 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout:20: finishing write tid 4 to smithi01785587-20 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 3 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout:21: finishing write tid 1 to smithi01785587-21 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout:21: finishing write tid 2 to smithi01785587-21 2024-09-21T09:38:02.205 INFO:tasks.rados.rados.0.smithi017.stdout:21: finishing write tid 3 to smithi01785587-21 2024-09-21T09:38:02.206 INFO:tasks.rados.rados.0.smithi017.stdout:21: finishing write tid 4 to smithi01785587-21 2024-09-21T09:38:02.206 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 6 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-21T09:38:02.206 INFO:tasks.rados.rados.0.smithi017.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:38:02.206 INFO:tasks.rados.rados.0.smithi017.stdout:22: finishing write tid 1 to smithi01785587-22 2024-09-21T09:38:02.206 INFO:tasks.rados.rados.0.smithi017.stdout:20: done (15 left) 2024-09-21T09:38:02.206 INFO:tasks.rados.rados.0.smithi017.stdout:21: done (14 left) 2024-09-21T09:38:02.206 INFO:tasks.rados.rados.0.smithi017.stdout:36: write initial oid 36 2024-09-21T09:38:02.206 INFO:tasks.rados.rados.0.smithi017.stdout:36: seq_num 35 ranges {588245=699127,1865105=489678} 2024-09-21T09:38:02.211 INFO:tasks.rados.rados.0.smithi017.stdout:36: writing smithi01785587-36 from 588245 to 1287372 tid 1 2024-09-21T09:38:02.217 INFO:tasks.rados.rados.0.smithi017.stdout:36: writing smithi01785587-36 from 1865105 to 2354783 tid 2 2024-09-21T09:38:02.217 INFO:tasks.rados.rados.0.smithi017.stdout:37: write initial oid 37 2024-09-21T09:38:02.218 INFO:tasks.rados.rados.0.smithi017.stdout:37: seq_num 36 ranges {605052=774376,1923587=633336} 2024-09-21T09:38:02.225 INFO:tasks.rados.rados.0.smithi017.stdout:37: writing smithi01785587-37 from 605052 to 1379428 tid 1 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:37: writing smithi01785587-37 from 1923587 to 2556923 tid 2 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:22: finishing write tid 2 to smithi01785587-22 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:22: finishing write tid 3 to smithi01785587-22 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:22: finishing write tid 4 to smithi01785587-22 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 9 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:23: finishing write tid 1 to smithi01785587-23 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:23: finishing write tid 2 to smithi01785587-23 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:22: done (15 left) 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:38: write initial oid 38 2024-09-21T09:38:02.233 INFO:tasks.rados.rados.0.smithi017.stdout:38: seq_num 37 ranges {621859=449625,1865716=756143} 2024-09-21T09:38:02.238 INFO:tasks.rados.rados.0.smithi017.stdout:38: writing smithi01785587-38 from 621859 to 1071484 tid 1 2024-09-21T09:38:02.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:01 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/674371772' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout:38: writing smithi01785587-38 from 1865716 to 2621859 tid 2 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout:23: finishing write tid 3 to smithi01785587-23 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout:23: finishing write tid 4 to smithi01785587-23 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 9 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout:24: finishing write tid 1 to smithi01785587-24 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout:24: finishing write tid 2 to smithi01785587-24 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout:24: finishing write tid 3 to smithi01785587-24 2024-09-21T09:38:02.248 INFO:tasks.rados.rados.0.smithi017.stdout:24: finishing write tid 4 to smithi01785587-24 2024-09-21T09:38:02.249 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 3 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-21T09:38:02.249 INFO:tasks.rados.rados.0.smithi017.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-09-21T09:38:02.249 INFO:tasks.rados.rados.0.smithi017.stdout:25: finishing write tid 1 to smithi01785587-25 2024-09-21T09:38:02.249 INFO:tasks.rados.rados.0.smithi017.stdout:23: done (15 left) 2024-09-21T09:38:02.249 INFO:tasks.rados.rados.0.smithi017.stdout:24: done (14 left) 2024-09-21T09:38:02.249 INFO:tasks.rados.rados.0.smithi017.stdout:39: write initial oid 39 2024-09-21T09:38:02.249 INFO:tasks.rados.rados.0.smithi017.stdout:39: seq_num 38 ranges {638666=524874,1924198=637005} 2024-09-21T09:38:02.254 INFO:tasks.rados.rados.0.smithi017.stdout:39: writing smithi01785587-39 from 638666 to 1163540 tid 1 2024-09-21T09:38:02.262 INFO:tasks.rados.rados.0.smithi017.stdout:39: writing smithi01785587-39 from 1924198 to 2561203 tid 2 2024-09-21T09:38:02.262 INFO:tasks.rados.rados.0.smithi017.stdout:40: write initial oid 40 2024-09-21T09:38:02.262 INFO:tasks.rados.rados.0.smithi017.stdout:40: seq_num 39 ranges {655473=716476,2099033=556440} 2024-09-21T09:38:02.269 INFO:tasks.rados.rados.0.smithi017.stdout:40: writing smithi01785587-40 from 655473 to 1371949 tid 1 2024-09-21T09:38:02.277 INFO:tasks.rados.rados.0.smithi017.stdout:40: writing smithi01785587-40 from 2099033 to 2655473 tid 2 2024-09-21T09:38:02.277 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.277 INFO:tasks.rados.rados.0.smithi017.stdout:25: finishing write tid 2 to smithi01785587-25 2024-09-21T09:38:02.277 INFO:tasks.rados.rados.0.smithi017.stdout:25: finishing write tid 3 to smithi01785587-25 2024-09-21T09:38:02.277 INFO:tasks.rados.rados.0.smithi017.stdout:25: finishing write tid 4 to smithi01785587-25 2024-09-21T09:38:02.277 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 6 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-09-21T09:38:02.277 INFO:tasks.rados.rados.0.smithi017.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:26: finishing write tid 1 to smithi01785587-26 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:26: finishing write tid 2 to smithi01785587-26 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:26: finishing write tid 3 to smithi01785587-26 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:26: finishing write tid 4 to smithi01785587-26 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 12 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:27: finishing write tid 1 to smithi01785587-27 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:25: done (15 left) 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:26: done (14 left) 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:41: write initial oid 41 2024-09-21T09:38:02.278 INFO:tasks.rados.rados.0.smithi017.stdout:41: seq_num 40 ranges {672280=791725,2157515=514765} 2024-09-21T09:38:02.285 INFO:tasks.rados.rados.0.smithi017.stdout:41: writing smithi01785587-41 from 672280 to 1464005 tid 1 2024-09-21T09:38:02.292 INFO:tasks.rados.rados.0.smithi017.stdout:41: writing smithi01785587-41 from 2157515 to 2672280 tid 2 2024-09-21T09:38:02.292 INFO:tasks.rados.rados.0.smithi017.stdout:42: write initial oid 42 2024-09-21T09:38:02.292 INFO:tasks.rados.rados.0.smithi017.stdout:42: seq_num 41 ranges {689087=466974,1699644=784332} 2024-09-21T09:38:02.298 INFO:tasks.rados.rados.0.smithi017.stdout:42: writing smithi01785587-42 from 689087 to 1156061 tid 1 2024-09-21T09:38:02.306 INFO:tasks.rados.rados.0.smithi017.stdout:42: writing smithi01785587-42 from 1699644 to 2483976 tid 2 2024-09-21T09:38:02.306 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.306 INFO:tasks.rados.rados.0.smithi017.stdout:27: finishing write tid 2 to smithi01785587-27 2024-09-21T09:38:02.306 INFO:tasks.rados.rados.0.smithi017.stdout:27: finishing write tid 3 to smithi01785587-27 2024-09-21T09:38:02.306 INFO:tasks.rados.rados.0.smithi017.stdout:27: finishing write tid 4 to smithi01785587-27 2024-09-21T09:38:02.306 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 6 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:38:02.306 INFO:tasks.rados.rados.0.smithi017.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:28: finishing write tid 1 to smithi01785587-28 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:28: finishing write tid 2 to smithi01785587-28 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:28: finishing write tid 3 to smithi01785587-28 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:28: finishing write tid 4 to smithi01785587-28 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 15 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:29: finishing write tid 1 to smithi01785587-29 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:27: done (15 left) 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:28: done (14 left) 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:43: write initial oid 43 2024-09-21T09:38:02.307 INFO:tasks.rados.rados.0.smithi017.stdout:43: seq_num 42 ranges {705894=542223,1758126=644343} 2024-09-21T09:38:02.312 INFO:tasks.rados.rados.0.smithi017.stdout:43: writing smithi01785587-43 from 705894 to 1248117 tid 1 2024-09-21T09:38:02.319 INFO:tasks.rados.rados.0.smithi017.stdout:43: writing smithi01785587-43 from 1758126 to 2402469 tid 2 2024-09-21T09:38:02.319 INFO:tasks.rados.rados.0.smithi017.stdout:44: write initial oid 44 2024-09-21T09:38:02.319 INFO:tasks.rados.rados.0.smithi017.stdout:44: seq_num 43 ranges {722701=617472,1816608=788001} 2024-09-21T09:38:02.326 INFO:tasks.rados.rados.0.smithi017.stdout:44: writing smithi01785587-44 from 722701 to 1340173 tid 1 2024-09-21T09:38:02.333 INFO:tasks.rados.rados.0.smithi017.stdout:44: writing smithi01785587-44 from 1816608 to 2604609 tid 2 2024-09-21T09:38:02.333 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.333 INFO:tasks.rados.rados.0.smithi017.stdout:29: finishing write tid 2 to smithi01785587-29 2024-09-21T09:38:02.333 INFO:tasks.rados.rados.0.smithi017.stdout:29: finishing write tid 3 to smithi01785587-29 2024-09-21T09:38:02.334 INFO:tasks.rados.rados.0.smithi017.stdout:29: finishing write tid 4 to smithi01785587-29 2024-09-21T09:38:02.334 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 9 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-09-21T09:38:02.334 INFO:tasks.rados.rados.0.smithi017.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-09-21T09:38:02.334 INFO:tasks.rados.rados.0.smithi017.stdout:30: finishing write tid 1 to smithi01785587-30 2024-09-21T09:38:02.334 INFO:tasks.rados.rados.0.smithi017.stdout:29: done (15 left) 2024-09-21T09:38:02.334 INFO:tasks.rados.rados.0.smithi017.stdout:45: write initial oid 45 2024-09-21T09:38:02.334 INFO:tasks.rados.rados.0.smithi017.stdout:45: seq_num 44 ranges {739508=692721,1875090=648012} 2024-09-21T09:38:02.341 INFO:tasks.rados.rados.0.smithi017.stdout:45: writing smithi01785587-45 from 739508 to 1432229 tid 1 2024-09-21T09:38:02.347 INFO:tasks.rados.rados.0.smithi017.stdout:45: writing smithi01785587-45 from 1875090 to 2523102 tid 2 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout:30: finishing write tid 2 to smithi01785587-30 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout:30: finishing write tid 3 to smithi01785587-30 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout:30: finishing write tid 4 to smithi01785587-30 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 9 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout:31: finishing write tid 1 to smithi01785587-31 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout:30: done (15 left) 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout:46: write initial oid 46 2024-09-21T09:38:02.348 INFO:tasks.rados.rados.0.smithi017.stdout:46: seq_num 45 ranges {756315=767970,1933572=791670} 2024-09-21T09:38:02.355 INFO:tasks.rados.rados.0.smithi017.stdout:46: writing smithi01785587-46 from 756315 to 1524285 tid 1 2024-09-21T09:38:02.362 INFO:tasks.rados.rados.0.smithi017.stdout:46: writing smithi01785587-46 from 1933572 to 2725242 tid 2 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:31: finishing write tid 2 to smithi01785587-31 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:31: finishing write tid 3 to smithi01785587-31 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:31: finishing write tid 4 to smithi01785587-31 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 12 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:32: finishing write tid 1 to smithi01785587-32 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:32: finishing write tid 2 to smithi01785587-32 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:31: done (15 left) 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:47: write initial oid 47 2024-09-21T09:38:02.363 INFO:tasks.rados.rados.0.smithi017.stdout:47: seq_num 46 ranges {773122=559572,1992054=651681} 2024-09-21T09:38:02.370 INFO:tasks.rados.rados.0.smithi017.stdout:47: writing smithi01785587-47 from 773122 to 1332694 tid 1 2024-09-21T09:38:02.377 INFO:tasks.rados.rados.0.smithi017.stdout:47: writing smithi01785587-47 from 1992054 to 2643735 tid 2 2024-09-21T09:38:02.377 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:32: finishing write tid 3 to smithi01785587-32 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:32: finishing write tid 4 to smithi01785587-32 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 12 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:33: finishing write tid 1 to smithi01785587-33 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:33: finishing write tid 2 to smithi01785587-33 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:33: finishing write tid 3 to smithi01785587-33 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:33: finishing write tid 4 to smithi01785587-33 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 18 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:34: finishing write tid 1 to smithi01785587-34 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:32: done (15 left) 2024-09-21T09:38:02.378 INFO:tasks.rados.rados.0.smithi017.stdout:33: done (14 left) 2024-09-21T09:38:02.379 INFO:tasks.rados.rados.0.smithi017.stdout:48: write initial oid 48 2024-09-21T09:38:02.379 INFO:tasks.rados.rados.0.smithi017.stdout:48: seq_num 47 ranges {789929=634821,2050536=739393} 2024-09-21T09:38:02.385 INFO:tasks.rados.rados.0.smithi017.stdout:48: writing smithi01785587-48 from 789929 to 1424750 tid 1 2024-09-21T09:38:02.392 INFO:tasks.rados.rados.0.smithi017.stdout:48: writing smithi01785587-48 from 2050536 to 2789929 tid 2 2024-09-21T09:38:02.393 INFO:tasks.rados.rados.0.smithi017.stdout:49: write initial oid 49 2024-09-21T09:38:02.393 INFO:tasks.rados.rados.0.smithi017.stdout:49: seq_num 48 ranges {406736=710070,1709018=655350} 2024-09-21T09:38:02.399 INFO:tasks.rados.rados.0.smithi017.stdout:49: writing smithi01785587-49 from 406736 to 1116806 tid 1 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout:49: writing smithi01785587-49 from 1709018 to 2364368 tid 2 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout:34: finishing write tid 2 to smithi01785587-34 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout:34: finishing write tid 3 to smithi01785587-34 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout:34: finishing write tid 4 to smithi01785587-34 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 9 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout:35: finishing write tid 1 to smithi01785587-35 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout:35: finishing write tid 2 to smithi01785587-35 2024-09-21T09:38:02.407 INFO:tasks.rados.rados.0.smithi017.stdout:35: finishing write tid 3 to smithi01785587-35 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:35: finishing write tid 4 to smithi01785587-35 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 3 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:36: finishing write tid 1 to smithi01785587-36 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:36: finishing write tid 2 to smithi01785587-36 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:36: finishing write tid 3 to smithi01785587-36 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:36: finishing write tid 4 to smithi01785587-36 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 3 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:37: finishing write tid 1 to smithi01785587-37 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:37: finishing write tid 2 to smithi01785587-37 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:37: finishing write tid 3 to smithi01785587-37 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:37: finishing write tid 4 to smithi01785587-37 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 6 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:38:02.408 INFO:tasks.rados.rados.0.smithi017.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:38:02.409 INFO:tasks.rados.rados.0.smithi017.stdout:38: finishing write tid 1 to smithi01785587-38 2024-09-21T09:38:02.409 INFO:tasks.rados.rados.0.smithi017.stdout:34: done (15 left) 2024-09-21T09:38:02.409 INFO:tasks.rados.rados.0.smithi017.stdout:35: done (14 left) 2024-09-21T09:38:02.409 INFO:tasks.rados.rados.0.smithi017.stdout:36: done (13 left) 2024-09-21T09:38:02.409 INFO:tasks.rados.rados.0.smithi017.stdout:37: done (12 left) 2024-09-21T09:38:02.409 INFO:tasks.rados.rados.0.smithi017.stdout:50: write initial oid 50 2024-09-21T09:38:02.409 INFO:tasks.rados.rados.0.smithi017.stdout:50: seq_num 49 ranges {423543=785319,1767500=799008} 2024-09-21T09:38:02.412 INFO:tasks.rados.rados.0.smithi017.stdout:50: writing smithi01785587-50 from 423543 to 1208862 tid 1 2024-09-21T09:38:02.420 INFO:tasks.rados.rados.0.smithi017.stdout:50: writing smithi01785587-50 from 1767500 to 2566508 tid 2 2024-09-21T09:38:02.420 INFO:tasks.rados.rados.0.smithi017.stdout:51: write oid 8 current snap is 0 2024-09-21T09:38:02.420 INFO:tasks.rados.rados.0.smithi017.stdout:38: finishing write tid 2 to smithi01785587-38 2024-09-21T09:38:02.420 INFO:tasks.rados.rados.0.smithi017.stdout:38: finishing write tid 3 to smithi01785587-38 2024-09-21T09:38:02.420 INFO:tasks.rados.rados.0.smithi017.stdout:51: seq_num 50 ranges {440350=460568,1309629=542666,2323973=516377} 2024-09-21T09:38:02.425 INFO:tasks.rados.rados.0.smithi017.stdout:51: writing smithi01785587-8 from 440350 to 900918 tid 1 2024-09-21T09:38:02.430 INFO:tasks.rados.rados.0.smithi017.stdout:51: writing smithi01785587-8 from 1309629 to 1852295 tid 2 2024-09-21T09:38:02.436 INFO:tasks.rados.rados.0.smithi017.stdout:51: writing smithi01785587-8 from 2323973 to 2840350 tid 3 2024-09-21T09:38:02.436 INFO:tasks.rados.rados.0.smithi017.stdout:52: rollback oid 16 current snap is 0 2024-09-21T09:38:02.436 INFO:tasks.rados.rados.0.smithi017.stdout:52: done (14 left) 2024-09-21T09:38:02.436 INFO:tasks.rados.rados.0.smithi017.stdout:53: write oid 16 current snap is 0 2024-09-21T09:38:02.436 INFO:tasks.rados.rados.0.smithi017.stdout:38: finishing write tid 4 to smithi01785587-38 2024-09-21T09:38:02.436 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 6 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-21T09:38:02.437 INFO:tasks.rados.rados.0.smithi017.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-09-21T09:38:02.437 INFO:tasks.rados.rados.0.smithi017.stdout:39: finishing write tid 1 to smithi01785587-39 2024-09-21T09:38:02.437 INFO:tasks.rados.rados.0.smithi017.stdout:53: seq_num 51 ranges {457157=535817,1768111=402677} 2024-09-21T09:38:02.442 INFO:tasks.rados.rados.0.smithi017.stdout:53: writing smithi01785587-16 from 457157 to 992974 tid 1 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:53: writing smithi01785587-16 from 1768111 to 2170788 tid 2 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:38: done (14 left) 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:54: write oid 21 current snap is 0 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:39: finishing write tid 2 to smithi01785587-39 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:39: finishing write tid 3 to smithi01785587-39 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:39: finishing write tid 4 to smithi01785587-39 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 9 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:40: finishing write tid 1 to smithi01785587-40 2024-09-21T09:38:02.449 INFO:tasks.rados.rados.0.smithi017.stdout:54: seq_num 52 ranges {473964=611066,1826593=546335,2778819=95145} 2024-09-21T09:38:02.454 INFO:tasks.rados.rados.0.smithi017.stdout:54: writing smithi01785587-21 from 473964 to 1085030 tid 1 2024-09-21T09:38:02.461 INFO:tasks.rados.rados.0.smithi017.stdout:54: writing smithi01785587-21 from 1826593 to 2372928 tid 2 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:54: writing smithi01785587-21 from 2778819 to 2873964 tid 3 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:39: done (14 left) 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:55: delete oid 10 current snap is 0 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:40: finishing write tid 2 to smithi01785587-40 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:40: finishing write tid 3 to smithi01785587-40 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:40: finishing write tid 4 to smithi01785587-40 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 12 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:41: finishing write tid 1 to smithi01785587-41 2024-09-21T09:38:02.468 INFO:tasks.rados.rados.0.smithi017.stdout:41: finishing write tid 2 to smithi01785587-41 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:41: finishing write tid 3 to smithi01785587-41 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:41: finishing write tid 4 to smithi01785587-41 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 6 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:42: finishing write tid 1 to smithi01785587-42 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:42: finishing write tid 2 to smithi01785587-42 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:42: finishing write tid 3 to smithi01785587-42 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:42: finishing write tid 4 to smithi01785587-42 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 6 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:43: finishing write tid 1 to smithi01785587-43 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:43: finishing write tid 2 to smithi01785587-43 2024-09-21T09:38:02.469 INFO:tasks.rados.rados.0.smithi017.stdout:43: finishing write tid 3 to smithi01785587-43 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:43: finishing write tid 4 to smithi01785587-43 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 9 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:44: finishing write tid 1 to smithi01785587-44 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:44: finishing write tid 2 to smithi01785587-44 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:44: finishing write tid 3 to smithi01785587-44 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:44: finishing write tid 4 to smithi01785587-44 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 15 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:45: finishing write tid 1 to smithi01785587-45 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:45: finishing write tid 2 to smithi01785587-45 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:45: finishing write tid 3 to smithi01785587-45 2024-09-21T09:38:02.470 INFO:tasks.rados.rados.0.smithi017.stdout:45: finishing write tid 4 to smithi01785587-45 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 9 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:46: finishing write tid 1 to smithi01785587-46 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:46: finishing write tid 2 to smithi01785587-46 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:46: finishing write tid 3 to smithi01785587-46 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:46: finishing write tid 4 to smithi01785587-46 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 15 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:47: finishing write tid 1 to smithi01785587-47 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:47: finishing write tid 2 to smithi01785587-47 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:47: finishing write tid 3 to smithi01785587-47 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:47: finishing write tid 4 to smithi01785587-47 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 9 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:38:02.471 INFO:tasks.rados.rados.0.smithi017.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:48: finishing write tid 1 to smithi01785587-48 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:48: finishing write tid 2 to smithi01785587-48 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:48: finishing write tid 3 to smithi01785587-48 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:48: finishing write tid 4 to smithi01785587-48 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 12 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:49: finishing write tid 1 to smithi01785587-49 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:49: finishing write tid 2 to smithi01785587-49 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:49: finishing write tid 3 to smithi01785587-49 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:49: finishing write tid 4 to smithi01785587-49 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 12 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:50: finishing write tid 1 to smithi01785587-50 2024-09-21T09:38:02.472 INFO:tasks.rados.rados.0.smithi017.stdout:50: finishing write tid 2 to smithi01785587-50 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:50: finishing write tid 3 to smithi01785587-50 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:50: finishing write tid 4 to smithi01785587-50 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 9 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:51: finishing write tid 1 to smithi01785587-8 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:51: finishing write tid 2 to smithi01785587-8 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:51: finishing write tid 3 to smithi01785587-8 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:51: finishing write tid 4 to smithi01785587-8 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:51: finishing write tid 5 to smithi01785587-8 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 19 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:51: left oid 8 (ObjNum 50 snap 0 seq_num 50) 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:53: finishing write tid 1 to smithi01785587-16 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:53: finishing write tid 2 to smithi01785587-16 2024-09-21T09:38:02.473 INFO:tasks.rados.rados.0.smithi017.stdout:53: finishing write tid 3 to smithi01785587-16 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:53: finishing write tid 4 to smithi01785587-16 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 21 (ObjNum 51 snap 0 seq_num 51) dirty exists 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:53: left oid 16 (ObjNum 51 snap 0 seq_num 51) 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:54: finishing write tid 1 to smithi01785587-21 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:54: finishing write tid 2 to smithi01785587-21 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:40: done (14 left) 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:41: done (13 left) 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:42: done (12 left) 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:43: done (11 left) 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:44: done (10 left) 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:45: done (9 left) 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:46: done (8 left) 2024-09-21T09:38:02.474 INFO:tasks.rados.rados.0.smithi017.stdout:47: done (7 left) 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:48: done (6 left) 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:49: done (5 left) 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:50: done (4 left) 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:51: done (3 left) 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:53: done (2 left) 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:55: done (1 left) 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:56: read oid 15 snap -1 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:57: delete oid 13 current snap is 0 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:54: finishing write tid 3 to smithi01785587-21 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:54: finishing write tid 4 to smithi01785587-21 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:54: finishing write tid 5 to smithi01785587-21 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 16 (ObjNum 52 snap 0 seq_num 52) dirty exists 2024-09-21T09:38:02.475 INFO:tasks.rados.rados.0.smithi017.stdout:54: left oid 21 (ObjNum 52 snap 0 seq_num 52) 2024-09-21T09:38:02.476 INFO:tasks.rados.rados.0.smithi017.stdout:54: done (2 left) 2024-09-21T09:38:02.476 INFO:tasks.rados.rados.0.smithi017.stdout:57: done (1 left) 2024-09-21T09:38:02.476 INFO:tasks.rados.rados.0.smithi017.stdout:58: read oid 3 snap -1 2024-09-21T09:38:02.476 INFO:tasks.rados.rados.0.smithi017.stdout:58: expect (ObjNum 2 snap 0 seq_num 2) 2024-09-21T09:38:02.476 INFO:tasks.rados.rados.0.smithi017.stdout:59: read oid 28 snap -1 2024-09-21T09:38:02.476 INFO:tasks.rados.rados.0.smithi017.stdout:59: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:38:02.476 INFO:tasks.rados.rados.0.smithi017.stdout:60: read oid 34 snap -1 2024-09-21T09:38:02.476 INFO:tasks.rados.rados.0.smithi017.stdout:60: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:38:02.476 INFO:tasks.rados.rados.0.smithi017.stdout:61: delete oid 2 current snap is 0 2024-09-21T09:38:02.477 INFO:tasks.rados.rados.0.smithi017.stdout:61: done (4 left) 2024-09-21T09:38:02.477 INFO:tasks.rados.rados.0.smithi017.stdout:62: write oid 7 current snap is 0 2024-09-21T09:38:02.477 INFO:tasks.rados.rados.0.smithi017.stdout:62: seq_num 56 ranges {541192=628415,1660521=553673,2888511=52681} 2024-09-21T09:38:02.477 INFO:tasks.rados.rados.0.smithi017.stdout:62: writing smithi01785587-7 from 541192 to 1169607 tid 1 2024-09-21T09:38:02.480 INFO:tasks.rados.rados.0.smithi017.stdout:62: writing smithi01785587-7 from 1660521 to 2214194 tid 2 2024-09-21T09:38:02.484 INFO:tasks.rados.rados.0.smithi017.stdout:62: writing smithi01785587-7 from 2888511 to 2941192 tid 3 2024-09-21T09:38:02.485 INFO:tasks.rados.rados.0.smithi017.stdout:63: write oid 23 current snap is 0 2024-09-21T09:38:02.485 INFO:tasks.rados.rados.0.smithi017.stdout:63: seq_num 57 ranges {557999=703664,1719003=413684,2632287=325712} 2024-09-21T09:38:02.491 INFO:tasks.rados.rados.0.smithi017.stdout:63: writing smithi01785587-23 from 557999 to 1261663 tid 1 2024-09-21T09:38:02.496 INFO:tasks.rados.rados.0.smithi017.stdout:63: writing smithi01785587-23 from 1719003 to 2132687 tid 2 2024-09-21T09:38:02.501 INFO:tasks.rados.rados.0.smithi017.stdout:63: writing smithi01785587-23 from 2632287 to 2957999 tid 3 2024-09-21T09:38:02.501 INFO:tasks.rados.rados.0.smithi017.stdout:64: write oid 38 current snap is 0 2024-09-21T09:38:02.501 INFO:tasks.rados.rados.0.smithi017.stdout:64: seq_num 58 ranges {574806=778913,2061132=557342} 2024-09-21T09:38:02.508 INFO:tasks.rados.rados.0.smithi017.stdout:64: writing smithi01785587-38 from 574806 to 1353719 tid 1 2024-09-21T09:38:02.516 INFO:tasks.rados.rados.0.smithi017.stdout:64: writing smithi01785587-38 from 2061132 to 2618474 tid 2 2024-09-21T09:38:02.516 INFO:tasks.rados.rados.0.smithi017.stdout:65: rollback oid 25 current snap is 0 2024-09-21T09:38:02.516 INFO:tasks.rados.rados.0.smithi017.stdout:65: done (7 left) 2024-09-21T09:38:02.516 INFO:tasks.rados.rados.0.smithi017.stdout:66: read oid 29 snap -1 2024-09-21T09:38:02.516 INFO:tasks.rados.rados.0.smithi017.stdout:66: expect (ObjNum 28 snap 0 seq_num 28) 2024-09-21T09:38:02.516 INFO:tasks.rados.rados.0.smithi017.stdout:67: write oid 44 current snap is 0 2024-09-21T09:38:02.516 INFO:tasks.rados.rados.0.smithi017.stdout:58: expect (ObjNum 2 snap 0 seq_num 2) 2024-09-21T09:38:02.583 INFO:tasks.rados.rados.0.smithi017.stdout:60: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:38:02.670 INFO:tasks.rados.rados.0.smithi017.stdout:62: finishing write tid 1 to smithi01785587-7 2024-09-21T09:38:02.670 INFO:tasks.rados.rados.0.smithi017.stdout:59: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:38:02.750 INFO:tasks.rados.rados.0.smithi017.stdout:62: finishing write tid 2 to smithi01785587-7 2024-09-21T09:38:02.750 INFO:tasks.rados.rados.0.smithi017.stdout:62: finishing write tid 3 to smithi01785587-7 2024-09-21T09:38:02.750 INFO:tasks.rados.rados.0.smithi017.stdout:62: finishing write tid 4 to smithi01785587-7 2024-09-21T09:38:02.751 INFO:tasks.rados.rados.0.smithi017.stdout:62: finishing write tid 5 to smithi01785587-7 2024-09-21T09:38:02.751 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 20 (ObjNum 56 snap 0 seq_num 56) dirty exists 2024-09-21T09:38:02.751 INFO:tasks.rados.rados.0.smithi017.stdout:62: left oid 7 (ObjNum 56 snap 0 seq_num 56) 2024-09-21T09:38:02.751 INFO:tasks.rados.rados.0.smithi017.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:38:02.826 INFO:tasks.rados.rados.0.smithi017.stdout:63: finishing write tid 1 to smithi01785587-23 2024-09-21T09:38:02.826 INFO:tasks.rados.rados.0.smithi017.stdout:63: finishing write tid 2 to smithi01785587-23 2024-09-21T09:38:02.827 INFO:tasks.rados.rados.0.smithi017.stdout:63: finishing write tid 3 to smithi01785587-23 2024-09-21T09:38:02.827 INFO:tasks.rados.rados.0.smithi017.stdout:63: finishing write tid 4 to smithi01785587-23 2024-09-21T09:38:02.827 INFO:tasks.rados.rados.0.smithi017.stdout:63: finishing write tid 5 to smithi01785587-23 2024-09-21T09:38:02.827 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 23 (ObjNum 57 snap 0 seq_num 57) dirty exists 2024-09-21T09:38:02.827 INFO:tasks.rados.rados.0.smithi017.stdout:63: left oid 23 (ObjNum 57 snap 0 seq_num 57) 2024-09-21T09:38:02.827 INFO:tasks.rados.rados.0.smithi017.stdout:64: finishing write tid 1 to smithi01785587-38 2024-09-21T09:38:02.827 INFO:tasks.rados.rados.0.smithi017.stdout:67: seq_num 59 ranges {591613=454162,1719614=417353,2570780=420833} 2024-09-21T09:38:02.832 INFO:tasks.rados.rados.0.smithi017.stdout:67: writing smithi01785587-44 from 591613 to 1045775 tid 1 2024-09-21T09:38:02.838 INFO:tasks.rados.rados.0.smithi017.stdout:67: writing smithi01785587-44 from 1719614 to 2136967 tid 2 2024-09-21T09:38:02.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:02.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:38:02.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: Cluster is now healthy 2024-09-21T09:38:02.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/674371772' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:38:02.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:38:02.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:02.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:02.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:38:02.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:38:02.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:02.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:02.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:38:02.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: Cluster is now healthy 2024-09-21T09:38:02.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/674371772' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:38:02.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:38:02.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:02.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:02.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:38:02.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:38:02.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:02 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:02.843 INFO:tasks.rados.rados.0.smithi017.stdout:67: writing smithi01785587-44 from 2570780 to 2991613 tid 3 2024-09-21T09:38:02.843 INFO:tasks.rados.rados.0.smithi017.stdout:56: done (8 left) 2024-09-21T09:38:02.843 INFO:tasks.rados.rados.0.smithi017.stdout:58: done (7 left) 2024-09-21T09:38:02.843 INFO:tasks.rados.rados.0.smithi017.stdout:59: done (6 left) 2024-09-21T09:38:02.844 INFO:tasks.rados.rados.0.smithi017.stdout:60: done (5 left) 2024-09-21T09:38:02.844 INFO:tasks.rados.rados.0.smithi017.stdout:62: done (4 left) 2024-09-21T09:38:02.844 INFO:tasks.rados.rados.0.smithi017.stdout:63: done (3 left) 2024-09-21T09:38:02.845 INFO:tasks.rados.rados.0.smithi017.stdout:68: write oid 40 current snap is 0 2024-09-21T09:38:02.845 INFO:tasks.rados.rados.0.smithi017.stdout:64: finishing write tid 2 to smithi01785587-38 2024-09-21T09:38:02.845 INFO:tasks.rados.rados.0.smithi017.stdout:64: finishing write tid 3 to smithi01785587-38 2024-09-21T09:38:02.845 INFO:tasks.rados.rados.0.smithi017.stdout:64: finishing write tid 4 to smithi01785587-38 2024-09-21T09:38:02.845 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 12 (ObjNum 58 snap 0 seq_num 58) dirty exists 2024-09-21T09:38:02.845 INFO:tasks.rados.rados.0.smithi017.stdout:64: left oid 38 (ObjNum 58 snap 0 seq_num 58) 2024-09-21T09:38:02.845 INFO:tasks.rados.rados.0.smithi017.stdout:68: seq_num 60 ranges {608420=529411,1778096=561011,2881850=126570} 2024-09-21T09:38:02.850 INFO:tasks.rados.rados.0.smithi017.stdout:68: writing smithi01785587-40 from 608420 to 1137831 tid 1 2024-09-21T09:38:02.857 INFO:tasks.rados.rados.0.smithi017.stdout:68: writing smithi01785587-40 from 1778096 to 2339107 tid 2 2024-09-21T09:38:02.861 INFO:tasks.rados.rados.0.smithi017.stdout:68: writing smithi01785587-40 from 2881850 to 3008420 tid 3 2024-09-21T09:38:02.861 INFO:tasks.rados.rados.0.smithi017.stdout:64: done (3 left) 2024-09-21T09:38:02.861 INFO:tasks.rados.rados.0.smithi017.stdout:69: write oid 10 current snap is 0 2024-09-21T09:38:02.861 INFO:tasks.rados.rados.0.smithi017.stdout:66: expect (ObjNum 28 snap 0 seq_num 28) 2024-09-21T09:38:02.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch ps' 2024-09-21T09:38:02.958 INFO:tasks.rados.rados.0.smithi017.stdout:67: finishing write tid 1 to smithi01785587-44 2024-09-21T09:38:02.959 INFO:tasks.rados.rados.0.smithi017.stdout:67: finishing write tid 2 to smithi01785587-44 2024-09-21T09:38:02.959 INFO:tasks.rados.rados.0.smithi017.stdout:67: finishing write tid 3 to smithi01785587-44 2024-09-21T09:38:02.959 INFO:tasks.rados.rados.0.smithi017.stdout:67: finishing write tid 4 to smithi01785587-44 2024-09-21T09:38:02.959 INFO:tasks.rados.rados.0.smithi017.stdout:67: finishing write tid 5 to smithi01785587-44 2024-09-21T09:38:02.959 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 19 (ObjNum 59 snap 0 seq_num 59) dirty exists 2024-09-21T09:38:02.959 INFO:tasks.rados.rados.0.smithi017.stdout:67: left oid 44 (ObjNum 59 snap 0 seq_num 59) 2024-09-21T09:38:02.959 INFO:tasks.rados.rados.0.smithi017.stdout:68: finishing write tid 1 to smithi01785587-40 2024-09-21T09:38:02.959 INFO:tasks.rados.rados.0.smithi017.stdout:69: seq_num 61 ranges {625227=721013,1952931=704669} 2024-09-21T09:38:02.968 INFO:tasks.rados.rados.0.smithi017.stdout:69: writing smithi01785587-10 from 625227 to 1346240 tid 1 2024-09-21T09:38:02.977 INFO:tasks.rados.rados.0.smithi017.stdout:69: writing smithi01785587-10 from 1952931 to 2657600 tid 2 2024-09-21T09:38:02.978 INFO:tasks.rados.rados.0.smithi017.stdout:66: done (3 left) 2024-09-21T09:38:02.978 INFO:tasks.rados.rados.0.smithi017.stdout:67: done (2 left) 2024-09-21T09:38:02.978 INFO:tasks.rados.rados.0.smithi017.stdout:70: write oid 11 current snap is 0 2024-09-21T09:38:02.979 INFO:tasks.rados.rados.0.smithi017.stdout:70: seq_num 62 ranges {642034=796262,1895060=564680} 2024-09-21T09:38:02.988 INFO:tasks.rados.rados.0.smithi017.stdout:70: writing smithi01785587-11 from 642034 to 1438296 tid 1 2024-09-21T09:38:02.996 INFO:tasks.rados.rados.0.smithi017.stdout:70: writing smithi01785587-11 from 1895060 to 2459740 tid 2 2024-09-21T09:38:02.996 INFO:tasks.rados.rados.0.smithi017.stdout:71: snap_create 2024-09-21T09:38:02.996 INFO:tasks.rados.rados.0.smithi017.stdout:68: finishing write tid 2 to smithi01785587-40 2024-09-21T09:38:02.996 INFO:tasks.rados.rados.0.smithi017.stdout:68: finishing write tid 3 to smithi01785587-40 2024-09-21T09:38:02.996 INFO:tasks.rados.rados.0.smithi017.stdout:68: finishing write tid 4 to smithi01785587-40 2024-09-21T09:38:02.996 INFO:tasks.rados.rados.0.smithi017.stdout:68: finishing write tid 5 to smithi01785587-40 2024-09-21T09:38:02.997 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 16 (ObjNum 60 snap 0 seq_num 60) dirty exists 2024-09-21T09:38:02.997 INFO:tasks.rados.rados.0.smithi017.stdout:68: left oid 40 (ObjNum 60 snap 0 seq_num 60) 2024-09-21T09:38:02.997 INFO:tasks.rados.rados.0.smithi017.stdout:69: finishing write tid 1 to smithi01785587-10 2024-09-21T09:38:02.997 INFO:tasks.rados.rados.0.smithi017.stdout:69: finishing write tid 2 to smithi01785587-10 2024-09-21T09:38:02.997 INFO:tasks.rados.rados.0.smithi017.stdout:69: finishing write tid 3 to smithi01785587-10 2024-09-21T09:38:02.997 INFO:tasks.rados.rados.0.smithi017.stdout:69: finishing write tid 4 to smithi01785587-10 2024-09-21T09:38:02.997 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 7 (ObjNum 61 snap 0 seq_num 61) dirty exists 2024-09-21T09:38:02.997 INFO:tasks.rados.rados.0.smithi017.stdout:69: left oid 10 (ObjNum 61 snap 0 seq_num 61) 2024-09-21T09:38:02.997 INFO:tasks.rados.rados.0.smithi017.stdout:70: finishing write tid 1 to smithi01785587-11 2024-09-21T09:38:02.998 INFO:tasks.rados.rados.0.smithi017.stdout:70: finishing write tid 2 to smithi01785587-11 2024-09-21T09:38:02.998 INFO:tasks.rados.rados.0.smithi017.stdout:70: finishing write tid 3 to smithi01785587-11 2024-09-21T09:38:02.998 INFO:tasks.rados.rados.0.smithi017.stdout:70: finishing write tid 4 to smithi01785587-11 2024-09-21T09:38:02.998 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 26 (ObjNum 62 snap 0 seq_num 62) dirty exists 2024-09-21T09:38:02.998 INFO:tasks.rados.rados.0.smithi017.stdout:70: left oid 11 (ObjNum 62 snap 0 seq_num 62) 2024-09-21T09:38:03.185 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:38:03.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:38:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: Cluster is now healthy 2024-09-21T09:38:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/674371772' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:38:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:38:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:38:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:38:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:02 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:03.777 INFO:tasks.rados.rados.0.smithi017.stdout:68: done (3 left) 2024-09-21T09:38:03.777 INFO:tasks.rados.rados.0.smithi017.stdout:69: done (2 left) 2024-09-21T09:38:03.777 INFO:tasks.rados.rados.0.smithi017.stdout:70: done (1 left) 2024-09-21T09:38:03.777 INFO:tasks.rados.rados.0.smithi017.stdout:71: done (0 left) 2024-09-21T09:38:03.777 INFO:tasks.rados.rados.0.smithi017.stdout:72: write oid 50 current snap is 1 2024-09-21T09:38:03.777 INFO:tasks.rados.rados.0.smithi017.stdout:72: seq_num 63 ranges {658841=471511,1553542=708338,2964119=94722} 2024-09-21T09:38:03.783 INFO:tasks.rados.rados.0.smithi017.stdout:72: writing smithi01785587-50 from 658841 to 1130352 tid 1 2024-09-21T09:38:03.790 INFO:tasks.rados.rados.0.smithi017.stdout:72: writing smithi01785587-50 from 1553542 to 2261880 tid 2 2024-09-21T09:38:03.795 INFO:tasks.rados.rados.0.smithi017.stdout:72: writing smithi01785587-50 from 2964119 to 3058841 tid 3 2024-09-21T09:38:03.795 INFO:tasks.rados.rados.0.smithi017.stdout:73: rollback oid 38 current snap is 1 2024-09-21T09:38:03.795 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 0 2024-09-21T09:38:03.795 INFO:tasks.rados.rados.0.smithi017.stdout:72: finishing write tid 1 to smithi01785587-50 2024-09-21T09:38:03.795 INFO:tasks.rados.rados.0.smithi017.stdout:72: finishing write tid 2 to smithi01785587-50 2024-09-21T09:38:03.795 INFO:tasks.rados.rados.0.smithi017.stdout:74: write oid 33 current snap is 1 2024-09-21T09:38:03.796 INFO:tasks.rados.rados.0.smithi017.stdout:74: seq_num 64 ranges {675648=546760,2012024=568349} 2024-09-21T09:38:03.801 INFO:tasks.rados.rados.0.smithi017.stdout:74: writing smithi01785587-33 from 675648 to 1222408 tid 1 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:74: writing smithi01785587-33 from 2012024 to 2580373 tid 2 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:75: rollback oid 37 current snap is 1 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 0 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:72: finishing write tid 3 to smithi01785587-50 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:72: finishing write tid 4 to smithi01785587-50 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:72: finishing write tid 5 to smithi01785587-50 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 17 (ObjNum 63 snap 1 seq_num 63) dirty exists 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:72: left oid 50 (ObjNum 63 snap 1 seq_num 63) 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:73: finishing rollback tid 0 to smithi01785587-38 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:72: done (3 left) 2024-09-21T09:38:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:76: snap_create 2024-09-21T09:38:03.811 INFO:tasks.rados.rados.0.smithi017.stdout:73: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:38:03.811 INFO:tasks.rados.rados.0.smithi017.stdout:73: finishing rollback tid 2 to smithi01785587-38 2024-09-21T09:38:03.811 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 21 (ObjNum 58 snap 0 seq_num 58) dirty exists 2024-09-21T09:38:03.811 INFO:tasks.rados.rados.0.smithi017.stdout:74: finishing write tid 1 to smithi01785587-33 2024-09-21T09:38:03.812 INFO:tasks.rados.rados.0.smithi017.stdout:75: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:38:03.812 INFO:tasks.rados.rados.0.smithi017.stdout:75: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:38:03.812 INFO:tasks.rados.rados.0.smithi017.stdout:75: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:38:03.812 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 20 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:38:03.812 INFO:tasks.rados.rados.0.smithi017.stdout:74: finishing write tid 2 to smithi01785587-33 2024-09-21T09:38:03.812 INFO:tasks.rados.rados.0.smithi017.stdout:74: finishing write tid 3 to smithi01785587-33 2024-09-21T09:38:03.813 INFO:tasks.rados.rados.0.smithi017.stdout:74: finishing write tid 4 to smithi01785587-33 2024-09-21T09:38:03.813 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 25 (ObjNum 64 snap 1 seq_num 64) dirty exists 2024-09-21T09:38:03.813 INFO:tasks.rados.rados.0.smithi017.stdout:74: left oid 33 (ObjNum 64 snap 1 seq_num 64) 2024-09-21T09:38:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:03 smithi017 ceph-mon[29380]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:03 smithi017 ceph-mon[29380]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:03 smithi017 ceph-mon[29380]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:03 smithi017 ceph-mon[40737]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:03 smithi017 ceph-mon[40737]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:03 smithi017 ceph-mon[40737]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:04.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:03 smithi134 ceph-mon[33541]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:04.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:03 smithi134 ceph-mon[33541]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:04.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:03 smithi134 ceph-mon[33541]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:04.780 INFO:tasks.rados.rados.0.smithi017.stdout:73: done (3 left) 2024-09-21T09:38:04.780 INFO:tasks.rados.rados.0.smithi017.stdout:74: done (2 left) 2024-09-21T09:38:04.780 INFO:tasks.rados.rados.0.smithi017.stdout:75: done (1 left) 2024-09-21T09:38:04.780 INFO:tasks.rados.rados.0.smithi017.stdout:76: done (0 left) 2024-09-21T09:38:04.781 INFO:tasks.rados.rados.0.smithi017.stdout:77: read oid 33 snap -1 2024-09-21T09:38:04.781 INFO:tasks.rados.rados.0.smithi017.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2024-09-21T09:38:04.781 INFO:tasks.rados.rados.0.smithi017.stdout:78: snap_create 2024-09-21T09:38:04.789 INFO:tasks.rados.rados.0.smithi017.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2024-09-21T09:38:04.878 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:04 smithi017 ceph-mon[29380]: pgmap v101: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:04.879 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:04 smithi017 ceph-mon[29380]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:38:04.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:04 smithi017 ceph-mon[40737]: pgmap v101: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:04.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:04 smithi017 ceph-mon[40737]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:38:05.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:04 smithi134 ceph-mon[33541]: pgmap v101: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:05.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:04 smithi134 ceph-mon[33541]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (87s) 23s ago 2m 15.6M - 0.25.0 c8568f914cd2 b40d1bacbd50 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi134 *:3000 running (2m) 58s ago 2m 79.2M - 9.4.7 954c08fa6188 f7936515cf4e 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.coodep smithi017 running (71s) 23s ago 70s 13.0M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 81acf97499eb 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi134.bwovvb smithi134 running (68s) 58s ago 68s 13.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be bf739a2a98de 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi134 *:8443 running (6m) 58s ago 6m 399M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c61de7d53222 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:9283,8765 running (8m) 23s ago 8m 443M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be cd2cad7817e1 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (8m) 23s ago 8m 48.7M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 52c5c6bf4934 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi134 running (6m) 58s ago 6m 38.9M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 7eb897902574 2024-09-21T09:38:05.333 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (6m) 23s ago 6m 38.6M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 316af690367d 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (2m) 23s ago 118s 16.0M - 1.5.0 0da6a335fe13 c1d4600d9989 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi134 *:9100 running (114s) 58s ago 114s 7872k - 1.5.0 0da6a335fe13 a405017b5317 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (6m) 23s ago 6m 44.6M 4017M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 958bb6e5f55d 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (5m) 23s ago 5m 42.7M 4017M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5e9c6f25d8a5 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (5m) 23s ago 5m 41.1M 4017M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b87d82e4dc74 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (4m) 23s ago 4m 40.2M 4017M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 44d3b78acaa4 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi134 running (4m) 58s ago 4m 39.6M 4016M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 7ece30084c41 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi134 running (4m) 58s ago 4m 39.4M 4016M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5dec2255fcd7 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi134 running (3m) 58s ago 3m 39.9M 4016M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 80d4436a0b8a 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi134 running (3m) 58s ago 3m 41.1M 4016M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed2e26ef5698 2024-09-21T09:38:05.334 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi134 *:9095 running (81s) 58s ago 2m 37.0M - 2.43.0 a07b618ecd1d b50e22f97e5d 2024-09-21T09:38:05.785 INFO:tasks.rados.rados.0.smithi017.stdout:77: done (1 left) 2024-09-21T09:38:05.785 INFO:tasks.rados.rados.0.smithi017.stdout:78: done (0 left) 2024-09-21T09:38:05.785 INFO:tasks.rados.rados.0.smithi017.stdout:79: rollback oid 9 current snap is 3 2024-09-21T09:38:05.785 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 9 to 0 2024-09-21T09:38:05.785 INFO:tasks.rados.rados.0.smithi017.stdout:80: snap_create 2024-09-21T09:38:05.788 INFO:tasks.rados.rados.0.smithi017.stdout:79: finishing rollback tid 0 to smithi01785587-9 2024-09-21T09:38:05.788 INFO:tasks.rados.rados.0.smithi017.stdout:79: finishing rollback tid 1 to smithi01785587-9 2024-09-21T09:38:05.789 INFO:tasks.rados.rados.0.smithi017.stdout:79: finishing rollback tid 2 to smithi01785587-9 2024-09-21T09:38:05.789 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 11 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:38:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:05 smithi017 ceph-mon[29380]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:38:05.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:05 smithi017 ceph-mon[40737]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:38:06.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-21T09:38:06.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:05 smithi134 ceph-mon[33541]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:38:06.418 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:38:06.790 INFO:tasks.rados.rados.0.smithi017.stdout:79: done (1 left) 2024-09-21T09:38:06.790 INFO:tasks.rados.rados.0.smithi017.stdout:80: done (0 left) 2024-09-21T09:38:06.790 INFO:tasks.rados.rados.0.smithi017.stdout:81: read oid 26 snap -1 2024-09-21T09:38:06.790 INFO:tasks.rados.rados.0.smithi017.stdout:81: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-21T09:38:06.790 INFO:tasks.rados.rados.0.smithi017.stdout:82: read oid 36 snap -1 2024-09-21T09:38:06.791 INFO:tasks.rados.rados.0.smithi017.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2024-09-21T09:38:06.791 INFO:tasks.rados.rados.0.smithi017.stdout:83: snap_create 2024-09-21T09:38:06.795 INFO:tasks.rados.rados.0.smithi017.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2024-09-21T09:38:06.807 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:06 smithi017 ceph-mon[29380]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:06.807 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:06 smithi017 ceph-mon[29380]: pgmap v104: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 12 MiB/s wr, 70 op/s 2024-09-21T09:38:06.807 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:06 smithi017 ceph-mon[29380]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:38:06.808 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:06 smithi017 ceph-mon[40737]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:06.808 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:06 smithi017 ceph-mon[40737]: pgmap v104: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 12 MiB/s wr, 70 op/s 2024-09-21T09:38:06.808 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:06 smithi017 ceph-mon[40737]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:38:06.894 INFO:tasks.rados.rados.0.smithi017.stdout:81: expect (ObjNum 25 snap 0 seq_num 25) 2024-09-21T09:38:07.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:06 smithi134 ceph-mon[33541]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:06 smithi134 ceph-mon[33541]: pgmap v104: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 12 MiB/s wr, 70 op/s 2024-09-21T09:38:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:06 smithi134 ceph-mon[33541]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:38:07.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:38:06 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:38:07.795 INFO:tasks.rados.rados.0.smithi017.stdout:81: done (2 left) 2024-09-21T09:38:07.795 INFO:tasks.rados.rados.0.smithi017.stdout:82: done (1 left) 2024-09-21T09:38:07.795 INFO:tasks.rados.rados.0.smithi017.stdout:83: done (0 left) 2024-09-21T09:38:07.795 INFO:tasks.rados.rados.0.smithi017.stdout:84: delete oid 7 current snap is 5 2024-09-21T09:38:07.798 INFO:tasks.rados.rados.0.smithi017.stdout:84: done (0 left) 2024-09-21T09:38:07.798 INFO:tasks.rados.rados.0.smithi017.stdout:85: write oid 48 current snap is 5 2024-09-21T09:38:07.798 INFO:tasks.rados.rados.0.smithi017.stdout:85: seq_num 66 ranges {709262=697258,2012635=572018,3046388=62874} 2024-09-21T09:38:07.806 INFO:tasks.rados.rados.0.smithi017.stdout:85: writing smithi01785587-48 from 709262 to 1406520 tid 1 2024-09-21T09:38:07.812 INFO:tasks.rados.rados.0.smithi017.stdout:85: writing smithi01785587-48 from 2012635 to 2584653 tid 2 2024-09-21T09:38:07.816 INFO:tasks.rados.rados.0.smithi017.stdout:85: writing smithi01785587-48 from 3046388 to 3109262 tid 3 2024-09-21T09:38:07.816 INFO:tasks.rados.rados.0.smithi017.stdout:86: snap_create 2024-09-21T09:38:07.816 INFO:tasks.rados.rados.0.smithi017.stdout:85: finishing write tid 1 to smithi01785587-48 2024-09-21T09:38:07.816 INFO:tasks.rados.rados.0.smithi017.stdout:85: finishing write tid 2 to smithi01785587-48 2024-09-21T09:38:07.817 INFO:tasks.rados.rados.0.smithi017.stdout:85: finishing write tid 3 to smithi01785587-48 2024-09-21T09:38:07.818 INFO:tasks.rados.rados.0.smithi017.stdout:85: finishing write tid 4 to smithi01785587-48 2024-09-21T09:38:07.818 INFO:tasks.rados.rados.0.smithi017.stdout:85: finishing write tid 5 to smithi01785587-48 2024-09-21T09:38:07.818 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 18 (ObjNum 66 snap 5 seq_num 66) dirty exists 2024-09-21T09:38:07.818 INFO:tasks.rados.rados.0.smithi017.stdout:85: left oid 48 (ObjNum 66 snap 5 seq_num 66) 2024-09-21T09:38:08.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:07 smithi017 ceph-mon[29380]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:38:08.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:07 smithi017 ceph-mon[40737]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:38:08.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:07 smithi134 ceph-mon[33541]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:38:08.419 INFO:teuthology.orchestra.run.smithi017.stdout:wait for minority of mons to upgrade 2024-09-21T09:38:08.799 INFO:tasks.rados.rados.0.smithi017.stdout:85: done (1 left) 2024-09-21T09:38:08.799 INFO:tasks.rados.rados.0.smithi017.stdout:86: done (0 left) 2024-09-21T09:38:08.799 INFO:tasks.rados.rados.0.smithi017.stdout:87: rollback oid 10 current snap is 6 2024-09-21T09:38:08.799 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 2 2024-09-21T09:38:08.799 INFO:tasks.rados.rados.0.smithi017.stdout:88: snap_create 2024-09-21T09:38:08.801 INFO:tasks.rados.rados.0.smithi017.stdout:87: finishing rollback tid 0 to smithi01785587-10 2024-09-21T09:38:08.801 INFO:tasks.rados.rados.0.smithi017.stdout:87: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:38:08.802 INFO:tasks.rados.rados.0.smithi017.stdout:87: finishing rollback tid 2 to smithi01785587-10 2024-09-21T09:38:08.802 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 11 (ObjNum 61 snap 0 seq_num 61) dirty exists 2024-09-21T09:38:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:08 smithi017 ceph-mon[29380]: pgmap v107: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 17 MiB/s wr, 104 op/s 2024-09-21T09:38:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:08 smithi017 ceph-mon[29380]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:38:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:08 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:08 smithi017 ceph-mon[40737]: pgmap v107: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 17 MiB/s wr, 104 op/s 2024-09-21T09:38:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:08 smithi017 ceph-mon[40737]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:38:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:08 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:09.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-21T09:38:09.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:08 smithi134 ceph-mon[33541]: pgmap v107: 89 pgs: 89 active+clean; 120 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 17 MiB/s wr, 104 op/s 2024-09-21T09:38:09.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:08 smithi134 ceph-mon[33541]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:38:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:08 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:09.530 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:38:09.806 INFO:tasks.rados.rados.0.smithi017.stdout:87: done (1 left) 2024-09-21T09:38:09.806 INFO:tasks.rados.rados.0.smithi017.stdout:88: done (0 left) 2024-09-21T09:38:09.806 INFO:tasks.rados.rados.0.smithi017.stdout:89: delete oid 23 current snap is 7 2024-09-21T09:38:09.808 INFO:tasks.rados.rados.0.smithi017.stdout:89: done (0 left) 2024-09-21T09:38:09.809 INFO:tasks.rados.rados.0.smithi017.stdout:90: read oid 27 snap 3 2024-09-21T09:38:09.809 INFO:tasks.rados.rados.0.smithi017.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:38:09.809 INFO:tasks.rados.rados.0.smithi017.stdout:91: write oid 12 current snap is 7 2024-09-21T09:38:09.809 INFO:tasks.rados.rados.0.smithi017.stdout:91: seq_num 68 ranges {742876=447756,1729599=575687,3101234=41642} 2024-09-21T09:38:09.816 INFO:tasks.rados.rados.0.smithi017.stdout:91: writing smithi01785587-12 from 742876 to 1190632 tid 1 2024-09-21T09:38:09.822 INFO:tasks.rados.rados.0.smithi017.stdout:91: writing smithi01785587-12 from 1729599 to 2305286 tid 2 2024-09-21T09:38:09.827 INFO:tasks.rados.rados.0.smithi017.stdout:91: writing smithi01785587-12 from 3101234 to 3142876 tid 3 2024-09-21T09:38:09.827 INFO:tasks.rados.rados.0.smithi017.stdout:92: snap_create 2024-09-21T09:38:09.827 INFO:tasks.rados.rados.0.smithi017.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:38:09.922 INFO:tasks.rados.rados.0.smithi017.stdout:91: finishing write tid 1 to smithi01785587-12 2024-09-21T09:38:09.922 INFO:tasks.rados.rados.0.smithi017.stdout:91: finishing write tid 2 to smithi01785587-12 2024-09-21T09:38:09.922 INFO:tasks.rados.rados.0.smithi017.stdout:91: finishing write tid 3 to smithi01785587-12 2024-09-21T09:38:09.922 INFO:tasks.rados.rados.0.smithi017.stdout:91: finishing write tid 4 to smithi01785587-12 2024-09-21T09:38:09.922 INFO:tasks.rados.rados.0.smithi017.stdout:91: finishing write tid 5 to smithi01785587-12 2024-09-21T09:38:09.922 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 14 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-09-21T09:38:09.922 INFO:tasks.rados.rados.0.smithi017.stdout:91: left oid 12 (ObjNum 68 snap 7 seq_num 68) 2024-09-21T09:38:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:09 smithi017 ceph-mon[29380]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:38:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:09 smithi017 ceph-mon[40737]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:38:10.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:09 smithi134 ceph-mon[33541]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:38:10.448 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:38:10 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:10] "GET /metrics HTTP/1.1" 200 242813 "" "Prometheus/2.43.0" 2024-09-21T09:38:10.809 INFO:tasks.rados.rados.0.smithi017.stdout:90: done (2 left) 2024-09-21T09:38:10.809 INFO:tasks.rados.rados.0.smithi017.stdout:91: done (1 left) 2024-09-21T09:38:10.809 INFO:tasks.rados.rados.0.smithi017.stdout:92: done (0 left) 2024-09-21T09:38:10.809 INFO:tasks.rados.rados.0.smithi017.stdout:93: snap_remove snap 7 2024-09-21T09:38:11.161 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:10 smithi017 ceph-mon[29380]: pgmap v110: 89 pgs: 89 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-09-21T09:38:11.161 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:10 smithi017 ceph-mon[29380]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:38:11.161 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:10 smithi017 ceph-mon[40737]: pgmap v110: 89 pgs: 89 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-09-21T09:38:11.161 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:10 smithi017 ceph-mon[40737]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:38:11.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:10 smithi134 ceph-mon[33541]: pgmap v110: 89 pgs: 89 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-09-21T09:38:11.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:10 smithi134 ceph-mon[33541]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:38:11.816 INFO:tasks.rados.rados.0.smithi017.stdout:93: done (0 left) 2024-09-21T09:38:11.817 INFO:tasks.rados.rados.0.smithi017.stdout:94: write oid 15 current snap is 8 2024-09-21T09:38:11.817 INFO:tasks.rados.rados.0.smithi017.stdout:94: seq_num 69 ranges {759683=639358,1904434=719345,3128657=31026} 2024-09-21T09:38:11.824 INFO:tasks.rados.rados.0.smithi017.stdout:94: writing smithi01785587-15 from 759683 to 1399041 tid 1 2024-09-21T09:38:11.832 INFO:tasks.rados.rados.0.smithi017.stdout:94: writing smithi01785587-15 from 1904434 to 2623779 tid 2 2024-09-21T09:38:11.835 INFO:tasks.rados.rados.0.smithi017.stdout:94: writing smithi01785587-15 from 3128657 to 3159683 tid 3 2024-09-21T09:38:11.835 INFO:tasks.rados.rados.0.smithi017.stdout:95: snap_create 2024-09-21T09:38:11.835 INFO:tasks.rados.rados.0.smithi017.stdout:94: finishing write tid 1 to smithi01785587-15 2024-09-21T09:38:11.837 INFO:tasks.rados.rados.0.smithi017.stdout:94: finishing write tid 2 to smithi01785587-15 2024-09-21T09:38:11.837 INFO:tasks.rados.rados.0.smithi017.stdout:94: finishing write tid 3 to smithi01785587-15 2024-09-21T09:38:11.837 INFO:tasks.rados.rados.0.smithi017.stdout:94: finishing write tid 4 to smithi01785587-15 2024-09-21T09:38:11.837 INFO:tasks.rados.rados.0.smithi017.stdout:94: finishing write tid 5 to smithi01785587-15 2024-09-21T09:38:11.837 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 11 (ObjNum 69 snap 8 seq_num 69) dirty exists 2024-09-21T09:38:11.837 INFO:tasks.rados.rados.0.smithi017.stdout:94: left oid 15 (ObjNum 69 snap 8 seq_num 69) 2024-09-21T09:38:12.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:11 smithi017 ceph-mon[29380]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:38:12.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:11 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1354764231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:12.111 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:12.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:11 smithi017 ceph-mon[40737]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:38:12.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:11 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1354764231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:12.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:11 smithi134 ceph-mon[33541]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:38:12.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:11 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1354764231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:12.820 INFO:tasks.rados.rados.0.smithi017.stdout:94: done (1 left) 2024-09-21T09:38:12.820 INFO:tasks.rados.rados.0.smithi017.stdout:95: done (0 left) 2024-09-21T09:38:12.820 INFO:tasks.rados.rados.0.smithi017.stdout:96: rollback oid 42 current snap is 9 2024-09-21T09:38:12.820 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 5 2024-09-21T09:38:12.820 INFO:tasks.rados.rados.0.smithi017.stdout:97: write oid 3 current snap is 9 2024-09-21T09:38:12.820 INFO:tasks.rados.rados.0.smithi017.stdout:97: seq_num 70 ranges {776490=714607,2246563=579356} 2024-09-21T09:38:12.828 INFO:tasks.rados.rados.0.smithi017.stdout:97: writing smithi01785587-3 from 776490 to 1491097 tid 1 2024-09-21T09:38:12.836 INFO:tasks.rados.rados.0.smithi017.stdout:97: writing smithi01785587-3 from 2246563 to 2825919 tid 2 2024-09-21T09:38:12.836 INFO:tasks.rados.rados.0.smithi017.stdout:98: delete oid 4 current snap is 9 2024-09-21T09:38:12.836 INFO:tasks.rados.rados.0.smithi017.stdout:96: finishing rollback tid 0 to smithi01785587-42 2024-09-21T09:38:12.836 INFO:tasks.rados.rados.0.smithi017.stdout:96: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:38:12.836 INFO:tasks.rados.rados.0.smithi017.stdout:96: finishing rollback tid 2 to smithi01785587-42 2024-09-21T09:38:12.836 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 18 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-21T09:38:12.836 INFO:tasks.rados.rados.0.smithi017.stdout:97: finishing write tid 1 to smithi01785587-3 2024-09-21T09:38:12.838 INFO:tasks.rados.rados.0.smithi017.stdout:96: done (2 left) 2024-09-21T09:38:12.838 INFO:tasks.rados.rados.0.smithi017.stdout:98: done (1 left) 2024-09-21T09:38:12.838 INFO:tasks.rados.rados.0.smithi017.stdout:99: read oid 46 snap -1 2024-09-21T09:38:12.838 INFO:tasks.rados.rados.0.smithi017.stdout:99: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:38:12.838 INFO:tasks.rados.rados.0.smithi017.stdout:100: write oid 36 current snap is 9 2024-09-21T09:38:12.838 INFO:tasks.rados.rados.0.smithi017.stdout:100: seq_num 72 ranges {410104=465105,1563527=583025,2810926=399178} 2024-09-21T09:38:12.843 INFO:tasks.rados.rados.0.smithi017.stdout:100: writing smithi01785587-36 from 410104 to 875209 tid 1 2024-09-21T09:38:12.850 INFO:tasks.rados.rados.0.smithi017.stdout:100: writing smithi01785587-36 from 1563527 to 2146552 tid 2 2024-09-21T09:38:12.856 INFO:tasks.rados.rados.0.smithi017.stdout:100: writing smithi01785587-36 from 2810926 to 3210104 tid 3 2024-09-21T09:38:12.856 INFO:tasks.rados.rados.0.smithi017.stdout:101: snap_remove snap 4 2024-09-21T09:38:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:12 smithi017 ceph-mon[29380]: pgmap v113: 89 pgs: 89 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-09-21T09:38:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:12 smithi017 ceph-mon[29380]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:38:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:12 smithi017 ceph-mon[29380]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:12 smithi017 ceph-mon[40737]: pgmap v113: 89 pgs: 89 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-09-21T09:38:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:12 smithi017 ceph-mon[40737]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:38:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:12 smithi017 ceph-mon[40737]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:13.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:12 smithi134 ceph-mon[33541]: pgmap v113: 89 pgs: 89 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-09-21T09:38:13.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:12 smithi134 ceph-mon[33541]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:38:13.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:12 smithi134 ceph-mon[33541]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:13.826 INFO:tasks.rados.rados.0.smithi017.stdout:101: done (3 left) 2024-09-21T09:38:13.826 INFO:tasks.rados.rados.0.smithi017.stdout:102: write oid 41 current snap is 9 2024-09-21T09:38:13.826 INFO:tasks.rados.rados.0.smithi017.stdout:102: seq_num 73 ranges {426911=540354,1622009=726683,3121996=104915} 2024-09-21T09:38:13.831 INFO:tasks.rados.rados.0.smithi017.stdout:102: writing smithi01785587-41 from 426911 to 967265 tid 1 2024-09-21T09:38:13.839 INFO:tasks.rados.rados.0.smithi017.stdout:102: writing smithi01785587-41 from 1622009 to 2348692 tid 2 2024-09-21T09:38:13.844 INFO:tasks.rados.rados.0.smithi017.stdout:102: writing smithi01785587-41 from 3121996 to 3226911 tid 3 2024-09-21T09:38:13.844 INFO:tasks.rados.rados.0.smithi017.stdout:103: snap_remove snap 5 2024-09-21T09:38:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:13 smithi017 ceph-mon[29380]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:13 smithi017 ceph-mon[29380]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:38:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:13 smithi017 ceph-mon[40737]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:13 smithi017 ceph-mon[40737]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:38:14.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:13 smithi134 ceph-mon[33541]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:14.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:13 smithi134 ceph-mon[33541]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:38:14.833 INFO:tasks.rados.rados.0.smithi017.stdout:103: done (4 left) 2024-09-21T09:38:14.834 INFO:tasks.rados.rados.0.smithi017.stdout:104: delete oid 15 current snap is 9 2024-09-21T09:38:14.834 INFO:tasks.rados.rados.0.smithi017.stdout:97: finishing write tid 2 to smithi01785587-3 2024-09-21T09:38:14.834 INFO:tasks.rados.rados.0.smithi017.stdout:97: finishing write tid 3 to smithi01785587-3 2024-09-21T09:38:14.834 INFO:tasks.rados.rados.0.smithi017.stdout:97: finishing write tid 4 to smithi01785587-3 2024-09-21T09:38:14.834 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 23 (ObjNum 70 snap 9 seq_num 70) dirty exists 2024-09-21T09:38:14.834 INFO:tasks.rados.rados.0.smithi017.stdout:97: left oid 3 (ObjNum 70 snap 9 seq_num 70) 2024-09-21T09:38:14.834 INFO:tasks.rados.rados.0.smithi017.stdout:99: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:38:14.936 INFO:tasks.rados.rados.0.smithi017.stdout:100: finishing write tid 1 to smithi01785587-36 2024-09-21T09:38:14.936 INFO:tasks.rados.rados.0.smithi017.stdout:100: finishing write tid 2 to smithi01785587-36 2024-09-21T09:38:14.936 INFO:tasks.rados.rados.0.smithi017.stdout:100: finishing write tid 3 to smithi01785587-36 2024-09-21T09:38:14.936 INFO:tasks.rados.rados.0.smithi017.stdout:97: done (4 left) 2024-09-21T09:38:14.937 INFO:tasks.rados.rados.0.smithi017.stdout:99: done (3 left) 2024-09-21T09:38:14.937 INFO:tasks.rados.rados.0.smithi017.stdout:104: done (2 left) 2024-09-21T09:38:14.937 INFO:tasks.rados.rados.0.smithi017.stdout:105: write oid 43 current snap is 9 2024-09-21T09:38:14.937 INFO:tasks.rados.rados.0.smithi017.stdout:100: finishing write tid 4 to smithi01785587-36 2024-09-21T09:38:14.937 INFO:tasks.rados.rados.0.smithi017.stdout:105: seq_num 75 ranges {460525=690852,1622620=730352,3060489=200036} 2024-09-21T09:38:14.942 INFO:tasks.rados.rados.0.smithi017.stdout:105: writing smithi01785587-43 from 460525 to 1151377 tid 1 2024-09-21T09:38:14.949 INFO:tasks.rados.rados.0.smithi017.stdout:105: writing smithi01785587-43 from 1622620 to 2352972 tid 2 2024-09-21T09:38:14.955 INFO:tasks.rados.rados.0.smithi017.stdout:105: writing smithi01785587-43 from 3060489 to 3260525 tid 3 2024-09-21T09:38:14.955 INFO:tasks.rados.rados.0.smithi017.stdout:106: rollback oid 47 current snap is 9 2024-09-21T09:38:14.955 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 47 to 2 2024-09-21T09:38:14.955 INFO:tasks.rados.rados.0.smithi017.stdout:100: finishing write tid 5 to smithi01785587-36 2024-09-21T09:38:14.955 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 25 (ObjNum 72 snap 9 seq_num 72) dirty exists 2024-09-21T09:38:14.955 INFO:tasks.rados.rados.0.smithi017.stdout:100: left oid 36 (ObjNum 72 snap 9 seq_num 72) 2024-09-21T09:38:14.955 INFO:tasks.rados.rados.0.smithi017.stdout:102: finishing write tid 1 to smithi01785587-41 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:102: finishing write tid 2 to smithi01785587-41 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:102: finishing write tid 3 to smithi01785587-41 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:102: finishing write tid 4 to smithi01785587-41 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:102: finishing write tid 5 to smithi01785587-41 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 18 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:102: left oid 41 (ObjNum 73 snap 9 seq_num 73) 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:105: finishing write tid 1 to smithi01785587-43 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:105: finishing write tid 2 to smithi01785587-43 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:100: done (3 left) 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:102: done (2 left) 2024-09-21T09:38:14.956 INFO:tasks.rados.rados.0.smithi017.stdout:107: delete oid 10 current snap is 9 2024-09-21T09:38:14.957 INFO:tasks.rados.rados.0.smithi017.stdout:107: done (2 left) 2024-09-21T09:38:14.957 INFO:tasks.rados.rados.0.smithi017.stdout:108: rollback oid 35 current snap is 9 2024-09-21T09:38:14.957 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 2 2024-09-21T09:38:14.957 INFO:tasks.rados.rados.0.smithi017.stdout:109: write oid 13 current snap is 9 2024-09-21T09:38:14.957 INFO:tasks.rados.rados.0.smithi017.stdout:109: seq_num 77 ranges {494139=557703,1455937=734021,2948041=346098} 2024-09-21T09:38:14.963 INFO:tasks.rados.rados.0.smithi017.stdout:109: writing smithi01785587-13 from 494139 to 1051842 tid 1 2024-09-21T09:38:14.969 INFO:tasks.rados.rados.0.smithi017.stdout:109: writing smithi01785587-13 from 1455937 to 2189958 tid 2 2024-09-21T09:38:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:109: writing smithi01785587-13 from 2948041 to 3294139 tid 3 2024-09-21T09:38:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:110: write oid 24 current snap is 9 2024-09-21T09:38:14.977 INFO:tasks.rados.rados.0.smithi017.stdout:110: seq_num 78 ranges {510946=632952,1798066=477679,2742758=457217} 2024-09-21T09:38:14.983 INFO:tasks.rados.rados.0.smithi017.stdout:110: writing smithi01785587-24 from 510946 to 1143898 tid 1 2024-09-21T09:38:14.990 INFO:tasks.rados.rados.0.smithi017.stdout:110: writing smithi01785587-24 from 1798066 to 2275745 tid 2 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:110: writing smithi01785587-24 from 2742758 to 3199975 tid 3 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:111: snap_create 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:105: finishing write tid 3 to smithi01785587-43 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:105: finishing write tid 4 to smithi01785587-43 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:106: finishing rollback tid 0 to smithi01785587-47 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:105: finishing write tid 5 to smithi01785587-43 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 14 (ObjNum 75 snap 9 seq_num 75) dirty exists 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:105: left oid 43 (ObjNum 75 snap 9 seq_num 75) 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:106: finishing rollback tid 1 to smithi01785587-47 2024-09-21T09:38:14.996 INFO:tasks.rados.rados.0.smithi017.stdout:106: finishing rollback tid 2 to smithi01785587-47 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 30 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:108: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:108: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:108: finishing rollback tid 2 to smithi01785587-35 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 7 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:109: finishing write tid 1 to smithi01785587-13 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:109: finishing write tid 2 to smithi01785587-13 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:109: finishing write tid 3 to smithi01785587-13 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:109: finishing write tid 4 to smithi01785587-13 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:109: finishing write tid 5 to smithi01785587-13 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 24 (ObjNum 77 snap 9 seq_num 77) dirty exists 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:109: left oid 13 (ObjNum 77 snap 9 seq_num 77) 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:110: finishing write tid 1 to smithi01785587-24 2024-09-21T09:38:14.997 INFO:tasks.rados.rados.0.smithi017.stdout:110: finishing write tid 2 to smithi01785587-24 2024-09-21T09:38:14.998 INFO:tasks.rados.rados.0.smithi017.stdout:110: finishing write tid 3 to smithi01785587-24 2024-09-21T09:38:14.998 INFO:tasks.rados.rados.0.smithi017.stdout:110: finishing write tid 4 to smithi01785587-24 2024-09-21T09:38:14.998 INFO:tasks.rados.rados.0.smithi017.stdout:110: finishing write tid 5 to smithi01785587-24 2024-09-21T09:38:14.998 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 26 (ObjNum 78 snap 9 seq_num 78) dirty exists 2024-09-21T09:38:14.998 INFO:tasks.rados.rados.0.smithi017.stdout:110: left oid 24 (ObjNum 78 snap 9 seq_num 78) 2024-09-21T09:38:15.137 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:15.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:14 smithi017 ceph-mon[29380]: pgmap v116: 89 pgs: 89 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:15.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:14 smithi017 ceph-mon[29380]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:38:15.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:14 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3648225625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:15.137 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:14 smithi017 ceph-mon[40737]: pgmap v116: 89 pgs: 89 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:15.137 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:14 smithi017 ceph-mon[40737]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:38:15.138 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:14 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3648225625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:15.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:14 smithi134 ceph-mon[33541]: pgmap v116: 89 pgs: 89 active+clean; 122 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:15.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:14 smithi134 ceph-mon[33541]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:38:15.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:14 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3648225625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:15.834 INFO:tasks.rados.rados.0.smithi017.stdout:105: done (5 left) 2024-09-21T09:38:15.834 INFO:tasks.rados.rados.0.smithi017.stdout:106: done (4 left) 2024-09-21T09:38:15.834 INFO:tasks.rados.rados.0.smithi017.stdout:108: done (3 left) 2024-09-21T09:38:15.834 INFO:tasks.rados.rados.0.smithi017.stdout:109: done (2 left) 2024-09-21T09:38:15.835 INFO:tasks.rados.rados.0.smithi017.stdout:110: done (1 left) 2024-09-21T09:38:15.835 INFO:tasks.rados.rados.0.smithi017.stdout:111: done (0 left) 2024-09-21T09:38:15.835 INFO:tasks.rados.rados.0.smithi017.stdout:112: write oid 47 current snap is 10 2024-09-21T09:38:15.835 INFO:tasks.rados.rados.0.smithi017.stdout:112: seq_num 79 ranges {527753=708201,1856548=737690,3286534=41219} 2024-09-21T09:38:15.841 INFO:tasks.rados.rados.0.smithi017.stdout:112: writing smithi01785587-47 from 527753 to 1235954 tid 1 2024-09-21T09:38:15.849 INFO:tasks.rados.rados.0.smithi017.stdout:112: writing smithi01785587-47 from 1856548 to 2594238 tid 2 2024-09-21T09:38:15.853 INFO:tasks.rados.rados.0.smithi017.stdout:112: writing smithi01785587-47 from 3286534 to 3327753 tid 3 2024-09-21T09:38:15.853 INFO:tasks.rados.rados.0.smithi017.stdout:113: delete oid 17 current snap is 10 2024-09-21T09:38:15.853 INFO:tasks.rados.rados.0.smithi017.stdout:112: finishing write tid 1 to smithi01785587-47 2024-09-21T09:38:15.853 INFO:tasks.rados.rados.0.smithi017.stdout:112: finishing write tid 2 to smithi01785587-47 2024-09-21T09:38:15.854 INFO:tasks.rados.rados.0.smithi017.stdout:112: finishing write tid 3 to smithi01785587-47 2024-09-21T09:38:15.855 INFO:tasks.rados.rados.0.smithi017.stdout:113: done (1 left) 2024-09-21T09:38:15.855 INFO:tasks.rados.rados.0.smithi017.stdout:114: snap_remove snap 2 2024-09-21T09:38:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:15 smithi017 ceph-mon[29380]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:38:16.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:15 smithi017 ceph-mon[40737]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:38:16.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:15 smithi134 ceph-mon[33541]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:38:16.844 INFO:tasks.rados.rados.0.smithi017.stdout:114: done (1 left) 2024-09-21T09:38:16.844 INFO:tasks.rados.rados.0.smithi017.stdout:115: write oid 1 current snap is 10 2024-09-21T09:38:16.844 INFO:tasks.rados.rados.0.smithi017.stdout:112: finishing write tid 4 to smithi01785587-47 2024-09-21T09:38:16.844 INFO:tasks.rados.rados.0.smithi017.stdout:112: finishing write tid 5 to smithi01785587-47 2024-09-21T09:38:16.844 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 35 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-21T09:38:16.844 INFO:tasks.rados.rados.0.smithi017.stdout:112: left oid 47 (ObjNum 79 snap 10 seq_num 79) 2024-09-21T09:38:16.844 INFO:tasks.rados.rados.0.smithi017.stdout:115: seq_num 81 ranges {561367=458699,1573512=741359,2941380=419987} 2024-09-21T09:38:16.850 INFO:tasks.rados.rados.0.smithi017.stdout:115: writing smithi01785587-1 from 561367 to 1020066 tid 1 2024-09-21T09:38:16.857 INFO:tasks.rados.rados.0.smithi017.stdout:115: writing smithi01785587-1 from 1573512 to 2314871 tid 2 2024-09-21T09:38:16.864 INFO:tasks.rados.rados.0.smithi017.stdout:115: writing smithi01785587-1 from 2941380 to 3361367 tid 3 2024-09-21T09:38:16.864 INFO:tasks.rados.rados.0.smithi017.stdout:112: done (1 left) 2024-09-21T09:38:16.864 INFO:tasks.rados.rados.0.smithi017.stdout:116: read oid 12 snap -1 2024-09-21T09:38:16.864 INFO:tasks.rados.rados.0.smithi017.stdout:116: expect (ObjNum 68 snap 7 seq_num 68) 2024-09-21T09:38:16.864 INFO:tasks.rados.rados.0.smithi017.stdout:117: rollback oid 38 current snap is 10 2024-09-21T09:38:16.864 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 1 2024-09-21T09:38:16.864 INFO:tasks.rados.rados.0.smithi017.stdout:115: finishing write tid 1 to smithi01785587-1 2024-09-21T09:38:16.864 INFO:tasks.rados.rados.0.smithi017.stdout:115: finishing write tid 2 to smithi01785587-1 2024-09-21T09:38:16.865 INFO:tasks.rados.rados.0.smithi017.stdout:118: delete oid 39 current snap is 10 2024-09-21T09:38:16.866 INFO:tasks.rados.rados.0.smithi017.stdout:118: done (3 left) 2024-09-21T09:38:16.866 INFO:tasks.rados.rados.0.smithi017.stdout:119: read oid 14 snap -1 2024-09-21T09:38:16.866 INFO:tasks.rados.rados.0.smithi017.stdout:119: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:38:16.867 INFO:tasks.rados.rados.0.smithi017.stdout:120: read oid 15 snap -1 2024-09-21T09:38:16.867 INFO:tasks.rados.rados.0.smithi017.stdout:120: expect deleted 2024-09-21T09:38:16.867 INFO:tasks.rados.rados.0.smithi017.stdout:121: read oid 45 snap -1 2024-09-21T09:38:16.867 INFO:tasks.rados.rados.0.smithi017.stdout:121: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-21T09:38:16.867 INFO:tasks.rados.rados.0.smithi017.stdout:122: snap_remove snap 9 2024-09-21T09:38:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:16 smithi017 ceph-mon[29380]: from='client.24749 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:16 smithi017 ceph-mon[29380]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:16 smithi017 ceph-mon[29380]: pgmap v119: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:38:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:16 smithi017 ceph-mon[29380]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:38:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:16 smithi017 ceph-mon[40737]: from='client.24749 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:16 smithi017 ceph-mon[40737]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:16 smithi017 ceph-mon[40737]: pgmap v119: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:38:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:16 smithi017 ceph-mon[40737]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:38:17.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:38:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:38:17.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:16 smithi134 ceph-mon[33541]: from='client.24749 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:16 smithi134 ceph-mon[33541]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:16 smithi134 ceph-mon[33541]: pgmap v119: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:38:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:16 smithi134 ceph-mon[33541]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:38:17.663 INFO:tasks.rados.rados.0.smithi017.stdout:122: done (6 left) 2024-09-21T09:38:17.663 INFO:tasks.rados.rados.0.smithi017.stdout:123: write oid 20 current snap is 10 2024-09-21T09:38:17.663 INFO:tasks.rados.rados.0.smithi017.stdout:123: seq_num 83 ranges {594981=609197,1974123=745028,3279873=115108} 2024-09-21T09:38:17.670 INFO:tasks.rados.rados.0.smithi017.stdout:123: writing smithi01785587-20 from 594981 to 1204178 tid 1 2024-09-21T09:38:17.678 INFO:tasks.rados.rados.0.smithi017.stdout:123: writing smithi01785587-20 from 1974123 to 2719151 tid 2 2024-09-21T09:38:17.683 INFO:tasks.rados.rados.0.smithi017.stdout:123: writing smithi01785587-20 from 3279873 to 3394981 tid 3 2024-09-21T09:38:17.683 INFO:tasks.rados.rados.0.smithi017.stdout:124: write oid 39 current snap is 10 2024-09-21T09:38:17.683 INFO:tasks.rados.rados.0.smithi017.stdout:124: seq_num 84 ranges {611788=400799,1748958=488686,2907296=504492} 2024-09-21T09:38:17.688 INFO:tasks.rados.rados.0.smithi017.stdout:124: writing smithi01785587-39 from 611788 to 1012587 tid 1 2024-09-21T09:38:17.695 INFO:tasks.rados.rados.0.smithi017.stdout:124: writing smithi01785587-39 from 1748958 to 2237644 tid 2 2024-09-21T09:38:17.702 INFO:tasks.rados.rados.0.smithi017.stdout:124: writing smithi01785587-39 from 2907296 to 3411788 tid 3 2024-09-21T09:38:17.702 INFO:tasks.rados.rados.0.smithi017.stdout:125: write oid 27 current snap is 10 2024-09-21T09:38:17.702 INFO:tasks.rados.rados.0.smithi017.stdout:125: seq_num 85 ranges {628595=476048,1807440=632344,2934719=452474} 2024-09-21T09:38:17.708 INFO:tasks.rados.rados.0.smithi017.stdout:125: writing smithi01785587-27 from 628595 to 1104643 tid 1 2024-09-21T09:38:17.716 INFO:tasks.rados.rados.0.smithi017.stdout:125: writing smithi01785587-27 from 1807440 to 2439784 tid 2 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:125: writing smithi01785587-27 from 2934719 to 3387193 tid 3 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:126: snap_create 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:117: finishing rollback tid 0 to smithi01785587-38 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:115: finishing write tid 3 to smithi01785587-1 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:115: finishing write tid 4 to smithi01785587-1 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:115: finishing write tid 5 to smithi01785587-1 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 21 (ObjNum 81 snap 10 seq_num 81) dirty exists 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:115: left oid 1 (ObjNum 81 snap 10 seq_num 81) 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:117: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:117: finishing rollback tid 2 to smithi01785587-38 2024-09-21T09:38:17.722 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 32 (ObjNum 58 snap 0 seq_num 58) dirty exists 2024-09-21T09:38:17.723 INFO:tasks.rados.rados.0.smithi017.stdout:119: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:38:17.805 INFO:tasks.rados.rados.0.smithi017.stdout:116: expect (ObjNum 68 snap 7 seq_num 68) 2024-09-21T09:38:17.963 INFO:tasks.rados.rados.0.smithi017.stdout:121: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-21T09:38:18.059 INFO:tasks.rados.rados.0.smithi017.stdout:123: finishing write tid 1 to smithi01785587-20 2024-09-21T09:38:18.059 INFO:tasks.rados.rados.0.smithi017.stdout:123: finishing write tid 2 to smithi01785587-20 2024-09-21T09:38:18.059 INFO:tasks.rados.rados.0.smithi017.stdout:123: finishing write tid 3 to smithi01785587-20 2024-09-21T09:38:18.059 INFO:tasks.rados.rados.0.smithi017.stdout:123: finishing write tid 4 to smithi01785587-20 2024-09-21T09:38:18.059 INFO:tasks.rados.rados.0.smithi017.stdout:123: finishing write tid 5 to smithi01785587-20 2024-09-21T09:38:18.059 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 14 (ObjNum 83 snap 10 seq_num 83) dirty exists 2024-09-21T09:38:18.059 INFO:tasks.rados.rados.0.smithi017.stdout:123: left oid 20 (ObjNum 83 snap 10 seq_num 83) 2024-09-21T09:38:18.059 INFO:tasks.rados.rados.0.smithi017.stdout:124: finishing write tid 1 to smithi01785587-39 2024-09-21T09:38:18.059 INFO:tasks.rados.rados.0.smithi017.stdout:124: finishing write tid 2 to smithi01785587-39 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:124: finishing write tid 3 to smithi01785587-39 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:124: finishing write tid 4 to smithi01785587-39 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:124: finishing write tid 5 to smithi01785587-39 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 39 (ObjNum 84 snap 10 seq_num 84) dirty exists 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:124: left oid 39 (ObjNum 84 snap 10 seq_num 84) 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:125: finishing write tid 1 to smithi01785587-27 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:125: finishing write tid 2 to smithi01785587-27 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:125: finishing write tid 3 to smithi01785587-27 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:125: finishing write tid 4 to smithi01785587-27 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:125: finishing write tid 5 to smithi01785587-27 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 20 (ObjNum 85 snap 10 seq_num 85) dirty exists 2024-09-21T09:38:18.060 INFO:tasks.rados.rados.0.smithi017.stdout:125: left oid 27 (ObjNum 85 snap 10 seq_num 85) 2024-09-21T09:38:18.151 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:18.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:17 smithi017 ceph-mon[29380]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:38:18.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:17 smithi017 ceph-mon[29380]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:38:18.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:17 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2352286786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:18.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:17 smithi017 ceph-mon[40737]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:38:18.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:17 smithi017 ceph-mon[40737]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:38:18.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:17 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2352286786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:18.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:17 smithi134 ceph-mon[33541]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:38:18.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:17 smithi134 ceph-mon[33541]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:38:18.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:17 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2352286786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:18.663 INFO:tasks.rados.rados.0.smithi017.stdout:115: done (9 left) 2024-09-21T09:38:18.663 INFO:tasks.rados.rados.0.smithi017.stdout:116: done (8 left) 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:117: done (7 left) 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:119: done (6 left) 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:120: done (5 left) 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:121: done (4 left) 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:123: done (3 left) 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:124: done (2 left) 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:125: done (1 left) 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:126: done (0 left) 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:127: write oid 32 current snap is 11 2024-09-21T09:38:18.664 INFO:tasks.rados.rados.0.smithi017.stdout:127: seq_num 86 ranges {645402=551297,1749569=492355,2845789=599613} 2024-09-21T09:38:18.670 INFO:tasks.rados.rados.0.smithi017.stdout:127: writing smithi01785587-32 from 645402 to 1196699 tid 1 2024-09-21T09:38:18.676 INFO:tasks.rados.rados.0.smithi017.stdout:127: writing smithi01785587-32 from 1749569 to 2241924 tid 2 2024-09-21T09:38:18.683 INFO:tasks.rados.rados.0.smithi017.stdout:127: writing smithi01785587-32 from 2845789 to 3445402 tid 3 2024-09-21T09:38:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:128: read oid 35 snap -1 2024-09-21T09:38:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:128: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-21T09:38:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:129: read oid 11 snap -1 2024-09-21T09:38:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:129: expect (ObjNum 62 snap 0 seq_num 62) 2024-09-21T09:38:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:130: write oid 42 current snap is 11 2024-09-21T09:38:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:127: finishing write tid 1 to smithi01785587-32 2024-09-21T09:38:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:127: finishing write tid 2 to smithi01785587-32 2024-09-21T09:38:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:130: seq_num 87 ranges {662209=626546,1808051=636013,2873212=588997} 2024-09-21T09:38:18.691 INFO:tasks.rados.rados.0.smithi017.stdout:130: writing smithi01785587-42 from 662209 to 1288755 tid 1 2024-09-21T09:38:18.697 INFO:tasks.rados.rados.0.smithi017.stdout:130: writing smithi01785587-42 from 1808051 to 2444064 tid 2 2024-09-21T09:38:18.703 INFO:tasks.rados.rados.0.smithi017.stdout:130: writing smithi01785587-42 from 2873212 to 3462209 tid 3 2024-09-21T09:38:18.703 INFO:tasks.rados.rados.0.smithi017.stdout:131: snap_remove snap 10 2024-09-21T09:38:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:18 smithi017 ceph-mon[29380]: pgmap v123: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:38:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:18 smithi017 ceph-mon[29380]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:38:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:18 smithi017 ceph-mon[40737]: pgmap v123: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:38:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:18 smithi017 ceph-mon[40737]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:38:19.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:18 smithi134 ceph-mon[33541]: pgmap v123: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:38:19.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:18 smithi134 ceph-mon[33541]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:38:19.666 INFO:tasks.rados.rados.0.smithi017.stdout:131: done (4 left) 2024-09-21T09:38:19.666 INFO:tasks.rados.rados.0.smithi017.stdout:132: read oid 16 snap -1 2024-09-21T09:38:19.666 INFO:tasks.rados.rados.0.smithi017.stdout:132: expect (ObjNum 51 snap 0 seq_num 51) 2024-09-21T09:38:19.666 INFO:tasks.rados.rados.0.smithi017.stdout:133: snap_remove snap 3 2024-09-21T09:38:20.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:38:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:20] "GET /metrics HTTP/1.1" 200 242813 "" "Prometheus/2.43.0" 2024-09-21T09:38:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:19 smithi017 ceph-mon[29380]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:19 smithi017 ceph-mon[29380]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:19 smithi017 ceph-mon[29380]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:38:20.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:19 smithi017 ceph-mon[40737]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:20.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:19 smithi017 ceph-mon[40737]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:19 smithi017 ceph-mon[40737]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:38:20.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:19 smithi134 ceph-mon[33541]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:20.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:19 smithi134 ceph-mon[33541]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:20.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:19 smithi134 ceph-mon[33541]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:38:20.674 INFO:tasks.rados.rados.0.smithi017.stdout:133: done (5 left) 2024-09-21T09:38:20.674 INFO:tasks.rados.rados.0.smithi017.stdout:134: write oid 20 current snap is 11 2024-09-21T09:38:20.674 INFO:tasks.rados.rados.0.smithi017.stdout:134: seq_num 88 ranges {679016=701795,1866533=496024,2900635=402643} 2024-09-21T09:38:20.682 INFO:tasks.rados.rados.0.smithi017.stdout:134: writing smithi01785587-20 from 679016 to 1380811 tid 1 2024-09-21T09:38:20.687 INFO:tasks.rados.rados.0.smithi017.stdout:134: writing smithi01785587-20 from 1866533 to 2362557 tid 2 2024-09-21T09:38:20.693 INFO:tasks.rados.rados.0.smithi017.stdout:134: writing smithi01785587-20 from 2900635 to 3303278 tid 3 2024-09-21T09:38:20.693 INFO:tasks.rados.rados.0.smithi017.stdout:135: read oid 10 snap -1 2024-09-21T09:38:20.693 INFO:tasks.rados.rados.0.smithi017.stdout:135: expect deleted 2024-09-21T09:38:20.693 INFO:tasks.rados.rados.0.smithi017.stdout:136: write oid 33 current snap is 11 2024-09-21T09:38:20.693 INFO:tasks.rados.rados.0.smithi017.stdout:136: seq_num 89 ranges {695823=777044,1925015=639682,3328058=167765} 2024-09-21T09:38:20.701 INFO:tasks.rados.rados.0.smithi017.stdout:136: writing smithi01785587-33 from 695823 to 1472867 tid 1 2024-09-21T09:38:20.707 INFO:tasks.rados.rados.0.smithi017.stdout:136: writing smithi01785587-33 from 1925015 to 2564697 tid 2 2024-09-21T09:38:20.713 INFO:tasks.rados.rados.0.smithi017.stdout:136: writing smithi01785587-33 from 3328058 to 3495823 tid 3 2024-09-21T09:38:20.713 INFO:tasks.rados.rados.0.smithi017.stdout:137: delete oid 1 current snap is 11 2024-09-21T09:38:20.713 INFO:tasks.rados.rados.0.smithi017.stdout:127: finishing write tid 3 to smithi01785587-32 2024-09-21T09:38:20.713 INFO:tasks.rados.rados.0.smithi017.stdout:127: finishing write tid 4 to smithi01785587-32 2024-09-21T09:38:20.713 INFO:tasks.rados.rados.0.smithi017.stdout:127: finishing write tid 5 to smithi01785587-32 2024-09-21T09:38:20.713 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 39 (ObjNum 86 snap 11 seq_num 86) dirty exists 2024-09-21T09:38:20.713 INFO:tasks.rados.rados.0.smithi017.stdout:127: left oid 32 (ObjNum 86 snap 11 seq_num 86) 2024-09-21T09:38:20.713 INFO:tasks.rados.rados.0.smithi017.stdout:129: expect (ObjNum 62 snap 0 seq_num 62) 2024-09-21T09:38:20.862 INFO:tasks.rados.rados.0.smithi017.stdout:128: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-21T09:38:20.957 INFO:tasks.rados.rados.0.smithi017.stdout:130: finishing write tid 1 to smithi01785587-42 2024-09-21T09:38:20.957 INFO:tasks.rados.rados.0.smithi017.stdout:130: finishing write tid 2 to smithi01785587-42 2024-09-21T09:38:20.957 INFO:tasks.rados.rados.0.smithi017.stdout:130: finishing write tid 3 to smithi01785587-42 2024-09-21T09:38:20.957 INFO:tasks.rados.rados.0.smithi017.stdout:130: finishing write tid 4 to smithi01785587-42 2024-09-21T09:38:20.957 INFO:tasks.rados.rados.0.smithi017.stdout:130: finishing write tid 5 to smithi01785587-42 2024-09-21T09:38:20.957 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 48 (ObjNum 87 snap 11 seq_num 87) dirty exists 2024-09-21T09:38:20.958 INFO:tasks.rados.rados.0.smithi017.stdout:130: left oid 42 (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:20.958 INFO:tasks.rados.rados.0.smithi017.stdout:132: expect (ObjNum 51 snap 0 seq_num 51) 2024-09-21T09:38:21.103 INFO:tasks.rados.rados.0.smithi017.stdout:134: finishing write tid 1 to smithi01785587-20 2024-09-21T09:38:21.103 INFO:tasks.rados.rados.0.smithi017.stdout:134: finishing write tid 2 to smithi01785587-20 2024-09-21T09:38:21.103 INFO:tasks.rados.rados.0.smithi017.stdout:127: done (8 left) 2024-09-21T09:38:21.103 INFO:tasks.rados.rados.0.smithi017.stdout:128: done (7 left) 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:129: done (6 left) 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:130: done (5 left) 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:132: done (4 left) 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:137: done (3 left) 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:138: snap_create 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:134: finishing write tid 3 to smithi01785587-20 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:134: finishing write tid 4 to smithi01785587-20 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:134: finishing write tid 5 to smithi01785587-20 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 20 (ObjNum 88 snap 11 seq_num 88) dirty exists 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:134: left oid 20 (ObjNum 88 snap 11 seq_num 88) 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:136: finishing write tid 1 to smithi01785587-33 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:136: finishing write tid 2 to smithi01785587-33 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:136: finishing write tid 3 to smithi01785587-33 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:136: finishing write tid 4 to smithi01785587-33 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:136: finishing write tid 5 to smithi01785587-33 2024-09-21T09:38:21.104 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 40 (ObjNum 89 snap 11 seq_num 89) dirty exists 2024-09-21T09:38:21.105 INFO:tasks.rados.rados.0.smithi017.stdout:136: left oid 33 (ObjNum 89 snap 11 seq_num 89) 2024-09-21T09:38:21.147 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:21.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:20 smithi017 ceph-mon[29380]: pgmap v126: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.9 MiB/s wr, 31 op/s 2024-09-21T09:38:21.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:20 smithi017 ceph-mon[29380]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:38:21.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:20 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1216147249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:21.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:20 smithi017 ceph-mon[40737]: pgmap v126: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.9 MiB/s wr, 31 op/s 2024-09-21T09:38:21.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:20 smithi017 ceph-mon[40737]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:38:21.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:20 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1216147249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:21.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:20 smithi134 ceph-mon[33541]: pgmap v126: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.9 MiB/s wr, 31 op/s 2024-09-21T09:38:21.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:20 smithi134 ceph-mon[33541]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:38:21.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:20 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1216147249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:21.850 INFO:tasks.rados.rados.0.smithi017.stdout:134: done (3 left) 2024-09-21T09:38:21.850 INFO:tasks.rados.rados.0.smithi017.stdout:135: done (2 left) 2024-09-21T09:38:21.850 INFO:tasks.rados.rados.0.smithi017.stdout:136: done (1 left) 2024-09-21T09:38:21.851 INFO:tasks.rados.rados.0.smithi017.stdout:138: done (0 left) 2024-09-21T09:38:21.851 INFO:tasks.rados.rados.0.smithi017.stdout:139: snap_remove snap 1 2024-09-21T09:38:22.662 INFO:tasks.rados.rados.0.smithi017.stdout:139: done (0 left) 2024-09-21T09:38:22.663 INFO:tasks.rados.rados.0.smithi017.stdout:140: snap_create 2024-09-21T09:38:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[40737]: from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[40737]: from='client.24791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[40737]: pgmap v128: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.8 MiB/s wr, 29 op/s 2024-09-21T09:38:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[40737]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:38:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[40737]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:38:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[29380]: from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[29380]: from='client.24791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:23.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[29380]: pgmap v128: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.8 MiB/s wr, 29 op/s 2024-09-21T09:38:23.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[29380]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:38:23.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:22 smithi017 ceph-mon[29380]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:38:23.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:22 smithi134 ceph-mon[33541]: from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:23.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:22 smithi134 ceph-mon[33541]: from='client.24791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:22 smithi134 ceph-mon[33541]: pgmap v128: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.8 MiB/s wr, 29 op/s 2024-09-21T09:38:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:22 smithi134 ceph-mon[33541]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:38:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:22 smithi134 ceph-mon[33541]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:38:23.662 INFO:tasks.rados.rados.0.smithi017.stdout:140: done (0 left) 2024-09-21T09:38:23.662 INFO:tasks.rados.rados.0.smithi017.stdout:141: read oid 31 snap -1 2024-09-21T09:38:23.662 INFO:tasks.rados.rados.0.smithi017.stdout:141: expect (ObjNum 30 snap 0 seq_num 30) 2024-09-21T09:38:23.662 INFO:tasks.rados.rados.0.smithi017.stdout:142: read oid 13 snap -1 2024-09-21T09:38:23.663 INFO:tasks.rados.rados.0.smithi017.stdout:142: expect (ObjNum 77 snap 9 seq_num 77) 2024-09-21T09:38:23.663 INFO:tasks.rados.rados.0.smithi017.stdout:143: read oid 7 snap -1 2024-09-21T09:38:23.663 INFO:tasks.rados.rados.0.smithi017.stdout:143: expect deleted 2024-09-21T09:38:23.663 INFO:tasks.rados.rados.0.smithi017.stdout:144: snap_create 2024-09-21T09:38:23.667 INFO:tasks.rados.rados.0.smithi017.stdout:141: expect (ObjNum 30 snap 0 seq_num 30) 2024-09-21T09:38:23.763 INFO:tasks.rados.rados.0.smithi017.stdout:142: expect (ObjNum 77 snap 9 seq_num 77) 2024-09-21T09:38:24.159 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:24.667 INFO:tasks.rados.rados.0.smithi017.stdout:141: done (3 left) 2024-09-21T09:38:24.667 INFO:tasks.rados.rados.0.smithi017.stdout:142: done (2 left) 2024-09-21T09:38:24.667 INFO:tasks.rados.rados.0.smithi017.stdout:143: done (1 left) 2024-09-21T09:38:24.667 INFO:tasks.rados.rados.0.smithi017.stdout:144: done (0 left) 2024-09-21T09:38:24.667 INFO:tasks.rados.rados.0.smithi017.stdout:145: snap_create 2024-09-21T09:38:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:24 smithi017 ceph-mon[29380]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:38:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:24 smithi017 ceph-mon[29380]: pgmap v132: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:24 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:24 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/4262437277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:24 smithi017 ceph-mon[40737]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:38:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:24 smithi017 ceph-mon[40737]: pgmap v132: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:24 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:24.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:24 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/4262437277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:24.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:24 smithi134 ceph-mon[33541]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:38:24.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:24 smithi134 ceph-mon[33541]: pgmap v132: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:24 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:24 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/4262437277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:25.668 INFO:tasks.rados.rados.0.smithi017.stdout:145: done (0 left) 2024-09-21T09:38:25.668 INFO:tasks.rados.rados.0.smithi017.stdout:146: write oid 9 current snap is 15 2024-09-21T09:38:25.668 INFO:tasks.rados.rados.0.smithi017.stdout:146: seq_num 91 ranges {729437=527542,1925626=643351,3266551=262886} 2024-09-21T09:38:25.675 INFO:tasks.rados.rados.0.smithi017.stdout:146: writing smithi01785587-9 from 729437 to 1256979 tid 1 2024-09-21T09:38:25.683 INFO:tasks.rados.rados.0.smithi017.stdout:146: writing smithi01785587-9 from 1925626 to 2568977 tid 2 2024-09-21T09:38:25.689 INFO:tasks.rados.rados.0.smithi017.stdout:146: writing smithi01785587-9 from 3266551 to 3529437 tid 3 2024-09-21T09:38:25.689 INFO:tasks.rados.rados.0.smithi017.stdout:147: rollback oid 14 current snap is 15 2024-09-21T09:38:25.689 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 13 2024-09-21T09:38:25.689 INFO:tasks.rados.rados.0.smithi017.stdout:146: finishing write tid 1 to smithi01785587-9 2024-09-21T09:38:25.689 INFO:tasks.rados.rados.0.smithi017.stdout:146: finishing write tid 2 to smithi01785587-9 2024-09-21T09:38:25.689 INFO:tasks.rados.rados.0.smithi017.stdout:148: write oid 6 current snap is 15 2024-09-21T09:38:25.689 INFO:tasks.rados.rados.0.smithi017.stdout:148: seq_num 92 ranges {746244=719144,2100461=503362,3126680=419564} 2024-09-21T09:38:25.697 INFO:tasks.rados.rados.0.smithi017.stdout:148: writing smithi01785587-6 from 746244 to 1465388 tid 1 2024-09-21T09:38:25.704 INFO:tasks.rados.rados.0.smithi017.stdout:148: writing smithi01785587-6 from 2100461 to 2603823 tid 2 2024-09-21T09:38:25.710 INFO:tasks.rados.rados.0.smithi017.stdout:148: writing smithi01785587-6 from 3126680 to 3546244 tid 3 2024-09-21T09:38:25.710 INFO:tasks.rados.rados.0.smithi017.stdout:149: read oid 42 snap -1 2024-09-21T09:38:25.710 INFO:tasks.rados.rados.0.smithi017.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:25.710 INFO:tasks.rados.rados.0.smithi017.stdout:150: snap_create 2024-09-21T09:38:25.710 INFO:tasks.rados.rados.0.smithi017.stdout:147: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:38:25.710 INFO:tasks.rados.rados.0.smithi017.stdout:147: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:38:25.710 INFO:tasks.rados.rados.0.smithi017.stdout:146: finishing write tid 3 to smithi01785587-9 2024-09-21T09:38:25.710 INFO:tasks.rados.rados.0.smithi017.stdout:147: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:38:25.710 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 37 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:38:25.711 INFO:tasks.rados.rados.0.smithi017.stdout:146: finishing write tid 4 to smithi01785587-9 2024-09-21T09:38:25.711 INFO:tasks.rados.rados.0.smithi017.stdout:146: finishing write tid 5 to smithi01785587-9 2024-09-21T09:38:25.711 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 20 (ObjNum 91 snap 15 seq_num 91) dirty exists 2024-09-21T09:38:25.711 INFO:tasks.rados.rados.0.smithi017.stdout:146: left oid 9 (ObjNum 91 snap 15 seq_num 91) 2024-09-21T09:38:25.711 INFO:tasks.rados.rados.0.smithi017.stdout:148: finishing write tid 1 to smithi01785587-6 2024-09-21T09:38:25.711 INFO:tasks.rados.rados.0.smithi017.stdout:148: finishing write tid 2 to smithi01785587-6 2024-09-21T09:38:25.712 INFO:tasks.rados.rados.0.smithi017.stdout:148: finishing write tid 3 to smithi01785587-6 2024-09-21T09:38:25.712 INFO:tasks.rados.rados.0.smithi017.stdout:148: finishing write tid 4 to smithi01785587-6 2024-09-21T09:38:25.712 INFO:tasks.rados.rados.0.smithi017.stdout:148: finishing write tid 5 to smithi01785587-6 2024-09-21T09:38:25.712 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 37 (ObjNum 92 snap 15 seq_num 92) dirty exists 2024-09-21T09:38:25.712 INFO:tasks.rados.rados.0.smithi017.stdout:148: left oid 6 (ObjNum 92 snap 15 seq_num 92) 2024-09-21T09:38:25.717 INFO:tasks.rados.rados.0.smithi017.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:25 smithi017 ceph-mon[29380]: from='client.24803 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:25 smithi017 ceph-mon[29380]: from='client.24809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:25 smithi017 ceph-mon[29380]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:38:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:25 smithi017 ceph-mon[40737]: from='client.24803 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:25 smithi017 ceph-mon[40737]: from='client.24809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:25 smithi017 ceph-mon[40737]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:38:25.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:25 smithi134 ceph-mon[33541]: from='client.24803 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:25 smithi134 ceph-mon[33541]: from='client.24809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:25 smithi134 ceph-mon[33541]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:38:26.679 INFO:tasks.rados.rados.0.smithi017.stdout:146: done (4 left) 2024-09-21T09:38:26.679 INFO:tasks.rados.rados.0.smithi017.stdout:147: done (3 left) 2024-09-21T09:38:26.679 INFO:tasks.rados.rados.0.smithi017.stdout:148: done (2 left) 2024-09-21T09:38:26.679 INFO:tasks.rados.rados.0.smithi017.stdout:149: done (1 left) 2024-09-21T09:38:26.679 INFO:tasks.rados.rados.0.smithi017.stdout:150: done (0 left) 2024-09-21T09:38:26.679 INFO:tasks.rados.rados.0.smithi017.stdout:151: write oid 46 current snap is 16 2024-09-21T09:38:26.679 INFO:tasks.rados.rados.0.smithi017.stdout:151: seq_num 93 ranges {763051=794393,2158943=647020,3437750=125301} 2024-09-21T09:38:26.689 INFO:tasks.rados.rados.0.smithi017.stdout:151: writing smithi01785587-46 from 763051 to 1557444 tid 1 2024-09-21T09:38:26.697 INFO:tasks.rados.rados.0.smithi017.stdout:151: writing smithi01785587-46 from 2158943 to 2805963 tid 2 2024-09-21T09:38:26.702 INFO:tasks.rados.rados.0.smithi017.stdout:151: writing smithi01785587-46 from 3437750 to 3563051 tid 3 2024-09-21T09:38:26.702 INFO:tasks.rados.rados.0.smithi017.stdout:152: read oid 25 snap -1 2024-09-21T09:38:26.702 INFO:tasks.rados.rados.0.smithi017.stdout:152: expect (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:38:26.703 INFO:tasks.rados.rados.0.smithi017.stdout:153: write oid 50 current snap is 16 2024-09-21T09:38:26.703 INFO:tasks.rados.rados.0.smithi017.stdout:153: seq_num 94 ranges {779858=469642,1817425=507031,2781526=586628} 2024-09-21T09:38:26.711 INFO:tasks.rados.rados.0.smithi017.stdout:153: writing smithi01785587-50 from 779858 to 1249500 tid 1 2024-09-21T09:38:26.718 INFO:tasks.rados.rados.0.smithi017.stdout:153: writing smithi01785587-50 from 1817425 to 2324456 tid 2 2024-09-21T09:38:26.725 INFO:tasks.rados.rados.0.smithi017.stdout:153: writing smithi01785587-50 from 2781526 to 3368154 tid 3 2024-09-21T09:38:26.725 INFO:tasks.rados.rados.0.smithi017.stdout:154: read oid 6 snap 12 2024-09-21T09:38:26.725 INFO:tasks.rados.rados.0.smithi017.stdout:154: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:38:26.725 INFO:tasks.rados.rados.0.smithi017.stdout:151: finishing write tid 1 to smithi01785587-46 2024-09-21T09:38:26.725 INFO:tasks.rados.rados.0.smithi017.stdout:155: snap_remove snap 8 2024-09-21T09:38:26.725 INFO:tasks.rados.rados.0.smithi017.stdout:151: finishing write tid 2 to smithi01785587-46 2024-09-21T09:38:26.726 INFO:tasks.rados.rados.0.smithi017.stdout:151: finishing write tid 3 to smithi01785587-46 2024-09-21T09:38:26.726 INFO:tasks.rados.rados.0.smithi017.stdout:151: finishing write tid 4 to smithi01785587-46 2024-09-21T09:38:26.726 INFO:tasks.rados.rados.0.smithi017.stdout:151: finishing write tid 5 to smithi01785587-46 2024-09-21T09:38:26.726 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 54 (ObjNum 93 snap 16 seq_num 93) dirty exists 2024-09-21T09:38:26.726 INFO:tasks.rados.rados.0.smithi017.stdout:151: left oid 46 (ObjNum 93 snap 16 seq_num 93) 2024-09-21T09:38:26.726 INFO:tasks.rados.rados.0.smithi017.stdout:152: expect (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:38:26.823 INFO:tasks.rados.rados.0.smithi017.stdout:153: finishing write tid 1 to smithi01785587-50 2024-09-21T09:38:26.824 INFO:tasks.rados.rados.0.smithi017.stdout:153: finishing write tid 2 to smithi01785587-50 2024-09-21T09:38:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:26 smithi017 ceph-mon[29380]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:38:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:26 smithi017 ceph-mon[29380]: pgmap v135: 89 pgs: 89 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 811 KiB/s wr, 7 op/s 2024-09-21T09:38:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:26 smithi017 ceph-mon[40737]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:38:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:26 smithi017 ceph-mon[40737]: pgmap v135: 89 pgs: 89 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 811 KiB/s wr, 7 op/s 2024-09-21T09:38:26.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:26 smithi134 ceph-mon[33541]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:38:26.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:26 smithi134 ceph-mon[33541]: pgmap v135: 89 pgs: 89 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 811 KiB/s wr, 7 op/s 2024-09-21T09:38:27.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:38:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:38:27.356 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:27.666 INFO:tasks.rados.rados.0.smithi017.stdout:151: done (4 left) 2024-09-21T09:38:27.666 INFO:tasks.rados.rados.0.smithi017.stdout:152: done (3 left) 2024-09-21T09:38:27.666 INFO:tasks.rados.rados.0.smithi017.stdout:155: done (2 left) 2024-09-21T09:38:27.666 INFO:tasks.rados.rados.0.smithi017.stdout:156: snap_remove snap 6 2024-09-21T09:38:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:27 smithi017 ceph-mon[29380]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:38:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:27 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/143304939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:27 smithi017 ceph-mon[29380]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:38:27.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:27 smithi017 ceph-mon[40737]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:38:27.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:27 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/143304939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:27.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:27 smithi017 ceph-mon[40737]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:38:27.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:27 smithi134 ceph-mon[33541]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:38:27.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:27 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/143304939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:27 smithi134 ceph-mon[33541]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:38:28.665 INFO:tasks.rados.rados.0.smithi017.stdout:156: done (2 left) 2024-09-21T09:38:28.665 INFO:tasks.rados.rados.0.smithi017.stdout:157: delete oid 29 current snap is 16 2024-09-21T09:38:28.666 INFO:tasks.rados.rados.0.smithi017.stdout:153: finishing write tid 3 to smithi01785587-50 2024-09-21T09:38:28.666 INFO:tasks.rados.rados.0.smithi017.stdout:153: finishing write tid 4 to smithi01785587-50 2024-09-21T09:38:28.666 INFO:tasks.rados.rados.0.smithi017.stdout:153: finishing write tid 5 to smithi01785587-50 2024-09-21T09:38:28.666 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 47 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-09-21T09:38:28.666 INFO:tasks.rados.rados.0.smithi017.stdout:153: left oid 50 (ObjNum 94 snap 16 seq_num 94) 2024-09-21T09:38:28.667 INFO:tasks.rados.rados.0.smithi017.stdout:154: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:38:28.757 INFO:tasks.rados.rados.0.smithi017.stdout:153: done (2 left) 2024-09-21T09:38:28.758 INFO:tasks.rados.rados.0.smithi017.stdout:154: done (1 left) 2024-09-21T09:38:28.758 INFO:tasks.rados.rados.0.smithi017.stdout:157: done (0 left) 2024-09-21T09:38:28.758 INFO:tasks.rados.rados.0.smithi017.stdout:158: write oid 29 current snap is 16 2024-09-21T09:38:28.758 INFO:tasks.rados.rados.0.smithi017.stdout:158: seq_num 96 ranges {413472=620140,1818036=510700,3120019=493453} 2024-09-21T09:38:28.763 INFO:tasks.rados.rados.0.smithi017.stdout:158: writing smithi01785587-29 from 413472 to 1033612 tid 1 2024-09-21T09:38:28.772 INFO:tasks.rados.rados.0.smithi017.stdout:158: writing smithi01785587-29 from 1818036 to 2328736 tid 2 2024-09-21T09:38:28.780 INFO:tasks.rados.rados.0.smithi017.stdout:158: writing smithi01785587-29 from 3120019 to 3613472 tid 3 2024-09-21T09:38:28.780 INFO:tasks.rados.rados.0.smithi017.stdout:159: read oid 41 snap -1 2024-09-21T09:38:28.780 INFO:tasks.rados.rados.0.smithi017.stdout:159: expect (ObjNum 73 snap 9 seq_num 73) 2024-09-21T09:38:28.780 INFO:tasks.rados.rados.0.smithi017.stdout:160: delete oid 28 current snap is 16 2024-09-21T09:38:28.780 INFO:tasks.rados.rados.0.smithi017.stdout:158: finishing write tid 1 to smithi01785587-29 2024-09-21T09:38:28.780 INFO:tasks.rados.rados.0.smithi017.stdout:158: finishing write tid 2 to smithi01785587-29 2024-09-21T09:38:28.782 INFO:tasks.rados.rados.0.smithi017.stdout:160: done (2 left) 2024-09-21T09:38:28.782 INFO:tasks.rados.rados.0.smithi017.stdout:161: write oid 44 current snap is 16 2024-09-21T09:38:28.782 INFO:tasks.rados.rados.0.smithi017.stdout:161: seq_num 98 ranges {447086=770638,1935000=798016,3458512=188574} 2024-09-21T09:38:28.791 INFO:tasks.rados.rados.0.smithi017.stdout:161: writing smithi01785587-44 from 447086 to 1217724 tid 1 2024-09-21T09:38:28.801 INFO:tasks.rados.rados.0.smithi017.stdout:161: writing smithi01785587-44 from 1935000 to 2733016 tid 2 2024-09-21T09:38:28.807 INFO:tasks.rados.rados.0.smithi017.stdout:161: writing smithi01785587-44 from 3458512 to 3647086 tid 3 2024-09-21T09:38:28.807 INFO:tasks.rados.rados.0.smithi017.stdout:162: snap_remove snap 12 2024-09-21T09:38:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:28 smithi017 ceph-mon[29380]: from='client.14928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:28 smithi017 ceph-mon[29380]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:28 smithi017 ceph-mon[29380]: pgmap v138: 89 pgs: 89 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 778 KiB/s wr, 7 op/s 2024-09-21T09:38:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:28 smithi017 ceph-mon[29380]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:38:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:28 smithi017 ceph-mon[40737]: from='client.14928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:28 smithi017 ceph-mon[40737]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:28 smithi017 ceph-mon[40737]: pgmap v138: 89 pgs: 89 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 778 KiB/s wr, 7 op/s 2024-09-21T09:38:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:28 smithi017 ceph-mon[40737]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:38:28.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:28 smithi134 ceph-mon[33541]: from='client.14928 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:28.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:28 smithi134 ceph-mon[33541]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:28.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:28 smithi134 ceph-mon[33541]: pgmap v138: 89 pgs: 89 active+clean; 152 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 778 KiB/s wr, 7 op/s 2024-09-21T09:38:28.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:28 smithi134 ceph-mon[33541]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:38:29.689 INFO:tasks.rados.rados.0.smithi017.stdout:162: done (3 left) 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:163: rollback oid 15 current snap is 16 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 0 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:164: read oid 21 snap -1 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:164: expect (ObjNum 52 snap 0 seq_num 52) 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:165: read oid 17 snap 13 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:165: expect deleted 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:166: read oid 18 snap 13 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:166: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:167: write oid 24 current snap is 16 2024-09-21T09:38:29.690 INFO:tasks.rados.rados.0.smithi017.stdout:167: seq_num 99 ranges {463893=562240,1593482=658027,2685935=559341} 2024-09-21T09:38:29.695 INFO:tasks.rados.rados.0.smithi017.stdout:167: writing smithi01785587-24 from 463893 to 1026133 tid 1 2024-09-21T09:38:29.703 INFO:tasks.rados.rados.0.smithi017.stdout:167: writing smithi01785587-24 from 1593482 to 2251509 tid 2 2024-09-21T09:38:29.711 INFO:tasks.rados.rados.0.smithi017.stdout:167: writing smithi01785587-24 from 2685935 to 3245276 tid 3 2024-09-21T09:38:29.711 INFO:tasks.rados.rados.0.smithi017.stdout:168: write oid 30 current snap is 16 2024-09-21T09:38:29.711 INFO:tasks.rados.rados.0.smithi017.stdout:168: seq_num 100 ranges {480700=637489,1651964=401685,2713358=770613} 2024-09-21T09:38:29.717 INFO:tasks.rados.rados.0.smithi017.stdout:168: writing smithi01785587-30 from 480700 to 1118189 tid 1 2024-09-21T09:38:29.723 INFO:tasks.rados.rados.0.smithi017.stdout:168: writing smithi01785587-30 from 1651964 to 2053649 tid 2 2024-09-21T09:38:29.732 INFO:tasks.rados.rados.0.smithi017.stdout:168: writing smithi01785587-30 from 2713358 to 3483971 tid 3 2024-09-21T09:38:29.732 INFO:tasks.rados.rados.0.smithi017.stdout:169: write oid 26 current snap is 16 2024-09-21T09:38:29.732 INFO:tasks.rados.rados.0.smithi017.stdout:169: seq_num 101 ranges {497507=712738,1710446=661696,3140781=556726} 2024-09-21T09:38:29.740 INFO:tasks.rados.rados.0.smithi017.stdout:169: writing smithi01785587-26 from 497507 to 1210245 tid 1 2024-09-21T09:38:29.747 INFO:tasks.rados.rados.0.smithi017.stdout:169: writing smithi01785587-26 from 1710446 to 2372142 tid 2 2024-09-21T09:38:29.756 INFO:tasks.rados.rados.0.smithi017.stdout:169: writing smithi01785587-26 from 3140781 to 3697507 tid 3 2024-09-21T09:38:29.756 INFO:tasks.rados.rados.0.smithi017.stdout:170: write oid 25 current snap is 16 2024-09-21T09:38:29.756 INFO:tasks.rados.rados.0.smithi017.stdout:158: finishing write tid 3 to smithi01785587-29 2024-09-21T09:38:29.756 INFO:tasks.rados.rados.0.smithi017.stdout:158: finishing write tid 4 to smithi01785587-29 2024-09-21T09:38:29.756 INFO:tasks.rados.rados.0.smithi017.stdout:158: finishing write tid 5 to smithi01785587-29 2024-09-21T09:38:29.756 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 32 (ObjNum 96 snap 16 seq_num 96) dirty exists 2024-09-21T09:38:29.756 INFO:tasks.rados.rados.0.smithi017.stdout:158: left oid 29 (ObjNum 96 snap 16 seq_num 96) 2024-09-21T09:38:29.756 INFO:tasks.rados.rados.0.smithi017.stdout:159: expect (ObjNum 73 snap 9 seq_num 73) 2024-09-21T09:38:29.949 INFO:tasks.rados.rados.0.smithi017.stdout:161: finishing write tid 1 to smithi01785587-44 2024-09-21T09:38:29.949 INFO:tasks.rados.rados.0.smithi017.stdout:161: finishing write tid 2 to smithi01785587-44 2024-09-21T09:38:29.949 INFO:tasks.rados.rados.0.smithi017.stdout:161: finishing write tid 3 to smithi01785587-44 2024-09-21T09:38:29.949 INFO:tasks.rados.rados.0.smithi017.stdout:161: finishing write tid 4 to smithi01785587-44 2024-09-21T09:38:29.949 INFO:tasks.rados.rados.0.smithi017.stdout:161: finishing write tid 5 to smithi01785587-44 2024-09-21T09:38:29.949 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 50 (ObjNum 98 snap 16 seq_num 98) dirty exists 2024-09-21T09:38:29.949 INFO:tasks.rados.rados.0.smithi017.stdout:161: left oid 44 (ObjNum 98 snap 16 seq_num 98) 2024-09-21T09:38:29.949 INFO:tasks.rados.rados.0.smithi017.stdout:170: seq_num 102 ranges {514314=787987,1768928=405354,2768204=509510} 2024-09-21T09:38:29.957 INFO:tasks.rados.rados.0.smithi017.stdout:170: writing smithi01785587-25 from 514314 to 1302301 tid 1 2024-09-21T09:38:29.962 INFO:tasks.rados.rados.0.smithi017.stdout:170: writing smithi01785587-25 from 1768928 to 2174282 tid 2 2024-09-21T09:38:29.969 INFO:tasks.rados.rados.0.smithi017.stdout:170: writing smithi01785587-25 from 2768204 to 3277714 tid 3 2024-09-21T09:38:29.969 INFO:tasks.rados.rados.0.smithi017.stdout:158: done (10 left) 2024-09-21T09:38:29.969 INFO:tasks.rados.rados.0.smithi017.stdout:159: done (9 left) 2024-09-21T09:38:29.969 INFO:tasks.rados.rados.0.smithi017.stdout:161: done (8 left) 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:171: rollback oid 46 current snap is 16 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 15 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:163: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:163: finishing rollback tid 2 to smithi01785587-15 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 46 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:163: done (8 left) 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:165: done (7 left) 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:172: rollback oid 6 current snap is 16 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 14 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:173: write oid 1 current snap is 16 2024-09-21T09:38:29.970 INFO:tasks.rados.rados.0.smithi017.stdout:166: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:38:30.058 INFO:tasks.rados.rados.0.smithi017.stdout:164: expect (ObjNum 52 snap 0 seq_num 52) 2024-09-21T09:38:30.205 INFO:tasks.rados.rados.0.smithi017.stdout:167: finishing write tid 1 to smithi01785587-24 2024-09-21T09:38:30.205 INFO:tasks.rados.rados.0.smithi017.stdout:167: finishing write tid 2 to smithi01785587-24 2024-09-21T09:38:30.205 INFO:tasks.rados.rados.0.smithi017.stdout:173: seq_num 103 ranges {531121=463236,1711057=665365,3079274=651847} 2024-09-21T09:38:30.211 INFO:tasks.rados.rados.0.smithi017.stdout:173: writing smithi01785587-1 from 531121 to 994357 tid 1 2024-09-21T09:38:30.219 INFO:tasks.rados.rados.0.smithi017.stdout:173: writing smithi01785587-1 from 1711057 to 2376422 tid 2 2024-09-21T09:38:30.228 INFO:tasks.rados.rados.0.smithi017.stdout:173: writing smithi01785587-1 from 3079274 to 3731121 tid 3 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:164: done (8 left) 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:166: done (7 left) 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:174: read oid 44 snap -1 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:174: expect (ObjNum 98 snap 16 seq_num 98) 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:175: read oid 29 snap -1 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:175: expect (ObjNum 96 snap 16 seq_num 96) 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:167: finishing write tid 3 to smithi01785587-24 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:167: finishing write tid 4 to smithi01785587-24 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:167: finishing write tid 5 to smithi01785587-24 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 66 (ObjNum 99 snap 16 seq_num 99) dirty exists 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:167: left oid 24 (ObjNum 99 snap 16 seq_num 99) 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:168: finishing write tid 1 to smithi01785587-30 2024-09-21T09:38:30.229 INFO:tasks.rados.rados.0.smithi017.stdout:168: finishing write tid 2 to smithi01785587-30 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:168: finishing write tid 3 to smithi01785587-30 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:168: finishing write tid 4 to smithi01785587-30 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:168: finishing write tid 5 to smithi01785587-30 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 35 (ObjNum 100 snap 16 seq_num 100) dirty exists 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:168: left oid 30 (ObjNum 100 snap 16 seq_num 100) 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:169: finishing write tid 1 to smithi01785587-26 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:169: finishing write tid 2 to smithi01785587-26 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:169: finishing write tid 3 to smithi01785587-26 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:169: finishing write tid 4 to smithi01785587-26 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:169: finishing write tid 5 to smithi01785587-26 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 61 (ObjNum 101 snap 16 seq_num 101) dirty exists 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:169: left oid 26 (ObjNum 101 snap 16 seq_num 101) 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:170: finishing write tid 1 to smithi01785587-25 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:170: finishing write tid 2 to smithi01785587-25 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:172: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:38:30.230 INFO:tasks.rados.rados.0.smithi017.stdout:172: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:171: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:170: finishing write tid 3 to smithi01785587-25 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:172: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 55 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:171: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:170: finishing write tid 4 to smithi01785587-25 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:171: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 69 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:170: finishing write tid 5 to smithi01785587-25 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 39 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:170: left oid 25 (ObjNum 102 snap 16 seq_num 102) 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:173: finishing write tid 1 to smithi01785587-1 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:173: finishing write tid 2 to smithi01785587-1 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:167: done (8 left) 2024-09-21T09:38:30.231 INFO:tasks.rados.rados.0.smithi017.stdout:168: done (7 left) 2024-09-21T09:38:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:169: done (6 left) 2024-09-21T09:38:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:170: done (5 left) 2024-09-21T09:38:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:171: done (4 left) 2024-09-21T09:38:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:172: done (3 left) 2024-09-21T09:38:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:176: rollback oid 35 current snap is 16 2024-09-21T09:38:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 15 2024-09-21T09:38:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:177: write oid 18 current snap is 16 2024-09-21T09:38:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:177: seq_num 104 ranges {547928=538485,1769539=409023,2706697=532054} 2024-09-21T09:38:30.235 INFO:tasks.rados.rados.0.smithi017.stdout:177: writing smithi01785587-18 from 547928 to 1086413 tid 1 2024-09-21T09:38:30.241 INFO:tasks.rados.rados.0.smithi017.stdout:177: writing smithi01785587-18 from 1769539 to 2178562 tid 2 2024-09-21T09:38:30.248 INFO:tasks.rados.rados.0.smithi017.stdout:177: writing smithi01785587-18 from 2706697 to 3238751 tid 3 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:178: read oid 21 snap 11 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:178: expect (ObjNum 52 snap 0 seq_num 52) 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:179: rollback oid 43 current snap is 16 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 14 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:173: finishing write tid 3 to smithi01785587-1 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:176: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:173: finishing write tid 4 to smithi01785587-1 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:173: finishing write tid 5 to smithi01785587-1 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 59 (ObjNum 103 snap 16 seq_num 103) dirty exists 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:173: left oid 1 (ObjNum 103 snap 16 seq_num 103) 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:176: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:173: done (6 left) 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:180: rollback oid 8 current snap is 16 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 11 2024-09-21T09:38:30.249 INFO:tasks.rados.rados.0.smithi017.stdout:176: finishing rollback tid 2 to smithi01785587-35 2024-09-21T09:38:30.250 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 21 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-21T09:38:30.250 INFO:tasks.rados.rados.0.smithi017.stdout:174: expect (ObjNum 98 snap 16 seq_num 98) 2024-09-21T09:38:30.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:38:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:30] "GET /metrics HTTP/1.1" 200 243019 "" "Prometheus/2.43.0" 2024-09-21T09:38:30.461 INFO:tasks.rados.rados.0.smithi017.stdout:177: finishing write tid 1 to smithi01785587-18 2024-09-21T09:38:30.461 INFO:tasks.rados.rados.0.smithi017.stdout:177: finishing write tid 2 to smithi01785587-18 2024-09-21T09:38:30.461 INFO:tasks.rados.rados.0.smithi017.stdout:175: expect (ObjNum 96 snap 16 seq_num 96) 2024-09-21T09:38:30.572 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:178: expect (ObjNum 52 snap 0 seq_num 52) 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:177: finishing write tid 3 to smithi01785587-18 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:177: finishing write tid 4 to smithi01785587-18 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:180: finishing rollback tid 0 to smithi01785587-8 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:177: finishing write tid 5 to smithi01785587-18 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 8 (ObjNum 104 snap 16 seq_num 104) dirty exists 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:177: left oid 18 (ObjNum 104 snap 16 seq_num 104) 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:180: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:174: done (6 left) 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:175: done (5 left) 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:176: done (4 left) 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:177: done (3 left) 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:178: done (2 left) 2024-09-21T09:38:30.726 INFO:tasks.rados.rados.0.smithi017.stdout:181: read oid 32 snap 14 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:181: expect (ObjNum 86 snap 11 seq_num 86) 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:182: read oid 26 snap -1 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:182: expect (ObjNum 101 snap 16 seq_num 101) 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:183: write oid 11 current snap is 16 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:179: finishing rollback tid 0 to smithi01785587-43 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:179: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:180: finishing rollback tid 2 to smithi01785587-8 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 73 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:179: finishing rollback tid 2 to smithi01785587-43 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 63 (ObjNum 75 snap 9 seq_num 75) dirty exists 2024-09-21T09:38:30.727 INFO:tasks.rados.rados.0.smithi017.stdout:183: seq_num 105 ranges {564735=613734,1828021=669034,3134120=630615} 2024-09-21T09:38:30.732 INFO:tasks.rados.rados.0.smithi017.stdout:183: writing smithi01785587-11 from 564735 to 1178469 tid 1 2024-09-21T09:38:30.739 INFO:tasks.rados.rados.0.smithi017.stdout:183: writing smithi01785587-11 from 1828021 to 2497055 tid 2 2024-09-21T09:38:30.747 INFO:tasks.rados.rados.0.smithi017.stdout:183: writing smithi01785587-11 from 3134120 to 3764735 tid 3 2024-09-21T09:38:30.747 INFO:tasks.rados.rados.0.smithi017.stdout:179: done (4 left) 2024-09-21T09:38:30.747 INFO:tasks.rados.rados.0.smithi017.stdout:180: done (3 left) 2024-09-21T09:38:30.747 INFO:tasks.rados.rados.0.smithi017.stdout:184: write oid 36 current snap is 16 2024-09-21T09:38:30.747 INFO:tasks.rados.rados.0.smithi017.stdout:184: seq_num 106 ranges {581542=688983,1886503=412692,2761543=670951} 2024-09-21T09:38:30.753 INFO:tasks.rados.rados.0.smithi017.stdout:184: writing smithi01785587-36 from 581542 to 1270525 tid 1 2024-09-21T09:38:30.759 INFO:tasks.rados.rados.0.smithi017.stdout:184: writing smithi01785587-36 from 1886503 to 2299195 tid 2 2024-09-21T09:38:30.766 INFO:tasks.rados.rados.0.smithi017.stdout:184: writing smithi01785587-36 from 2761543 to 3432494 tid 3 2024-09-21T09:38:30.766 INFO:tasks.rados.rados.0.smithi017.stdout:185: delete oid 14 current snap is 16 2024-09-21T09:38:30.766 INFO:tasks.rados.rados.0.smithi017.stdout:181: expect (ObjNum 86 snap 11 seq_num 86) 2024-09-21T09:38:30.919 INFO:tasks.rados.rados.0.smithi017.stdout:183: finishing write tid 1 to smithi01785587-11 2024-09-21T09:38:30.919 INFO:tasks.rados.rados.0.smithi017.stdout:182: expect (ObjNum 101 snap 16 seq_num 101) 2024-09-21T09:38:30.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:30 smithi017 ceph-mon[29380]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:38:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:30 smithi017 ceph-mon[29380]: pgmap v140: 89 pgs: 89 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 19 op/s 2024-09-21T09:38:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:30 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/4161014090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:30 smithi017 ceph-mon[40737]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:38:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:30 smithi017 ceph-mon[40737]: pgmap v140: 89 pgs: 89 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 19 op/s 2024-09-21T09:38:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:30 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/4161014090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:30.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:30 smithi134 ceph-mon[33541]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:38:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:30 smithi134 ceph-mon[33541]: pgmap v140: 89 pgs: 89 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 19 op/s 2024-09-21T09:38:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:30 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/4161014090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:31.077 INFO:tasks.rados.rados.0.smithi017.stdout:183: finishing write tid 2 to smithi01785587-11 2024-09-21T09:38:31.077 INFO:tasks.rados.rados.0.smithi017.stdout:183: finishing write tid 3 to smithi01785587-11 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:183: finishing write tid 4 to smithi01785587-11 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:183: finishing write tid 5 to smithi01785587-11 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 78 (ObjNum 105 snap 16 seq_num 105) dirty exists 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:183: left oid 11 (ObjNum 105 snap 16 seq_num 105) 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:184: finishing write tid 1 to smithi01785587-36 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:184: finishing write tid 2 to smithi01785587-36 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:181: done (4 left) 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:182: done (3 left) 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:183: done (2 left) 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:185: done (1 left) 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:186: delete oid 34 current snap is 16 2024-09-21T09:38:31.078 INFO:tasks.rados.rados.0.smithi017.stdout:184: finishing write tid 3 to smithi01785587-36 2024-09-21T09:38:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:184: finishing write tid 4 to smithi01785587-36 2024-09-21T09:38:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:184: finishing write tid 5 to smithi01785587-36 2024-09-21T09:38:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 60 (ObjNum 106 snap 16 seq_num 106) dirty exists 2024-09-21T09:38:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:184: left oid 36 (ObjNum 106 snap 16 seq_num 106) 2024-09-21T09:38:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:184: done (1 left) 2024-09-21T09:38:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:186: done (0 left) 2024-09-21T09:38:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:187: snap_create 2024-09-21T09:38:31.697 INFO:tasks.rados.rados.0.smithi017.stdout:187: done (0 left) 2024-09-21T09:38:31.698 INFO:tasks.rados.rados.0.smithi017.stdout:188: read oid 42 snap -1 2024-09-21T09:38:31.698 INFO:tasks.rados.rados.0.smithi017.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:31.698 INFO:tasks.rados.rados.0.smithi017.stdout:189: rollback oid 30 current snap is 17 2024-09-21T09:38:31.698 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 14 2024-09-21T09:38:31.698 INFO:tasks.rados.rados.0.smithi017.stdout:190: delete oid 20 current snap is 17 2024-09-21T09:38:31.701 INFO:tasks.rados.rados.0.smithi017.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:31.855 INFO:tasks.rados.rados.0.smithi017.stdout:189: finishing rollback tid 0 to smithi01785587-30 2024-09-21T09:38:31.856 INFO:tasks.rados.rados.0.smithi017.stdout:189: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:38:31.856 INFO:tasks.rados.rados.0.smithi017.stdout:189: finishing rollback tid 2 to smithi01785587-30 2024-09-21T09:38:31.856 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 39 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:38:31.856 INFO:tasks.rados.rados.0.smithi017.stdout:188: done (2 left) 2024-09-21T09:38:31.856 INFO:tasks.rados.rados.0.smithi017.stdout:189: done (1 left) 2024-09-21T09:38:31.856 INFO:tasks.rados.rados.0.smithi017.stdout:190: done (0 left) 2024-09-21T09:38:31.856 INFO:tasks.rados.rados.0.smithi017.stdout:191: snap_remove snap 0 2024-09-21T09:38:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:31 smithi017 ceph-mon[29380]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:31 smithi017 ceph-mon[29380]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:31 smithi017 ceph-mon[40737]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:31 smithi017 ceph-mon[40737]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:31.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:31 smithi134 ceph-mon[33541]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:31 smithi134 ceph-mon[33541]: from='client.14952 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:32.664 INFO:tasks.rados.rados.0.smithi017.stdout:191: done (0 left) 2024-09-21T09:38:32.664 INFO:tasks.rados.rados.0.smithi017.stdout:192: snap_remove snap 15 2024-09-21T09:38:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[40737]: pgmap v142: 89 pgs: 89 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-09-21T09:38:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[40737]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:38:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[40737]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:38:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[29380]: pgmap v142: 89 pgs: 89 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-09-21T09:38:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[29380]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:38:32.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:32.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:32.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:32.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:32 smithi017 ceph-mon[29380]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:38:32.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:32 smithi134 ceph-mon[33541]: pgmap v142: 89 pgs: 89 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-09-21T09:38:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:32 smithi134 ceph-mon[33541]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:38:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:32 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:32 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:32 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:38:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:32 smithi134 ceph-mon[33541]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:38:33.475 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:33.665 INFO:tasks.rados.rados.0.smithi017.stdout:192: done (0 left) 2024-09-21T09:38:33.665 INFO:tasks.rados.rados.0.smithi017.stdout:193: read oid 1 snap 11 2024-09-21T09:38:33.665 INFO:tasks.rados.rados.0.smithi017.stdout:193: expect deleted 2024-09-21T09:38:33.665 INFO:tasks.rados.rados.0.smithi017.stdout:194: snap_create 2024-09-21T09:38:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[29380]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:38:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[29380]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:38:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[29380]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:38:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[29380]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi134 2024-09-21T09:38:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1205655913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[29380]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:38:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[40737]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:38:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[40737]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:38:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[40737]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:38:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[40737]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi134 2024-09-21T09:38:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1205655913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:33 smithi017 ceph-mon[40737]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:38:33.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:33 smithi134 ceph-mon[33541]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:38:33.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:33 smithi134 ceph-mon[33541]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:38:33.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:33 smithi134 ceph-mon[33541]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:38:33.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:33 smithi134 ceph-mon[33541]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi134 2024-09-21T09:38:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:33 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1205655913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:33 smithi134 ceph-mon[33541]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:38:34.670 INFO:tasks.rados.rados.0.smithi017.stdout:193: done (1 left) 2024-09-21T09:38:34.670 INFO:tasks.rados.rados.0.smithi017.stdout:194: done (0 left) 2024-09-21T09:38:34.670 INFO:tasks.rados.rados.0.smithi017.stdout:195: delete oid 45 current snap is 18 2024-09-21T09:38:34.672 INFO:tasks.rados.rados.0.smithi017.stdout:195: done (0 left) 2024-09-21T09:38:34.673 INFO:tasks.rados.rados.0.smithi017.stdout:196: read oid 12 snap 13 2024-09-21T09:38:34.673 INFO:tasks.rados.rados.0.smithi017.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2024-09-21T09:38:34.673 INFO:tasks.rados.rados.0.smithi017.stdout:197: rollback oid 44 current snap is 18 2024-09-21T09:38:34.673 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 17 2024-09-21T09:38:34.673 INFO:tasks.rados.rados.0.smithi017.stdout:198: snap_remove snap 14 2024-09-21T09:38:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:34 smithi017 ceph-mon[40737]: from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:34 smithi017 ceph-mon[40737]: pgmap v146: 89 pgs: 89 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:34 smithi017 ceph-mon[40737]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:34 smithi017 ceph-mon[40737]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:38:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:34 smithi017 ceph-mon[29380]: from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:34 smithi017 ceph-mon[29380]: pgmap v146: 89 pgs: 89 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:34 smithi017 ceph-mon[29380]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:34 smithi017 ceph-mon[29380]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:38:34.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:34 smithi134 ceph-mon[33541]: from='client.14964 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:34.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:34 smithi134 ceph-mon[33541]: pgmap v146: 89 pgs: 89 active+clean; 163 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:34.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:34 smithi134 ceph-mon[33541]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:34.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:34 smithi134 ceph-mon[33541]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:38:35.677 INFO:tasks.rados.rados.0.smithi017.stdout:198: done (2 left) 2024-09-21T09:38:35.677 INFO:tasks.rados.rados.0.smithi017.stdout:199: delete oid 24 current snap is 18 2024-09-21T09:38:35.677 INFO:tasks.rados.rados.0.smithi017.stdout:197: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:38:35.677 INFO:tasks.rados.rados.0.smithi017.stdout:197: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:38:35.677 INFO:tasks.rados.rados.0.smithi017.stdout:197: finishing rollback tid 2 to smithi01785587-44 2024-09-21T09:38:35.677 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 70 (ObjNum 98 snap 16 seq_num 98) dirty exists 2024-09-21T09:38:35.677 INFO:tasks.rados.rados.0.smithi017.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2024-09-21T09:38:35.820 INFO:tasks.rados.rados.0.smithi017.stdout:196: done (2 left) 2024-09-21T09:38:35.820 INFO:tasks.rados.rados.0.smithi017.stdout:197: done (1 left) 2024-09-21T09:38:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:199: done (0 left) 2024-09-21T09:38:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:200: read oid 9 snap 13 2024-09-21T09:38:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:38:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:201: rollback oid 45 current snap is 18 2024-09-21T09:38:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 45 to 16 2024-09-21T09:38:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:202: rollback oid 14 current snap is 18 2024-09-21T09:38:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 13 2024-09-21T09:38:35.822 INFO:tasks.rados.rados.0.smithi017.stdout:203: write oid 34 current snap is 18 2024-09-21T09:38:35.822 INFO:tasks.rados.rados.0.smithi017.stdout:203: seq_num 112 ranges {682384=456830,1721042=423699,2809728=454936,3731357=151027} 2024-09-21T09:38:35.826 INFO:tasks.rados.rados.0.smithi017.stdout:203: writing smithi01785587-34 from 682384 to 1139214 tid 1 2024-09-21T09:38:35.832 INFO:tasks.rados.rados.0.smithi017.stdout:203: writing smithi01785587-34 from 1721042 to 2144741 tid 2 2024-09-21T09:38:35.838 INFO:tasks.rados.rados.0.smithi017.stdout:203: writing smithi01785587-34 from 2809728 to 3264664 tid 3 2024-09-21T09:38:35.842 INFO:tasks.rados.rados.0.smithi017.stdout:203: writing smithi01785587-34 from 3731357 to 3882384 tid 4 2024-09-21T09:38:35.842 INFO:tasks.rados.rados.0.smithi017.stdout:204: snap_create 2024-09-21T09:38:35.842 INFO:tasks.rados.rados.0.smithi017.stdout:202: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:38:35.842 INFO:tasks.rados.rados.0.smithi017.stdout:201: finishing rollback tid 1 to smithi01785587-45 2024-09-21T09:38:35.842 INFO:tasks.rados.rados.0.smithi017.stdout:201: finishing rollback tid 2 to smithi01785587-45 2024-09-21T09:38:35.842 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 46 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-21T09:38:35.842 INFO:tasks.rados.rados.0.smithi017.stdout:202: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:38:35.842 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 76 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:38:35.842 INFO:tasks.rados.rados.0.smithi017.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:38:35.914 INFO:tasks.rados.rados.0.smithi017.stdout:203: finishing write tid 1 to smithi01785587-34 2024-09-21T09:38:35.914 INFO:tasks.rados.rados.0.smithi017.stdout:203: finishing write tid 2 to smithi01785587-34 2024-09-21T09:38:35.914 INFO:tasks.rados.rados.0.smithi017.stdout:203: finishing write tid 3 to smithi01785587-34 2024-09-21T09:38:35.914 INFO:tasks.rados.rados.0.smithi017.stdout:203: finishing write tid 4 to smithi01785587-34 2024-09-21T09:38:35.914 INFO:tasks.rados.rados.0.smithi017.stdout:203: finishing write tid 5 to smithi01785587-34 2024-09-21T09:38:35.915 INFO:tasks.rados.rados.0.smithi017.stdout:203: finishing write tid 6 to smithi01785587-34 2024-09-21T09:38:35.915 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 45 (ObjNum 112 snap 18 seq_num 112) dirty exists 2024-09-21T09:38:35.915 INFO:tasks.rados.rados.0.smithi017.stdout:203: left oid 34 (ObjNum 112 snap 18 seq_num 112) 2024-09-21T09:38:36.355 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:36.675 INFO:tasks.rados.rados.0.smithi017.stdout:200: done (4 left) 2024-09-21T09:38:36.675 INFO:tasks.rados.rados.0.smithi017.stdout:201: done (3 left) 2024-09-21T09:38:36.675 INFO:tasks.rados.rados.0.smithi017.stdout:202: done (2 left) 2024-09-21T09:38:36.675 INFO:tasks.rados.rados.0.smithi017.stdout:203: done (1 left) 2024-09-21T09:38:36.675 INFO:tasks.rados.rados.0.smithi017.stdout:204: done (0 left) 2024-09-21T09:38:36.675 INFO:tasks.rados.rados.0.smithi017.stdout:205: write oid 49 current snap is 19 2024-09-21T09:38:36.675 INFO:tasks.rados.rados.0.smithi017.stdout:205: seq_num 113 ranges {699191=532079,1779524=567357,2837151=666208} 2024-09-21T09:38:36.681 INFO:tasks.rados.rados.0.smithi017.stdout:205: writing smithi01785587-49 from 699191 to 1231270 tid 1 2024-09-21T09:38:36.687 INFO:tasks.rados.rados.0.smithi017.stdout:205: writing smithi01785587-49 from 1779524 to 2346881 tid 2 2024-09-21T09:38:36.694 INFO:tasks.rados.rados.0.smithi017.stdout:205: writing smithi01785587-49 from 2837151 to 3503359 tid 3 2024-09-21T09:38:36.694 INFO:tasks.rados.rados.0.smithi017.stdout:206: snap_remove snap 18 2024-09-21T09:38:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:36 smithi017 ceph-mon[40737]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:38:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:36 smithi017 ceph-mon[40737]: pgmap v149: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.4 MiB/s wr, 33 op/s 2024-09-21T09:38:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:36 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2291592845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:36 smithi017 ceph-mon[29380]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:38:36.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:36 smithi017 ceph-mon[29380]: pgmap v149: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.4 MiB/s wr, 33 op/s 2024-09-21T09:38:36.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:36 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2291592845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:36.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:36 smithi134 ceph-mon[33541]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:38:36.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:36 smithi134 ceph-mon[33541]: pgmap v149: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.4 MiB/s wr, 33 op/s 2024-09-21T09:38:36.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:36 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2291592845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:37.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:38:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:38:37.666 INFO:tasks.rados.rados.0.smithi017.stdout:206: done (1 left) 2024-09-21T09:38:37.666 INFO:tasks.rados.rados.0.smithi017.stdout:207: rollback oid 12 current snap is 19 2024-09-21T09:38:37.666 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 12 to 11 2024-09-21T09:38:37.666 INFO:tasks.rados.rados.0.smithi017.stdout:208: snap_remove snap 13 2024-09-21T09:38:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:37 smithi017 ceph-mon[29380]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:37 smithi017 ceph-mon[29380]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:37 smithi017 ceph-mon[29380]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:38:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:37 smithi017 ceph-mon[29380]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:38:37.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:37 smithi017 ceph-mon[40737]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:37 smithi017 ceph-mon[40737]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:37 smithi017 ceph-mon[40737]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:38:37.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:37 smithi017 ceph-mon[40737]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:38:37.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:37 smithi134 ceph-mon[33541]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:37 smithi134 ceph-mon[33541]: from='client.14988 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:37 smithi134 ceph-mon[33541]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:38:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:37 smithi134 ceph-mon[33541]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:208: done (2 left) 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:209: delete oid 18 current snap is 19 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:205: finishing write tid 1 to smithi01785587-49 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:205: finishing write tid 2 to smithi01785587-49 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:205: finishing write tid 3 to smithi01785587-49 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:205: finishing write tid 4 to smithi01785587-49 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:205: finishing write tid 5 to smithi01785587-49 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 50 (ObjNum 113 snap 19 seq_num 113) dirty exists 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:205: left oid 49 (ObjNum 113 snap 19 seq_num 113) 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:207: finishing rollback tid 0 to smithi01785587-12 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:207: finishing rollback tid 1 to smithi01785587-12 2024-09-21T09:38:38.667 INFO:tasks.rados.rados.0.smithi017.stdout:207: finishing rollback tid 2 to smithi01785587-12 2024-09-21T09:38:38.668 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 82 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-09-21T09:38:38.673 INFO:tasks.rados.rados.0.smithi017.stdout:205: done (2 left) 2024-09-21T09:38:38.673 INFO:tasks.rados.rados.0.smithi017.stdout:207: done (1 left) 2024-09-21T09:38:38.673 INFO:tasks.rados.rados.0.smithi017.stdout:209: done (0 left) 2024-09-21T09:38:38.673 INFO:tasks.rados.rados.0.smithi017.stdout:210: read oid 19 snap -1 2024-09-21T09:38:38.673 INFO:tasks.rados.rados.0.smithi017.stdout:210: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-21T09:38:38.673 INFO:tasks.rados.rados.0.smithi017.stdout:211: snap_remove snap 11 2024-09-21T09:38:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:38 smithi017 ceph-mon[29380]: pgmap v152: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.4 MiB/s wr, 32 op/s 2024-09-21T09:38:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:38 smithi017 ceph-mon[29380]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:38:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:38 smithi017 ceph-mon[40737]: pgmap v152: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.4 MiB/s wr, 32 op/s 2024-09-21T09:38:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:38 smithi017 ceph-mon[40737]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:38:38.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:38 smithi134 ceph-mon[33541]: pgmap v152: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.4 MiB/s wr, 32 op/s 2024-09-21T09:38:38.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:38 smithi134 ceph-mon[33541]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:38:39.243 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:39.679 INFO:tasks.rados.rados.0.smithi017.stdout:211: done (1 left) 2024-09-21T09:38:39.679 INFO:tasks.rados.rados.0.smithi017.stdout:212: delete oid 2 current snap is 19 2024-09-21T09:38:39.679 INFO:tasks.rados.rados.0.smithi017.stdout:210: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-21T09:38:39.757 INFO:tasks.rados.rados.0.smithi017.stdout:210: done (1 left) 2024-09-21T09:38:39.757 INFO:tasks.rados.rados.0.smithi017.stdout:212: done (0 left) 2024-09-21T09:38:39.757 INFO:tasks.rados.rados.0.smithi017.stdout:213: snap_create 2024-09-21T09:38:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:39 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:39 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3278021423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:39 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:39 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3278021423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:39.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:39 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:39.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:39 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3278021423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:40.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:38:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:40] "GET /metrics HTTP/1.1" 200 243142 "" "Prometheus/2.43.0" 2024-09-21T09:38:40.690 INFO:tasks.rados.rados.0.smithi017.stdout:213: done (0 left) 2024-09-21T09:38:40.690 INFO:tasks.rados.rados.0.smithi017.stdout:214: delete oid 46 current snap is 20 2024-09-21T09:38:40.693 INFO:tasks.rados.rados.0.smithi017.stdout:214: done (0 left) 2024-09-21T09:38:40.693 INFO:tasks.rados.rados.0.smithi017.stdout:215: write oid 16 current snap is 20 2024-09-21T09:38:40.693 INFO:tasks.rados.rados.0.smithi017.stdout:215: seq_num 117 ranges {766419=549428,2013452=574695,3346843=427649} 2024-09-21T09:38:40.700 INFO:tasks.rados.rados.0.smithi017.stdout:215: writing smithi01785587-16 from 766419 to 1315847 tid 1 2024-09-21T09:38:40.707 INFO:tasks.rados.rados.0.smithi017.stdout:215: writing smithi01785587-16 from 2013452 to 2588147 tid 2 2024-09-21T09:38:40.714 INFO:tasks.rados.rados.0.smithi017.stdout:215: writing smithi01785587-16 from 3346843 to 3774492 tid 3 2024-09-21T09:38:40.714 INFO:tasks.rados.rados.0.smithi017.stdout:216: read oid 30 snap -1 2024-09-21T09:38:40.714 INFO:tasks.rados.rados.0.smithi017.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:38:40.714 INFO:tasks.rados.rados.0.smithi017.stdout:217: snap_remove snap 16 2024-09-21T09:38:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:40 smithi017 ceph-mon[29380]: from='client.24893 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:40 smithi017 ceph-mon[29380]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:40 smithi017 ceph-mon[29380]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:38:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:40 smithi017 ceph-mon[29380]: pgmap v155: 89 pgs: 89 active+clean; 158 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 795 KiB/s wr, 12 op/s 2024-09-21T09:38:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:40 smithi017 ceph-mon[40737]: from='client.24893 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:40 smithi017 ceph-mon[40737]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:40 smithi017 ceph-mon[40737]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:38:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:40 smithi017 ceph-mon[40737]: pgmap v155: 89 pgs: 89 active+clean; 158 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 795 KiB/s wr, 12 op/s 2024-09-21T09:38:40.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:40 smithi134 ceph-mon[33541]: from='client.24893 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:40.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:40 smithi134 ceph-mon[33541]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:40.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:40 smithi134 ceph-mon[33541]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:38:40.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:40 smithi134 ceph-mon[33541]: pgmap v155: 89 pgs: 89 active+clean; 158 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 795 KiB/s wr, 12 op/s 2024-09-21T09:38:41.694 INFO:tasks.rados.rados.0.smithi017.stdout:215: finishing write tid 1 to smithi01785587-16 2024-09-21T09:38:41.695 INFO:tasks.rados.rados.0.smithi017.stdout:215: finishing write tid 2 to smithi01785587-16 2024-09-21T09:38:41.695 INFO:tasks.rados.rados.0.smithi017.stdout:215: finishing write tid 3 to smithi01785587-16 2024-09-21T09:38:41.695 INFO:tasks.rados.rados.0.smithi017.stdout:215: finishing write tid 4 to smithi01785587-16 2024-09-21T09:38:41.695 INFO:tasks.rados.rados.0.smithi017.stdout:215: finishing write tid 5 to smithi01785587-16 2024-09-21T09:38:41.695 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 90 (ObjNum 117 snap 20 seq_num 117) dirty exists 2024-09-21T09:38:41.695 INFO:tasks.rados.rados.0.smithi017.stdout:215: left oid 16 (ObjNum 117 snap 20 seq_num 117) 2024-09-21T09:38:41.695 INFO:tasks.rados.rados.0.smithi017.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:38:41.792 INFO:tasks.rados.rados.0.smithi017.stdout:215: done (2 left) 2024-09-21T09:38:41.792 INFO:tasks.rados.rados.0.smithi017.stdout:216: done (1 left) 2024-09-21T09:38:41.793 INFO:tasks.rados.rados.0.smithi017.stdout:217: done (0 left) 2024-09-21T09:38:41.793 INFO:tasks.rados.rados.0.smithi017.stdout:218: snap_remove snap 17 2024-09-21T09:38:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:41 smithi017 ceph-mon[29380]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:38:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:41 smithi017 ceph-mon[40737]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:38:41.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:41 smithi134 ceph-mon[33541]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:38:42.129 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:42.665 INFO:tasks.rados.rados.0.smithi017.stdout:218: done (0 left) 2024-09-21T09:38:42.665 INFO:tasks.rados.rados.0.smithi017.stdout:219: write oid 11 current snap is 20 2024-09-21T09:38:42.665 INFO:tasks.rados.rados.0.smithi017.stdout:219: seq_num 118 ranges {783226=624677,2071934=434706,2974266=638921} 2024-09-21T09:38:42.674 INFO:tasks.rados.rados.0.smithi017.stdout:219: writing smithi01785587-11 from 783226 to 1407903 tid 1 2024-09-21T09:38:42.680 INFO:tasks.rados.rados.0.smithi017.stdout:219: writing smithi01785587-11 from 2071934 to 2506640 tid 2 2024-09-21T09:38:42.686 INFO:tasks.rados.rados.0.smithi017.stdout:219: writing smithi01785587-11 from 2974266 to 3613187 tid 3 2024-09-21T09:38:42.686 INFO:tasks.rados.rados.0.smithi017.stdout:220: rollback oid 12 current snap is 20 2024-09-21T09:38:42.686 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 12 to 19 2024-09-21T09:38:42.686 INFO:tasks.rados.rados.0.smithi017.stdout:219: finishing write tid 1 to smithi01785587-11 2024-09-21T09:38:42.686 INFO:tasks.rados.rados.0.smithi017.stdout:219: finishing write tid 2 to smithi01785587-11 2024-09-21T09:38:42.686 INFO:tasks.rados.rados.0.smithi017.stdout:221: snap_create 2024-09-21T09:38:42.689 INFO:tasks.rados.rados.0.smithi017.stdout:220: finishing rollback tid 0 to smithi01785587-12 2024-09-21T09:38:42.689 INFO:tasks.rados.rados.0.smithi017.stdout:220: finishing rollback tid 1 to smithi01785587-12 2024-09-21T09:38:42.689 INFO:tasks.rados.rados.0.smithi017.stdout:220: finishing rollback tid 2 to smithi01785587-12 2024-09-21T09:38:42.689 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 94 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-09-21T09:38:42.689 INFO:tasks.rados.rados.0.smithi017.stdout:219: finishing write tid 3 to smithi01785587-11 2024-09-21T09:38:42.690 INFO:tasks.rados.rados.0.smithi017.stdout:219: finishing write tid 4 to smithi01785587-11 2024-09-21T09:38:42.690 INFO:tasks.rados.rados.0.smithi017.stdout:219: finishing write tid 5 to smithi01785587-11 2024-09-21T09:38:42.690 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 123 (ObjNum 118 snap 20 seq_num 118) dirty exists 2024-09-21T09:38:42.690 INFO:tasks.rados.rados.0.smithi017.stdout:219: left oid 11 (ObjNum 118 snap 20 seq_num 118) 2024-09-21T09:38:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:42 smithi017 ceph-mon[29380]: pgmap v157: 89 pgs: 89 active+clean; 158 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 790 KiB/s wr, 12 op/s 2024-09-21T09:38:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:42 smithi017 ceph-mon[29380]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:38:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:42 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/604516983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:42 smithi017 ceph-mon[29380]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:38:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:42 smithi017 ceph-mon[40737]: pgmap v157: 89 pgs: 89 active+clean; 158 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 790 KiB/s wr, 12 op/s 2024-09-21T09:38:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:42 smithi017 ceph-mon[40737]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:38:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:42 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/604516983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:42 smithi017 ceph-mon[40737]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:38:42.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:42 smithi134 ceph-mon[33541]: pgmap v157: 89 pgs: 89 active+clean; 158 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 790 KiB/s wr, 12 op/s 2024-09-21T09:38:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:42 smithi134 ceph-mon[33541]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:38:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:42 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/604516983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:42 smithi134 ceph-mon[33541]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:38:43.667 INFO:tasks.rados.rados.0.smithi017.stdout:219: done (2 left) 2024-09-21T09:38:43.667 INFO:tasks.rados.rados.0.smithi017.stdout:220: done (1 left) 2024-09-21T09:38:43.667 INFO:tasks.rados.rados.0.smithi017.stdout:221: done (0 left) 2024-09-21T09:38:43.667 INFO:tasks.rados.rados.0.smithi017.stdout:222: delete oid 28 current snap is 21 2024-09-21T09:38:43.668 INFO:tasks.rados.rados.0.smithi017.stdout:222: done (0 left) 2024-09-21T09:38:43.668 INFO:tasks.rados.rados.0.smithi017.stdout:223: snap_create 2024-09-21T09:38:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:43 smithi017 ceph-mon[29380]: from='client.24911 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:43 smithi017 ceph-mon[29380]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:43 smithi017 ceph-mon[29380]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:38:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:43 smithi017 ceph-mon[40737]: from='client.24911 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:43 smithi017 ceph-mon[40737]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:43 smithi017 ceph-mon[40737]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:38:43.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:43 smithi134 ceph-mon[33541]: from='client.24911 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:43.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:43 smithi134 ceph-mon[33541]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:43.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:43 smithi134 ceph-mon[33541]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:38:44.671 INFO:tasks.rados.rados.0.smithi017.stdout:223: done (0 left) 2024-09-21T09:38:44.671 INFO:tasks.rados.rados.0.smithi017.stdout:224: read oid 49 snap -1 2024-09-21T09:38:44.671 INFO:tasks.rados.rados.0.smithi017.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2024-09-21T09:38:44.672 INFO:tasks.rados.rados.0.smithi017.stdout:225: read oid 23 snap 21 2024-09-21T09:38:44.672 INFO:tasks.rados.rados.0.smithi017.stdout:225: expect deleted 2024-09-21T09:38:44.672 INFO:tasks.rados.rados.0.smithi017.stdout:226: write oid 7 current snap is 22 2024-09-21T09:38:44.672 INFO:tasks.rados.rados.0.smithi017.stdout:226: seq_num 120 ranges {416840=775175,1672545=344295} 2024-09-21T09:38:44.678 INFO:tasks.rados.rados.0.smithi017.stdout:226: writing smithi01785587-7 from 416840 to 1192015 tid 1 2024-09-21T09:38:44.683 INFO:tasks.rados.rados.0.smithi017.stdout:226: writing smithi01785587-7 from 1672545 to 2016840 tid 2 2024-09-21T09:38:44.683 INFO:tasks.rados.rados.0.smithi017.stdout:227: snap_create 2024-09-21T09:38:44.683 INFO:tasks.rados.rados.0.smithi017.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2024-09-21T09:38:44.870 INFO:tasks.rados.rados.0.smithi017.stdout:226: finishing write tid 1 to smithi01785587-7 2024-09-21T09:38:44.870 INFO:tasks.rados.rados.0.smithi017.stdout:226: finishing write tid 2 to smithi01785587-7 2024-09-21T09:38:44.870 INFO:tasks.rados.rados.0.smithi017.stdout:226: finishing write tid 3 to smithi01785587-7 2024-09-21T09:38:44.870 INFO:tasks.rados.rados.0.smithi017.stdout:226: finishing write tid 4 to smithi01785587-7 2024-09-21T09:38:44.870 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 67 (ObjNum 120 snap 22 seq_num 120) dirty exists 2024-09-21T09:38:44.870 INFO:tasks.rados.rados.0.smithi017.stdout:226: left oid 7 (ObjNum 120 snap 22 seq_num 120) 2024-09-21T09:38:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:44 smithi017 ceph-mon[29380]: pgmap v161: 89 pgs: 89 active+clean; 158 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:44 smithi017 ceph-mon[29380]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:38:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:44 smithi017 ceph-mon[40737]: pgmap v161: 89 pgs: 89 active+clean; 158 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:44 smithi017 ceph-mon[40737]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:38:44.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:44 smithi134 ceph-mon[33541]: pgmap v161: 89 pgs: 89 active+clean; 158 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:44.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:44 smithi134 ceph-mon[33541]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:38:45.015 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:45.673 INFO:tasks.rados.rados.0.smithi017.stdout:224: done (3 left) 2024-09-21T09:38:45.674 INFO:tasks.rados.rados.0.smithi017.stdout:225: done (2 left) 2024-09-21T09:38:45.674 INFO:tasks.rados.rados.0.smithi017.stdout:226: done (1 left) 2024-09-21T09:38:45.674 INFO:tasks.rados.rados.0.smithi017.stdout:227: done (0 left) 2024-09-21T09:38:45.674 INFO:tasks.rados.rados.0.smithi017.stdout:228: rollback oid 47 current snap is 23 2024-09-21T09:38:45.674 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 47 to 19 2024-09-21T09:38:45.674 INFO:tasks.rados.rados.0.smithi017.stdout:229: delete oid 28 current snap is 23 2024-09-21T09:38:45.677 INFO:tasks.rados.rados.0.smithi017.stdout:228: finishing rollback tid 0 to smithi01785587-47 2024-09-21T09:38:45.677 INFO:tasks.rados.rados.0.smithi017.stdout:229: done (1 left) 2024-09-21T09:38:45.677 INFO:tasks.rados.rados.0.smithi017.stdout:230: snap_remove snap 21 2024-09-21T09:38:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:45 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2439513432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:45 smithi017 ceph-mon[40737]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:45 smithi017 ceph-mon[40737]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:38:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:45 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2439513432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:45.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:45 smithi017 ceph-mon[29380]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:45.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:45 smithi017 ceph-mon[29380]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:38:45.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:45 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2439513432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:45.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:45 smithi134 ceph-mon[33541]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:45 smithi134 ceph-mon[33541]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:38:46.678 INFO:tasks.rados.rados.0.smithi017.stdout:230: done (1 left) 2024-09-21T09:38:46.678 INFO:tasks.rados.rados.0.smithi017.stdout:231: write oid 28 current snap is 23 2024-09-21T09:38:46.678 INFO:tasks.rados.rados.0.smithi017.stdout:228: finishing rollback tid 1 to smithi01785587-47 2024-09-21T09:38:46.678 INFO:tasks.rados.rados.0.smithi017.stdout:231: seq_num 122 ranges {450454=642026,1505862=544592} 2024-09-21T09:38:46.684 INFO:tasks.rados.rados.0.smithi017.stdout:231: writing smithi01785587-28 from 450454 to 1092480 tid 1 2024-09-21T09:38:46.690 INFO:tasks.rados.rados.0.smithi017.stdout:231: writing smithi01785587-28 from 1505862 to 2050454 tid 2 2024-09-21T09:38:46.690 INFO:tasks.rados.rados.0.smithi017.stdout:232: read oid 40 snap -1 2024-09-21T09:38:46.690 INFO:tasks.rados.rados.0.smithi017.stdout:232: expect (ObjNum 60 snap 0 seq_num 60) 2024-09-21T09:38:46.690 INFO:tasks.rados.rados.0.smithi017.stdout:233: read oid 46 snap -1 2024-09-21T09:38:46.690 INFO:tasks.rados.rados.0.smithi017.stdout:233: expect deleted 2024-09-21T09:38:46.690 INFO:tasks.rados.rados.0.smithi017.stdout:234: read oid 15 snap -1 2024-09-21T09:38:46.691 INFO:tasks.rados.rados.0.smithi017.stdout:234: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:38:46.691 INFO:tasks.rados.rados.0.smithi017.stdout:235: delete oid 27 current snap is 23 2024-09-21T09:38:46.691 INFO:tasks.rados.rados.0.smithi017.stdout:228: finishing rollback tid 2 to smithi01785587-47 2024-09-21T09:38:46.691 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 98 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-21T09:38:46.691 INFO:tasks.rados.rados.0.smithi017.stdout:231: finishing write tid 1 to smithi01785587-28 2024-09-21T09:38:46.693 INFO:tasks.rados.rados.0.smithi017.stdout:231: finishing write tid 2 to smithi01785587-28 2024-09-21T09:38:46.694 INFO:tasks.rados.rados.0.smithi017.stdout:231: finishing write tid 3 to smithi01785587-28 2024-09-21T09:38:46.694 INFO:tasks.rados.rados.0.smithi017.stdout:228: done (5 left) 2024-09-21T09:38:46.694 INFO:tasks.rados.rados.0.smithi017.stdout:235: done (4 left) 2024-09-21T09:38:46.694 INFO:tasks.rados.rados.0.smithi017.stdout:236: write oid 32 current snap is 23 2024-09-21T09:38:46.695 INFO:tasks.rados.rados.0.smithi017.stdout:236: seq_num 124 ranges {484068=792524,1906473=177595} 2024-09-21T09:38:46.703 INFO:tasks.rados.rados.0.smithi017.stdout:236: writing smithi01785587-32 from 484068 to 1276592 tid 1 2024-09-21T09:38:46.709 INFO:tasks.rados.rados.0.smithi017.stdout:236: writing smithi01785587-32 from 1906473 to 2084068 tid 2 2024-09-21T09:38:46.710 INFO:tasks.rados.rados.0.smithi017.stdout:237: write oid 1 current snap is 23 2024-09-21T09:38:46.710 INFO:tasks.rados.rados.0.smithi017.stdout:231: finishing write tid 4 to smithi01785587-28 2024-09-21T09:38:46.710 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 99 (ObjNum 122 snap 23 seq_num 122) dirty exists 2024-09-21T09:38:46.710 INFO:tasks.rados.rados.0.smithi017.stdout:231: left oid 28 (ObjNum 122 snap 23 seq_num 122) 2024-09-21T09:38:46.710 INFO:tasks.rados.rados.0.smithi017.stdout:232: expect (ObjNum 60 snap 0 seq_num 60) 2024-09-21T09:38:46.854 INFO:tasks.rados.rados.0.smithi017.stdout:234: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:38:46.930 INFO:tasks.rados.rados.0.smithi017.stdout:236: finishing write tid 1 to smithi01785587-32 2024-09-21T09:38:46.930 INFO:tasks.rados.rados.0.smithi017.stdout:236: finishing write tid 2 to smithi01785587-32 2024-09-21T09:38:46.930 INFO:tasks.rados.rados.0.smithi017.stdout:236: finishing write tid 3 to smithi01785587-32 2024-09-21T09:38:46.930 INFO:tasks.rados.rados.0.smithi017.stdout:236: finishing write tid 4 to smithi01785587-32 2024-09-21T09:38:46.930 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 127 (ObjNum 124 snap 23 seq_num 124) dirty exists 2024-09-21T09:38:46.931 INFO:tasks.rados.rados.0.smithi017.stdout:236: left oid 32 (ObjNum 124 snap 23 seq_num 124) 2024-09-21T09:38:46.931 INFO:tasks.rados.rados.0.smithi017.stdout:237: seq_num 125 ranges {500875=467773,1564955=535920} 2024-09-21T09:38:46.935 INFO:tasks.rados.rados.0.smithi017.stdout:237: writing smithi01785587-1 from 500875 to 968648 tid 1 2024-09-21T09:38:46.942 INFO:tasks.rados.rados.0.smithi017.stdout:237: writing smithi01785587-1 from 1564955 to 2100875 tid 2 2024-09-21T09:38:46.942 INFO:tasks.rados.rados.0.smithi017.stdout:231: done (5 left) 2024-09-21T09:38:46.942 INFO:tasks.rados.rados.0.smithi017.stdout:232: done (4 left) 2024-09-21T09:38:46.942 INFO:tasks.rados.rados.0.smithi017.stdout:233: done (3 left) 2024-09-21T09:38:46.942 INFO:tasks.rados.rados.0.smithi017.stdout:234: done (2 left) 2024-09-21T09:38:46.942 INFO:tasks.rados.rados.0.smithi017.stdout:236: done (1 left) 2024-09-21T09:38:46.942 INFO:tasks.rados.rados.0.smithi017.stdout:238: snap_create 2024-09-21T09:38:46.942 INFO:tasks.rados.rados.0.smithi017.stdout:237: finishing write tid 1 to smithi01785587-1 2024-09-21T09:38:46.944 INFO:tasks.rados.rados.0.smithi017.stdout:237: finishing write tid 2 to smithi01785587-1 2024-09-21T09:38:46.944 INFO:tasks.rados.rados.0.smithi017.stdout:237: finishing write tid 3 to smithi01785587-1 2024-09-21T09:38:46.945 INFO:tasks.rados.rados.0.smithi017.stdout:237: finishing write tid 4 to smithi01785587-1 2024-09-21T09:38:46.945 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 89 (ObjNum 125 snap 23 seq_num 125) dirty exists 2024-09-21T09:38:46.945 INFO:tasks.rados.rados.0.smithi017.stdout:237: left oid 1 (ObjNum 125 snap 23 seq_num 125) 2024-09-21T09:38:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:46 smithi017 ceph-mon[29380]: from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:46 smithi017 ceph-mon[29380]: pgmap v164: 89 pgs: 89 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:46 smithi017 ceph-mon[29380]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:38:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:46 smithi017 ceph-mon[40737]: from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:46 smithi017 ceph-mon[40737]: pgmap v164: 89 pgs: 89 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:46 smithi017 ceph-mon[40737]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:38:46.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:46 smithi134 ceph-mon[33541]: from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:46.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:46 smithi134 ceph-mon[33541]: pgmap v164: 89 pgs: 89 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:46.968 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:46 smithi134 ceph-mon[33541]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:38:47.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:38:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:38:47.669 INFO:tasks.rados.rados.0.smithi017.stdout:237: done (1 left) 2024-09-21T09:38:47.669 INFO:tasks.rados.rados.0.smithi017.stdout:238: done (0 left) 2024-09-21T09:38:47.669 INFO:tasks.rados.rados.0.smithi017.stdout:239: read oid 44 snap -1 2024-09-21T09:38:47.669 INFO:tasks.rados.rados.0.smithi017.stdout:239: expect (ObjNum 98 snap 16 seq_num 98) 2024-09-21T09:38:47.669 INFO:tasks.rados.rados.0.smithi017.stdout:240: write oid 22 current snap is 24 2024-09-21T09:38:47.669 INFO:tasks.rados.rados.0.smithi017.stdout:240: seq_num 126 ranges {517682=543022,1623437=494245} 2024-09-21T09:38:47.674 INFO:tasks.rados.rados.0.smithi017.stdout:240: writing smithi01785587-22 from 517682 to 1060704 tid 1 2024-09-21T09:38:47.681 INFO:tasks.rados.rados.0.smithi017.stdout:240: writing smithi01785587-22 from 1623437 to 2117682 tid 2 2024-09-21T09:38:47.681 INFO:tasks.rados.rados.0.smithi017.stdout:241: snap_create 2024-09-21T09:38:47.681 INFO:tasks.rados.rados.0.smithi017.stdout:240: finishing write tid 1 to smithi01785587-22 2024-09-21T09:38:47.681 INFO:tasks.rados.rados.0.smithi017.stdout:239: expect (ObjNum 98 snap 16 seq_num 98) 2024-09-21T09:38:47.892 INFO:tasks.rados.rados.0.smithi017.stdout:240: finishing write tid 2 to smithi01785587-22 2024-09-21T09:38:47.892 INFO:tasks.rados.rados.0.smithi017.stdout:240: finishing write tid 3 to smithi01785587-22 2024-09-21T09:38:47.892 INFO:tasks.rados.rados.0.smithi017.stdout:240: finishing write tid 4 to smithi01785587-22 2024-09-21T09:38:47.893 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 103 (ObjNum 126 snap 24 seq_num 126) dirty exists 2024-09-21T09:38:47.893 INFO:tasks.rados.rados.0.smithi017.stdout:240: left oid 22 (ObjNum 126 snap 24 seq_num 126) 2024-09-21T09:38:47.935 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:47.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:47 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3115490062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:47.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:47 smithi134 ceph-mon[33541]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:38:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:47 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3115490062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:47 smithi017 ceph-mon[40737]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:38:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:47 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3115490062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:47 smithi017 ceph-mon[29380]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:38:48.673 INFO:tasks.rados.rados.0.smithi017.stdout:239: done (2 left) 2024-09-21T09:38:48.673 INFO:tasks.rados.rados.0.smithi017.stdout:240: done (1 left) 2024-09-21T09:38:48.673 INFO:tasks.rados.rados.0.smithi017.stdout:241: done (0 left) 2024-09-21T09:38:48.673 INFO:tasks.rados.rados.0.smithi017.stdout:242: rollback oid 28 current snap is 25 2024-09-21T09:38:48.673 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 28 to 24 2024-09-21T09:38:48.673 INFO:tasks.rados.rados.0.smithi017.stdout:243: delete oid 29 current snap is 25 2024-09-21T09:38:48.677 INFO:tasks.rados.rados.0.smithi017.stdout:242: finishing rollback tid 0 to smithi01785587-28 2024-09-21T09:38:48.678 INFO:tasks.rados.rados.0.smithi017.stdout:242: finishing rollback tid 1 to smithi01785587-28 2024-09-21T09:38:48.678 INFO:tasks.rados.rados.0.smithi017.stdout:242: finishing rollback tid 2 to smithi01785587-28 2024-09-21T09:38:48.678 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 107 (ObjNum 122 snap 23 seq_num 122) dirty exists 2024-09-21T09:38:48.679 INFO:tasks.rados.rados.0.smithi017.stdout:242: done (1 left) 2024-09-21T09:38:48.679 INFO:tasks.rados.rados.0.smithi017.stdout:243: done (0 left) 2024-09-21T09:38:48.679 INFO:tasks.rados.rados.0.smithi017.stdout:244: rollback oid 3 current snap is 25 2024-09-21T09:38:48.679 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 24 2024-09-21T09:38:48.679 INFO:tasks.rados.rados.0.smithi017.stdout:245: rollback oid 41 current snap is 25 2024-09-21T09:38:48.680 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 20 2024-09-21T09:38:48.680 INFO:tasks.rados.rados.0.smithi017.stdout:246: snap_create 2024-09-21T09:38:48.682 INFO:tasks.rados.rados.0.smithi017.stdout:245: finishing rollback tid 0 to smithi01785587-41 2024-09-21T09:38:48.682 INFO:tasks.rados.rados.0.smithi017.stdout:245: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:38:48.682 INFO:tasks.rados.rados.0.smithi017.stdout:245: finishing rollback tid 2 to smithi01785587-41 2024-09-21T09:38:48.682 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 54 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-09-21T09:38:48.683 INFO:tasks.rados.rados.0.smithi017.stdout:244: finishing rollback tid 0 to smithi01785587-3 2024-09-21T09:38:48.684 INFO:tasks.rados.rados.0.smithi017.stdout:244: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:38:48.684 INFO:tasks.rados.rados.0.smithi017.stdout:244: finishing rollback tid 2 to smithi01785587-3 2024-09-21T09:38:48.684 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 92 (ObjNum 70 snap 9 seq_num 70) dirty exists 2024-09-21T09:38:48.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:48 smithi134 ceph-mon[33541]: pgmap v167: 89 pgs: 89 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:48.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:48 smithi134 ceph-mon[33541]: from='client.15054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:48.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:48 smithi134 ceph-mon[33541]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:38:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:48 smithi017 ceph-mon[29380]: pgmap v167: 89 pgs: 89 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:48 smithi017 ceph-mon[29380]: from='client.15054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:48 smithi017 ceph-mon[29380]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:38:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:48 smithi017 ceph-mon[40737]: pgmap v167: 89 pgs: 89 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:48 smithi017 ceph-mon[40737]: from='client.15054 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:48 smithi017 ceph-mon[40737]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:38:49.674 INFO:tasks.rados.rados.0.smithi017.stdout:244: done (2 left) 2024-09-21T09:38:49.674 INFO:tasks.rados.rados.0.smithi017.stdout:245: done (1 left) 2024-09-21T09:38:49.675 INFO:tasks.rados.rados.0.smithi017.stdout:246: done (0 left) 2024-09-21T09:38:49.675 INFO:tasks.rados.rados.0.smithi017.stdout:247: snap_create 2024-09-21T09:38:49.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:49 smithi134 ceph-mon[33541]: from='client.24953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:49.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:49 smithi134 ceph-mon[33541]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:38:50.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:49 smithi017 ceph-mon[29380]: from='client.24953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:50.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:49 smithi017 ceph-mon[29380]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:38:50.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:49 smithi017 ceph-mon[40737]: from='client.24953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:50.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:49 smithi017 ceph-mon[40737]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:38:50.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:38:50 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:50] "GET /metrics HTTP/1.1" 200 243142 "" "Prometheus/2.43.0" 2024-09-21T09:38:50.680 INFO:tasks.rados.rados.0.smithi017.stdout:247: done (0 left) 2024-09-21T09:38:50.680 INFO:tasks.rados.rados.0.smithi017.stdout:248: write oid 48 current snap is 27 2024-09-21T09:38:50.680 INFO:tasks.rados.rados.0.smithi017.stdout:248: seq_num 128 ranges {551296=693520,2024048=127248} 2024-09-21T09:38:50.686 INFO:tasks.rados.rados.0.smithi017.stdout:248: writing smithi01785587-48 from 551296 to 1244816 tid 1 2024-09-21T09:38:50.692 INFO:tasks.rados.rados.0.smithi017.stdout:248: writing smithi01785587-48 from 2024048 to 2151296 tid 2 2024-09-21T09:38:50.692 INFO:tasks.rados.rados.0.smithi017.stdout:249: write oid 44 current snap is 27 2024-09-21T09:38:50.692 INFO:tasks.rados.rados.0.smithi017.stdout:248: finishing write tid 1 to smithi01785587-48 2024-09-21T09:38:50.692 INFO:tasks.rados.rados.0.smithi017.stdout:249: seq_num 129 ranges {568103=768769,2082530=85573} 2024-09-21T09:38:50.699 INFO:tasks.rados.rados.0.smithi017.stdout:249: writing smithi01785587-44 from 568103 to 1336872 tid 1 2024-09-21T09:38:50.704 INFO:tasks.rados.rados.0.smithi017.stdout:249: writing smithi01785587-44 from 2082530 to 2168103 tid 2 2024-09-21T09:38:50.704 INFO:tasks.rados.rados.0.smithi017.stdout:250: snap_remove snap 26 2024-09-21T09:38:50.815 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:50 smithi134 ceph-mon[33541]: pgmap v170: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:38:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:50 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2010203055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:50 smithi134 ceph-mon[33541]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:38:51.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:50 smithi017 ceph-mon[40737]: pgmap v170: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:38:51.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:50 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2010203055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:51.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:50 smithi017 ceph-mon[40737]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:38:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:50 smithi017 ceph-mon[29380]: pgmap v170: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:38:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:50 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2010203055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:50 smithi017 ceph-mon[29380]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:38:51.681 INFO:tasks.rados.rados.0.smithi017.stdout:250: done (2 left) 2024-09-21T09:38:51.681 INFO:tasks.rados.rados.0.smithi017.stdout:251: write oid 8 current snap is 27 2024-09-21T09:38:51.681 INFO:tasks.rados.rados.0.smithi017.stdout:248: finishing write tid 2 to smithi01785587-48 2024-09-21T09:38:51.682 INFO:tasks.rados.rados.0.smithi017.stdout:251: seq_num 130 ranges {584910=560371,1857365=327545} 2024-09-21T09:38:51.687 INFO:tasks.rados.rados.0.smithi017.stdout:251: writing smithi01785587-8 from 584910 to 1145281 tid 1 2024-09-21T09:38:51.693 INFO:tasks.rados.rados.0.smithi017.stdout:251: writing smithi01785587-8 from 1857365 to 2184910 tid 2 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:252: snap_create 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:248: finishing write tid 3 to smithi01785587-48 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:248: finishing write tid 4 to smithi01785587-48 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 57 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:248: left oid 48 (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:249: finishing write tid 1 to smithi01785587-44 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:249: finishing write tid 2 to smithi01785587-44 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:249: finishing write tid 3 to smithi01785587-44 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:249: finishing write tid 4 to smithi01785587-44 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 96 (ObjNum 129 snap 27 seq_num 129) dirty exists 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:249: left oid 44 (ObjNum 129 snap 27 seq_num 129) 2024-09-21T09:38:51.694 INFO:tasks.rados.rados.0.smithi017.stdout:251: finishing write tid 1 to smithi01785587-8 2024-09-21T09:38:51.695 INFO:tasks.rados.rados.0.smithi017.stdout:251: finishing write tid 2 to smithi01785587-8 2024-09-21T09:38:51.695 INFO:tasks.rados.rados.0.smithi017.stdout:251: finishing write tid 3 to smithi01785587-8 2024-09-21T09:38:51.696 INFO:tasks.rados.rados.0.smithi017.stdout:251: finishing write tid 4 to smithi01785587-8 2024-09-21T09:38:51.696 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 131 (ObjNum 130 snap 27 seq_num 130) dirty exists 2024-09-21T09:38:51.696 INFO:tasks.rados.rados.0.smithi017.stdout:251: left oid 8 (ObjNum 130 snap 27 seq_num 130) 2024-09-21T09:38:51.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:51 smithi134 ceph-mon[33541]: from='client.24965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:51.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:51 smithi134 ceph-mon[33541]: from='client.24971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:51.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:51 smithi134 ceph-mon[33541]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:38:52.051 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:51 smithi017 ceph-mon[29380]: from='client.24965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:52.051 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:51 smithi017 ceph-mon[29380]: from='client.24971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:52.051 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:51 smithi017 ceph-mon[29380]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:38:52.051 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:51 smithi017 ceph-mon[40737]: from='client.24965 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:52.051 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:51 smithi017 ceph-mon[40737]: from='client.24971 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:52.051 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:51 smithi017 ceph-mon[40737]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:38:52.670 INFO:tasks.rados.rados.0.smithi017.stdout:248: done (3 left) 2024-09-21T09:38:52.670 INFO:tasks.rados.rados.0.smithi017.stdout:249: done (2 left) 2024-09-21T09:38:52.670 INFO:tasks.rados.rados.0.smithi017.stdout:251: done (1 left) 2024-09-21T09:38:52.670 INFO:tasks.rados.rados.0.smithi017.stdout:252: done (0 left) 2024-09-21T09:38:52.670 INFO:tasks.rados.rados.0.smithi017.stdout:253: snap_remove snap 27 2024-09-21T09:38:52.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:52 smithi134 ceph-mon[33541]: pgmap v173: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:38:52.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:52 smithi134 ceph-mon[33541]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:38:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:52 smithi017 ceph-mon[40737]: pgmap v173: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:38:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:52 smithi017 ceph-mon[40737]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:38:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:52 smithi017 ceph-mon[29380]: pgmap v173: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:38:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:52 smithi017 ceph-mon[29380]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:38:53.671 INFO:tasks.rados.rados.0.smithi017.stdout:253: done (0 left) 2024-09-21T09:38:53.671 INFO:tasks.rados.rados.0.smithi017.stdout:254: read oid 29 snap -1 2024-09-21T09:38:53.671 INFO:tasks.rados.rados.0.smithi017.stdout:254: expect deleted 2024-09-21T09:38:53.672 INFO:tasks.rados.rados.0.smithi017.stdout:255: read oid 15 snap -1 2024-09-21T09:38:53.672 INFO:tasks.rados.rados.0.smithi017.stdout:255: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:38:53.672 INFO:tasks.rados.rados.0.smithi017.stdout:256: delete oid 36 current snap is 28 2024-09-21T09:38:53.675 INFO:tasks.rados.rados.0.smithi017.stdout:256: done (2 left) 2024-09-21T09:38:53.675 INFO:tasks.rados.rados.0.smithi017.stdout:257: delete oid 11 current snap is 28 2024-09-21T09:38:53.676 INFO:tasks.rados.rados.0.smithi017.stdout:255: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:38:53.709 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:53.754 INFO:tasks.rados.rados.0.smithi017.stdout:254: done (2 left) 2024-09-21T09:38:53.755 INFO:tasks.rados.rados.0.smithi017.stdout:255: done (1 left) 2024-09-21T09:38:53.755 INFO:tasks.rados.rados.0.smithi017.stdout:257: done (0 left) 2024-09-21T09:38:53.755 INFO:tasks.rados.rados.0.smithi017.stdout:258: read oid 29 snap -1 2024-09-21T09:38:53.755 INFO:tasks.rados.rados.0.smithi017.stdout:258: expect deleted 2024-09-21T09:38:53.755 INFO:tasks.rados.rados.0.smithi017.stdout:259: write oid 19 current snap is 28 2024-09-21T09:38:53.755 INFO:tasks.rados.rados.0.smithi017.stdout:259: seq_num 133 ranges {635331=786118,1916458=318873} 2024-09-21T09:38:53.762 INFO:tasks.rados.rados.0.smithi017.stdout:259: writing smithi01785587-19 from 635331 to 1421449 tid 1 2024-09-21T09:38:53.766 INFO:tasks.rados.rados.0.smithi017.stdout:259: writing smithi01785587-19 from 1916458 to 2235331 tid 2 2024-09-21T09:38:53.767 INFO:tasks.rados.rados.0.smithi017.stdout:260: snap_remove snap 19 2024-09-21T09:38:53.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:53 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1178683889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:53.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:53 smithi134 ceph-mon[33541]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:38:53.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:53 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:53 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1178683889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:53 smithi017 ceph-mon[29380]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:38:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:53 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:54.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:53 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1178683889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:53 smithi017 ceph-mon[40737]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:38:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:53 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:54.715 INFO:tasks.rados.rados.0.smithi017.stdout:259: finishing write tid 1 to smithi01785587-19 2024-09-21T09:38:54.716 INFO:tasks.rados.rados.0.smithi017.stdout:259: finishing write tid 2 to smithi01785587-19 2024-09-21T09:38:54.716 INFO:tasks.rados.rados.0.smithi017.stdout:259: finishing write tid 3 to smithi01785587-19 2024-09-21T09:38:54.716 INFO:tasks.rados.rados.0.smithi017.stdout:259: finishing write tid 4 to smithi01785587-19 2024-09-21T09:38:54.716 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 63 (ObjNum 133 snap 28 seq_num 133) dirty exists 2024-09-21T09:38:54.716 INFO:tasks.rados.rados.0.smithi017.stdout:259: left oid 19 (ObjNum 133 snap 28 seq_num 133) 2024-09-21T09:38:54.721 INFO:tasks.rados.rados.0.smithi017.stdout:258: done (2 left) 2024-09-21T09:38:54.721 INFO:tasks.rados.rados.0.smithi017.stdout:259: done (1 left) 2024-09-21T09:38:54.721 INFO:tasks.rados.rados.0.smithi017.stdout:260: done (0 left) 2024-09-21T09:38:54.721 INFO:tasks.rados.rados.0.smithi017.stdout:261: write oid 4 current snap is 28 2024-09-21T09:38:54.721 INFO:tasks.rados.rados.0.smithi017.stdout:261: seq_num 134 ranges {652138=461367,1574940=677198} 2024-09-21T09:38:54.723 INFO:tasks.rados.rados.0.smithi017.stdout:261: writing smithi01785587-4 from 652138 to 1113505 tid 1 2024-09-21T09:38:54.729 INFO:tasks.rados.rados.0.smithi017.stdout:261: writing smithi01785587-4 from 1574940 to 2252138 tid 2 2024-09-21T09:38:54.730 INFO:tasks.rados.rados.0.smithi017.stdout:262: read oid 39 snap -1 2024-09-21T09:38:54.730 INFO:tasks.rados.rados.0.smithi017.stdout:262: expect (ObjNum 84 snap 10 seq_num 84) 2024-09-21T09:38:54.730 INFO:tasks.rados.rados.0.smithi017.stdout:263: write oid 8 current snap is 28 2024-09-21T09:38:54.730 INFO:tasks.rados.rados.0.smithi017.stdout:261: finishing write tid 1 to smithi01785587-4 2024-09-21T09:38:54.730 INFO:tasks.rados.rados.0.smithi017.stdout:263: seq_num 135 ranges {668945=536616,1633422=491363} 2024-09-21T09:38:54.736 INFO:tasks.rados.rados.0.smithi017.stdout:263: writing smithi01785587-8 from 668945 to 1205561 tid 1 2024-09-21T09:38:54.742 INFO:tasks.rados.rados.0.smithi017.stdout:263: writing smithi01785587-8 from 1633422 to 2124785 tid 2 2024-09-21T09:38:54.742 INFO:tasks.rados.rados.0.smithi017.stdout:264: read oid 14 snap -1 2024-09-21T09:38:54.742 INFO:tasks.rados.rados.0.smithi017.stdout:264: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:38:54.742 INFO:tasks.rados.rados.0.smithi017.stdout:265: read oid 20 snap -1 2024-09-21T09:38:54.742 INFO:tasks.rados.rados.0.smithi017.stdout:265: expect deleted 2024-09-21T09:38:54.742 INFO:tasks.rados.rados.0.smithi017.stdout:266: write oid 19 current snap is 28 2024-09-21T09:38:54.742 INFO:tasks.rados.rados.0.smithi017.stdout:266: seq_num 136 ranges {685752=611865,1975551=310201} 2024-09-21T09:38:54.749 INFO:tasks.rados.rados.0.smithi017.stdout:266: writing smithi01785587-19 from 685752 to 1297617 tid 1 2024-09-21T09:38:54.755 INFO:tasks.rados.rados.0.smithi017.stdout:266: writing smithi01785587-19 from 1975551 to 2285752 tid 2 2024-09-21T09:38:54.755 INFO:tasks.rados.rados.0.smithi017.stdout:267: rollback oid 18 current snap is 28 2024-09-21T09:38:54.755 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 18 to 24 2024-09-21T09:38:54.755 INFO:tasks.rados.rados.0.smithi017.stdout:261: finishing write tid 2 to smithi01785587-4 2024-09-21T09:38:54.755 INFO:tasks.rados.rados.0.smithi017.stdout:261: finishing write tid 3 to smithi01785587-4 2024-09-21T09:38:54.755 INFO:tasks.rados.rados.0.smithi017.stdout:261: finishing write tid 4 to smithi01785587-4 2024-09-21T09:38:54.755 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 114 (ObjNum 134 snap 28 seq_num 134) dirty exists 2024-09-21T09:38:54.755 INFO:tasks.rados.rados.0.smithi017.stdout:261: left oid 4 (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:38:54.756 INFO:tasks.rados.rados.0.smithi017.stdout:262: expect (ObjNum 84 snap 10 seq_num 84) 2024-09-21T09:38:54.871 INFO:tasks.rados.rados.0.smithi017.stdout:263: finishing write tid 1 to smithi01785587-8 2024-09-21T09:38:54.871 INFO:tasks.rados.rados.0.smithi017.stdout:263: finishing write tid 2 to smithi01785587-8 2024-09-21T09:38:54.871 INFO:tasks.rados.rados.0.smithi017.stdout:263: finishing write tid 3 to smithi01785587-8 2024-09-21T09:38:54.871 INFO:tasks.rados.rados.0.smithi017.stdout:263: finishing write tid 4 to smithi01785587-8 2024-09-21T09:38:54.871 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 144 (ObjNum 135 snap 28 seq_num 135) dirty exists 2024-09-21T09:38:54.871 INFO:tasks.rados.rados.0.smithi017.stdout:263: left oid 8 (ObjNum 135 snap 28 seq_num 135) 2024-09-21T09:38:54.871 INFO:tasks.rados.rados.0.smithi017.stdout:264: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:38:54.946 INFO:tasks.rados.rados.0.smithi017.stdout:266: finishing write tid 1 to smithi01785587-19 2024-09-21T09:38:54.946 INFO:tasks.rados.rados.0.smithi017.stdout:267: finishing rollback tid 1 to smithi01785587-18 2024-09-21T09:38:54.946 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 0 (ObjNum 129 snap 0 seq_num 2399568584) dirty dne 2024-09-21T09:38:54.946 INFO:tasks.rados.rados.0.smithi017.stdout:266: finishing write tid 2 to smithi01785587-19 2024-09-21T09:38:54.946 INFO:tasks.rados.rados.0.smithi017.stdout:266: finishing write tid 3 to smithi01785587-19 2024-09-21T09:38:54.946 INFO:tasks.rados.rados.0.smithi017.stdout:266: finishing write tid 4 to smithi01785587-19 2024-09-21T09:38:54.946 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 70 (ObjNum 136 snap 28 seq_num 136) dirty exists 2024-09-21T09:38:54.946 INFO:tasks.rados.rados.0.smithi017.stdout:266: left oid 19 (ObjNum 136 snap 28 seq_num 136) 2024-09-21T09:38:54.946 INFO:tasks.rados.rados.0.smithi017.stdout:261: done (6 left) 2024-09-21T09:38:54.947 INFO:tasks.rados.rados.0.smithi017.stdout:262: done (5 left) 2024-09-21T09:38:54.947 INFO:tasks.rados.rados.0.smithi017.stdout:263: done (4 left) 2024-09-21T09:38:54.947 INFO:tasks.rados.rados.0.smithi017.stdout:264: done (3 left) 2024-09-21T09:38:54.947 INFO:tasks.rados.rados.0.smithi017.stdout:265: done (2 left) 2024-09-21T09:38:54.947 INFO:tasks.rados.rados.0.smithi017.stdout:266: done (1 left) 2024-09-21T09:38:54.947 INFO:tasks.rados.rados.0.smithi017.stdout:267: done (0 left) 2024-09-21T09:38:54.947 INFO:tasks.rados.rados.0.smithi017.stdout:268: write oid 14 current snap is 28 2024-09-21T09:38:54.947 INFO:tasks.rados.rados.0.smithi017.stdout:268: seq_num 137 ranges {702559=403467,1750386=495032} 2024-09-21T09:38:54.952 INFO:tasks.rados.rados.0.smithi017.stdout:268: writing smithi01785587-14 from 702559 to 1106026 tid 1 2024-09-21T09:38:54.958 INFO:tasks.rados.rados.0.smithi017.stdout:268: writing smithi01785587-14 from 1750386 to 2245418 tid 2 2024-09-21T09:38:54.958 INFO:tasks.rados.rados.0.smithi017.stdout:269: read oid 32 snap -1 2024-09-21T09:38:54.958 INFO:tasks.rados.rados.0.smithi017.stdout:269: expect (ObjNum 124 snap 23 seq_num 124) 2024-09-21T09:38:54.958 INFO:tasks.rados.rados.0.smithi017.stdout:270: rollback oid 8 current snap is 28 2024-09-21T09:38:54.958 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 24 2024-09-21T09:38:54.958 INFO:tasks.rados.rados.0.smithi017.stdout:268: finishing write tid 1 to smithi01785587-14 2024-09-21T09:38:54.959 INFO:tasks.rados.rados.0.smithi017.stdout:271: snap_create 2024-09-21T09:38:54.960 INFO:tasks.rados.rados.0.smithi017.stdout:268: finishing write tid 2 to smithi01785587-14 2024-09-21T09:38:54.960 INFO:tasks.rados.rados.0.smithi017.stdout:268: finishing write tid 3 to smithi01785587-14 2024-09-21T09:38:54.961 INFO:tasks.rados.rados.0.smithi017.stdout:268: finishing write tid 4 to smithi01785587-14 2024-09-21T09:38:54.961 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 106 (ObjNum 137 snap 28 seq_num 137) dirty exists 2024-09-21T09:38:54.961 INFO:tasks.rados.rados.0.smithi017.stdout:268: left oid 14 (ObjNum 137 snap 28 seq_num 137) 2024-09-21T09:38:54.962 INFO:tasks.rados.rados.0.smithi017.stdout:269: expect (ObjNum 124 snap 23 seq_num 124) 2024-09-21T09:38:54.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:54 smithi134 ceph-mon[33541]: pgmap v176: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:54.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:54 smithi134 ceph-mon[33541]: from='client.24983 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:54.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:54 smithi134 ceph-mon[33541]: from='client.15096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:55.088 INFO:tasks.rados.rados.0.smithi017.stdout:270: finishing rollback tid 0 to smithi01785587-8 2024-09-21T09:38:55.088 INFO:tasks.rados.rados.0.smithi017.stdout:270: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:38:55.088 INFO:tasks.rados.rados.0.smithi017.stdout:270: finishing rollback tid 2 to smithi01785587-8 2024-09-21T09:38:55.088 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 147 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-21T09:38:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:54 smithi017 ceph-mon[29380]: pgmap v176: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:54 smithi017 ceph-mon[29380]: from='client.24983 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:54 smithi017 ceph-mon[29380]: from='client.15096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:54 smithi017 ceph-mon[40737]: pgmap v176: 89 pgs: 89 active+clean; 130 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:38:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:54 smithi017 ceph-mon[40737]: from='client.24983 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:54 smithi017 ceph-mon[40737]: from='client.15096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:55.726 INFO:tasks.rados.rados.0.smithi017.stdout:268: done (3 left) 2024-09-21T09:38:55.726 INFO:tasks.rados.rados.0.smithi017.stdout:269: done (2 left) 2024-09-21T09:38:55.726 INFO:tasks.rados.rados.0.smithi017.stdout:270: done (1 left) 2024-09-21T09:38:55.726 INFO:tasks.rados.rados.0.smithi017.stdout:271: done (0 left) 2024-09-21T09:38:55.726 INFO:tasks.rados.rados.0.smithi017.stdout:272: snap_create 2024-09-21T09:38:55.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:55 smithi134 ceph-mon[33541]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:38:56.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:55 smithi017 ceph-mon[29380]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:38:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:55 smithi017 ceph-mon[40737]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:38:56.589 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:56.729 INFO:tasks.rados.rados.0.smithi017.stdout:272: done (0 left) 2024-09-21T09:38:56.729 INFO:tasks.rados.rados.0.smithi017.stdout:273: write oid 31 current snap is 30 2024-09-21T09:38:56.729 INFO:tasks.rados.rados.0.smithi017.stdout:273: seq_num 138 ranges {719366=478716,1808868=510498} 2024-09-21T09:38:56.735 INFO:tasks.rados.rados.0.smithi017.stdout:273: writing smithi01785587-31 from 719366 to 1198082 tid 1 2024-09-21T09:38:56.742 INFO:tasks.rados.rados.0.smithi017.stdout:273: writing smithi01785587-31 from 1808868 to 2319366 tid 2 2024-09-21T09:38:56.742 INFO:tasks.rados.rados.0.smithi017.stdout:274: delete oid 38 current snap is 30 2024-09-21T09:38:56.742 INFO:tasks.rados.rados.0.smithi017.stdout:273: finishing write tid 1 to smithi01785587-31 2024-09-21T09:38:56.744 INFO:tasks.rados.rados.0.smithi017.stdout:274: done (1 left) 2024-09-21T09:38:56.744 INFO:tasks.rados.rados.0.smithi017.stdout:275: rollback oid 6 current snap is 30 2024-09-21T09:38:56.745 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 25 2024-09-21T09:38:56.745 INFO:tasks.rados.rados.0.smithi017.stdout:276: snap_create 2024-09-21T09:38:56.745 INFO:tasks.rados.rados.0.smithi017.stdout:273: finishing write tid 2 to smithi01785587-31 2024-09-21T09:38:56.745 INFO:tasks.rados.rados.0.smithi017.stdout:273: finishing write tid 3 to smithi01785587-31 2024-09-21T09:38:56.745 INFO:tasks.rados.rados.0.smithi017.stdout:273: finishing write tid 4 to smithi01785587-31 2024-09-21T09:38:56.745 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 110 (ObjNum 138 snap 30 seq_num 138) dirty exists 2024-09-21T09:38:56.746 INFO:tasks.rados.rados.0.smithi017.stdout:273: left oid 31 (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:38:56.746 INFO:tasks.rados.rados.0.smithi017.stdout:275: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:38:56.747 INFO:tasks.rados.rados.0.smithi017.stdout:275: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:38:56.747 INFO:tasks.rados.rados.0.smithi017.stdout:275: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:38:56.747 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 95 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-21T09:38:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:56 smithi017 ceph-mon[40737]: pgmap v178: 89 pgs: 89 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 685 KiB/s rd, 893 KiB/s wr, 7 op/s 2024-09-21T09:38:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:56 smithi017 ceph-mon[40737]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:38:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:56 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2715928817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:56.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:56 smithi017 ceph-mon[29380]: pgmap v178: 89 pgs: 89 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 685 KiB/s rd, 893 KiB/s wr, 7 op/s 2024-09-21T09:38:56.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:56 smithi017 ceph-mon[29380]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:38:56.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:56 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2715928817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:56.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:56 smithi134 ceph-mon[33541]: pgmap v178: 89 pgs: 89 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 685 KiB/s rd, 893 KiB/s wr, 7 op/s 2024-09-21T09:38:56.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:56 smithi134 ceph-mon[33541]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:38:56.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:56 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2715928817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:56.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:38:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[35604]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:38:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:38:57.672 INFO:tasks.rados.rados.0.smithi017.stdout:273: done (2 left) 2024-09-21T09:38:57.672 INFO:tasks.rados.rados.0.smithi017.stdout:275: done (1 left) 2024-09-21T09:38:57.672 INFO:tasks.rados.rados.0.smithi017.stdout:276: done (0 left) 2024-09-21T09:38:57.672 INFO:tasks.rados.rados.0.smithi017.stdout:277: write oid 19 current snap is 31 2024-09-21T09:38:57.672 INFO:tasks.rados.rados.0.smithi017.stdout:277: seq_num 140 ranges {752980=629214,1809479=543501} 2024-09-21T09:38:57.679 INFO:tasks.rados.rados.0.smithi017.stdout:277: writing smithi01785587-19 from 752980 to 1382194 tid 1 2024-09-21T09:38:57.684 INFO:tasks.rados.rados.0.smithi017.stdout:277: writing smithi01785587-19 from 1809479 to 2352980 tid 2 2024-09-21T09:38:57.684 INFO:tasks.rados.rados.0.smithi017.stdout:278: snap_remove snap 29 2024-09-21T09:38:57.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:57 smithi134 ceph-mon[33541]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:57.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:57 smithi134 ceph-mon[33541]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:38:57.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:57 smithi134 ceph-mon[33541]: from='client.25007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:57 smithi134 ceph-mon[33541]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:38:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:57 smithi017 ceph-mon[29380]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:57 smithi017 ceph-mon[29380]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:38:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:57 smithi017 ceph-mon[29380]: from='client.25007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:57 smithi017 ceph-mon[29380]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:38:58.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:57 smithi017 ceph-mon[40737]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:58.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:57 smithi017 ceph-mon[40737]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:38:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:57 smithi017 ceph-mon[40737]: from='client.25007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:57 smithi017 ceph-mon[40737]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:38:58.675 INFO:tasks.rados.rados.0.smithi017.stdout:278: done (1 left) 2024-09-21T09:38:58.676 INFO:tasks.rados.rados.0.smithi017.stdout:279: delete oid 33 current snap is 31 2024-09-21T09:38:58.676 INFO:tasks.rados.rados.0.smithi017.stdout:277: finishing write tid 1 to smithi01785587-19 2024-09-21T09:38:58.676 INFO:tasks.rados.rados.0.smithi017.stdout:277: finishing write tid 2 to smithi01785587-19 2024-09-21T09:38:58.676 INFO:tasks.rados.rados.0.smithi017.stdout:277: finishing write tid 3 to smithi01785587-19 2024-09-21T09:38:58.676 INFO:tasks.rados.rados.0.smithi017.stdout:277: finishing write tid 4 to smithi01785587-19 2024-09-21T09:38:58.676 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 74 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-09-21T09:38:58.676 INFO:tasks.rados.rados.0.smithi017.stdout:277: left oid 19 (ObjNum 140 snap 31 seq_num 140) 2024-09-21T09:38:58.680 INFO:tasks.rados.rados.0.smithi017.stdout:277: done (1 left) 2024-09-21T09:38:58.680 INFO:tasks.rados.rados.0.smithi017.stdout:279: done (0 left) 2024-09-21T09:38:58.680 INFO:tasks.rados.rados.0.smithi017.stdout:280: snap_remove snap 24 2024-09-21T09:38:58.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:58 smithi134 ceph-mon[33541]: pgmap v182: 89 pgs: 89 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 687 KiB/s rd, 896 KiB/s wr, 7 op/s 2024-09-21T09:38:58.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:58 smithi134 ceph-mon[33541]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:38:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:58 smithi017 ceph-mon[29380]: pgmap v182: 89 pgs: 89 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 687 KiB/s rd, 896 KiB/s wr, 7 op/s 2024-09-21T09:38:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:58 smithi017 ceph-mon[29380]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:38:59.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:58 smithi017 ceph-mon[40737]: pgmap v182: 89 pgs: 89 active+clean; 133 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 687 KiB/s rd, 896 KiB/s wr, 7 op/s 2024-09-21T09:38:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:58 smithi017 ceph-mon[40737]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:38:59.471 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:38:59.676 INFO:tasks.rados.rados.0.smithi017.stdout:280: done (0 left) 2024-09-21T09:38:59.677 INFO:tasks.rados.rados.0.smithi017.stdout:281: rollback oid 49 current snap is 31 2024-09-21T09:38:59.677 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 22 2024-09-21T09:38:59.677 INFO:tasks.rados.rados.0.smithi017.stdout:282: snap_remove snap 25 2024-09-21T09:38:59.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:59 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/643826593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:38:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:38:59 smithi134 ceph-mon[33541]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:39:00.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:59 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/643826593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:00.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:38:59 smithi017 ceph-mon[40737]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:39:00.032 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:59 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/643826593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:00.032 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:38:59 smithi017 ceph-mon[29380]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:39:00.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:00 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:00] "GET /metrics HTTP/1.1" 200 243171 "" "Prometheus/2.43.0" 2024-09-21T09:39:00.682 INFO:tasks.rados.rados.0.smithi017.stdout:282: done (1 left) 2024-09-21T09:39:00.682 INFO:tasks.rados.rados.0.smithi017.stdout:283: read oid 9 snap -1 2024-09-21T09:39:00.682 INFO:tasks.rados.rados.0.smithi017.stdout:283: expect (ObjNum 91 snap 15 seq_num 91) 2024-09-21T09:39:00.682 INFO:tasks.rados.rados.0.smithi017.stdout:284: snap_create 2024-09-21T09:39:00.682 INFO:tasks.rados.rados.0.smithi017.stdout:281: finishing rollback tid 0 to smithi01785587-49 2024-09-21T09:39:00.682 INFO:tasks.rados.rados.0.smithi017.stdout:281: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:39:00.682 INFO:tasks.rados.rados.0.smithi017.stdout:281: finishing rollback tid 2 to smithi01785587-49 2024-09-21T09:39:00.682 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 71 (ObjNum 113 snap 19 seq_num 113) dirty exists 2024-09-21T09:39:00.694 INFO:tasks.rados.rados.0.smithi017.stdout:283: expect (ObjNum 91 snap 15 seq_num 91) 2024-09-21T09:39:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:00 smithi134 ceph-mon[33541]: from='client.25019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:01.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:00 smithi134 ceph-mon[33541]: pgmap v185: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:39:01.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:00 smithi134 ceph-mon[33541]: from='client.25025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:01.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:00 smithi134 ceph-mon[33541]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:39:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:00 smithi017 ceph-mon[29380]: from='client.25019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:00 smithi017 ceph-mon[29380]: pgmap v185: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:39:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:00 smithi017 ceph-mon[29380]: from='client.25025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:00 smithi017 ceph-mon[29380]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:39:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:00 smithi017 ceph-mon[40737]: from='client.25019 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:00 smithi017 ceph-mon[40737]: pgmap v185: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:39:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:00 smithi017 ceph-mon[40737]: from='client.25025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:00 smithi017 ceph-mon[40737]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:39:01.683 INFO:tasks.rados.rados.0.smithi017.stdout:281: done (2 left) 2024-09-21T09:39:01.683 INFO:tasks.rados.rados.0.smithi017.stdout:283: done (1 left) 2024-09-21T09:39:01.684 INFO:tasks.rados.rados.0.smithi017.stdout:284: done (0 left) 2024-09-21T09:39:01.684 INFO:tasks.rados.rados.0.smithi017.stdout:285: snap_create 2024-09-21T09:39:02.356 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:39:02.672 INFO:tasks.rados.rados.0.smithi017.stdout:285: done (0 left) 2024-09-21T09:39:02.672 INFO:tasks.rados.rados.0.smithi017.stdout:286: snap_create 2024-09-21T09:39:02.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:02 smithi134 ceph-mon[33541]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:39:02.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:02 smithi134 ceph-mon[33541]: pgmap v188: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:39:02.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:02 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1888493266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:02.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:02 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:02.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:02 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:02.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:02 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:02.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:02 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:39:02.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:02 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:02.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:02 smithi134 ceph-mon[33541]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:39:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[29380]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:39:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[29380]: pgmap v188: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:39:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1888493266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:39:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[29380]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[40737]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[40737]: pgmap v188: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1888493266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:02 smithi017 ceph-mon[40737]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:39:03.674 INFO:tasks.rados.rados.0.smithi017.stdout:286: done (0 left) 2024-09-21T09:39:03.675 INFO:tasks.rados.rados.0.smithi017.stdout:287: write oid 13 current snap is 34 2024-09-21T09:39:03.675 INFO:tasks.rados.rados.0.smithi017.stdout:287: seq_num 142 ranges {786594=779712,2326443=60151} 2024-09-21T09:39:03.683 INFO:tasks.rados.rados.0.smithi017.stdout:287: writing smithi01785587-13 from 786594 to 1566306 tid 1 2024-09-21T09:39:03.688 INFO:tasks.rados.rados.0.smithi017.stdout:287: writing smithi01785587-13 from 2326443 to 2386594 tid 2 2024-09-21T09:39:03.688 INFO:tasks.rados.rados.0.smithi017.stdout:288: write oid 29 current snap is 34 2024-09-21T09:39:03.688 INFO:tasks.rados.rados.0.smithi017.stdout:288: seq_num 143 ranges {403401=454961,1584925=506039} 2024-09-21T09:39:03.692 INFO:tasks.rados.rados.0.smithi017.stdout:288: writing smithi01785587-29 from 403401 to 858362 tid 1 2024-09-21T09:39:03.699 INFO:tasks.rados.rados.0.smithi017.stdout:288: writing smithi01785587-29 from 1584925 to 2090964 tid 2 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:289: read oid 30 snap -1 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:290: delete oid 22 current snap is 34 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:287: finishing write tid 1 to smithi01785587-13 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:287: finishing write tid 2 to smithi01785587-13 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:287: finishing write tid 3 to smithi01785587-13 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:287: finishing write tid 4 to smithi01785587-13 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 88 (ObjNum 142 snap 34 seq_num 142) dirty exists 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:287: left oid 13 (ObjNum 142 snap 34 seq_num 142) 2024-09-21T09:39:03.700 INFO:tasks.rados.rados.0.smithi017.stdout:288: finishing write tid 1 to smithi01785587-29 2024-09-21T09:39:03.702 INFO:tasks.rados.rados.0.smithi017.stdout:287: done (3 left) 2024-09-21T09:39:03.702 INFO:tasks.rados.rados.0.smithi017.stdout:290: done (2 left) 2024-09-21T09:39:03.702 INFO:tasks.rados.rados.0.smithi017.stdout:291: write oid 7 current snap is 34 2024-09-21T09:39:03.702 INFO:tasks.rados.rados.0.smithi017.stdout:291: seq_num 145 ranges {437015=721812,1701889=509708} 2024-09-21T09:39:03.708 INFO:tasks.rados.rados.0.smithi017.stdout:291: writing smithi01785587-7 from 437015 to 1158827 tid 1 2024-09-21T09:39:03.714 INFO:tasks.rados.rados.0.smithi017.stdout:291: writing smithi01785587-7 from 1701889 to 2211597 tid 2 2024-09-21T09:39:03.715 INFO:tasks.rados.rados.0.smithi017.stdout:292: delete oid 38 current snap is 34 2024-09-21T09:39:03.715 INFO:tasks.rados.rados.0.smithi017.stdout:288: finishing write tid 2 to smithi01785587-29 2024-09-21T09:39:03.715 INFO:tasks.rados.rados.0.smithi017.stdout:288: finishing write tid 3 to smithi01785587-29 2024-09-21T09:39:03.715 INFO:tasks.rados.rados.0.smithi017.stdout:288: finishing write tid 4 to smithi01785587-29 2024-09-21T09:39:03.715 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 82 (ObjNum 143 snap 34 seq_num 143) dirty exists 2024-09-21T09:39:03.715 INFO:tasks.rados.rados.0.smithi017.stdout:288: left oid 29 (ObjNum 143 snap 34 seq_num 143) 2024-09-21T09:39:03.715 INFO:tasks.rados.rados.0.smithi017.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:291: finishing write tid 1 to smithi01785587-7 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:291: finishing write tid 2 to smithi01785587-7 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:291: finishing write tid 3 to smithi01785587-7 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:291: finishing write tid 4 to smithi01785587-7 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 77 (ObjNum 145 snap 34 seq_num 145) dirty exists 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:291: left oid 7 (ObjNum 145 snap 34 seq_num 145) 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:288: done (3 left) 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:289: done (2 left) 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:291: done (1 left) 2024-09-21T09:39:03.800 INFO:tasks.rados.rados.0.smithi017.stdout:292: done (0 left) 2024-09-21T09:39:03.801 INFO:tasks.rados.rados.0.smithi017.stdout:293: write oid 28 current snap is 34 2024-09-21T09:39:03.801 INFO:tasks.rados.rados.0.smithi017.stdout:293: seq_num 147 ranges {470629=472310,1418853=513377} 2024-09-21T09:39:03.804 INFO:tasks.rados.rados.0.smithi017.stdout:293: writing smithi01785587-28 from 470629 to 942939 tid 1 2024-09-21T09:39:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:293: writing smithi01785587-28 from 1418853 to 1932230 tid 2 2024-09-21T09:39:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:294: write oid 26 current snap is 34 2024-09-21T09:39:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:293: finishing write tid 1 to smithi01785587-28 2024-09-21T09:39:03.810 INFO:tasks.rados.rados.0.smithi017.stdout:294: seq_num 148 ranges {487436=547559,1760982=657035} 2024-09-21T09:39:03.816 INFO:tasks.rados.rados.0.smithi017.stdout:294: writing smithi01785587-26 from 487436 to 1034995 tid 1 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:294: writing smithi01785587-26 from 1760982 to 2418017 tid 2 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:295: rollback oid 7 current snap is 34 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 30 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:293: finishing write tid 2 to smithi01785587-28 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:296: rollback oid 24 current snap is 34 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 24 to 20 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:293: finishing write tid 3 to smithi01785587-28 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:297: delete oid 5 current snap is 34 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:293: finishing write tid 4 to smithi01785587-28 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 128 (ObjNum 147 snap 34 seq_num 147) dirty exists 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:293: left oid 28 (ObjNum 147 snap 34 seq_num 147) 2024-09-21T09:39:03.824 INFO:tasks.rados.rados.0.smithi017.stdout:294: finishing write tid 1 to smithi01785587-26 2024-09-21T09:39:03.825 INFO:tasks.rados.rados.0.smithi017.stdout:296: finishing rollback tid 1 to smithi01785587-24 2024-09-21T09:39:03.825 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 0 (ObjNum 33 snap 0 seq_num 2399608336) dirty dne 2024-09-21T09:39:03.825 INFO:tasks.rados.rados.0.smithi017.stdout:295: finishing rollback tid 0 to smithi01785587-7 2024-09-21T09:39:03.826 INFO:tasks.rados.rados.0.smithi017.stdout:295: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:39:03.827 INFO:tasks.rados.rados.0.smithi017.stdout:295: finishing rollback tid 2 to smithi01785587-7 2024-09-21T09:39:03.827 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 80 (ObjNum 120 snap 22 seq_num 120) dirty exists 2024-09-21T09:39:03.827 INFO:tasks.rados.rados.0.smithi017.stdout:294: finishing write tid 2 to smithi01785587-26 2024-09-21T09:39:03.828 INFO:tasks.rados.rados.0.smithi017.stdout:293: done (4 left) 2024-09-21T09:39:03.828 INFO:tasks.rados.rados.0.smithi017.stdout:295: done (3 left) 2024-09-21T09:39:03.828 INFO:tasks.rados.rados.0.smithi017.stdout:296: done (2 left) 2024-09-21T09:39:03.828 INFO:tasks.rados.rados.0.smithi017.stdout:297: done (1 left) 2024-09-21T09:39:03.828 INFO:tasks.rados.rados.0.smithi017.stdout:298: snap_remove snap 28 2024-09-21T09:39:03.939 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:03 smithi134 ceph-mon[33541]: Upgrade: Updating mgr.x 2024-09-21T09:39:03.939 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:03 smithi134 ceph-mon[33541]: Deploying daemon mgr.x on smithi134 2024-09-21T09:39:03.939 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:03 smithi134 ceph-mon[33541]: from='client.25037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:03.939 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:03 smithi134 ceph-mon[33541]: from='client.25043 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:03.940 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:03 smithi134 ceph-mon[33541]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:39:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[40737]: Upgrade: Updating mgr.x 2024-09-21T09:39:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[40737]: Deploying daemon mgr.x on smithi134 2024-09-21T09:39:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[40737]: from='client.25037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[40737]: from='client.25043 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[40737]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:39:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[29380]: Upgrade: Updating mgr.x 2024-09-21T09:39:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[29380]: Deploying daemon mgr.x on smithi134 2024-09-21T09:39:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[29380]: from='client.25037 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[29380]: from='client.25043 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:03 smithi017 ceph-mon[29380]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:39:04.195 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:03 smithi134 systemd[1]: Stopping Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:04.493 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:04 smithi134 podman[68636]: 2024-09-21 09:39:04.193857046 +0000 UTC m=+0.112632047 container died c61de7d532224a9a89517a34a9e8405ebd8e1e677e83690ec54ba5de1a154449 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2) 2024-09-21T09:39:04.694 INFO:tasks.rados.rados.0.smithi017.stdout:298: done (1 left) 2024-09-21T09:39:04.694 INFO:tasks.rados.rados.0.smithi017.stdout:299: snap_remove snap 32 2024-09-21T09:39:04.948 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:04 smithi134 ceph-mon[33541]: pgmap v191: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:04.948 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:04 smithi134 podman[68636]: 2024-09-21 09:39:04.695942649 +0000 UTC m=+0.614717649 container cleanup c61de7d532224a9a89517a34a9e8405ebd8e1e677e83690ec54ba5de1a154449 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T09:39:04.949 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:04 smithi134 bash[68636]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:39:04.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:04 smithi017 ceph-mon[29380]: pgmap v191: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:04 smithi017 ceph-mon[40737]: pgmap v191: 89 pgs: 89 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:05.233 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:39:05.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:04 smithi134 podman[68653]: 2024-09-21 09:39:04.947407657 +0000 UTC m=+0.748282379 container remove c61de7d532224a9a89517a34a9e8405ebd8e1e677e83690ec54ba5de1a154449 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0) 2024-09-21T09:39:05.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:04 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:39:05.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:39:05.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 systemd[1]: Stopped Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:05.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.x.service: Consumed 10.116s CPU time. 2024-09-21T09:39:05.583 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 systemd[1]: Starting Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:05.583 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 podman[68785]: 2024-09-21 09:39:05.427482919 +0000 UTC m=+0.015862205 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:05.584 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 podman[68785]: 2024-09-21 09:39:05.582582261 +0000 UTC m=+0.170961529 container create 7b270a3ec585c2953579e3b64c75c8001885ca7cc5a5631f11eda2c4df20102d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T09:39:05.697 INFO:tasks.rados.rados.0.smithi017.stdout:299: done (1 left) 2024-09-21T09:39:05.697 INFO:tasks.rados.rados.0.smithi017.stdout:300: rollback oid 28 current snap is 34 2024-09-21T09:39:05.697 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 28 to 30 2024-09-21T09:39:05.698 INFO:tasks.rados.rados.0.smithi017.stdout:294: finishing write tid 3 to smithi01785587-26 2024-09-21T09:39:05.698 INFO:tasks.rados.rados.0.smithi017.stdout:294: finishing write tid 4 to smithi01785587-26 2024-09-21T09:39:05.698 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 132 (ObjNum 148 snap 34 seq_num 148) dirty exists 2024-09-21T09:39:05.698 INFO:tasks.rados.rados.0.smithi017.stdout:294: left oid 26 (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:05.698 INFO:tasks.rados.rados.0.smithi017.stdout:294: done (1 left) 2024-09-21T09:39:05.698 INFO:tasks.rados.rados.0.smithi017.stdout:301: snap_create 2024-09-21T09:39:05.701 INFO:tasks.rados.rados.0.smithi017.stdout:300: finishing rollback tid 0 to smithi01785587-28 2024-09-21T09:39:05.701 INFO:tasks.rados.rados.0.smithi017.stdout:300: finishing rollback tid 1 to smithi01785587-28 2024-09-21T09:39:05.701 INFO:tasks.rados.rados.0.smithi017.stdout:300: finishing rollback tid 2 to smithi01785587-28 2024-09-21T09:39:05.701 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 147 (ObjNum 122 snap 23 seq_num 122) dirty exists 2024-09-21T09:39:05.849 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:05 smithi134 ceph-mon[33541]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:39:05.849 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:05 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3657596608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:05.850 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 podman[68785]: 2024-09-21 09:39:05.818516249 +0000 UTC m=+0.406895535 container init 7b270a3ec585c2953579e3b64c75c8001885ca7cc5a5631f11eda2c4df20102d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T09:39:05.850 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 podman[68785]: 2024-09-21 09:39:05.823388237 +0000 UTC m=+0.411767500 container start 7b270a3ec585c2953579e3b64c75c8001885ca7cc5a5631f11eda2c4df20102d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-62da54b) 2024-09-21T09:39:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:05 smithi017 ceph-mon[29380]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:39:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:05 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3657596608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:05.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:05 smithi017 ceph-mon[40737]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:39:05.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:05 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3657596608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:06.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 ceph-mgr[68823]: -- 172.21.15.134:0/2072854208 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562f0d60f0e0 con 0x562f0d5ed400 2024-09-21T09:39:06.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 bash[68785]: 7b270a3ec585c2953579e3b64c75c8001885ca7cc5a5631f11eda2c4df20102d 2024-09-21T09:39:06.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:05 smithi134 systemd[1]: Started Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:06.704 INFO:tasks.rados.rados.0.smithi017.stdout:300: done (1 left) 2024-09-21T09:39:06.704 INFO:tasks.rados.rados.0.smithi017.stdout:301: done (0 left) 2024-09-21T09:39:06.704 INFO:tasks.rados.rados.0.smithi017.stdout:302: write oid 3 current snap is 35 2024-09-21T09:39:06.704 INFO:tasks.rados.rados.0.smithi017.stdout:302: seq_num 150 ranges {521050=698057,1877946=643104} 2024-09-21T09:39:06.710 INFO:tasks.rados.rados.0.smithi017.stdout:302: writing smithi01785587-3 from 521050 to 1219107 tid 1 2024-09-21T09:39:06.718 INFO:tasks.rados.rados.0.smithi017.stdout:302: writing smithi01785587-3 from 1877946 to 2521050 tid 2 2024-09-21T09:39:06.718 INFO:tasks.rados.rados.0.smithi017.stdout:303: read oid 25 snap 30 2024-09-21T09:39:06.718 INFO:tasks.rados.rados.0.smithi017.stdout:303: expect (ObjNum 102 snap 16 seq_num 102) 2024-09-21T09:39:06.718 INFO:tasks.rados.rados.0.smithi017.stdout:304: snap_create 2024-09-21T09:39:06.718 INFO:tasks.rados.rados.0.smithi017.stdout:302: finishing write tid 1 to smithi01785587-3 2024-09-21T09:39:06.720 INFO:tasks.rados.rados.0.smithi017.stdout:302: finishing write tid 2 to smithi01785587-3 2024-09-21T09:39:06.721 INFO:tasks.rados.rados.0.smithi017.stdout:302: finishing write tid 3 to smithi01785587-3 2024-09-21T09:39:06.721 INFO:tasks.rados.rados.0.smithi017.stdout:302: finishing write tid 4 to smithi01785587-3 2024-09-21T09:39:06.721 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 138 (ObjNum 150 snap 35 seq_num 150) dirty exists 2024-09-21T09:39:06.721 INFO:tasks.rados.rados.0.smithi017.stdout:302: left oid 3 (ObjNum 150 snap 35 seq_num 150) 2024-09-21T09:39:06.730 INFO:tasks.rados.rados.0.smithi017.stdout:303: expect (ObjNum 102 snap 16 seq_num 102) 2024-09-21T09:39:06.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:06 smithi134 ceph-mon[33541]: from='client.15162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:06.907 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:06 smithi134 ceph-mon[33541]: from='client.15168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:06.908 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:06 smithi134 ceph-mon[33541]: pgmap v193: 89 pgs: 89 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:06.908 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:06 smithi134 ceph-mon[33541]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:39:06.908 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:06 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:06.908 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:06 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:06.908 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:06 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[29380]: from='client.15162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[29380]: from='client.15168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[29380]: pgmap v193: 89 pgs: 89 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[29380]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:39:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[40737]: from='client.15162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[40737]: from='client.15168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[40737]: pgmap v193: 89 pgs: 89 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[40737]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:39:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:06 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:07.673 INFO:tasks.rados.rados.0.smithi017.stdout:302: done (2 left) 2024-09-21T09:39:07.673 INFO:tasks.rados.rados.0.smithi017.stdout:303: done (1 left) 2024-09-21T09:39:07.673 INFO:tasks.rados.rados.0.smithi017.stdout:304: done (0 left) 2024-09-21T09:39:07.674 INFO:tasks.rados.rados.0.smithi017.stdout:305: snap_remove snap 34 2024-09-21T09:39:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:07 smithi017 ceph-mon[40737]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:39:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:07 smithi017 ceph-mon[40737]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:39:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:07 smithi017 ceph-mon[29380]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:39:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:07 smithi017 ceph-mon[29380]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:39:07.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:07 smithi134 ceph-mon[33541]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:39:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:07 smithi134 ceph-mon[33541]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:39:08.109 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:39:08.677 INFO:tasks.rados.rados.0.smithi017.stdout:305: done (0 left) 2024-09-21T09:39:08.677 INFO:tasks.rados.rados.0.smithi017.stdout:306: read oid 30 snap -1 2024-09-21T09:39:08.677 INFO:tasks.rados.rados.0.smithi017.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:39:08.677 INFO:tasks.rados.rados.0.smithi017.stdout:307: write oid 9 current snap is 36 2024-09-21T09:39:08.677 INFO:tasks.rados.rados.0.smithi017.stdout:307: seq_num 151 ranges {537857=773306,1936428=520715} 2024-09-21T09:39:08.684 INFO:tasks.rados.rados.0.smithi017.stdout:307: writing smithi01785587-9 from 537857 to 1311163 tid 1 2024-09-21T09:39:08.691 INFO:tasks.rados.rados.0.smithi017.stdout:307: writing smithi01785587-9 from 1936428 to 2457143 tid 2 2024-09-21T09:39:08.691 INFO:tasks.rados.rados.0.smithi017.stdout:308: delete oid 1 current snap is 36 2024-09-21T09:39:08.691 INFO:tasks.rados.rados.0.smithi017.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:39:08.778 INFO:tasks.rados.rados.0.smithi017.stdout:307: finishing write tid 1 to smithi01785587-9 2024-09-21T09:39:08.778 INFO:tasks.rados.rados.0.smithi017.stdout:307: finishing write tid 2 to smithi01785587-9 2024-09-21T09:39:08.778 INFO:tasks.rados.rados.0.smithi017.stdout:307: finishing write tid 3 to smithi01785587-9 2024-09-21T09:39:08.778 INFO:tasks.rados.rados.0.smithi017.stdout:307: finishing write tid 4 to smithi01785587-9 2024-09-21T09:39:08.778 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 39 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-09-21T09:39:08.778 INFO:tasks.rados.rados.0.smithi017.stdout:307: left oid 9 (ObjNum 151 snap 36 seq_num 151) 2024-09-21T09:39:08.778 INFO:tasks.rados.rados.0.smithi017.stdout:306: done (2 left) 2024-09-21T09:39:08.778 INFO:tasks.rados.rados.0.smithi017.stdout:307: done (1 left) 2024-09-21T09:39:08.779 INFO:tasks.rados.rados.0.smithi017.stdout:308: done (0 left) 2024-09-21T09:39:08.779 INFO:tasks.rados.rados.0.smithi017.stdout:309: write oid 49 current snap is 36 2024-09-21T09:39:08.779 INFO:tasks.rados.rados.0.smithi017.stdout:309: seq_num 153 ranges {571471=640157,1653392=524384} 2024-09-21T09:39:08.784 INFO:tasks.rados.rados.0.smithi017.stdout:309: writing smithi01785587-49 from 571471 to 1211628 tid 1 2024-09-21T09:39:08.790 INFO:tasks.rados.rados.0.smithi017.stdout:309: writing smithi01785587-49 from 1653392 to 2177776 tid 2 2024-09-21T09:39:08.790 INFO:tasks.rados.rados.0.smithi017.stdout:310: snap_create 2024-09-21T09:39:08.790 INFO:tasks.rados.rados.0.smithi017.stdout:309: finishing write tid 1 to smithi01785587-49 2024-09-21T09:39:08.793 INFO:tasks.rados.rados.0.smithi017.stdout:309: finishing write tid 2 to smithi01785587-49 2024-09-21T09:39:08.793 INFO:tasks.rados.rados.0.smithi017.stdout:309: finishing write tid 3 to smithi01785587-49 2024-09-21T09:39:08.793 INFO:tasks.rados.rados.0.smithi017.stdout:309: finishing write tid 4 to smithi01785587-49 2024-09-21T09:39:08.793 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 90 (ObjNum 153 snap 36 seq_num 153) dirty exists 2024-09-21T09:39:08.793 INFO:tasks.rados.rados.0.smithi017.stdout:309: left oid 49 (ObjNum 153 snap 36 seq_num 153) 2024-09-21T09:39:08.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:08 smithi134 ceph-mon[33541]: pgmap v197: 89 pgs: 89 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:08 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2911669143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:08 smithi134 ceph-mon[33541]: from='client.25076 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:08 smithi134 ceph-mon[33541]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:39:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:08 smithi017 ceph-mon[29380]: pgmap v197: 89 pgs: 89 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:08 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2911669143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:08 smithi017 ceph-mon[29380]: from='client.25076 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:08 smithi017 ceph-mon[29380]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:39:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:08 smithi017 ceph-mon[40737]: pgmap v197: 89 pgs: 89 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:08 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2911669143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:08 smithi017 ceph-mon[40737]: from='client.25076 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:08 smithi017 ceph-mon[40737]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:39:09.710 INFO:tasks.rados.rados.0.smithi017.stdout:309: done (1 left) 2024-09-21T09:39:09.710 INFO:tasks.rados.rados.0.smithi017.stdout:310: done (0 left) 2024-09-21T09:39:09.710 INFO:tasks.rados.rados.0.smithi017.stdout:311: snap_create 2024-09-21T09:39:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:09 smithi017 ceph-mon[29380]: from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:09 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:09 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:09 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:09 smithi017 ceph-mon[40737]: from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:09 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:09 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:09 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:09.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:09 smithi134 ceph-mon[33541]: from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:09.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:09 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:09 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:09 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:10.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:10 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:10] "GET /metrics HTTP/1.1" 200 243215 "" "Prometheus/2.43.0" 2024-09-21T09:39:10.712 INFO:tasks.rados.rados.0.smithi017.stdout:311: done (0 left) 2024-09-21T09:39:10.712 INFO:tasks.rados.rados.0.smithi017.stdout:312: read oid 35 snap -1 2024-09-21T09:39:10.712 INFO:tasks.rados.rados.0.smithi017.stdout:312: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-21T09:39:10.712 INFO:tasks.rados.rados.0.smithi017.stdout:313: delete oid 29 current snap is 38 2024-09-21T09:39:10.714 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:10 smithi134 ceph-mon[33541]: pgmap v199: 89 pgs: 89 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 929 KiB/s wr, 8 op/s 2024-09-21T09:39:10.715 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:10 smithi134 ceph-mon[33541]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:39:10.715 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:10 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:10.717 INFO:tasks.rados.rados.0.smithi017.stdout:313: done (1 left) 2024-09-21T09:39:10.717 INFO:tasks.rados.rados.0.smithi017.stdout:314: snap_remove snap 22 2024-09-21T09:39:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:10 smithi017 ceph-mon[40737]: pgmap v199: 89 pgs: 89 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 929 KiB/s wr, 8 op/s 2024-09-21T09:39:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:10 smithi017 ceph-mon[40737]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:39:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:10 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:10 smithi017 ceph-mon[29380]: pgmap v199: 89 pgs: 89 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 929 KiB/s wr, 8 op/s 2024-09-21T09:39:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:10 smithi017 ceph-mon[29380]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:39:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:10 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:10.999 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:39:11.493 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:39:11] ENGINE Bus STARTING 2024-09-21T09:39:11.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: CherryPy Checker: 2024-09-21T09:39:11.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: The Application mounted at '' has an empty config. 2024-09-21T09:39:11.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:11.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:39:11] ENGINE Serving on http://:::9283 2024-09-21T09:39:11.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:39:11] ENGINE Bus STARTED 2024-09-21T09:39:11.720 INFO:tasks.rados.rados.0.smithi017.stdout:314: done (1 left) 2024-09-21T09:39:11.720 INFO:tasks.rados.rados.0.smithi017.stdout:315: snap_remove snap 30 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2340170136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: Standby manager daemon x restarted 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: Standby manager daemon x started 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:39:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:11.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:11.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:11 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:39:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2340170136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: Standby manager daemon x restarted 2024-09-21T09:39:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: Standby manager daemon x started 2024-09-21T09:39:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2340170136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: from='client.15201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: Standby manager daemon x restarted 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: Standby manager daemon x started 2024-09-21T09:39:12.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:39:12.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:12.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:39:12.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:12.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:12.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:11 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:12.676 INFO:tasks.rados.rados.0.smithi017.stdout:315: done (1 left) 2024-09-21T09:39:12.676 INFO:tasks.rados.rados.0.smithi017.stdout:316: write oid 47 current snap is 38 2024-09-21T09:39:12.676 INFO:tasks.rados.rados.0.smithi017.stdout:316: seq_num 155 ranges {605085=790655,2170356=434729} 2024-09-21T09:39:12.684 INFO:tasks.rados.rados.0.smithi017.stdout:316: writing smithi01785587-47 from 605085 to 1395740 tid 1 2024-09-21T09:39:12.691 INFO:tasks.rados.rados.0.smithi017.stdout:316: writing smithi01785587-47 from 2170356 to 2605085 tid 2 2024-09-21T09:39:12.691 INFO:tasks.rados.rados.0.smithi017.stdout:317: snap_create 2024-09-21T09:39:12.691 INFO:tasks.rados.rados.0.smithi017.stdout:312: expect (ObjNum 34 snap 0 seq_num 34) 2024-09-21T09:39:12.777 INFO:tasks.rados.rados.0.smithi017.stdout:316: finishing write tid 1 to smithi01785587-47 2024-09-21T09:39:12.777 INFO:tasks.rados.rados.0.smithi017.stdout:316: finishing write tid 2 to smithi01785587-47 2024-09-21T09:39:12.777 INFO:tasks.rados.rados.0.smithi017.stdout:316: finishing write tid 3 to smithi01785587-47 2024-09-21T09:39:12.777 INFO:tasks.rados.rados.0.smithi017.stdout:316: finishing write tid 4 to smithi01785587-47 2024-09-21T09:39:12.777 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 110 (ObjNum 155 snap 38 seq_num 155) dirty exists 2024-09-21T09:39:12.777 INFO:tasks.rados.rados.0.smithi017.stdout:316: left oid 47 (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:12.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:12 smithi134 ceph-mon[33541]: from='client.25100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:12.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:12 smithi134 ceph-mon[33541]: pgmap v202: 89 pgs: 89 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 923 KiB/s wr, 8 op/s 2024-09-21T09:39:12.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:12 smithi134 ceph-mon[33541]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:39:12.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:12 smithi134 ceph-mon[33541]: mgrmap e22: y(active, since 3m), standbys: x 2024-09-21T09:39:12.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:12 smithi134 ceph-mon[33541]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:39:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[29380]: from='client.25100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[29380]: pgmap v202: 89 pgs: 89 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 923 KiB/s wr, 8 op/s 2024-09-21T09:39:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[29380]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:39:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[29380]: mgrmap e22: y(active, since 3m), standbys: x 2024-09-21T09:39:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[29380]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:39:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[40737]: from='client.25100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:13.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[40737]: pgmap v202: 89 pgs: 89 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 923 KiB/s wr, 8 op/s 2024-09-21T09:39:13.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[40737]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:39:13.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[40737]: mgrmap e22: y(active, since 3m), standbys: x 2024-09-21T09:39:13.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:12 smithi017 ceph-mon[40737]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:39:13.677 INFO:tasks.rados.rados.0.smithi017.stdout:312: done (2 left) 2024-09-21T09:39:13.677 INFO:tasks.rados.rados.0.smithi017.stdout:316: done (1 left) 2024-09-21T09:39:13.677 INFO:tasks.rados.rados.0.smithi017.stdout:317: done (0 left) 2024-09-21T09:39:13.677 INFO:tasks.rados.rados.0.smithi017.stdout:318: write oid 7 current snap is 39 2024-09-21T09:39:13.677 INFO:tasks.rados.rados.0.smithi017.stdout:318: seq_num 156 ranges {621892=465904,1712485=671711} 2024-09-21T09:39:13.683 INFO:tasks.rados.rados.0.smithi017.stdout:318: writing smithi01785587-7 from 621892 to 1087796 tid 1 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:318: writing smithi01785587-7 from 1712485 to 2384196 tid 2 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:319: rollback oid 50 current snap is 39 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 50 to 36 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:318: finishing write tid 1 to smithi01785587-7 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:320: read oid 36 snap -1 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:320: expect deleted 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:321: rollback oid 44 current snap is 39 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 37 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:322: write oid 17 current snap is 39 2024-09-21T09:39:13.691 INFO:tasks.rados.rados.0.smithi017.stdout:322: seq_num 157 ranges {638699=541153,1770967=531722} 2024-09-21T09:39:13.697 INFO:tasks.rados.rados.0.smithi017.stdout:322: writing smithi01785587-17 from 638699 to 1179852 tid 1 2024-09-21T09:39:13.704 INFO:tasks.rados.rados.0.smithi017.stdout:322: writing smithi01785587-17 from 1770967 to 2302689 tid 2 2024-09-21T09:39:13.704 INFO:tasks.rados.rados.0.smithi017.stdout:323: snap_create 2024-09-21T09:39:13.704 INFO:tasks.rados.rados.0.smithi017.stdout:321: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:39:13.704 INFO:tasks.rados.rados.0.smithi017.stdout:321: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:39:13.704 INFO:tasks.rados.rados.0.smithi017.stdout:321: finishing rollback tid 2 to smithi01785587-44 2024-09-21T09:39:13.704 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 158 (ObjNum 129 snap 27 seq_num 129) dirty exists 2024-09-21T09:39:13.704 INFO:tasks.rados.rados.0.smithi017.stdout:319: finishing rollback tid 0 to smithi01785587-50 2024-09-21T09:39:13.705 INFO:tasks.rados.rados.0.smithi017.stdout:318: finishing write tid 2 to smithi01785587-7 2024-09-21T09:39:13.705 INFO:tasks.rados.rados.0.smithi017.stdout:319: finishing rollback tid 1 to smithi01785587-50 2024-09-21T09:39:13.705 INFO:tasks.rados.rados.0.smithi017.stdout:319: finishing rollback tid 2 to smithi01785587-50 2024-09-21T09:39:13.705 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 111 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-09-21T09:39:13.705 INFO:tasks.rados.rados.0.smithi017.stdout:318: finishing write tid 3 to smithi01785587-7 2024-09-21T09:39:13.705 INFO:tasks.rados.rados.0.smithi017.stdout:318: finishing write tid 4 to smithi01785587-7 2024-09-21T09:39:13.705 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 102 (ObjNum 156 snap 39 seq_num 156) dirty exists 2024-09-21T09:39:13.705 INFO:tasks.rados.rados.0.smithi017.stdout:318: left oid 7 (ObjNum 156 snap 39 seq_num 156) 2024-09-21T09:39:13.705 INFO:tasks.rados.rados.0.smithi017.stdout:322: finishing write tid 1 to smithi01785587-17 2024-09-21T09:39:13.707 INFO:tasks.rados.rados.0.smithi017.stdout:322: finishing write tid 2 to smithi01785587-17 2024-09-21T09:39:13.707 INFO:tasks.rados.rados.0.smithi017.stdout:322: finishing write tid 3 to smithi01785587-17 2024-09-21T09:39:13.707 INFO:tasks.rados.rados.0.smithi017.stdout:322: finishing write tid 4 to smithi01785587-17 2024-09-21T09:39:13.707 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 63 (ObjNum 157 snap 39 seq_num 157) dirty exists 2024-09-21T09:39:13.707 INFO:tasks.rados.rados.0.smithi017.stdout:322: left oid 17 (ObjNum 157 snap 39 seq_num 157) 2024-09-21T09:39:13.894 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:39:13.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:13 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3130468354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:13.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:13 smithi134 ceph-mon[33541]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:39:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:13 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3130468354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:13 smithi017 ceph-mon[29380]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:39:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:13 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3130468354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:13 smithi017 ceph-mon[40737]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:39:14.681 INFO:tasks.rados.rados.0.smithi017.stdout:318: done (5 left) 2024-09-21T09:39:14.681 INFO:tasks.rados.rados.0.smithi017.stdout:319: done (4 left) 2024-09-21T09:39:14.681 INFO:tasks.rados.rados.0.smithi017.stdout:320: done (3 left) 2024-09-21T09:39:14.682 INFO:tasks.rados.rados.0.smithi017.stdout:321: done (2 left) 2024-09-21T09:39:14.682 INFO:tasks.rados.rados.0.smithi017.stdout:322: done (1 left) 2024-09-21T09:39:14.682 INFO:tasks.rados.rados.0.smithi017.stdout:323: done (0 left) 2024-09-21T09:39:14.682 INFO:tasks.rados.rados.0.smithi017.stdout:324: write oid 40 current snap is 40 2024-09-21T09:39:14.682 INFO:tasks.rados.rados.0.smithi017.stdout:324: seq_num 158 ranges {655506=616402,1829449=675380} 2024-09-21T09:39:14.688 INFO:tasks.rados.rados.0.smithi017.stdout:324: writing smithi01785587-40 from 655506 to 1271908 tid 1 2024-09-21T09:39:14.695 INFO:tasks.rados.rados.0.smithi017.stdout:324: writing smithi01785587-40 from 1829449 to 2504829 tid 2 2024-09-21T09:39:14.695 INFO:tasks.rados.rados.0.smithi017.stdout:325: snap_create 2024-09-21T09:39:14.695 INFO:tasks.rados.rados.0.smithi017.stdout:324: finishing write tid 1 to smithi01785587-40 2024-09-21T09:39:14.698 INFO:tasks.rados.rados.0.smithi017.stdout:324: finishing write tid 2 to smithi01785587-40 2024-09-21T09:39:14.698 INFO:tasks.rados.rados.0.smithi017.stdout:324: finishing write tid 3 to smithi01785587-40 2024-09-21T09:39:14.698 INFO:tasks.rados.rados.0.smithi017.stdout:324: finishing write tid 4 to smithi01785587-40 2024-09-21T09:39:14.698 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 88 (ObjNum 158 snap 40 seq_num 158) dirty exists 2024-09-21T09:39:14.699 INFO:tasks.rados.rados.0.smithi017.stdout:324: left oid 40 (ObjNum 158 snap 40 seq_num 158) 2024-09-21T09:39:14.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:14 smithi134 ceph-mon[33541]: pgmap v206: 89 pgs: 89 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:14 smithi134 ceph-mon[33541]: from='client.25112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:14 smithi134 ceph-mon[33541]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:39:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:14 smithi017 ceph-mon[29380]: pgmap v206: 89 pgs: 89 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:14 smithi017 ceph-mon[29380]: from='client.25112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:14 smithi017 ceph-mon[29380]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:39:15.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:14 smithi017 ceph-mon[40737]: pgmap v206: 89 pgs: 89 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:15.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:14 smithi017 ceph-mon[40737]: from='client.25112 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:15.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:14 smithi017 ceph-mon[40737]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:39:15.684 INFO:tasks.rados.rados.0.smithi017.stdout:324: done (1 left) 2024-09-21T09:39:15.684 INFO:tasks.rados.rados.0.smithi017.stdout:325: done (0 left) 2024-09-21T09:39:15.684 INFO:tasks.rados.rados.0.smithi017.stdout:326: write oid 33 current snap is 41 2024-09-21T09:39:15.684 INFO:tasks.rados.rados.0.smithi017.stdout:326: seq_num 159 ranges {672313=691651,1887931=419038} 2024-09-21T09:39:15.691 INFO:tasks.rados.rados.0.smithi017.stdout:326: writing smithi01785587-33 from 672313 to 1363964 tid 1 2024-09-21T09:39:15.697 INFO:tasks.rados.rados.0.smithi017.stdout:326: writing smithi01785587-33 from 1887931 to 2306969 tid 2 2024-09-21T09:39:15.697 INFO:tasks.rados.rados.0.smithi017.stdout:327: read oid 18 snap 23 2024-09-21T09:39:15.697 INFO:tasks.rados.rados.0.smithi017.stdout:327: expect deleted 2024-09-21T09:39:15.697 INFO:tasks.rados.rados.0.smithi017.stdout:328: read oid 48 snap 40 2024-09-21T09:39:15.697 INFO:tasks.rados.rados.0.smithi017.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:39:15.697 INFO:tasks.rados.rados.0.smithi017.stdout:329: write oid 49 current snap is 41 2024-09-21T09:39:15.697 INFO:tasks.rados.rados.0.smithi017.stdout:329: seq_num 160 ranges {689120=483253,1946413=679049} 2024-09-21T09:39:15.703 INFO:tasks.rados.rados.0.smithi017.stdout:329: writing smithi01785587-49 from 689120 to 1172373 tid 1 2024-09-21T09:39:15.711 INFO:tasks.rados.rados.0.smithi017.stdout:329: writing smithi01785587-49 from 1946413 to 2625462 tid 2 2024-09-21T09:39:15.711 INFO:tasks.rados.rados.0.smithi017.stdout:330: snap_create 2024-09-21T09:39:15.711 INFO:tasks.rados.rados.0.smithi017.stdout:326: finishing write tid 1 to smithi01785587-33 2024-09-21T09:39:15.711 INFO:tasks.rados.rados.0.smithi017.stdout:326: finishing write tid 2 to smithi01785587-33 2024-09-21T09:39:15.712 INFO:tasks.rados.rados.0.smithi017.stdout:326: finishing write tid 3 to smithi01785587-33 2024-09-21T09:39:15.712 INFO:tasks.rados.rados.0.smithi017.stdout:326: finishing write tid 4 to smithi01785587-33 2024-09-21T09:39:15.712 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 166 (ObjNum 159 snap 41 seq_num 159) dirty exists 2024-09-21T09:39:15.712 INFO:tasks.rados.rados.0.smithi017.stdout:326: left oid 33 (ObjNum 159 snap 41 seq_num 159) 2024-09-21T09:39:15.712 INFO:tasks.rados.rados.0.smithi017.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:39:15.840 INFO:tasks.rados.rados.0.smithi017.stdout:329: finishing write tid 1 to smithi01785587-49 2024-09-21T09:39:15.840 INFO:tasks.rados.rados.0.smithi017.stdout:329: finishing write tid 2 to smithi01785587-49 2024-09-21T09:39:15.840 INFO:tasks.rados.rados.0.smithi017.stdout:329: finishing write tid 3 to smithi01785587-49 2024-09-21T09:39:15.840 INFO:tasks.rados.rados.0.smithi017.stdout:329: finishing write tid 4 to smithi01785587-49 2024-09-21T09:39:15.840 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 106 (ObjNum 160 snap 41 seq_num 160) dirty exists 2024-09-21T09:39:15.840 INFO:tasks.rados.rados.0.smithi017.stdout:329: left oid 49 (ObjNum 160 snap 41 seq_num 160) 2024-09-21T09:39:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:15 smithi134 ceph-mon[33541]: from='client.15225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:15 smithi134 ceph-mon[33541]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:39:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:15 smithi017 ceph-mon[29380]: from='client.15225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:15 smithi017 ceph-mon[29380]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:39:16.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:15 smithi017 ceph-mon[40737]: from='client.15225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:16.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:15 smithi017 ceph-mon[40737]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:39:16.687 INFO:tasks.rados.rados.0.smithi017.stdout:326: done (4 left) 2024-09-21T09:39:16.687 INFO:tasks.rados.rados.0.smithi017.stdout:327: done (3 left) 2024-09-21T09:39:16.687 INFO:tasks.rados.rados.0.smithi017.stdout:328: done (2 left) 2024-09-21T09:39:16.687 INFO:tasks.rados.rados.0.smithi017.stdout:329: done (1 left) 2024-09-21T09:39:16.687 INFO:tasks.rados.rados.0.smithi017.stdout:330: done (0 left) 2024-09-21T09:39:16.687 INFO:tasks.rados.rados.0.smithi017.stdout:331: snap_remove snap 39 2024-09-21T09:39:16.763 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:39:16.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:16 smithi134 ceph-mon[33541]: pgmap v209: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 843 KiB/s wr, 7 op/s 2024-09-21T09:39:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:16 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1252572727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:16 smithi134 ceph-mon[33541]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:39:16.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:39:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:16 smithi017 ceph-mon[29380]: pgmap v209: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 843 KiB/s wr, 7 op/s 2024-09-21T09:39:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:16 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1252572727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:16 smithi017 ceph-mon[29380]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:39:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:16 smithi017 ceph-mon[40737]: pgmap v209: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 843 KiB/s wr, 7 op/s 2024-09-21T09:39:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:16 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1252572727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:16 smithi017 ceph-mon[40737]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:39:17.677 INFO:tasks.rados.rados.0.smithi017.stdout:331: done (0 left) 2024-09-21T09:39:17.677 INFO:tasks.rados.rados.0.smithi017.stdout:332: rollback oid 33 current snap is 42 2024-09-21T09:39:17.677 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 38 2024-09-21T09:39:17.677 INFO:tasks.rados.rados.0.smithi017.stdout:333: snap_remove snap 33 2024-09-21T09:39:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:17 smithi134 ceph-mon[33541]: from='client.25130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:17 smithi134 ceph-mon[33541]: from='client.15243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:17 smithi134 ceph-mon[33541]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:39:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:17 smithi017 ceph-mon[29380]: from='client.25130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:17 smithi017 ceph-mon[29380]: from='client.15243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:17 smithi017 ceph-mon[29380]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:39:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:17 smithi017 ceph-mon[40737]: from='client.25130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:17 smithi017 ceph-mon[40737]: from='client.15243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:17 smithi017 ceph-mon[40737]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:39:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:333: done (1 left) 2024-09-21T09:39:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:334: rollback oid 2 current snap is 42 2024-09-21T09:39:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 2 to 37 2024-09-21T09:39:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:332: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:39:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:332: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:39:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 169 (ObjNum 129 snap 0 seq_num 2399590640) dirty dne 2024-09-21T09:39:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:332: done (1 left) 2024-09-21T09:39:18.684 INFO:tasks.rados.rados.0.smithi017.stdout:335: write oid 48 current snap is 42 2024-09-21T09:39:18.685 INFO:tasks.rados.rados.0.smithi017.stdout:335: seq_num 161 ranges {705927=558502,2004895=422707} 2024-09-21T09:39:18.692 INFO:tasks.rados.rados.0.smithi017.stdout:335: writing smithi01785587-48 from 705927 to 1264429 tid 1 2024-09-21T09:39:18.698 INFO:tasks.rados.rados.0.smithi017.stdout:335: writing smithi01785587-48 from 2004895 to 2427602 tid 2 2024-09-21T09:39:18.698 INFO:tasks.rados.rados.0.smithi017.stdout:336: snap_create 2024-09-21T09:39:18.698 INFO:tasks.rados.rados.0.smithi017.stdout:334: finishing rollback tid 1 to smithi01785587-2 2024-09-21T09:39:18.698 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 0 (ObjNum 129 snap 0 seq_num 2399661600) dirty dne 2024-09-21T09:39:18.698 INFO:tasks.rados.rados.0.smithi017.stdout:335: finishing write tid 1 to smithi01785587-48 2024-09-21T09:39:18.701 INFO:tasks.rados.rados.0.smithi017.stdout:335: finishing write tid 2 to smithi01785587-48 2024-09-21T09:39:18.701 INFO:tasks.rados.rados.0.smithi017.stdout:335: finishing write tid 3 to smithi01785587-48 2024-09-21T09:39:18.701 INFO:tasks.rados.rados.0.smithi017.stdout:335: finishing write tid 4 to smithi01785587-48 2024-09-21T09:39:18.701 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 110 (ObjNum 161 snap 42 seq_num 161) dirty exists 2024-09-21T09:39:18.701 INFO:tasks.rados.rados.0.smithi017.stdout:335: left oid 48 (ObjNum 161 snap 42 seq_num 161) 2024-09-21T09:39:18.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:18 smithi134 ceph-mon[33541]: pgmap v212: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 839 KiB/s wr, 7 op/s 2024-09-21T09:39:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:18 smithi134 ceph-mon[33541]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:39:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:18 smithi017 ceph-mon[40737]: pgmap v212: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 839 KiB/s wr, 7 op/s 2024-09-21T09:39:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:18 smithi017 ceph-mon[40737]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:39:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:18 smithi017 ceph-mon[29380]: pgmap v212: 89 pgs: 89 active+clean; 148 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 839 KiB/s wr, 7 op/s 2024-09-21T09:39:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:18 smithi017 ceph-mon[29380]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:39:19.638 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:39:19.683 INFO:tasks.rados.rados.0.smithi017.stdout:334: done (2 left) 2024-09-21T09:39:19.683 INFO:tasks.rados.rados.0.smithi017.stdout:335: done (1 left) 2024-09-21T09:39:19.683 INFO:tasks.rados.rados.0.smithi017.stdout:336: done (0 left) 2024-09-21T09:39:19.683 INFO:tasks.rados.rados.0.smithi017.stdout:337: read oid 36 snap -1 2024-09-21T09:39:19.683 INFO:tasks.rados.rados.0.smithi017.stdout:337: expect deleted 2024-09-21T09:39:19.683 INFO:tasks.rados.rados.0.smithi017.stdout:338: delete oid 35 current snap is 43 2024-09-21T09:39:19.686 INFO:tasks.rados.rados.0.smithi017.stdout:337: done (1 left) 2024-09-21T09:39:19.686 INFO:tasks.rados.rados.0.smithi017.stdout:338: done (0 left) 2024-09-21T09:39:19.686 INFO:tasks.rados.rados.0.smithi017.stdout:339: rollback oid 38 current snap is 43 2024-09-21T09:39:19.686 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 35 2024-09-21T09:39:19.686 INFO:tasks.rados.rados.0.smithi017.stdout:340: write oid 36 current snap is 43 2024-09-21T09:39:19.686 INFO:tasks.rados.rados.0.smithi017.stdout:340: seq_num 163 ranges {739541=709000,2121859=426376} 2024-09-21T09:39:19.694 INFO:tasks.rados.rados.0.smithi017.stdout:340: writing smithi01785587-36 from 739541 to 1448541 tid 1 2024-09-21T09:39:19.700 INFO:tasks.rados.rados.0.smithi017.stdout:340: writing smithi01785587-36 from 2121859 to 2548235 tid 2 2024-09-21T09:39:19.700 INFO:tasks.rados.rados.0.smithi017.stdout:341: rollback oid 14 current snap is 43 2024-09-21T09:39:19.700 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 40 2024-09-21T09:39:19.700 INFO:tasks.rados.rados.0.smithi017.stdout:339: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:39:19.701 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 96 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-09-21T09:39:19.701 INFO:tasks.rados.rados.0.smithi017.stdout:340: finishing write tid 1 to smithi01785587-36 2024-09-21T09:39:19.701 INFO:tasks.rados.rados.0.smithi017.stdout:339: done (2 left) 2024-09-21T09:39:19.701 INFO:tasks.rados.rados.0.smithi017.stdout:342: delete oid 25 current snap is 43 2024-09-21T09:39:19.703 INFO:tasks.rados.rados.0.smithi017.stdout:341: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:39:19.703 INFO:tasks.rados.rados.0.smithi017.stdout:342: done (2 left) 2024-09-21T09:39:19.703 INFO:tasks.rados.rados.0.smithi017.stdout:343: delete oid 42 current snap is 43 2024-09-21T09:39:19.703 INFO:tasks.rados.rados.0.smithi017.stdout:341: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:39:19.704 INFO:tasks.rados.rados.0.smithi017.stdout:340: finishing write tid 2 to smithi01785587-36 2024-09-21T09:39:19.704 INFO:tasks.rados.rados.0.smithi017.stdout:341: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:39:19.704 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 168 (ObjNum 137 snap 28 seq_num 137) dirty exists 2024-09-21T09:39:19.704 INFO:tasks.rados.rados.0.smithi017.stdout:340: finishing write tid 3 to smithi01785587-36 2024-09-21T09:39:19.704 INFO:tasks.rados.rados.0.smithi017.stdout:340: finishing write tid 4 to smithi01785587-36 2024-09-21T09:39:19.704 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 95 (ObjNum 163 snap 43 seq_num 163) dirty exists 2024-09-21T09:39:19.704 INFO:tasks.rados.rados.0.smithi017.stdout:340: left oid 36 (ObjNum 163 snap 43 seq_num 163) 2024-09-21T09:39:19.705 INFO:tasks.rados.rados.0.smithi017.stdout:340: done (2 left) 2024-09-21T09:39:19.705 INFO:tasks.rados.rados.0.smithi017.stdout:341: done (1 left) 2024-09-21T09:39:19.705 INFO:tasks.rados.rados.0.smithi017.stdout:343: done (0 left) 2024-09-21T09:39:19.705 INFO:tasks.rados.rados.0.smithi017.stdout:344: write oid 7 current snap is 43 2024-09-21T09:39:19.705 INFO:tasks.rados.rados.0.smithi017.stdout:344: seq_num 166 ranges {789962=534747,1780952=690056} 2024-09-21T09:39:19.712 INFO:tasks.rados.rados.0.smithi017.stdout:344: writing smithi01785587-7 from 789962 to 1324709 tid 1 2024-09-21T09:39:19.719 INFO:tasks.rados.rados.0.smithi017.stdout:344: writing smithi01785587-7 from 1780952 to 2471008 tid 2 2024-09-21T09:39:19.719 INFO:tasks.rados.rados.0.smithi017.stdout:345: read oid 31 snap -1 2024-09-21T09:39:19.719 INFO:tasks.rados.rados.0.smithi017.stdout:345: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:39:19.719 INFO:tasks.rados.rados.0.smithi017.stdout:346: write oid 12 current snap is 43 2024-09-21T09:39:19.720 INFO:tasks.rados.rados.0.smithi017.stdout:344: finishing write tid 1 to smithi01785587-7 2024-09-21T09:39:19.720 INFO:tasks.rados.rados.0.smithi017.stdout:346: seq_num 167 ranges {406769=609996,1439434=433714,2303522=503247} 2024-09-21T09:39:19.724 INFO:tasks.rados.rados.0.smithi017.stdout:346: writing smithi01785587-12 from 406769 to 1016765 tid 1 2024-09-21T09:39:19.729 INFO:tasks.rados.rados.0.smithi017.stdout:346: writing smithi01785587-12 from 1439434 to 1873148 tid 2 2024-09-21T09:39:19.736 INFO:tasks.rados.rados.0.smithi017.stdout:346: writing smithi01785587-12 from 2303522 to 2806769 tid 3 2024-09-21T09:39:19.736 INFO:tasks.rados.rados.0.smithi017.stdout:347: rollback oid 36 current snap is 43 2024-09-21T09:39:19.736 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 36 to 40 2024-09-21T09:39:19.737 INFO:tasks.rados.rados.0.smithi017.stdout:344: finishing write tid 2 to smithi01785587-7 2024-09-21T09:39:19.737 INFO:tasks.rados.rados.0.smithi017.stdout:344: finishing write tid 3 to smithi01785587-7 2024-09-21T09:39:19.737 INFO:tasks.rados.rados.0.smithi017.stdout:344: finishing write tid 4 to smithi01785587-7 2024-09-21T09:39:19.737 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 116 (ObjNum 166 snap 43 seq_num 166) dirty exists 2024-09-21T09:39:19.737 INFO:tasks.rados.rados.0.smithi017.stdout:344: left oid 7 (ObjNum 166 snap 43 seq_num 166) 2024-09-21T09:39:19.737 INFO:tasks.rados.rados.0.smithi017.stdout:345: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:39:19.839 INFO:tasks.rados.rados.0.smithi017.stdout:346: finishing write tid 1 to smithi01785587-12 2024-09-21T09:39:19.839 INFO:tasks.rados.rados.0.smithi017.stdout:346: finishing write tid 2 to smithi01785587-12 2024-09-21T09:39:19.839 INFO:tasks.rados.rados.0.smithi017.stdout:344: done (3 left) 2024-09-21T09:39:19.839 INFO:tasks.rados.rados.0.smithi017.stdout:345: done (2 left) 2024-09-21T09:39:19.839 INFO:tasks.rados.rados.0.smithi017.stdout:348: snap_create 2024-09-21T09:39:19.839 INFO:tasks.rados.rados.0.smithi017.stdout:347: finishing rollback tid 0 to smithi01785587-36 2024-09-21T09:39:19.839 INFO:tasks.rados.rados.0.smithi017.stdout:347: finishing rollback tid 1 to smithi01785587-36 2024-09-21T09:39:19.840 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 97 (ObjNum 97 snap 0 seq_num 3571343968) dirty dne 2024-09-21T09:39:19.840 INFO:tasks.rados.rados.0.smithi017.stdout:346: finishing write tid 3 to smithi01785587-12 2024-09-21T09:39:19.840 INFO:tasks.rados.rados.0.smithi017.stdout:346: finishing write tid 4 to smithi01785587-12 2024-09-21T09:39:19.840 INFO:tasks.rados.rados.0.smithi017.stdout:346: finishing write tid 5 to smithi01785587-12 2024-09-21T09:39:19.840 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 119 (ObjNum 167 snap 43 seq_num 167) dirty exists 2024-09-21T09:39:19.840 INFO:tasks.rados.rados.0.smithi017.stdout:346: left oid 12 (ObjNum 167 snap 43 seq_num 167) 2024-09-21T09:39:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:19 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3268438686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:19 smithi017 ceph-mon[29380]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:39:19.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:19 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3268438686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:19.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:19 smithi017 ceph-mon[40737]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:39:19.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:19 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3268438686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:19 smithi134 ceph-mon[33541]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:39:20.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:20] "GET /metrics HTTP/1.1" 200 243215 "" "Prometheus/2.43.0" 2024-09-21T09:39:20.739 INFO:tasks.rados.rados.0.smithi017.stdout:346: done (2 left) 2024-09-21T09:39:20.740 INFO:tasks.rados.rados.0.smithi017.stdout:347: done (1 left) 2024-09-21T09:39:20.740 INFO:tasks.rados.rados.0.smithi017.stdout:348: done (0 left) 2024-09-21T09:39:20.740 INFO:tasks.rados.rados.0.smithi017.stdout:349: write oid 13 current snap is 44 2024-09-21T09:39:20.740 INFO:tasks.rados.rados.0.smithi017.stdout:349: seq_num 168 ranges {423576=401598,1497916=693725} 2024-09-21T09:39:20.743 INFO:tasks.rados.rados.0.smithi017.stdout:349: writing smithi01785587-13 from 423576 to 825174 tid 1 2024-09-21T09:39:20.751 INFO:tasks.rados.rados.0.smithi017.stdout:349: writing smithi01785587-13 from 1497916 to 2191641 tid 2 2024-09-21T09:39:20.751 INFO:tasks.rados.rados.0.smithi017.stdout:350: write oid 44 current snap is 44 2024-09-21T09:39:20.751 INFO:tasks.rados.rados.0.smithi017.stdout:350: seq_num 169 ranges {440383=476847,1556398=437383,2474721=365662} 2024-09-21T09:39:20.756 INFO:tasks.rados.rados.0.smithi017.stdout:350: writing smithi01785587-44 from 440383 to 917230 tid 1 2024-09-21T09:39:20.762 INFO:tasks.rados.rados.0.smithi017.stdout:350: writing smithi01785587-44 from 1556398 to 1993781 tid 2 2024-09-21T09:39:20.767 INFO:tasks.rados.rados.0.smithi017.stdout:350: writing smithi01785587-44 from 2474721 to 2840383 tid 3 2024-09-21T09:39:20.767 INFO:tasks.rados.rados.0.smithi017.stdout:351: write oid 21 current snap is 44 2024-09-21T09:39:20.767 INFO:tasks.rados.rados.0.smithi017.stdout:349: finishing write tid 1 to smithi01785587-13 2024-09-21T09:39:20.767 INFO:tasks.rados.rados.0.smithi017.stdout:351: seq_num 170 ranges {457190=552096,1614880=581041,2785791=71399} 2024-09-21T09:39:20.772 INFO:tasks.rados.rados.0.smithi017.stdout:351: writing smithi01785587-21 from 457190 to 1009286 tid 1 2024-09-21T09:39:20.779 INFO:tasks.rados.rados.0.smithi017.stdout:351: writing smithi01785587-21 from 1614880 to 2195921 tid 2 2024-09-21T09:39:20.782 INFO:tasks.rados.rados.0.smithi017.stdout:351: writing smithi01785587-21 from 2785791 to 2857190 tid 3 2024-09-21T09:39:20.782 INFO:tasks.rados.rados.0.smithi017.stdout:352: write oid 14 current snap is 44 2024-09-21T09:39:20.783 INFO:tasks.rados.rados.0.smithi017.stdout:352: seq_num 171 ranges {473997=627345,1673362=441052,2529567=344430} 2024-09-21T09:39:20.788 INFO:tasks.rados.rados.0.smithi017.stdout:352: writing smithi01785587-14 from 473997 to 1101342 tid 1 2024-09-21T09:39:20.794 INFO:tasks.rados.rados.0.smithi017.stdout:352: writing smithi01785587-14 from 1673362 to 2114414 tid 2 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:352: writing smithi01785587-14 from 2529567 to 2873997 tid 3 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:353: snap_create 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:349: finishing write tid 2 to smithi01785587-13 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:349: finishing write tid 3 to smithi01785587-13 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:349: finishing write tid 4 to smithi01785587-13 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 114 (ObjNum 168 snap 44 seq_num 168) dirty exists 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:349: left oid 13 (ObjNum 168 snap 44 seq_num 168) 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:350: finishing write tid 1 to smithi01785587-44 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:350: finishing write tid 2 to smithi01785587-44 2024-09-21T09:39:20.799 INFO:tasks.rados.rados.0.smithi017.stdout:350: finishing write tid 3 to smithi01785587-44 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:350: finishing write tid 4 to smithi01785587-44 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:350: finishing write tid 5 to smithi01785587-44 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 173 (ObjNum 169 snap 44 seq_num 169) dirty exists 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:350: left oid 44 (ObjNum 169 snap 44 seq_num 169) 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:351: finishing write tid 1 to smithi01785587-21 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:351: finishing write tid 2 to smithi01785587-21 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:351: finishing write tid 3 to smithi01785587-21 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:351: finishing write tid 4 to smithi01785587-21 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:351: finishing write tid 5 to smithi01785587-21 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 121 (ObjNum 170 snap 44 seq_num 170) dirty exists 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:351: left oid 21 (ObjNum 170 snap 44 seq_num 170) 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:352: finishing write tid 1 to smithi01785587-14 2024-09-21T09:39:20.800 INFO:tasks.rados.rados.0.smithi017.stdout:352: finishing write tid 2 to smithi01785587-14 2024-09-21T09:39:20.801 INFO:tasks.rados.rados.0.smithi017.stdout:352: finishing write tid 3 to smithi01785587-14 2024-09-21T09:39:20.801 INFO:tasks.rados.rados.0.smithi017.stdout:352: finishing write tid 4 to smithi01785587-14 2024-09-21T09:39:20.801 INFO:tasks.rados.rados.0.smithi017.stdout:352: finishing write tid 5 to smithi01785587-14 2024-09-21T09:39:20.801 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 178 (ObjNum 171 snap 44 seq_num 171) dirty exists 2024-09-21T09:39:20.801 INFO:tasks.rados.rados.0.smithi017.stdout:352: left oid 14 (ObjNum 171 snap 44 seq_num 171) 2024-09-21T09:39:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:20 smithi017 ceph-mon[29380]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:20 smithi017 ceph-mon[29380]: pgmap v215: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:39:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:20 smithi017 ceph-mon[29380]: from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:20 smithi017 ceph-mon[40737]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:20 smithi017 ceph-mon[40737]: pgmap v215: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:39:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:20 smithi017 ceph-mon[40737]: from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:20 smithi134 ceph-mon[33541]: from='client.15255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:20.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:20 smithi134 ceph-mon[33541]: pgmap v215: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:39:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:20 smithi134 ceph-mon[33541]: from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:21.746 INFO:tasks.rados.rados.0.smithi017.stdout:349: done (4 left) 2024-09-21T09:39:21.746 INFO:tasks.rados.rados.0.smithi017.stdout:350: done (3 left) 2024-09-21T09:39:21.746 INFO:tasks.rados.rados.0.smithi017.stdout:351: done (2 left) 2024-09-21T09:39:21.746 INFO:tasks.rados.rados.0.smithi017.stdout:352: done (1 left) 2024-09-21T09:39:21.746 INFO:tasks.rados.rados.0.smithi017.stdout:353: done (0 left) 2024-09-21T09:39:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:354: rollback oid 19 current snap is 45 2024-09-21T09:39:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 19 to 38 2024-09-21T09:39:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:355: read oid 36 snap -1 2024-09-21T09:39:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:355: expect deleted 2024-09-21T09:39:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:356: write oid 27 current snap is 45 2024-09-21T09:39:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:356: seq_num 172 ranges {490804=702594,1615491=584710,2724284=166520} 2024-09-21T09:39:21.752 INFO:tasks.rados.rados.0.smithi017.stdout:356: writing smithi01785587-27 from 490804 to 1193398 tid 1 2024-09-21T09:39:21.758 INFO:tasks.rados.rados.0.smithi017.stdout:356: writing smithi01785587-27 from 1615491 to 2200201 tid 2 2024-09-21T09:39:21.762 INFO:tasks.rados.rados.0.smithi017.stdout:356: writing smithi01785587-27 from 2724284 to 2890804 tid 3 2024-09-21T09:39:21.762 INFO:tasks.rados.rados.0.smithi017.stdout:357: snap_create 2024-09-21T09:39:21.763 INFO:tasks.rados.rados.0.smithi017.stdout:354: finishing rollback tid 0 to smithi01785587-19 2024-09-21T09:39:21.763 INFO:tasks.rados.rados.0.smithi017.stdout:354: finishing rollback tid 1 to smithi01785587-19 2024-09-21T09:39:21.763 INFO:tasks.rados.rados.0.smithi017.stdout:354: finishing rollback tid 2 to smithi01785587-19 2024-09-21T09:39:21.763 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 118 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-09-21T09:39:21.763 INFO:tasks.rados.rados.0.smithi017.stdout:356: finishing write tid 1 to smithi01785587-27 2024-09-21T09:39:21.763 INFO:tasks.rados.rados.0.smithi017.stdout:356: finishing write tid 2 to smithi01785587-27 2024-09-21T09:39:21.764 INFO:tasks.rados.rados.0.smithi017.stdout:356: finishing write tid 3 to smithi01785587-27 2024-09-21T09:39:21.764 INFO:tasks.rados.rados.0.smithi017.stdout:356: finishing write tid 4 to smithi01785587-27 2024-09-21T09:39:21.764 INFO:tasks.rados.rados.0.smithi017.stdout:356: finishing write tid 5 to smithi01785587-27 2024-09-21T09:39:21.764 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 67 (ObjNum 172 snap 45 seq_num 172) dirty exists 2024-09-21T09:39:21.764 INFO:tasks.rados.rados.0.smithi017.stdout:356: left oid 27 (ObjNum 172 snap 45 seq_num 172) 2024-09-21T09:39:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:21 smithi017 ceph-mon[29380]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:39:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:21 smithi017 ceph-mon[40737]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:39:22.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:21 smithi134 ceph-mon[33541]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:39:22.520 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:39:22.678 INFO:tasks.rados.rados.0.smithi017.stdout:354: done (3 left) 2024-09-21T09:39:22.678 INFO:tasks.rados.rados.0.smithi017.stdout:355: done (2 left) 2024-09-21T09:39:22.678 INFO:tasks.rados.rados.0.smithi017.stdout:356: done (1 left) 2024-09-21T09:39:22.678 INFO:tasks.rados.rados.0.smithi017.stdout:357: done (0 left) 2024-09-21T09:39:22.679 INFO:tasks.rados.rados.0.smithi017.stdout:358: write oid 28 current snap is 46 2024-09-21T09:39:22.679 INFO:tasks.rados.rados.0.smithi017.stdout:358: seq_num 173 ranges {507611=777843,2073973=444721} 2024-09-21T09:39:22.685 INFO:tasks.rados.rados.0.smithi017.stdout:358: writing smithi01785587-28 from 507611 to 1285454 tid 1 2024-09-21T09:39:22.692 INFO:tasks.rados.rados.0.smithi017.stdout:358: writing smithi01785587-28 from 2073973 to 2518694 tid 2 2024-09-21T09:39:22.692 INFO:tasks.rados.rados.0.smithi017.stdout:359: snap_create 2024-09-21T09:39:22.692 INFO:tasks.rados.rados.0.smithi017.stdout:358: finishing write tid 1 to smithi01785587-28 2024-09-21T09:39:22.694 INFO:tasks.rados.rados.0.smithi017.stdout:358: finishing write tid 2 to smithi01785587-28 2024-09-21T09:39:22.695 INFO:tasks.rados.rados.0.smithi017.stdout:358: finishing write tid 3 to smithi01785587-28 2024-09-21T09:39:22.695 INFO:tasks.rados.rados.0.smithi017.stdout:358: finishing write tid 4 to smithi01785587-28 2024-09-21T09:39:22.695 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 179 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-09-21T09:39:22.695 INFO:tasks.rados.rados.0.smithi017.stdout:358: left oid 28 (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:22 smithi017 ceph-mon[29380]: pgmap v217: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:39:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:22 smithi017 ceph-mon[29380]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:39:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:22 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3206454770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:22 smithi017 ceph-mon[29380]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:39:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:22 smithi017 ceph-mon[40737]: pgmap v217: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:39:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:22 smithi017 ceph-mon[40737]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:39:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:22 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3206454770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:22 smithi017 ceph-mon[40737]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:39:22.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:22 smithi134 ceph-mon[33541]: pgmap v217: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:39:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:22 smithi134 ceph-mon[33541]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:39:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:22 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3206454770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:22 smithi134 ceph-mon[33541]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:39:23.679 INFO:tasks.rados.rados.0.smithi017.stdout:358: done (1 left) 2024-09-21T09:39:23.680 INFO:tasks.rados.rados.0.smithi017.stdout:359: done (0 left) 2024-09-21T09:39:23.680 INFO:tasks.rados.rados.0.smithi017.stdout:360: snap_remove snap 45 2024-09-21T09:39:23.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:23 smithi134 ceph-mon[33541]: from='client.25166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:23 smithi134 ceph-mon[33541]: from='client.25172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:23 smithi134 ceph-mon[33541]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:39:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:23 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:23 smithi017 ceph-mon[29380]: from='client.25166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:23 smithi017 ceph-mon[29380]: from='client.25172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:23 smithi017 ceph-mon[29380]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:39:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:23 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:24.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:23 smithi017 ceph-mon[40737]: from='client.25166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:24.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:23 smithi017 ceph-mon[40737]: from='client.25172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:24.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:23 smithi017 ceph-mon[40737]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:39:24.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:23 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:24.685 INFO:tasks.rados.rados.0.smithi017.stdout:360: done (0 left) 2024-09-21T09:39:24.685 INFO:tasks.rados.rados.0.smithi017.stdout:361: read oid 32 snap -1 2024-09-21T09:39:24.685 INFO:tasks.rados.rados.0.smithi017.stdout:361: expect (ObjNum 124 snap 23 seq_num 124) 2024-09-21T09:39:24.685 INFO:tasks.rados.rados.0.smithi017.stdout:362: read oid 10 snap 43 2024-09-21T09:39:24.685 INFO:tasks.rados.rados.0.smithi017.stdout:362: expect deleted 2024-09-21T09:39:24.685 INFO:tasks.rados.rados.0.smithi017.stdout:363: read oid 26 snap -1 2024-09-21T09:39:24.685 INFO:tasks.rados.rados.0.smithi017.stdout:363: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:24.685 INFO:tasks.rados.rados.0.smithi017.stdout:364: read oid 2 snap -1 2024-09-21T09:39:24.686 INFO:tasks.rados.rados.0.smithi017.stdout:364: expect deleted 2024-09-21T09:39:24.686 INFO:tasks.rados.rados.0.smithi017.stdout:365: delete oid 40 current snap is 47 2024-09-21T09:39:24.687 INFO:tasks.rados.rados.0.smithi017.stdout:361: expect (ObjNum 124 snap 23 seq_num 124) 2024-09-21T09:39:24.816 INFO:tasks.rados.rados.0.smithi017.stdout:361: done (4 left) 2024-09-21T09:39:24.816 INFO:tasks.rados.rados.0.smithi017.stdout:362: done (3 left) 2024-09-21T09:39:24.816 INFO:tasks.rados.rados.0.smithi017.stdout:365: done (2 left) 2024-09-21T09:39:24.817 INFO:tasks.rados.rados.0.smithi017.stdout:366: read oid 36 snap -1 2024-09-21T09:39:24.817 INFO:tasks.rados.rados.0.smithi017.stdout:366: expect deleted 2024-09-21T09:39:24.817 INFO:tasks.rados.rados.0.smithi017.stdout:367: rollback oid 44 current snap is 47 2024-09-21T09:39:24.817 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 38 2024-09-21T09:39:24.817 INFO:tasks.rados.rados.0.smithi017.stdout:363: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:24.856 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:24 smithi134 ceph-mon[33541]: pgmap v221: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:24.856 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:24 smithi134 ceph-mon[33541]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:39:24.965 INFO:tasks.rados.rados.0.smithi017.stdout:367: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:367: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:367: finishing rollback tid 2 to smithi01785587-44 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 182 (ObjNum 129 snap 27 seq_num 129) dirty exists 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:363: done (3 left) 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:364: done (2 left) 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:366: done (1 left) 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:367: done (0 left) 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:368: rollback oid 34 current snap is 47 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 43 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:369: write oid 41 current snap is 47 2024-09-21T09:39:24.966 INFO:tasks.rados.rados.0.smithi017.stdout:369: seq_num 175 ranges {541225=644694,1907290=448390} 2024-09-21T09:39:24.972 INFO:tasks.rados.rados.0.smithi017.stdout:369: writing smithi01785587-41 from 541225 to 1185919 tid 1 2024-09-21T09:39:24.979 INFO:tasks.rados.rados.0.smithi017.stdout:369: writing smithi01785587-41 from 1907290 to 2355680 tid 2 2024-09-21T09:39:24.980 INFO:tasks.rados.rados.0.smithi017.stdout:370: write oid 27 current snap is 47 2024-09-21T09:39:24.980 INFO:tasks.rados.rados.0.smithi017.stdout:368: finishing rollback tid 0 to smithi01785587-34 2024-09-21T09:39:24.980 INFO:tasks.rados.rados.0.smithi017.stdout:370: seq_num 176 ranges {558032=719943,1849419=592048} 2024-09-21T09:39:24.988 INFO:tasks.rados.rados.0.smithi017.stdout:370: writing smithi01785587-27 from 558032 to 1277975 tid 1 2024-09-21T09:39:24.996 INFO:tasks.rados.rados.0.smithi017.stdout:370: writing smithi01785587-27 from 1849419 to 2441467 tid 2 2024-09-21T09:39:24.996 INFO:tasks.rados.rados.0.smithi017.stdout:371: read oid 19 snap -1 2024-09-21T09:39:24.996 INFO:tasks.rados.rados.0.smithi017.stdout:371: expect (ObjNum 140 snap 31 seq_num 140) 2024-09-21T09:39:24.996 INFO:tasks.rados.rados.0.smithi017.stdout:372: delete oid 7 current snap is 47 2024-09-21T09:39:24.996 INFO:tasks.rados.rados.0.smithi017.stdout:368: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:39:24.997 INFO:tasks.rados.rados.0.smithi017.stdout:368: finishing rollback tid 2 to smithi01785587-34 2024-09-21T09:39:24.997 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 125 (ObjNum 112 snap 18 seq_num 112) dirty exists 2024-09-21T09:39:24.997 INFO:tasks.rados.rados.0.smithi017.stdout:369: finishing write tid 1 to smithi01785587-41 2024-09-21T09:39:24.997 INFO:tasks.rados.rados.0.smithi017.stdout:369: finishing write tid 2 to smithi01785587-41 2024-09-21T09:39:24.997 INFO:tasks.rados.rados.0.smithi017.stdout:369: finishing write tid 3 to smithi01785587-41 2024-09-21T09:39:24.997 INFO:tasks.rados.rados.0.smithi017.stdout:369: finishing write tid 4 to smithi01785587-41 2024-09-21T09:39:24.997 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 64 (ObjNum 175 snap 47 seq_num 175) dirty exists 2024-09-21T09:39:24.997 INFO:tasks.rados.rados.0.smithi017.stdout:369: left oid 41 (ObjNum 175 snap 47 seq_num 175) 2024-09-21T09:39:24.997 INFO:tasks.rados.rados.0.smithi017.stdout:370: finishing write tid 1 to smithi01785587-27 2024-09-21T09:39:25.000 INFO:tasks.rados.rados.0.smithi017.stdout:370: finishing write tid 2 to smithi01785587-27 2024-09-21T09:39:25.000 INFO:tasks.rados.rados.0.smithi017.stdout:368: done (4 left) 2024-09-21T09:39:25.000 INFO:tasks.rados.rados.0.smithi017.stdout:369: done (3 left) 2024-09-21T09:39:25.000 INFO:tasks.rados.rados.0.smithi017.stdout:372: done (2 left) 2024-09-21T09:39:25.000 INFO:tasks.rados.rados.0.smithi017.stdout:373: write oid 11 current snap is 47 2024-09-21T09:39:25.000 INFO:tasks.rados.rados.0.smithi017.stdout:373: seq_num 178 ranges {591646=470441,1566383=595717,2888822=102824} 2024-09-21T09:39:25.006 INFO:tasks.rados.rados.0.smithi017.stdout:373: writing smithi01785587-11 from 591646 to 1062087 tid 1 2024-09-21T09:39:25.014 INFO:tasks.rados.rados.0.smithi017.stdout:373: writing smithi01785587-11 from 1566383 to 2162100 tid 2 2024-09-21T09:39:25.019 INFO:tasks.rados.rados.0.smithi017.stdout:373: writing smithi01785587-11 from 2888822 to 2991646 tid 3 2024-09-21T09:39:25.019 INFO:tasks.rados.rados.0.smithi017.stdout:374: write oid 45 current snap is 47 2024-09-21T09:39:25.019 INFO:tasks.rados.rados.0.smithi017.stdout:370: finishing write tid 3 to smithi01785587-27 2024-09-21T09:39:25.019 INFO:tasks.rados.rados.0.smithi017.stdout:374: seq_num 179 ranges {608453=545690,1624865=455728,2632598=375855} 2024-09-21T09:39:25.026 INFO:tasks.rados.rados.0.smithi017.stdout:374: writing smithi01785587-45 from 608453 to 1154143 tid 1 2024-09-21T09:39:25.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:24 smithi017 ceph-mon[29380]: pgmap v221: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:25.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:24 smithi017 ceph-mon[29380]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:39:25.029 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:24 smithi017 ceph-mon[40737]: pgmap v221: 89 pgs: 89 active+clean; 154 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:25.029 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:24 smithi017 ceph-mon[40737]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:39:25.032 INFO:tasks.rados.rados.0.smithi017.stdout:374: writing smithi01785587-45 from 1624865 to 2080593 tid 2 2024-09-21T09:39:25.039 INFO:tasks.rados.rados.0.smithi017.stdout:374: writing smithi01785587-45 from 2632598 to 3008453 tid 3 2024-09-21T09:39:25.039 INFO:tasks.rados.rados.0.smithi017.stdout:375: rollback oid 40 current snap is 47 2024-09-21T09:39:25.039 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 46 2024-09-21T09:39:25.039 INFO:tasks.rados.rados.0.smithi017.stdout:370: finishing write tid 4 to smithi01785587-27 2024-09-21T09:39:25.039 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 71 (ObjNum 176 snap 47 seq_num 176) dirty exists 2024-09-21T09:39:25.039 INFO:tasks.rados.rados.0.smithi017.stdout:370: left oid 27 (ObjNum 176 snap 47 seq_num 176) 2024-09-21T09:39:25.039 INFO:tasks.rados.rados.0.smithi017.stdout:371: expect (ObjNum 140 snap 31 seq_num 140) 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:373: finishing write tid 1 to smithi01785587-11 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:373: finishing write tid 2 to smithi01785587-11 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:370: done (4 left) 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:371: done (3 left) 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:376: delete oid 15 current snap is 47 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:373: finishing write tid 3 to smithi01785587-11 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:373: finishing write tid 4 to smithi01785587-11 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:373: finishing write tid 5 to smithi01785587-11 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 165 (ObjNum 178 snap 47 seq_num 178) dirty exists 2024-09-21T09:39:25.175 INFO:tasks.rados.rados.0.smithi017.stdout:373: left oid 11 (ObjNum 178 snap 47 seq_num 178) 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:374: finishing write tid 1 to smithi01785587-45 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:374: finishing write tid 2 to smithi01785587-45 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:375: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:375: finishing rollback tid 2 to smithi01785587-40 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 101 (ObjNum 158 snap 40 seq_num 158) dirty exists 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:374: finishing write tid 3 to smithi01785587-45 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:374: finishing write tid 4 to smithi01785587-45 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:374: finishing write tid 5 to smithi01785587-45 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 76 (ObjNum 179 snap 47 seq_num 179) dirty exists 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:374: left oid 45 (ObjNum 179 snap 47 seq_num 179) 2024-09-21T09:39:25.176 INFO:tasks.rados.rados.0.smithi017.stdout:373: done (3 left) 2024-09-21T09:39:25.177 INFO:tasks.rados.rados.0.smithi017.stdout:374: done (2 left) 2024-09-21T09:39:25.177 INFO:tasks.rados.rados.0.smithi017.stdout:375: done (1 left) 2024-09-21T09:39:25.177 INFO:tasks.rados.rados.0.smithi017.stdout:376: done (0 left) 2024-09-21T09:39:25.177 INFO:tasks.rados.rados.0.smithi017.stdout:377: snap_remove snap 37 2024-09-21T09:39:25.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:39:24] ENGINE Bus STOPPING 2024-09-21T09:39:25.309 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ignoring --setuser ceph since I am not root 2024-09-21T09:39:25.309 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ignoring --setgroup ceph since I am not root 2024-09-21T09:39:25.310 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mgr[29655]: -- 172.21.15.17:0/3264504174 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564a9ce22340 con 0x564a9ce2f000 2024-09-21T09:39:25.310 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:25.146+0000 7fe4454d4ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:39:25.310 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:25.205+0000 7fe4454d4ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:39:25.310 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:25.307+0000 7fe4454d4ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:39:25.581 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:25.437+0000 7fe4454d4ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:39:25.581 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:25.579+0000 7fe4454d4ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:39:25.743 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:39:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:39:25.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:39:25] ENGINE Bus STOPPED 2024-09-21T09:39:25.991 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENOTSUP: Warning: due to ceph-mgr restart, some PG states may not be up to date 2024-09-21T09:39:25.991 INFO:teuthology.orchestra.run.smithi017.stderr:Module 'orchestrator' is not enabled/loaded (required by command 'orch upgrade status'): use `ceph mgr module enable orchestrator` to enable it 2024-09-21T09:39:25.997 INFO:tasks.rados.rados.0.smithi017.stdout:377: done (0 left) 2024-09-21T09:39:25.998 INFO:tasks.rados.rados.0.smithi017.stdout:378: read oid 7 snap -1 2024-09-21T09:39:25.998 INFO:tasks.rados.rados.0.smithi017.stdout:378: expect deleted 2024-09-21T09:39:25.998 INFO:tasks.rados.rados.0.smithi017.stdout:379: delete oid 3 current snap is 47 2024-09-21T09:39:26.004 INFO:tasks.rados.rados.0.smithi017.stdout:378: done (1 left) 2024-09-21T09:39:26.004 INFO:tasks.rados.rados.0.smithi017.stdout:379: done (0 left) 2024-09-21T09:39:26.004 INFO:tasks.rados.rados.0.smithi017.stdout:380: snap_create 2024-09-21T09:39:26.098 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:25.845+0000 7fe4454d4ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:39:26.098 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:25.908+0000 7fe4454d4ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:39:26.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:26.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:26.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:26.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:26.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:26.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:26.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:26.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: Activating manager daemon x 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.coodep"}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:39:26.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: mgrmap e23: x(active, starting, since 0.0104519s) 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1235980989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:26.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[29380]: Manager daemon x is now available 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: Activating manager daemon x 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:39:26.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.coodep"}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: mgrmap e23: x(active, starting, since 0.0104519s) 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:39:26.103 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:39:26.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:39:26.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:39:26.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:39:26.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:39:26.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1235980989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:26.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:25 smithi017 ceph-mon[40737]: Manager daemon x is now available 2024-09-21T09:39:26.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:25.988+0000 7f67035bc640 -1 mgr.server reply reply (95) Operation not supported Warning: due to ceph-mgr restart, some PG states may not be up to date 2024-09-21T09:39:26.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: Module 'orchestrator' is not enabled/loaded (required by command 'orch upgrade status'): use `ceph mgr module enable orchestrator` to enable it 2024-09-21T09:39:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' 2024-09-21T09:39:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24332 172.21.15.17:0/3058825117' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: Activating manager daemon x 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.coodep"}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:39:26.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: mgrmap e23: x(active, starting, since 0.0104519s) 2024-09-21T09:39:26.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:39:26.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:39:26.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:39:26.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:39:26.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:39:26.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:39:26.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:39:26.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1235980989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:39:26.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:25 smithi134 ceph-mon[33541]: Manager daemon x is now available 2024-09-21T09:39:26.380 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:26.096+0000 7fe4454d4ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:39:26.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch ps' 2024-09-21T09:39:26.723 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:39:26] ENGINE Bus STARTING 2024-09-21T09:39:26.723 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.415+0000 7f66dad6b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.723 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.415+0000 7f66dad6b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.723 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.415+0000 7f66dad6b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.724 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.415+0000 7f66dad6b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.724 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.415+0000 7f66dad6b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.724 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.416+0000 7f66d7564640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.724 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.416+0000 7f66d7564640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.724 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.416+0000 7f66d7564640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.724 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.416+0000 7f66d7564640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.724 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:26.416+0000 7f66d7564640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:26.724 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:39:26] ENGINE Serving on http://:::9283 2024-09-21T09:39:26.724 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:39:26] ENGINE Bus STARTED 2024-09-21T09:39:26.995 INFO:tasks.rados.rados.0.smithi017.stdout:380: done (0 left) 2024-09-21T09:39:26.996 INFO:tasks.rados.rados.0.smithi017.stdout:381: read oid 34 snap -1 2024-09-21T09:39:26.996 INFO:tasks.rados.rados.0.smithi017.stdout:381: expect (ObjNum 112 snap 18 seq_num 112) 2024-09-21T09:39:26.996 INFO:tasks.rados.rados.0.smithi017.stdout:382: read oid 47 snap -1 2024-09-21T09:39:26.996 INFO:tasks.rados.rados.0.smithi017.stdout:382: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:26.996 INFO:tasks.rados.rados.0.smithi017.stdout:383: write oid 25 current snap is 48 2024-09-21T09:39:26.996 INFO:tasks.rados.rados.0.smithi017.stdout:383: seq_num 182 ranges {658874=771437,2083958=603055} 2024-09-21T09:39:26.998 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:39:27.004 INFO:tasks.rados.rados.0.smithi017.stdout:383: writing smithi01785587-25 from 658874 to 1430311 tid 1 2024-09-21T09:39:27.011 INFO:tasks.rados.rados.0.smithi017.stdout:383: writing smithi01785587-25 from 2083958 to 2687013 tid 2 2024-09-21T09:39:27.012 INFO:tasks.rados.rados.0.smithi017.stdout:384: rollback oid 1 current snap is 48 2024-09-21T09:39:27.012 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 1 to 35 2024-09-21T09:39:27.012 INFO:tasks.rados.rados.0.smithi017.stdout:382: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:27.132 INFO:tasks.rados.rados.0.smithi017.stdout:381: expect (ObjNum 112 snap 18 seq_num 112) 2024-09-21T09:39:27.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:26] "GET /metrics HTTP/1.1" 200 35903 "" "Prometheus/2.43.0" 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: mgrmap e24: x(active, since 1.00996s) 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: Checking for cert/key for grafana.a 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: Migrating grafana.a cert to cert store 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: Migrating grafana.a key to cert store 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:27.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:27.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:27.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:27.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:27.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:27.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:27.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:26 smithi134 ceph-mon[33541]: Deploying cephadm binary to smithi017 2024-09-21T09:39:27.266 INFO:tasks.rados.rados.0.smithi017.stdout:383: finishing write tid 1 to smithi01785587-25 2024-09-21T09:39:27.266 INFO:tasks.rados.rados.0.smithi017.stdout:384: finishing rollback tid 1 to smithi01785587-1 2024-09-21T09:39:27.266 INFO:tasks.rados.rados.0.smithi017.stdout:383: finishing write tid 2 to smithi01785587-25 2024-09-21T09:39:27.266 INFO:tasks.rados.rados.0.smithi017.stdout:383: finishing write tid 3 to smithi01785587-25 2024-09-21T09:39:27.266 INFO:tasks.rados.rados.0.smithi017.stdout:384: finishing rollback tid 2 to smithi01785587-1 2024-09-21T09:39:27.266 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 115 (ObjNum 125 snap 23 seq_num 125) dirty exists 2024-09-21T09:39:27.266 INFO:tasks.rados.rados.0.smithi017.stdout:383: finishing write tid 4 to smithi01785587-25 2024-09-21T09:39:27.266 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 95 (ObjNum 182 snap 48 seq_num 182) dirty exists 2024-09-21T09:39:27.266 INFO:tasks.rados.rados.0.smithi017.stdout:383: left oid 25 (ObjNum 182 snap 48 seq_num 182) 2024-09-21T09:39:27.267 INFO:tasks.rados.rados.0.smithi017.stdout:381: done (3 left) 2024-09-21T09:39:27.267 INFO:tasks.rados.rados.0.smithi017.stdout:382: done (2 left) 2024-09-21T09:39:27.267 INFO:tasks.rados.rados.0.smithi017.stdout:383: done (1 left) 2024-09-21T09:39:27.267 INFO:tasks.rados.rados.0.smithi017.stdout:384: done (0 left) 2024-09-21T09:39:27.267 INFO:tasks.rados.rados.0.smithi017.stdout:385: write oid 11 current snap is 48 2024-09-21T09:39:27.267 INFO:tasks.rados.rados.0.smithi017.stdout:385: seq_num 183 ranges {675681=563039,1858793=746713,3025937=49744} 2024-09-21T09:39:27.273 INFO:tasks.rados.rados.0.smithi017.stdout:385: writing smithi01785587-11 from 675681 to 1238720 tid 1 2024-09-21T09:39:27.280 INFO:tasks.rados.rados.0.smithi017.stdout:385: writing smithi01785587-11 from 1858793 to 2605506 tid 2 2024-09-21T09:39:27.284 INFO:tasks.rados.rados.0.smithi017.stdout:385: writing smithi01785587-11 from 3025937 to 3075681 tid 3 2024-09-21T09:39:27.284 INFO:tasks.rados.rados.0.smithi017.stdout:386: snap_create 2024-09-21T09:39:27.284 INFO:tasks.rados.rados.0.smithi017.stdout:385: finishing write tid 1 to smithi01785587-11 2024-09-21T09:39:27.284 INFO:tasks.rados.rados.0.smithi017.stdout:385: finishing write tid 2 to smithi01785587-11 2024-09-21T09:39:27.285 INFO:tasks.rados.rados.0.smithi017.stdout:385: finishing write tid 3 to smithi01785587-11 2024-09-21T09:39:27.285 INFO:tasks.rados.rados.0.smithi017.stdout:385: finishing write tid 4 to smithi01785587-11 2024-09-21T09:39:27.286 INFO:tasks.rados.rados.0.smithi017.stdout:385: finishing write tid 5 to smithi01785587-11 2024-09-21T09:39:27.286 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 170 (ObjNum 183 snap 48 seq_num 183) dirty exists 2024-09-21T09:39:27.286 INFO:tasks.rados.rados.0.smithi017.stdout:385: left oid 11 (ObjNum 183 snap 48 seq_num 183) 2024-09-21T09:39:27.316 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:27.042+0000 7fe4454d4ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:27.113+0000 7fe4454d4ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: mgrmap e24: x(active, since 1.00996s) 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: Checking for cert/key for grafana.a 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: Migrating grafana.a cert to cert store 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: Migrating grafana.a key to cert store 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[29380]: Deploying cephadm binary to smithi017 2024-09-21T09:39:27.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: mgrmap e24: x(active, since 1.00996s) 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: Checking for cert/key for grafana.a 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: Migrating grafana.a cert to cert store 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: Migrating grafana.a key to cert store 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:27.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:27.320 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:27.320 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:26 smithi017 ceph-mon[40737]: Deploying cephadm binary to smithi017 2024-09-21T09:39:27.678 INFO:tasks.rados.rados.0.smithi017.stdout:385: done (1 left) 2024-09-21T09:39:27.679 INFO:tasks.rados.rados.0.smithi017.stdout:386: done (0 left) 2024-09-21T09:39:27.679 INFO:tasks.rados.rados.0.smithi017.stdout:387: read oid 39 snap -1 2024-09-21T09:39:27.679 INFO:tasks.rados.rados.0.smithi017.stdout:387: expect (ObjNum 84 snap 10 seq_num 84) 2024-09-21T09:39:27.679 INFO:tasks.rados.rados.0.smithi017.stdout:388: delete oid 22 current snap is 49 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:388: done (1 left) 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:389: rollback oid 29 current snap is 49 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 29 to 38 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:390: read oid 28 snap -1 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:390: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:391: read oid 33 snap -1 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:391: expect deleted 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:392: rollback oid 20 current snap is 49 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 48 2024-09-21T09:39:27.680 INFO:tasks.rados.rados.0.smithi017.stdout:393: snap_remove snap 23 2024-09-21T09:39:27.889 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:27.887+0000 7fe4454d4ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:39:28.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[29380]: Deploying cephadm binary to smithi134 2024-09-21T09:39:28.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[29380]: pgmap v5: 89 pgs: 89 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 287 MiB/s rd, 95 MiB/s wr, 1.05k op/s; 577 KiB/s, 2 objects/s recovering 2024-09-21T09:39:28.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[29380]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:39:28.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[29380]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:39:28.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[29380]: [21/Sep/2024:09:39:27] ENGINE Bus STARTING 2024-09-21T09:39:28.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[40737]: Deploying cephadm binary to smithi134 2024-09-21T09:39:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[40737]: pgmap v5: 89 pgs: 89 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 287 MiB/s rd, 95 MiB/s wr, 1.05k op/s; 577 KiB/s, 2 objects/s recovering 2024-09-21T09:39:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[40737]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:39:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[40737]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:39:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:27 smithi017 ceph-mon[40737]: [21/Sep/2024:09:39:27] ENGINE Bus STARTING 2024-09-21T09:39:28.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.007+0000 7fe4454d4ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:39:28.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.135+0000 7fe4454d4ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:39:28.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.197+0000 7fe4454d4ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:39:28.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:27 smithi134 ceph-mon[33541]: Deploying cephadm binary to smithi134 2024-09-21T09:39:28.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:27 smithi134 ceph-mon[33541]: pgmap v5: 89 pgs: 89 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 287 MiB/s rd, 95 MiB/s wr, 1.05k op/s; 577 KiB/s, 2 objects/s recovering 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:27 smithi134 ceph-mon[33541]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:27 smithi134 ceph-mon[33541]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:27 smithi134 ceph-mon[33541]: [21/Sep/2024:09:39:27] ENGINE Bus STARTING 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:28.044+0000 7f66bdab2640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:39:28] ENGINE Error in HTTPServer.serve 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: Traceback (most recent call last): 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: self._connections.run(self.expiration_interval) 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: self._run(expiration_interval) 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:39:28.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: s = self.context.wrap_socket( 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: return self.sslsocket_class._create( 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: self.do_handshake() 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: self._sslobj.do_handshake() 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:39:28.245 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:39:28.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.344+0000 7fe4454d4ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:39:28.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.421+0000 7fe4454d4ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:39:28.681 INFO:tasks.rados.rados.0.smithi017.stdout:393: done (5 left) 2024-09-21T09:39:28.682 INFO:tasks.rados.rados.0.smithi017.stdout:394: write oid 17 current snap is 49 2024-09-21T09:39:28.682 INFO:tasks.rados.rados.0.smithi017.stdout:392: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:39:28.682 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 0 (ObjNum 33 snap 0 seq_num 2384147832) dirty dne 2024-09-21T09:39:28.682 INFO:tasks.rados.rados.0.smithi017.stdout:389: finishing rollback tid 1 to smithi01785587-29 2024-09-21T09:39:28.682 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 84 (ObjNum 33 snap 0 seq_num 2399706784) dirty dne 2024-09-21T09:39:28.682 INFO:tasks.rados.rados.0.smithi017.stdout:387: expect (ObjNum 84 snap 10 seq_num 84) 2024-09-21T09:39:28.712 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.483+0000 7fe4454d4ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:39:28.713 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.630+0000 7fe4454d4ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:39:28.713 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.706+0000 7fe4454d4ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:39:28.806 INFO:tasks.rados.rados.0.smithi017.stdout:390: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:28.939 INFO:tasks.rados.rados.0.smithi017.stdout:394: seq_num 185 ranges {709295=713537,1859404=750382} 2024-09-21T09:39:28.947 INFO:tasks.rados.rados.0.smithi017.stdout:394: writing smithi01785587-17 from 709295 to 1422832 tid 1 2024-09-21T09:39:28.954 INFO:tasks.rados.rados.0.smithi017.stdout:394: writing smithi01785587-17 from 1859404 to 2609786 tid 2 2024-09-21T09:39:28.954 INFO:tasks.rados.rados.0.smithi017.stdout:387: done (5 left) 2024-09-21T09:39:28.954 INFO:tasks.rados.rados.0.smithi017.stdout:389: done (4 left) 2024-09-21T09:39:28.954 INFO:tasks.rados.rados.0.smithi017.stdout:390: done (3 left) 2024-09-21T09:39:28.954 INFO:tasks.rados.rados.0.smithi017.stdout:391: done (2 left) 2024-09-21T09:39:28.954 INFO:tasks.rados.rados.0.smithi017.stdout:392: done (1 left) 2024-09-21T09:39:28.954 INFO:tasks.rados.rados.0.smithi017.stdout:395: write oid 23 current snap is 49 2024-09-21T09:39:28.954 INFO:tasks.rados.rados.0.smithi017.stdout:394: finishing write tid 1 to smithi01785587-17 2024-09-21T09:39:28.954 INFO:tasks.rados.rados.0.smithi017.stdout:395: seq_num 186 ranges {726102=788786,1917886=610393,3108206=17896} 2024-09-21T09:39:28.962 INFO:tasks.rados.rados.0.smithi017.stdout:395: writing smithi01785587-23 from 726102 to 1514888 tid 1 2024-09-21T09:39:28.968 INFO:tasks.rados.rados.0.smithi017.stdout:395: writing smithi01785587-23 from 1917886 to 2528279 tid 2 2024-09-21T09:39:28.971 INFO:tasks.rados.rados.0.smithi017.stdout:395: writing smithi01785587-23 from 3108206 to 3126102 tid 3 2024-09-21T09:39:28.971 INFO:tasks.rados.rados.0.smithi017.stdout:396: write oid 35 current snap is 49 2024-09-21T09:39:28.972 INFO:tasks.rados.rados.0.smithi017.stdout:394: finishing write tid 2 to smithi01785587-17 2024-09-21T09:39:28.972 INFO:tasks.rados.rados.0.smithi017.stdout:396: seq_num 187 ranges {742909=464035,1976368=754051} 2024-09-21T09:39:28.978 INFO:tasks.rados.rados.0.smithi017.stdout:396: writing smithi01785587-35 from 742909 to 1206944 tid 1 2024-09-21T09:39:28.987 INFO:tasks.rados.rados.0.smithi017.stdout:396: writing smithi01785587-35 from 1976368 to 2730419 tid 2 2024-09-21T09:39:28.987 INFO:tasks.rados.rados.0.smithi017.stdout:397: read oid 15 snap 47 2024-09-21T09:39:28.987 INFO:tasks.rados.rados.0.smithi017.stdout:397: expect deleted 2024-09-21T09:39:28.987 INFO:tasks.rados.rados.0.smithi017.stdout:398: snap_create 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:394: finishing write tid 3 to smithi01785587-17 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:394: finishing write tid 4 to smithi01785587-17 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 84 (ObjNum 185 snap 49 seq_num 185) dirty exists 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:394: left oid 17 (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:395: finishing write tid 1 to smithi01785587-23 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:395: finishing write tid 2 to smithi01785587-23 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:395: finishing write tid 3 to smithi01785587-23 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:395: finishing write tid 4 to smithi01785587-23 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:395: finishing write tid 5 to smithi01785587-23 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 178 (ObjNum 186 snap 49 seq_num 186) dirty exists 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:395: left oid 23 (ObjNum 186 snap 49 seq_num 186) 2024-09-21T09:39:28.988 INFO:tasks.rados.rados.0.smithi017.stdout:396: finishing write tid 1 to smithi01785587-35 2024-09-21T09:39:28.990 INFO:tasks.rados.rados.0.smithi017.stdout:396: finishing write tid 2 to smithi01785587-35 2024-09-21T09:39:28.990 INFO:tasks.rados.rados.0.smithi017.stdout:396: finishing write tid 3 to smithi01785587-35 2024-09-21T09:39:28.991 INFO:tasks.rados.rados.0.smithi017.stdout:396: finishing write tid 4 to smithi01785587-35 2024-09-21T09:39:28.991 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 40 (ObjNum 187 snap 49 seq_num 187) dirty exists 2024-09-21T09:39:28.991 INFO:tasks.rados.rados.0.smithi017.stdout:396: left oid 35 (ObjNum 187 snap 49 seq_num 187) 2024-09-21T09:39:29.001 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.777+0000 7fe4454d4ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:39:29.001 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.862+0000 7fe4454d4ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:39:29.001 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.928+0000 7fe4454d4ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:39:29.001 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:28.999+0000 7fe4454d4ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:39:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:29 smithi134 ceph-mon[33541]: [21/Sep/2024:09:39:27] ENGINE Serving on http://172.21.15.134:8765 2024-09-21T09:39:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:29 smithi134 ceph-mon[33541]: mgrmap e25: x(active, since 3s) 2024-09-21T09:39:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:29 smithi134 ceph-mon[33541]: [21/Sep/2024:09:39:28] ENGINE Serving on https://172.21.15.134:7150 2024-09-21T09:39:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:29 smithi134 ceph-mon[33541]: [21/Sep/2024:09:39:28] ENGINE Bus STARTED 2024-09-21T09:39:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:29 smithi134 ceph-mon[33541]: [21/Sep/2024:09:39:28] ENGINE Error in HTTPServer.serve 2024-09-21T09:39:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: Traceback (most recent call last): 2024-09-21T09:39:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:39:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:39:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._run(expiration_interval) 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: s = self.context.wrap_socket( 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: return self.sslsocket_class._create( 2024-09-21T09:39:29.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:39:29.246 INFO:journalctl@ceph.mon.b.smithi134.stdout: self.do_handshake() 2024-09-21T09:39:29.246 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:39:29.246 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._sslobj.do_handshake() 2024-09-21T09:39:29.246 INFO:journalctl@ceph.mon.b.smithi134.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:39:29.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:29 smithi134 ceph-mon[33541]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[29380]: [21/Sep/2024:09:39:27] ENGINE Serving on http://172.21.15.134:8765 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[29380]: mgrmap e25: x(active, since 3s) 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[29380]: [21/Sep/2024:09:39:28] ENGINE Serving on https://172.21.15.134:7150 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[29380]: [21/Sep/2024:09:39:28] ENGINE Bus STARTED 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[29380]: [21/Sep/2024:09:39:28] ENGINE Error in HTTPServer.serve 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:39:29.279 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[29380]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[40737]: [21/Sep/2024:09:39:27] ENGINE Serving on http://172.21.15.134:8765 2024-09-21T09:39:29.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[40737]: mgrmap e25: x(active, since 3s) 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[40737]: [21/Sep/2024:09:39:28] ENGINE Serving on https://172.21.15.134:7150 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[40737]: [21/Sep/2024:09:39:28] ENGINE Bus STARTED 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[40737]: [21/Sep/2024:09:39:28] ENGINE Error in HTTPServer.serve 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:39:29.281 INFO:journalctl@ceph.mon.c.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:39:29.282 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:39:29.282 INFO:journalctl@ceph.mon.c.smithi017.stdout: self.do_handshake() 2024-09-21T09:39:29.282 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:39:29.282 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:39:29.282 INFO:journalctl@ceph.mon.c.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:39:29.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-mon[40737]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:39:29.298 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:39:29.298 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (2m) 20s ago 3m 25.7M - 0.25.0 c8568f914cd2 b40d1bacbd50 2024-09-21T09:39:29.298 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi134 *:3000 running (3m) 18s ago 3m 87.3M - 9.4.7 954c08fa6188 f7936515cf4e 2024-09-21T09:39:29.298 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.coodep smithi017 running (2m) 20s ago 2m 13.8M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 81acf97499eb 2024-09-21T09:39:29.298 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi134.bwovvb smithi134 running (2m) 18s ago 2m 14.4M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be bf739a2a98de 2024-09-21T09:39:29.298 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi134 *:8443,9283 running (23s) 18s ago 7m 70.9M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 7b270a3ec585 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:9283,8765 running (9m) 20s ago 9m 451M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be cd2cad7817e1 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (9m) 20s ago 9m 48.0M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 52c5c6bf4934 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi134 running (8m) 18s ago 8m 37.5M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 7eb897902574 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (8m) 20s ago 8m 40.2M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 316af690367d 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (3m) 20s ago 3m 16.0M - 1.5.0 0da6a335fe13 c1d4600d9989 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi134 *:9100 running (3m) 18s ago 3m 17.0M - 1.5.0 0da6a335fe13 a405017b5317 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (7m) 20s ago 7m 61.3M 4017M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 958bb6e5f55d 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (7m) 20s ago 7m 51.7M 4017M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5e9c6f25d8a5 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (6m) 20s ago 6m 64.7M 4017M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b87d82e4dc74 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (6m) 20s ago 6m 54.6M 4017M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 44d3b78acaa4 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi134 running (5m) 18s ago 5m 60.0M 4016M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 7ece30084c41 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi134 running (5m) 18s ago 5m 58.6M 4016M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5dec2255fcd7 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi134 running (5m) 18s ago 5m 47.5M 4016M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 80d4436a0b8a 2024-09-21T09:39:29.299 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi134 running (4m) 18s ago 4m 54.9M 4016M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed2e26ef5698 2024-09-21T09:39:29.300 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi134 *:9095 running (2m) 18s ago 4m 51.3M - 2.43.0 a07b618ecd1d b50e22f97e5d 2024-09-21T09:39:29.683 INFO:tasks.rados.rados.0.smithi017.stdout:394: done (4 left) 2024-09-21T09:39:29.683 INFO:tasks.rados.rados.0.smithi017.stdout:395: done (3 left) 2024-09-21T09:39:29.683 INFO:tasks.rados.rados.0.smithi017.stdout:396: done (2 left) 2024-09-21T09:39:29.683 INFO:tasks.rados.rados.0.smithi017.stdout:397: done (1 left) 2024-09-21T09:39:29.683 INFO:tasks.rados.rados.0.smithi017.stdout:398: done (0 left) 2024-09-21T09:39:29.683 INFO:tasks.rados.rados.0.smithi017.stdout:399: read oid 47 snap 46 2024-09-21T09:39:29.683 INFO:tasks.rados.rados.0.smithi017.stdout:399: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:29.683 INFO:tasks.rados.rados.0.smithi017.stdout:400: rollback oid 24 current snap is 50 2024-09-21T09:39:29.683 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 24 to 20 2024-09-21T09:39:29.684 INFO:tasks.rados.rados.0.smithi017.stdout:401: delete oid 8 current snap is 50 2024-09-21T09:39:29.686 INFO:tasks.rados.rados.0.smithi017.stdout:399: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:29.810 INFO:tasks.rados.rados.0.smithi017.stdout:399: done (2 left) 2024-09-21T09:39:29.810 INFO:tasks.rados.rados.0.smithi017.stdout:401: done (1 left) 2024-09-21T09:39:29.810 INFO:tasks.rados.rados.0.smithi017.stdout:402: snap_remove snap 35 2024-09-21T09:39:30.161 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:29.828+0000 7fe4454d4ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:39:30.161 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:39:29] ENGINE Bus STARTING 2024-09-21T09:39:30.161 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: CherryPy Checker: 2024-09-21T09:39:30.161 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: The Application mounted at '' has an empty config. 2024-09-21T09:39:30.161 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: 2024-09-21T09:39:30.161 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:39:29] ENGINE Serving on http://:::9283 2024-09-21T09:39:30.162 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: [21/Sep/2024:09:39:29] ENGINE Bus STARTED 2024-09-21T09:39:30.162 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:39:30.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade pause' 2024-09-21T09:39:30.464 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:39:30.689 INFO:tasks.rados.rados.0.smithi017.stdout:402: done (1 left) 2024-09-21T09:39:30.689 INFO:tasks.rados.rados.0.smithi017.stdout:403: write oid 8 current snap is 50 2024-09-21T09:39:30.689 INFO:tasks.rados.rados.0.smithi017.stdout:403: seq_num 189 ranges {776523=614533,1976979=757720} 2024-09-21T09:39:30.698 INFO:tasks.rados.rados.0.smithi017.stdout:403: writing smithi01785587-8 from 776523 to 1391056 tid 1 2024-09-21T09:39:30.705 INFO:tasks.rados.rados.0.smithi017.stdout:403: writing smithi01785587-8 from 1976979 to 2734699 tid 2 2024-09-21T09:39:30.705 INFO:tasks.rados.rados.0.smithi017.stdout:404: write oid 39 current snap is 50 2024-09-21T09:39:30.705 INFO:tasks.rados.rados.0.smithi017.stdout:404: seq_num 190 ranges {793330=689782,2035461=617731,3101545=91785} 2024-09-21T09:39:30.713 INFO:tasks.rados.rados.0.smithi017.stdout:404: writing smithi01785587-39 from 793330 to 1483112 tid 1 2024-09-21T09:39:30.720 INFO:tasks.rados.rados.0.smithi017.stdout:404: writing smithi01785587-39 from 2035461 to 2653192 tid 2 2024-09-21T09:39:30.724 INFO:tasks.rados.rados.0.smithi017.stdout:404: writing smithi01785587-39 from 3101545 to 3193330 tid 3 2024-09-21T09:39:30.724 INFO:tasks.rados.rados.0.smithi017.stdout:405: write oid 43 current snap is 50 2024-09-21T09:39:30.724 INFO:tasks.rados.rados.0.smithi017.stdout:400: finishing rollback tid 1 to smithi01785587-24 2024-09-21T09:39:30.724 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 0 (ObjNum 129 snap 0 seq_num 2399634600) dirty dne 2024-09-21T09:39:30.724 INFO:tasks.rados.rados.0.smithi017.stdout:403: finishing write tid 1 to smithi01785587-8 2024-09-21T09:39:30.724 INFO:tasks.rados.rados.0.smithi017.stdout:405: seq_num 191 ranges {410137=481384,1410296=761389,2728968=481169} 2024-09-21T09:39:30.728 INFO:tasks.rados.rados.0.smithi017.stdout:405: writing smithi01785587-43 from 410137 to 891521 tid 1 2024-09-21T09:39:30.735 INFO:tasks.rados.rados.0.smithi017.stdout:405: writing smithi01785587-43 from 1410296 to 2171685 tid 2 2024-09-21T09:39:30.741 INFO:tasks.rados.rados.0.smithi017.stdout:405: writing smithi01785587-43 from 2728968 to 3210137 tid 3 2024-09-21T09:39:30.741 INFO:tasks.rados.rados.0.smithi017.stdout:400: done (3 left) 2024-09-21T09:39:30.741 INFO:tasks.rados.rados.0.smithi017.stdout:406: rollback oid 25 current snap is 50 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 38 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:403: finishing write tid 2 to smithi01785587-8 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:403: finishing write tid 3 to smithi01785587-8 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:403: finishing write tid 4 to smithi01785587-8 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 185 (ObjNum 189 snap 50 seq_num 189) dirty exists 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:403: left oid 8 (ObjNum 189 snap 50 seq_num 189) 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:404: finishing write tid 1 to smithi01785587-39 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:404: finishing write tid 2 to smithi01785587-39 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:403: done (3 left) 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:407: write oid 44 current snap is 50 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:404: finishing write tid 3 to smithi01785587-39 2024-09-21T09:39:30.742 INFO:tasks.rados.rados.0.smithi017.stdout:407: seq_num 192 ranges {426944=556633,1468778=621400,2872744=354200} 2024-09-21T09:39:30.746 INFO:tasks.rados.rados.0.smithi017.stdout:407: writing smithi01785587-44 from 426944 to 983577 tid 1 2024-09-21T09:39:30.752 INFO:tasks.rados.rados.0.smithi017.stdout:407: writing smithi01785587-44 from 1468778 to 2090178 tid 2 2024-09-21T09:39:30.759 INFO:tasks.rados.rados.0.smithi017.stdout:407: writing smithi01785587-44 from 2872744 to 3226944 tid 3 2024-09-21T09:39:30.759 INFO:tasks.rados.rados.0.smithi017.stdout:408: snap_create 2024-09-21T09:39:30.759 INFO:tasks.rados.rados.0.smithi017.stdout:404: finishing write tid 4 to smithi01785587-39 2024-09-21T09:39:30.759 INFO:tasks.rados.rados.0.smithi017.stdout:404: finishing write tid 5 to smithi01785587-39 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 114 (ObjNum 190 snap 50 seq_num 190) dirty exists 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:404: left oid 39 (ObjNum 190 snap 50 seq_num 190) 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:405: finishing write tid 1 to smithi01785587-43 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:405: finishing write tid 2 to smithi01785587-43 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:406: finishing rollback tid 0 to smithi01785587-25 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:406: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:405: finishing write tid 3 to smithi01785587-43 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:405: finishing write tid 4 to smithi01785587-43 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:406: finishing rollback tid 2 to smithi01785587-25 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 108 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:405: finishing write tid 5 to smithi01785587-43 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 126 (ObjNum 191 snap 50 seq_num 191) dirty exists 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:405: left oid 43 (ObjNum 191 snap 50 seq_num 191) 2024-09-21T09:39:30.760 INFO:tasks.rados.rados.0.smithi017.stdout:407: finishing write tid 1 to smithi01785587-44 2024-09-21T09:39:30.761 INFO:tasks.rados.rados.0.smithi017.stdout:407: finishing write tid 2 to smithi01785587-44 2024-09-21T09:39:30.761 INFO:tasks.rados.rados.0.smithi017.stdout:407: finishing write tid 3 to smithi01785587-44 2024-09-21T09:39:30.761 INFO:tasks.rados.rados.0.smithi017.stdout:407: finishing write tid 4 to smithi01785587-44 2024-09-21T09:39:30.761 INFO:tasks.rados.rados.0.smithi017.stdout:407: finishing write tid 5 to smithi01785587-44 2024-09-21T09:39:30.761 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 209 (ObjNum 192 snap 50 seq_num 192) dirty exists 2024-09-21T09:39:30.761 INFO:tasks.rados.rados.0.smithi017.stdout:407: left oid 44 (ObjNum 192 snap 50 seq_num 192) 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: pgmap v9: 89 pgs: 89 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 96 MiB/s rd, 31 MiB/s wr, 351 op/s; 192 KiB/s, 0 objects/s recovering 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: from='client.25232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: Standby manager daemon y started 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-21T09:39:30.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: pgmap v9: 89 pgs: 89 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 96 MiB/s rd, 31 MiB/s wr, 351 op/s; 192 KiB/s, 0 objects/s recovering 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: from='client.25232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: Standby manager daemon y started 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-21T09:39:30.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:30 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: pgmap v9: 89 pgs: 89 active+clean; 177 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 96 MiB/s rd, 31 MiB/s wr, 351 op/s; 192 KiB/s, 0 objects/s recovering 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: from='client.25232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: Standby manager daemon y started 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.17:0/3378216662' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-21T09:39:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:30 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:39:31.693 INFO:tasks.rados.rados.0.smithi017.stdout:404: done (4 left) 2024-09-21T09:39:31.693 INFO:tasks.rados.rados.0.smithi017.stdout:405: done (3 left) 2024-09-21T09:39:31.693 INFO:tasks.rados.rados.0.smithi017.stdout:406: done (2 left) 2024-09-21T09:39:31.693 INFO:tasks.rados.rados.0.smithi017.stdout:407: done (1 left) 2024-09-21T09:39:31.693 INFO:tasks.rados.rados.0.smithi017.stdout:408: done (0 left) 2024-09-21T09:39:31.693 INFO:tasks.rados.rados.0.smithi017.stdout:409: snap_create 2024-09-21T09:39:31.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:31 smithi017 ceph-mon[29380]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:39:31.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:31 smithi017 ceph-mon[40737]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:39:31.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:31 smithi134 ceph-mon[33541]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:39:32.695 INFO:tasks.rados.rados.0.smithi017.stdout:409: done (0 left) 2024-09-21T09:39:32.695 INFO:tasks.rados.rados.0.smithi017.stdout:410: write oid 8 current snap is 52 2024-09-21T09:39:32.696 INFO:tasks.rados.rados.0.smithi017.stdout:410: seq_num 193 ranges {443751=631882,1810907=765058,3067461=176290} 2024-09-21T09:39:32.701 INFO:tasks.rados.rados.0.smithi017.stdout:410: writing smithi01785587-8 from 443751 to 1075633 tid 1 2024-09-21T09:39:32.710 INFO:tasks.rados.rados.0.smithi017.stdout:410: writing smithi01785587-8 from 1810907 to 2575965 tid 2 2024-09-21T09:39:32.714 INFO:tasks.rados.rados.0.smithi017.stdout:410: writing smithi01785587-8 from 3067461 to 3243751 tid 3 2024-09-21T09:39:32.714 INFO:tasks.rados.rados.0.smithi017.stdout:411: snap_create 2024-09-21T09:39:32.714 INFO:tasks.rados.rados.0.smithi017.stdout:410: finishing write tid 1 to smithi01785587-8 2024-09-21T09:39:32.714 INFO:tasks.rados.rados.0.smithi017.stdout:410: finishing write tid 2 to smithi01785587-8 2024-09-21T09:39:32.716 INFO:tasks.rados.rados.0.smithi017.stdout:410: finishing write tid 3 to smithi01785587-8 2024-09-21T09:39:32.716 INFO:tasks.rados.rados.0.smithi017.stdout:410: finishing write tid 4 to smithi01785587-8 2024-09-21T09:39:32.716 INFO:tasks.rados.rados.0.smithi017.stdout:410: finishing write tid 5 to smithi01785587-8 2024-09-21T09:39:32.716 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 190 (ObjNum 193 snap 52 seq_num 193) dirty exists 2024-09-21T09:39:32.716 INFO:tasks.rados.rados.0.smithi017.stdout:410: left oid 8 (ObjNum 193 snap 52 seq_num 193) 2024-09-21T09:39:32.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:32 smithi017 ceph-mon[29380]: pgmap v12: 89 pgs: 89 active+clean; 200 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.7 MiB/s wr, 42 op/s 2024-09-21T09:39:32.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:32 smithi017 ceph-mon[29380]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:39:32.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:32 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:32.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:32 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:32.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:32 smithi017 ceph-mon[40737]: pgmap v12: 89 pgs: 89 active+clean; 200 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.7 MiB/s wr, 42 op/s 2024-09-21T09:39:32.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:32 smithi017 ceph-mon[40737]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:39:32.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:32 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:32.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:32 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:32.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:32 smithi134 ceph-mon[33541]: pgmap v12: 89 pgs: 89 active+clean; 200 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.7 MiB/s wr, 42 op/s 2024-09-21T09:39:32.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:32 smithi134 ceph-mon[33541]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:39:32.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:32 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:32 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:33.135 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:33.704 INFO:tasks.rados.rados.0.smithi017.stdout:410: done (1 left) 2024-09-21T09:39:33.704 INFO:tasks.rados.rados.0.smithi017.stdout:411: done (0 left) 2024-09-21T09:39:33.704 INFO:tasks.rados.rados.0.smithi017.stdout:412: snap_create 2024-09-21T09:39:33.831 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:33 smithi134 ceph-mon[33541]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:39:33.831 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:33 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:33.831 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:33 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:33.831 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:33 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:33 smithi017 ceph-mon[29380]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:39:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:33 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:33 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:33 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:33 smithi017 ceph-mon[40737]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:39:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:33 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:33 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:33 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:34.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'sleep 60' 2024-09-21T09:39:34.271 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:39:34.705 INFO:tasks.rados.rados.0.smithi017.stdout:412: done (0 left) 2024-09-21T09:39:34.705 INFO:tasks.rados.rados.0.smithi017.stdout:413: snap_remove snap 46 2024-09-21T09:39:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[40737]: pgmap v15: 89 pgs: 89 active+clean; 202 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.1 MiB/s wr, 44 op/s 2024-09-21T09:39:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[40737]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[40737]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[40737]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:39:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[29380]: pgmap v15: 89 pgs: 89 active+clean; 202 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.1 MiB/s wr, 44 op/s 2024-09-21T09:39:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[29380]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[29380]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[29380]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:39:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:34 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:34.983 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:34 smithi134 ceph-mon[33541]: pgmap v15: 89 pgs: 89 active+clean; 202 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 3.1 MiB/s wr, 44 op/s 2024-09-21T09:39:34.983 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:34 smithi134 ceph-mon[33541]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:34.983 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:34 smithi134 ceph-mon[33541]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:34.983 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:34 smithi134 ceph-mon[33541]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:39:34.983 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:34 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:34.983 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:34 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:35.708 INFO:tasks.rados.rados.0.smithi017.stdout:413: done (0 left) 2024-09-21T09:39:35.709 INFO:tasks.rados.rados.0.smithi017.stdout:414: rollback oid 9 current snap is 54 2024-09-21T09:39:35.709 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 9 to 49 2024-09-21T09:39:35.709 INFO:tasks.rados.rados.0.smithi017.stdout:415: rollback oid 11 current snap is 54 2024-09-21T09:39:35.709 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 40 2024-09-21T09:39:35.709 INFO:tasks.rados.rados.0.smithi017.stdout:416: write oid 25 current snap is 54 2024-09-21T09:39:35.709 INFO:tasks.rados.rados.0.smithi017.stdout:416: seq_num 194 ranges {460558=707131,1869389=508716,3094884=165674} 2024-09-21T09:39:35.716 INFO:tasks.rados.rados.0.smithi017.stdout:416: writing smithi01785587-25 from 460558 to 1167689 tid 1 2024-09-21T09:39:35.723 INFO:tasks.rados.rados.0.smithi017.stdout:416: writing smithi01785587-25 from 1869389 to 2378105 tid 2 2024-09-21T09:39:35.729 INFO:tasks.rados.rados.0.smithi017.stdout:416: writing smithi01785587-25 from 3094884 to 3260558 tid 3 2024-09-21T09:39:35.729 INFO:tasks.rados.rados.0.smithi017.stdout:417: write oid 5 current snap is 54 2024-09-21T09:39:35.729 INFO:tasks.rados.rados.0.smithi017.stdout:415: finishing rollback tid 0 to smithi01785587-11 2024-09-21T09:39:35.729 INFO:tasks.rados.rados.0.smithi017.stdout:417: seq_num 195 ranges {477365=782380,1927871=768727,3122307=155058} 2024-09-21T09:39:35.735 INFO:tasks.rados.rados.0.smithi017.stdout:417: writing smithi01785587-5 from 477365 to 1259745 tid 1 2024-09-21T09:39:35.743 INFO:tasks.rados.rados.0.smithi017.stdout:417: writing smithi01785587-5 from 1927871 to 2696598 tid 2 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:417: writing smithi01785587-5 from 3122307 to 3277365 tid 3 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:418: read oid 6 snap -1 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:418: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:419: write oid 35 current snap is 54 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:415: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 195 (ObjNum 305 snap 0 seq_num 1) dirty dne 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:414: finishing rollback tid 0 to smithi01785587-9 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:414: finishing rollback tid 1 to smithi01785587-9 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:414: finishing rollback tid 2 to smithi01785587-9 2024-09-21T09:39:35.748 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 54 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-09-21T09:39:35.749 INFO:tasks.rados.rados.0.smithi017.stdout:416: finishing write tid 1 to smithi01785587-25 2024-09-21T09:39:35.749 INFO:tasks.rados.rados.0.smithi017.stdout:416: finishing write tid 2 to smithi01785587-25 2024-09-21T09:39:35.749 INFO:tasks.rados.rados.0.smithi017.stdout:419: seq_num 196 ranges {494172=457629,1586353=512385,2749730=544442} 2024-09-21T09:39:35.753 INFO:tasks.rados.rados.0.smithi017.stdout:419: writing smithi01785587-35 from 494172 to 951801 tid 1 2024-09-21T09:39:35.759 INFO:tasks.rados.rados.0.smithi017.stdout:419: writing smithi01785587-35 from 1586353 to 2098738 tid 2 2024-09-21T09:39:35.768 INFO:tasks.rados.rados.0.smithi017.stdout:419: writing smithi01785587-35 from 2749730 to 3294172 tid 3 2024-09-21T09:39:35.768 INFO:tasks.rados.rados.0.smithi017.stdout:414: done (5 left) 2024-09-21T09:39:35.768 INFO:tasks.rados.rados.0.smithi017.stdout:415: done (4 left) 2024-09-21T09:39:35.769 INFO:tasks.rados.rados.0.smithi017.stdout:420: rollback oid 28 current snap is 54 2024-09-21T09:39:35.769 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 28 to 36 2024-09-21T09:39:35.769 INFO:tasks.rados.rados.0.smithi017.stdout:416: finishing write tid 3 to smithi01785587-25 2024-09-21T09:39:35.769 INFO:tasks.rados.rados.0.smithi017.stdout:416: finishing write tid 4 to smithi01785587-25 2024-09-21T09:39:35.769 INFO:tasks.rados.rados.0.smithi017.stdout:416: finishing write tid 5 to smithi01785587-25 2024-09-21T09:39:35.769 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 113 (ObjNum 194 snap 54 seq_num 194) dirty exists 2024-09-21T09:39:35.769 INFO:tasks.rados.rados.0.smithi017.stdout:416: left oid 25 (ObjNum 194 snap 54 seq_num 194) 2024-09-21T09:39:35.770 INFO:tasks.rados.rados.0.smithi017.stdout:417: finishing write tid 1 to smithi01785587-5 2024-09-21T09:39:35.770 INFO:tasks.rados.rados.0.smithi017.stdout:417: finishing write tid 2 to smithi01785587-5 2024-09-21T09:39:35.770 INFO:tasks.rados.rados.0.smithi017.stdout:416: done (4 left) 2024-09-21T09:39:35.770 INFO:tasks.rados.rados.0.smithi017.stdout:421: write oid 20 current snap is 54 2024-09-21T09:39:35.770 INFO:tasks.rados.rados.0.smithi017.stdout:417: finishing write tid 3 to smithi01785587-5 2024-09-21T09:39:35.770 INFO:tasks.rados.rados.0.smithi017.stdout:417: finishing write tid 4 to smithi01785587-5 2024-09-21T09:39:35.771 INFO:tasks.rados.rados.0.smithi017.stdout:417: finishing write tid 5 to smithi01785587-5 2024-09-21T09:39:35.771 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 217 (ObjNum 195 snap 54 seq_num 195) dirty exists 2024-09-21T09:39:35.771 INFO:tasks.rados.rados.0.smithi017.stdout:417: left oid 5 (ObjNum 195 snap 54 seq_num 195) 2024-09-21T09:39:35.771 INFO:tasks.rados.rados.0.smithi017.stdout:421: seq_num 197 ranges {510979=532878,1528482=772396,3060800=250179} 2024-09-21T09:39:35.774 INFO:tasks.rados.rados.0.smithi017.stdout:421: writing smithi01785587-20 from 510979 to 1043857 tid 1 2024-09-21T09:39:35.779 INFO:tasks.rados.rados.0.smithi017.stdout:421: writing smithi01785587-20 from 1528482 to 2300878 tid 2 2024-09-21T09:39:35.784 INFO:tasks.rados.rados.0.smithi017.stdout:421: writing smithi01785587-20 from 3060800 to 3310979 tid 3 2024-09-21T09:39:35.784 INFO:tasks.rados.rados.0.smithi017.stdout:417: done (4 left) 2024-09-21T09:39:35.784 INFO:tasks.rados.rados.0.smithi017.stdout:422: snap_create 2024-09-21T09:39:35.784 INFO:tasks.rados.rados.0.smithi017.stdout:418: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:39:35.856 INFO:tasks.rados.rados.0.smithi017.stdout:419: finishing write tid 1 to smithi01785587-35 2024-09-21T09:39:35.856 INFO:tasks.rados.rados.0.smithi017.stdout:419: finishing write tid 2 to smithi01785587-35 2024-09-21T09:39:35.856 INFO:tasks.rados.rados.0.smithi017.stdout:420: finishing rollback tid 0 to smithi01785587-28 2024-09-21T09:39:35.856 INFO:tasks.rados.rados.0.smithi017.stdout:419: finishing write tid 3 to smithi01785587-35 2024-09-21T09:39:35.856 INFO:tasks.rados.rados.0.smithi017.stdout:420: finishing rollback tid 1 to smithi01785587-28 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:419: finishing write tid 4 to smithi01785587-35 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:420: finishing rollback tid 2 to smithi01785587-28 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 198 (ObjNum 122 snap 23 seq_num 122) dirty exists 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:419: finishing write tid 5 to smithi01785587-35 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 47 (ObjNum 196 snap 54 seq_num 196) dirty exists 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:419: left oid 35 (ObjNum 196 snap 54 seq_num 196) 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:421: finishing write tid 1 to smithi01785587-20 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:421: finishing write tid 2 to smithi01785587-20 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:421: finishing write tid 3 to smithi01785587-20 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:421: finishing write tid 4 to smithi01785587-20 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:421: finishing write tid 5 to smithi01785587-20 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 117 (ObjNum 197 snap 54 seq_num 197) dirty exists 2024-09-21T09:39:35.857 INFO:tasks.rados.rados.0.smithi017.stdout:421: left oid 20 (ObjNum 197 snap 54 seq_num 197) 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:35 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:35.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:39:35.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:35.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:35.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:35.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.001 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.001 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.001 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.001 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:35 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:36.721 INFO:tasks.rados.rados.0.smithi017.stdout:418: done (4 left) 2024-09-21T09:39:36.721 INFO:tasks.rados.rados.0.smithi017.stdout:419: done (3 left) 2024-09-21T09:39:36.721 INFO:tasks.rados.rados.0.smithi017.stdout:420: done (2 left) 2024-09-21T09:39:36.721 INFO:tasks.rados.rados.0.smithi017.stdout:421: done (1 left) 2024-09-21T09:39:36.721 INFO:tasks.rados.rados.0.smithi017.stdout:422: done (0 left) 2024-09-21T09:39:36.721 INFO:tasks.rados.rados.0.smithi017.stdout:423: rollback oid 48 current snap is 55 2024-09-21T09:39:36.721 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 38 2024-09-21T09:39:36.722 INFO:tasks.rados.rados.0.smithi017.stdout:424: snap_create 2024-09-21T09:39:36.724 INFO:tasks.rados.rados.0.smithi017.stdout:423: finishing rollback tid 0 to smithi01785587-48 2024-09-21T09:39:36.725 INFO:tasks.rados.rados.0.smithi017.stdout:423: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:39:36.725 INFO:tasks.rados.rados.0.smithi017.stdout:423: finishing rollback tid 2 to smithi01785587-48 2024-09-21T09:39:36.725 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 140 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-09-21T09:39:36.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: pgmap v18: 89 pgs: 89 active+clean; 202 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 374 KiB/s wr, 1 op/s 2024-09-21T09:39:36.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi017 to 2481M 2024-09-21T09:39:36.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:39:36.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:36.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:36.969 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:36.970 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: pgmap v18: 89 pgs: 89 active+clean; 202 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 374 KiB/s wr, 1 op/s 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi017 to 2481M 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: pgmap v18: 89 pgs: 89 active+clean; 202 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 374 KiB/s wr, 1 op/s 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi017 to 2481M 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:37.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:36 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:37.243 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:36] "GET /metrics HTTP/1.1" 200 35903 "" "Prometheus/2.43.0" 2024-09-21T09:39:37.682 INFO:tasks.rados.rados.0.smithi017.stdout:423: done (1 left) 2024-09-21T09:39:37.682 INFO:tasks.rados.rados.0.smithi017.stdout:424: done (0 left) 2024-09-21T09:39:37.682 INFO:tasks.rados.rados.0.smithi017.stdout:425: snap_remove snap 41 2024-09-21T09:39:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: Adjusting osd_memory_target on smithi134 to 2480M 2024-09-21T09:39:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:39:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:39:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:37.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:37 smithi134 ceph-mon[33541]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:39:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: Adjusting osd_memory_target on smithi134 to 2480M 2024-09-21T09:39:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:39:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:39:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:39:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[40737]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: Adjusting osd_memory_target on smithi134 to 2480M 2024-09-21T09:39:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:38.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:37 smithi017 ceph-mon[29380]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:39:38.687 INFO:tasks.rados.rados.0.smithi017.stdout:425: done (0 left) 2024-09-21T09:39:38.687 INFO:tasks.rados.rados.0.smithi017.stdout:426: delete oid 27 current snap is 56 2024-09-21T09:39:38.693 INFO:tasks.rados.rados.0.smithi017.stdout:426: done (0 left) 2024-09-21T09:39:38.693 INFO:tasks.rados.rados.0.smithi017.stdout:427: rollback oid 33 current snap is 56 2024-09-21T09:39:38.693 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 31 2024-09-21T09:39:38.693 INFO:tasks.rados.rados.0.smithi017.stdout:428: read oid 45 snap -1 2024-09-21T09:39:38.693 INFO:tasks.rados.rados.0.smithi017.stdout:428: expect (ObjNum 179 snap 47 seq_num 179) 2024-09-21T09:39:38.693 INFO:tasks.rados.rados.0.smithi017.stdout:429: read oid 26 snap -1 2024-09-21T09:39:38.693 INFO:tasks.rados.rados.0.smithi017.stdout:429: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:38.693 INFO:tasks.rados.rados.0.smithi017.stdout:430: write oid 43 current snap is 56 2024-09-21T09:39:38.693 INFO:tasks.rados.rados.0.smithi017.stdout:430: seq_num 199 ranges {544593=799729,1761799=776065,2948352=396241} 2024-09-21T09:39:38.701 INFO:tasks.rados.rados.0.smithi017.stdout:430: writing smithi01785587-43 from 544593 to 1344322 tid 1 2024-09-21T09:39:38.708 INFO:tasks.rados.rados.0.smithi017.stdout:430: writing smithi01785587-43 from 1761799 to 2537864 tid 2 2024-09-21T09:39:38.713 INFO:tasks.rados.rados.0.smithi017.stdout:430: writing smithi01785587-43 from 2948352 to 3344593 tid 3 2024-09-21T09:39:38.713 INFO:tasks.rados.rados.0.smithi017.stdout:431: read oid 47 snap -1 2024-09-21T09:39:38.713 INFO:tasks.rados.rados.0.smithi017.stdout:431: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:38.713 INFO:tasks.rados.rados.0.smithi017.stdout:432: read oid 31 snap -1 2024-09-21T09:39:38.713 INFO:tasks.rados.rados.0.smithi017.stdout:432: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:39:38.713 INFO:tasks.rados.rados.0.smithi017.stdout:433: write oid 30 current snap is 56 2024-09-21T09:39:38.713 INFO:tasks.rados.rados.0.smithi017.stdout:433: seq_num 200 ranges {561400=474978,1820281=519723,2859422=457579} 2024-09-21T09:39:38.719 INFO:tasks.rados.rados.0.smithi017.stdout:433: writing smithi01785587-30 from 561400 to 1036378 tid 1 2024-09-21T09:39:38.726 INFO:tasks.rados.rados.0.smithi017.stdout:433: writing smithi01785587-30 from 1820281 to 2340004 tid 2 2024-09-21T09:39:38.731 INFO:tasks.rados.rados.0.smithi017.stdout:433: writing smithi01785587-30 from 2859422 to 3317001 tid 3 2024-09-21T09:39:38.731 INFO:tasks.rados.rados.0.smithi017.stdout:434: write oid 21 current snap is 56 2024-09-21T09:39:38.731 INFO:tasks.rados.rados.0.smithi017.stdout:434: seq_num 201 ranges {578207=550227,1762410=779734,3286845=91362} 2024-09-21T09:39:38.737 INFO:tasks.rados.rados.0.smithi017.stdout:434: writing smithi01785587-21 from 578207 to 1128434 tid 1 2024-09-21T09:39:38.745 INFO:tasks.rados.rados.0.smithi017.stdout:434: writing smithi01785587-21 from 1762410 to 2542144 tid 2 2024-09-21T09:39:38.750 INFO:tasks.rados.rados.0.smithi017.stdout:434: writing smithi01785587-21 from 3286845 to 3378207 tid 3 2024-09-21T09:39:38.750 INFO:tasks.rados.rados.0.smithi017.stdout:435: rollback oid 1 current snap is 56 2024-09-21T09:39:38.750 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 1 to 47 2024-09-21T09:39:38.750 INFO:tasks.rados.rados.0.smithi017.stdout:428: expect (ObjNum 179 snap 47 seq_num 179) 2024-09-21T09:39:38.882 INFO:tasks.rados.rados.0.smithi017.stdout:427: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:39:38.882 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 169 (ObjNum 49 snap 0 seq_num 1) dirty dne 2024-09-21T09:39:38.882 INFO:tasks.rados.rados.0.smithi017.stdout:429: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:38.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:38 smithi134 ceph-mon[33541]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:39:38.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:38 smithi134 ceph-mon[33541]: Deploying daemon alertmanager.a on smithi017 2024-09-21T09:39:38.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:38 smithi134 ceph-mon[33541]: pgmap v21: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:39:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:38 smithi134 ceph-mon[33541]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:39:39.028 INFO:tasks.rados.rados.0.smithi017.stdout:430: finishing write tid 1 to smithi01785587-43 2024-09-21T09:39:39.028 INFO:tasks.rados.rados.0.smithi017.stdout:430: finishing write tid 2 to smithi01785587-43 2024-09-21T09:39:39.028 INFO:tasks.rados.rados.0.smithi017.stdout:427: done (8 left) 2024-09-21T09:39:39.028 INFO:tasks.rados.rados.0.smithi017.stdout:428: done (7 left) 2024-09-21T09:39:39.029 INFO:tasks.rados.rados.0.smithi017.stdout:429: done (6 left) 2024-09-21T09:39:39.029 INFO:tasks.rados.rados.0.smithi017.stdout:436: snap_remove snap 53 2024-09-21T09:39:39.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:38 smithi017 ceph-mon[29380]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:39:39.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:38 smithi017 ceph-mon[29380]: Deploying daemon alertmanager.a on smithi017 2024-09-21T09:39:39.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:38 smithi017 ceph-mon[29380]: pgmap v21: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:39:39.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:38 smithi017 ceph-mon[29380]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:39:39.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:38 smithi017 ceph-mon[40737]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:39:39.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:38 smithi017 ceph-mon[40737]: Deploying daemon alertmanager.a on smithi017 2024-09-21T09:39:39.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:38 smithi017 ceph-mon[40737]: pgmap v21: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:39:39.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:38 smithi017 ceph-mon[40737]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:39:39.729 INFO:tasks.rados.rados.0.smithi017.stdout:436: done (6 left) 2024-09-21T09:39:39.729 INFO:tasks.rados.rados.0.smithi017.stdout:437: write oid 37 current snap is 56 2024-09-21T09:39:39.729 INFO:tasks.rados.rados.0.smithi017.stdout:437: seq_num 202 ranges {595014=625476,1820892=523392,2797915=596476} 2024-09-21T09:39:39.736 INFO:tasks.rados.rados.0.smithi017.stdout:437: writing smithi01785587-37 from 595014 to 1220490 tid 1 2024-09-21T09:39:39.744 INFO:tasks.rados.rados.0.smithi017.stdout:437: writing smithi01785587-37 from 1820892 to 2344284 tid 2 2024-09-21T09:39:39.750 INFO:tasks.rados.rados.0.smithi017.stdout:437: writing smithi01785587-37 from 2797915 to 3394391 tid 3 2024-09-21T09:39:39.750 INFO:tasks.rados.rados.0.smithi017.stdout:438: write oid 13 current snap is 56 2024-09-21T09:39:39.750 INFO:tasks.rados.rados.0.smithi017.stdout:438: seq_num 203 ranges {611821=700725,1879374=783403,3341691=70130} 2024-09-21T09:39:39.758 INFO:tasks.rados.rados.0.smithi017.stdout:438: writing smithi01785587-13 from 611821 to 1312546 tid 1 2024-09-21T09:39:39.765 INFO:tasks.rados.rados.0.smithi017.stdout:438: writing smithi01785587-13 from 1879374 to 2662777 tid 2 2024-09-21T09:39:39.770 INFO:tasks.rados.rados.0.smithi017.stdout:438: writing smithi01785587-13 from 3341691 to 3411821 tid 3 2024-09-21T09:39:39.770 INFO:tasks.rados.rados.0.smithi017.stdout:439: snap_create 2024-09-21T09:39:39.770 INFO:tasks.rados.rados.0.smithi017.stdout:432: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:39:39.875 INFO:tasks.rados.rados.0.smithi017.stdout:431: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:430: finishing write tid 3 to smithi01785587-43 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:430: finishing write tid 4 to smithi01785587-43 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:430: finishing write tid 5 to smithi01785587-43 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 135 (ObjNum 199 snap 56 seq_num 199) dirty exists 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:430: left oid 43 (ObjNum 199 snap 56 seq_num 199) 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:433: finishing write tid 1 to smithi01785587-30 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:433: finishing write tid 2 to smithi01785587-30 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:433: finishing write tid 3 to smithi01785587-30 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:433: finishing write tid 4 to smithi01785587-30 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:433: finishing write tid 5 to smithi01785587-30 2024-09-21T09:39:39.999 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 151 (ObjNum 200 snap 56 seq_num 200) dirty exists 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:433: left oid 30 (ObjNum 200 snap 56 seq_num 200) 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:434: finishing write tid 1 to smithi01785587-21 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:434: finishing write tid 2 to smithi01785587-21 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:434: finishing write tid 3 to smithi01785587-21 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:434: finishing write tid 4 to smithi01785587-21 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:434: finishing write tid 5 to smithi01785587-21 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 166 (ObjNum 201 snap 56 seq_num 201) dirty exists 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:434: left oid 21 (ObjNum 201 snap 56 seq_num 201) 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:435: finishing rollback tid 0 to smithi01785587-1 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:435: finishing rollback tid 1 to smithi01785587-1 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 138 (ObjNum 33 snap 0 seq_num 3570409741) dirty dne 2024-09-21T09:39:40.000 INFO:tasks.rados.rados.0.smithi017.stdout:437: finishing write tid 1 to smithi01785587-37 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:437: finishing write tid 2 to smithi01785587-37 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:437: finishing write tid 3 to smithi01785587-37 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:437: finishing write tid 4 to smithi01785587-37 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:437: finishing write tid 5 to smithi01785587-37 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 127 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:437: left oid 37 (ObjNum 202 snap 56 seq_num 202) 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:438: finishing write tid 1 to smithi01785587-13 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:438: finishing write tid 2 to smithi01785587-13 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:438: finishing write tid 3 to smithi01785587-13 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:438: finishing write tid 4 to smithi01785587-13 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:438: finishing write tid 5 to smithi01785587-13 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 160 (ObjNum 203 snap 56 seq_num 203) dirty exists 2024-09-21T09:39:40.001 INFO:tasks.rados.rados.0.smithi017.stdout:438: left oid 13 (ObjNum 203 snap 56 seq_num 203) 2024-09-21T09:39:40.368 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:430: done (8 left) 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:431: done (7 left) 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:432: done (6 left) 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:433: done (5 left) 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:434: done (4 left) 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:435: done (3 left) 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:437: done (2 left) 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:438: done (1 left) 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:439: done (0 left) 2024-09-21T09:39:40.734 INFO:tasks.rados.rados.0.smithi017.stdout:440: snap_remove snap 44 2024-09-21T09:39:40.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:40 smithi134 ceph-mon[33541]: pgmap v24: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:39:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:40 smithi134 ceph-mon[33541]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:39:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:40 smithi017 ceph-mon[29380]: pgmap v24: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:39:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:40 smithi017 ceph-mon[29380]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:39:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:40 smithi017 ceph-mon[40737]: pgmap v24: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:39:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:40 smithi017 ceph-mon[40737]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:39:41.737 INFO:tasks.rados.rados.0.smithi017.stdout:440: done (0 left) 2024-09-21T09:39:41.737 INFO:tasks.rados.rados.0.smithi017.stdout:441: read oid 36 snap -1 2024-09-21T09:39:41.737 INFO:tasks.rados.rados.0.smithi017.stdout:441: expect deleted 2024-09-21T09:39:41.738 INFO:tasks.rados.rados.0.smithi017.stdout:442: read oid 35 snap 49 2024-09-21T09:39:41.738 INFO:tasks.rados.rados.0.smithi017.stdout:442: expect (ObjNum 187 snap 49 seq_num 187) 2024-09-21T09:39:41.738 INFO:tasks.rados.rados.0.smithi017.stdout:443: snap_remove snap 56 2024-09-21T09:39:41.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:41 smithi017 ceph-mon[29380]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:39:41.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:41 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:41.778 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:41 smithi017 ceph-mon[40737]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:39:41.778 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:41 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:41.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:41 smithi134 ceph-mon[33541]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:39:41.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:41 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:42.687 INFO:tasks.rados.rados.0.smithi017.stdout:443: done (2 left) 2024-09-21T09:39:42.687 INFO:tasks.rados.rados.0.smithi017.stdout:444: rollback oid 37 current snap is 57 2024-09-21T09:39:42.687 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 48 2024-09-21T09:39:42.687 INFO:tasks.rados.rados.0.smithi017.stdout:442: expect (ObjNum 187 snap 49 seq_num 187) 2024-09-21T09:39:42.799 INFO:tasks.rados.rados.0.smithi017.stdout:444: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:39:42.799 INFO:tasks.rados.rados.0.smithi017.stdout:444: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:39:42.799 INFO:tasks.rados.rados.0.smithi017.stdout:441: done (2 left) 2024-09-21T09:39:42.800 INFO:tasks.rados.rados.0.smithi017.stdout:442: done (1 left) 2024-09-21T09:39:42.800 INFO:tasks.rados.rados.0.smithi017.stdout:445: snap_remove snap 47 2024-09-21T09:39:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:42 smithi134 ceph-mon[33541]: pgmap v27: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-21T09:39:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:42 smithi134 ceph-mon[33541]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:39:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:42 smithi134 ceph-mon[33541]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:39:43.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:42 smithi017 ceph-mon[29380]: pgmap v27: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-21T09:39:43.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:42 smithi017 ceph-mon[29380]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:39:43.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:42 smithi017 ceph-mon[29380]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:39:43.146 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:42 smithi017 ceph-mon[40737]: pgmap v27: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-21T09:39:43.146 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:42 smithi017 ceph-mon[40737]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:39:43.147 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:42 smithi017 ceph-mon[40737]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:39:43.751 INFO:tasks.rados.rados.0.smithi017.stdout:445: done (1 left) 2024-09-21T09:39:43.751 INFO:tasks.rados.rados.0.smithi017.stdout:446: read oid 25 snap -1 2024-09-21T09:39:43.751 INFO:tasks.rados.rados.0.smithi017.stdout:446: expect (ObjNum 194 snap 54 seq_num 194) 2024-09-21T09:39:43.751 INFO:tasks.rados.rados.0.smithi017.stdout:447: read oid 4 snap -1 2024-09-21T09:39:43.751 INFO:tasks.rados.rados.0.smithi017.stdout:447: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:43.751 INFO:tasks.rados.rados.0.smithi017.stdout:448: delete oid 16 current snap is 57 2024-09-21T09:39:43.751 INFO:tasks.rados.rados.0.smithi017.stdout:444: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:39:43.752 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 136 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:39:43.764 INFO:tasks.rados.rados.0.smithi017.stdout:447: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:43.838 INFO:tasks.rados.rados.0.smithi017.stdout:446: expect (ObjNum 194 snap 54 seq_num 194) 2024-09-21T09:39:44.026 INFO:tasks.rados.rados.0.smithi017.stdout:444: done (3 left) 2024-09-21T09:39:44.026 INFO:tasks.rados.rados.0.smithi017.stdout:446: done (2 left) 2024-09-21T09:39:44.026 INFO:tasks.rados.rados.0.smithi017.stdout:447: done (1 left) 2024-09-21T09:39:44.026 INFO:tasks.rados.rados.0.smithi017.stdout:448: done (0 left) 2024-09-21T09:39:44.026 INFO:tasks.rados.rados.0.smithi017.stdout:449: snap_remove snap 48 2024-09-21T09:39:44.668 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:44 smithi017 systemd[1]: Stopping Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:44.755 INFO:tasks.rados.rados.0.smithi017.stdout:449: done (0 left) 2024-09-21T09:39:44.755 INFO:tasks.rados.rados.0.smithi017.stdout:450: snap_create 2024-09-21T09:39:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:44 smithi017 ceph-mon[40737]: pgmap v30: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-21T09:39:44.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:44 smithi017 ceph-mon[40737]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:39:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:44 smithi017 ceph-mon[29380]: pgmap v30: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-21T09:39:44.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:44 smithi017 ceph-mon[29380]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:39:44.950 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[77416]: ts=2024-09-21T09:39:44.666Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:39:44.950 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:44 smithi017 podman[91124]: 2024-09-21 09:39:44.718365821 +0000 UTC m=+0.284494231 container died b40d1bacbd5033c71dc3ad74e2060dea5980b1cab98a0ce482f308564473cb83 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:45.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:44 smithi134 ceph-mon[33541]: pgmap v30: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-09-21T09:39:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:44 smithi134 ceph-mon[33541]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:39:45.383 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:45 smithi017 podman[91124]: 2024-09-21 09:39:45.098438303 +0000 UTC m=+0.664566722 container cleanup b40d1bacbd5033c71dc3ad74e2060dea5980b1cab98a0ce482f308564473cb83 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:45.384 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:45 smithi017 bash[91124]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:39:45.638 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:45 smithi017 podman[91141]: 2024-09-21 09:39:45.38281745 +0000 UTC m=+0.659336646 container remove b40d1bacbd5033c71dc3ad74e2060dea5980b1cab98a0ce482f308564473cb83 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:45.638 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:45 smithi017 podman[91141]: 2024-09-21 09:39:45.483767855 +0000 UTC m=+0.760287051 volume remove 751396c7b39e6ee1080396c9ad6fe24fd418729ac2bd208b1a39990d931afbae 2024-09-21T09:39:45.759 INFO:tasks.rados.rados.0.smithi017.stdout:450: done (0 left) 2024-09-21T09:39:45.759 INFO:tasks.rados.rados.0.smithi017.stdout:451: delete oid 44 current snap is 58 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:451: done (0 left) 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:452: rollback oid 27 current snap is 58 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 40 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:453: rollback oid 7 current snap is 58 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 38 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:454: read oid 25 snap -1 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:454: expect (ObjNum 194 snap 54 seq_num 194) 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:455: read oid 48 snap -1 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:455: expect (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:456: write oid 8 current snap is 58 2024-09-21T09:39:45.765 INFO:tasks.rados.rados.0.smithi017.stdout:456: seq_num 206 ranges {662242=642825,2054820=530730,3307607=154635} 2024-09-21T09:39:45.772 INFO:tasks.rados.rados.0.smithi017.stdout:456: writing smithi01785587-8 from 662242 to 1305067 tid 1 2024-09-21T09:39:45.779 INFO:tasks.rados.rados.0.smithi017.stdout:456: writing smithi01785587-8 from 2054820 to 2585550 tid 2 2024-09-21T09:39:45.785 INFO:tasks.rados.rados.0.smithi017.stdout:456: writing smithi01785587-8 from 3307607 to 3462242 tid 3 2024-09-21T09:39:45.785 INFO:tasks.rados.rados.0.smithi017.stdout:457: snap_remove snap 57 2024-09-21T09:39:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:45 smithi017 ceph-mon[29380]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:39:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:45 smithi017 ceph-mon[40737]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:39:45.951 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:45 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:39:45.951 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:45 smithi017 systemd[1]: Stopped Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:45.951 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:45 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 1.177s CPU time. 2024-09-21T09:39:45.951 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:45 smithi017 systemd[1]: Starting Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:46.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:45 smithi134 ceph-mon[33541]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:39:46.360 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 podman[91277]: 2024-09-21 09:39:45.968970934 +0000 UTC m=+0.016625652 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-21T09:39:46.360 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 podman[91277]: 2024-09-21 09:39:46.077053941 +0000 UTC m=+0.124708696 volume create c3af648b4c0a7b19a59d0a7955985fa50e7c12b73f49f92dc9c6b58bab72f555 2024-09-21T09:39:46.360 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 podman[91277]: 2024-09-21 09:39:46.16889022 +0000 UTC m=+0.216544931 container create 0a4d8256f8d7818423b62865207ade6ae97fa0fdff7283789ad1763bd54affb3 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:46.757 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 podman[91277]: 2024-09-21 09:39:46.480311168 +0000 UTC m=+0.527965888 container init 0a4d8256f8d7818423b62865207ade6ae97fa0fdff7283789ad1763bd54affb3 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:46.757 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 podman[91277]: 2024-09-21 09:39:46.48554033 +0000 UTC m=+0.533195043 container start 0a4d8256f8d7818423b62865207ade6ae97fa0fdff7283789ad1763bd54affb3 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:46.757 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:46.496Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-21T09:39:46.757 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:46.496Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-21T09:39:46.758 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:46.497Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.17 port=9094 2024-09-21T09:39:46.758 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:46.499Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:39:46.758 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:46.520Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:39:46.758 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:46.521Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:39:46.758 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:46.523Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:39:46.758 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:46.523Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:39:46.758 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 bash[91277]: 0a4d8256f8d7818423b62865207ade6ae97fa0fdff7283789ad1763bd54affb3 2024-09-21T09:39:46.758 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:46 smithi017 systemd[1]: Started Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:46.769 INFO:tasks.rados.rados.0.smithi017.stdout:457: done (5 left) 2024-09-21T09:39:46.769 INFO:tasks.rados.rados.0.smithi017.stdout:458: snap_remove snap 36 2024-09-21T09:39:47.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-mon[29380]: pgmap v33: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:47.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-mon[29380]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:39:47.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:47.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:47.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-mon[40737]: pgmap v33: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:47.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-mon[40737]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:39:47.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:47.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:46 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:46 smithi134 ceph-mon[33541]: pgmap v33: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:46 smithi134 ceph-mon[33541]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:39:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:46 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:46 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:47.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:46] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.43.0" 2024-09-21T09:39:47.415 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:47 smithi017 systemd[1]: Stopping Ceph node-exporter.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:47.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:47 smithi017 podman[91440]: 2024-09-21 09:39:47.414101614 +0000 UTC m=+0.115994782 container died c1d4600d9989ee43391f76460eaa0db194787373f329a876cf128112192592b1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:47.777 INFO:tasks.rados.rados.0.smithi017.stdout:458: done (5 left) 2024-09-21T09:39:47.777 INFO:tasks.rados.rados.0.smithi017.stdout:459: snap_create 2024-09-21T09:39:47.777 INFO:tasks.rados.rados.0.smithi017.stdout:453: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:39:47.777 INFO:tasks.rados.rados.0.smithi017.stdout:453: finishing rollback tid 2 to smithi01785587-7 2024-09-21T09:39:47.777 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 180 (ObjNum 120 snap 22 seq_num 120) dirty exists 2024-09-21T09:39:47.777 INFO:tasks.rados.rados.0.smithi017.stdout:452: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:39:47.777 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 98 (ObjNum 33 snap 0 seq_num 2399717808) dirty dne 2024-09-21T09:39:47.777 INFO:tasks.rados.rados.0.smithi017.stdout:456: finishing write tid 1 to smithi01785587-8 2024-09-21T09:39:47.778 INFO:tasks.rados.rados.0.smithi017.stdout:455: expect (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:39:47.919 INFO:tasks.rados.rados.0.smithi017.stdout:456: finishing write tid 2 to smithi01785587-8 2024-09-21T09:39:47.920 INFO:tasks.rados.rados.0.smithi017.stdout:454: expect (ObjNum 194 snap 54 seq_num 194) 2024-09-21T09:39:48.109 INFO:tasks.rados.rados.0.smithi017.stdout:456: finishing write tid 3 to smithi01785587-8 2024-09-21T09:39:48.109 INFO:tasks.rados.rados.0.smithi017.stdout:456: finishing write tid 4 to smithi01785587-8 2024-09-21T09:39:48.109 INFO:tasks.rados.rados.0.smithi017.stdout:456: finishing write tid 5 to smithi01785587-8 2024-09-21T09:39:48.109 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 214 (ObjNum 206 snap 58 seq_num 206) dirty exists 2024-09-21T09:39:48.109 INFO:tasks.rados.rados.0.smithi017.stdout:456: left oid 8 (ObjNum 206 snap 58 seq_num 206) 2024-09-21T09:39:48.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:47 smithi017 podman[91440]: 2024-09-21 09:39:47.784656582 +0000 UTC m=+0.486549746 container cleanup c1d4600d9989ee43391f76460eaa0db194787373f329a876cf128112192592b1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:48.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:47 smithi017 bash[91440]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T09:39:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:47 smithi017 ceph-mon[29380]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T09:39:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:47 smithi017 ceph-mon[29380]: Deploying daemon node-exporter.a on smithi017 2024-09-21T09:39:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:47 smithi017 ceph-mon[29380]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:39:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:47 smithi017 ceph-mon[40737]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T09:39:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:47 smithi017 ceph-mon[40737]: Deploying daemon node-exporter.a on smithi017 2024-09-21T09:39:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:47 smithi017 ceph-mon[40737]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:39:48.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:47 smithi134 ceph-mon[33541]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T09:39:48.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:47 smithi134 ceph-mon[33541]: Deploying daemon node-exporter.a on smithi017 2024-09-21T09:39:48.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:47 smithi134 ceph-mon[33541]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:39:48.700 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:48 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:48.499Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000489168s 2024-09-21T09:39:48.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:48 smithi017 podman[91458]: 2024-09-21 09:39:48.40365183 +0000 UTC m=+0.991426442 container remove c1d4600d9989ee43391f76460eaa0db194787373f329a876cf128112192592b1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:48.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:48 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:39:48.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:48 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T09:39:48.701 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:48 smithi017 systemd[1]: Stopped Ceph node-exporter.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:48.701 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:48 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 2.366s CPU time. 2024-09-21T09:39:48.775 INFO:tasks.rados.rados.0.smithi017.stdout:452: done (5 left) 2024-09-21T09:39:48.776 INFO:tasks.rados.rados.0.smithi017.stdout:453: done (4 left) 2024-09-21T09:39:48.776 INFO:tasks.rados.rados.0.smithi017.stdout:454: done (3 left) 2024-09-21T09:39:48.776 INFO:tasks.rados.rados.0.smithi017.stdout:455: done (2 left) 2024-09-21T09:39:48.776 INFO:tasks.rados.rados.0.smithi017.stdout:456: done (1 left) 2024-09-21T09:39:48.776 INFO:tasks.rados.rados.0.smithi017.stdout:459: done (0 left) 2024-09-21T09:39:48.776 INFO:tasks.rados.rados.0.smithi017.stdout:460: delete oid 40 current snap is 59 2024-09-21T09:39:48.779 INFO:tasks.rados.rados.0.smithi017.stdout:460: done (0 left) 2024-09-21T09:39:48.779 INFO:tasks.rados.rados.0.smithi017.stdout:461: rollback oid 3 current snap is 59 2024-09-21T09:39:48.779 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 55 2024-09-21T09:39:48.779 INFO:tasks.rados.rados.0.smithi017.stdout:462: rollback oid 19 current snap is 59 2024-09-21T09:39:48.779 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 19 to 42 2024-09-21T09:39:48.779 INFO:tasks.rados.rados.0.smithi017.stdout:463: snap_remove snap 38 2024-09-21T09:39:49.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:48 smithi017 ceph-mon[29380]: pgmap v36: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 325 KiB/s wr, 12 op/s 2024-09-21T09:39:49.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:48 smithi017 ceph-mon[29380]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:39:49.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:48 smithi017 ceph-mon[40737]: pgmap v36: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 325 KiB/s wr, 12 op/s 2024-09-21T09:39:49.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:48 smithi017 ceph-mon[40737]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:39:49.041 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:48 smithi017 systemd[1]: Starting Ceph node-exporter.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:49.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:48 smithi134 ceph-mon[33541]: pgmap v36: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 325 KiB/s wr, 12 op/s 2024-09-21T09:39:49.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:48 smithi134 ceph-mon[33541]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:39:49.419 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:49 smithi017 bash[91587]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:39:49.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:49 smithi017 bash[91587]: Getting image source signatures 2024-09-21T09:39:49.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:49 smithi017 bash[91587]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:39:49.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:49 smithi017 bash[91587]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:39:49.700 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:49 smithi017 bash[91587]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:39:49.790 INFO:tasks.rados.rados.0.smithi017.stdout:463: done (2 left) 2024-09-21T09:39:49.791 INFO:tasks.rados.rados.0.smithi017.stdout:464: write oid 12 current snap is 59 2024-09-21T09:39:49.791 INFO:tasks.rados.rados.0.smithi017.stdout:464: seq_num 208 ranges {695856=793323,2171784=534399,3362453=133403} 2024-09-21T09:39:49.799 INFO:tasks.rados.rados.0.smithi017.stdout:464: writing smithi01785587-12 from 695856 to 1489179 tid 1 2024-09-21T09:39:49.807 INFO:tasks.rados.rados.0.smithi017.stdout:464: writing smithi01785587-12 from 2171784 to 2706183 tid 2 2024-09-21T09:39:49.812 INFO:tasks.rados.rados.0.smithi017.stdout:464: writing smithi01785587-12 from 3362453 to 3495856 tid 3 2024-09-21T09:39:49.812 INFO:tasks.rados.rados.0.smithi017.stdout:465: read oid 28 snap -1 2024-09-21T09:39:49.812 INFO:tasks.rados.rados.0.smithi017.stdout:465: expect (ObjNum 122 snap 23 seq_num 122) 2024-09-21T09:39:49.812 INFO:tasks.rados.rados.0.smithi017.stdout:466: delete oid 13 current snap is 59 2024-09-21T09:39:49.812 INFO:tasks.rados.rados.0.smithi017.stdout:461: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:39:49.812 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 184 (ObjNum 97 snap 0 seq_num 0) dirty dne 2024-09-21T09:39:49.812 INFO:tasks.rados.rados.0.smithi017.stdout:462: finishing rollback tid 0 to smithi01785587-19 2024-09-21T09:39:49.812 INFO:tasks.rados.rados.0.smithi017.stdout:462: finishing rollback tid 1 to smithi01785587-19 2024-09-21T09:39:49.813 INFO:tasks.rados.rados.0.smithi017.stdout:462: finishing rollback tid 2 to smithi01785587-19 2024-09-21T09:39:49.813 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 192 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-09-21T09:39:49.813 INFO:tasks.rados.rados.0.smithi017.stdout:464: finishing write tid 1 to smithi01785587-12 2024-09-21T09:39:49.813 INFO:tasks.rados.rados.0.smithi017.stdout:464: finishing write tid 2 to smithi01785587-12 2024-09-21T09:39:49.814 INFO:tasks.rados.rados.0.smithi017.stdout:464: finishing write tid 3 to smithi01785587-12 2024-09-21T09:39:49.814 INFO:tasks.rados.rados.0.smithi017.stdout:461: done (4 left) 2024-09-21T09:39:49.814 INFO:tasks.rados.rados.0.smithi017.stdout:462: done (3 left) 2024-09-21T09:39:49.814 INFO:tasks.rados.rados.0.smithi017.stdout:466: done (2 left) 2024-09-21T09:39:49.815 INFO:tasks.rados.rados.0.smithi017.stdout:467: snap_remove snap 51 2024-09-21T09:39:50.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:49 smithi017 ceph-mon[29380]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:39:50.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:49 smithi017 ceph-mon[40737]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:39:50.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:39:50 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:39:50.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:49 smithi134 ceph-mon[33541]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:39:50.793 INFO:tasks.rados.rados.0.smithi017.stdout:467: done (2 left) 2024-09-21T09:39:50.793 INFO:tasks.rados.rados.0.smithi017.stdout:468: snap_remove snap 50 2024-09-21T09:39:51.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 bash[91587]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:39:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:50 smithi017 ceph-mon[29380]: pgmap v39: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 325 KiB/s wr, 12 op/s 2024-09-21T09:39:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:50 smithi017 ceph-mon[29380]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:39:51.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:50 smithi017 ceph-mon[40737]: pgmap v39: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 325 KiB/s wr, 12 op/s 2024-09-21T09:39:51.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:50 smithi017 ceph-mon[40737]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:39:51.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:50 smithi134 ceph-mon[33541]: pgmap v39: 89 pgs: 89 active+clean; 220 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 325 KiB/s wr, 12 op/s 2024-09-21T09:39:51.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:50 smithi134 ceph-mon[33541]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:39:51.668 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 bash[91587]: Writing manifest to image destination 2024-09-21T09:39:51.797 INFO:tasks.rados.rados.0.smithi017.stdout:468: done (2 left) 2024-09-21T09:39:51.797 INFO:tasks.rados.rados.0.smithi017.stdout:469: snap_create 2024-09-21T09:39:51.798 INFO:tasks.rados.rados.0.smithi017.stdout:464: finishing write tid 4 to smithi01785587-12 2024-09-21T09:39:51.798 INFO:tasks.rados.rados.0.smithi017.stdout:464: finishing write tid 5 to smithi01785587-12 2024-09-21T09:39:51.798 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 156 (ObjNum 208 snap 59 seq_num 208) dirty exists 2024-09-21T09:39:51.798 INFO:tasks.rados.rados.0.smithi017.stdout:464: left oid 12 (ObjNum 208 snap 59 seq_num 208) 2024-09-21T09:39:51.798 INFO:tasks.rados.rados.0.smithi017.stdout:465: expect (ObjNum 122 snap 23 seq_num 122) 2024-09-21T09:39:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-mon[29380]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:39:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-mon[40737]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:39:51.950 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 podman[91587]: 2024-09-21 09:39:51.574148163 +0000 UTC m=+2.630262330 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:39:51.950 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 podman[91587]: 2024-09-21 09:39:51.694414575 +0000 UTC m=+2.750528732 container create 057b63034a9e6600976eb2e71da4800bc610f1fbbb790a2399666c1da3353819 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:52.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:51 smithi134 ceph-mon[33541]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:39:52.450 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 podman[91587]: 2024-09-21 09:39:51.950413299 +0000 UTC m=+3.006527456 container init 057b63034a9e6600976eb2e71da4800bc610f1fbbb790a2399666c1da3353819 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:52.450 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 podman[91587]: 2024-09-21 09:39:51.955493681 +0000 UTC m=+3.011607839 container start 057b63034a9e6600976eb2e71da4800bc610f1fbbb790a2399666c1da3353819 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:52.450 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.958Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:39:52.450 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.958Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.959Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.959Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.959Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.959Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:39:52.451 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:39:52.452 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:39:52.453 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:39:52.454 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:39:52.455 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:39:52.455 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a[91660]: ts=2024-09-21T09:39:51.960Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:39:52.455 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:52 smithi017 bash[91587]: 057b63034a9e6600976eb2e71da4800bc610f1fbbb790a2399666c1da3353819 2024-09-21T09:39:52.455 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 09:39:52 smithi017 systemd[1]: Started Ceph node-exporter.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:52.689 INFO:tasks.rados.rados.0.smithi017.stdout:464: done (2 left) 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:465: done (1 left) 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:469: done (0 left) 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:470: read oid 27 snap -1 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:470: expect deleted 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:471: rollback oid 48 current snap is 60 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 59 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:472: read oid 17 snap 49 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:472: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:473: read oid 14 snap -1 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:473: expect (ObjNum 171 snap 44 seq_num 171) 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:474: write oid 46 current snap is 60 2024-09-21T09:39:52.690 INFO:tasks.rados.rados.0.smithi017.stdout:474: seq_num 210 ranges {729470=543821,1772395=538068,2900946=403005} 2024-09-21T09:39:52.697 INFO:tasks.rados.rados.0.smithi017.stdout:474: writing smithi01785587-46 from 729470 to 1273291 tid 1 2024-09-21T09:39:52.703 INFO:tasks.rados.rados.0.smithi017.stdout:474: writing smithi01785587-46 from 1772395 to 2310463 tid 2 2024-09-21T09:39:52.708 INFO:tasks.rados.rados.0.smithi017.stdout:474: writing smithi01785587-46 from 2900946 to 3303951 tid 3 2024-09-21T09:39:52.709 INFO:tasks.rados.rados.0.smithi017.stdout:475: snap_remove snap 55 2024-09-21T09:39:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[40737]: pgmap v42: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 357 KiB/s wr, 5 op/s 2024-09-21T09:39:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[40737]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:39:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[40737]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:39:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[29380]: pgmap v42: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 357 KiB/s wr, 5 op/s 2024-09-21T09:39:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[29380]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:39:53.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:53.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:53.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:52 smithi017 ceph-mon[29380]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:39:53.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:52 smithi134 ceph-mon[33541]: pgmap v42: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 357 KiB/s wr, 5 op/s 2024-09-21T09:39:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:52 smithi134 ceph-mon[33541]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:39:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:52 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:52 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:39:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:52 smithi134 ceph-mon[33541]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:39:53.692 INFO:tasks.rados.rados.0.smithi017.stdout:471: finishing rollback tid 0 to smithi01785587-48 2024-09-21T09:39:53.692 INFO:tasks.rados.rados.0.smithi017.stdout:471: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:39:53.692 INFO:tasks.rados.rados.0.smithi017.stdout:471: finishing rollback tid 2 to smithi01785587-48 2024-09-21T09:39:53.692 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 222 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-09-21T09:39:53.692 INFO:tasks.rados.rados.0.smithi017.stdout:473: expect (ObjNum 171 snap 44 seq_num 171) 2024-09-21T09:39:53.858 INFO:tasks.rados.rados.0.smithi017.stdout:474: finishing write tid 1 to smithi01785587-46 2024-09-21T09:39:53.858 INFO:tasks.rados.rados.0.smithi017.stdout:472: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:39:54.002 INFO:tasks.rados.rados.0.smithi017.stdout:474: finishing write tid 2 to smithi01785587-46 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:474: finishing write tid 3 to smithi01785587-46 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:470: done (5 left) 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:471: done (4 left) 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:472: done (3 left) 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:473: done (2 left) 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:475: done (1 left) 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:476: read oid 40 snap -1 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:476: expect deleted 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:477: rollback oid 21 current snap is 60 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 52 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:474: finishing write tid 4 to smithi01785587-46 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:474: finishing write tid 5 to smithi01785587-46 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 232 (ObjNum 210 snap 60 seq_num 210) dirty exists 2024-09-21T09:39:54.003 INFO:tasks.rados.rados.0.smithi017.stdout:474: left oid 46 (ObjNum 210 snap 60 seq_num 210) 2024-09-21T09:39:54.004 INFO:tasks.rados.rados.0.smithi017.stdout:474: done (2 left) 2024-09-21T09:39:54.004 INFO:tasks.rados.rados.0.smithi017.stdout:478: rollback oid 11 current snap is 60 2024-09-21T09:39:54.004 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 58 2024-09-21T09:39:54.004 INFO:tasks.rados.rados.0.smithi017.stdout:479: rollback oid 6 current snap is 60 2024-09-21T09:39:54.004 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 52 2024-09-21T09:39:54.004 INFO:tasks.rados.rados.0.smithi017.stdout:480: snap_remove snap 40 2024-09-21T09:39:54.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:53 smithi017 ceph-mon[40737]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:39:54.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:53 smithi017 ceph-mon[40737]: Deploying daemon prometheus.a on smithi134 2024-09-21T09:39:54.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:53 smithi017 ceph-mon[40737]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:39:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:53 smithi017 ceph-mon[29380]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:39:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:53 smithi017 ceph-mon[29380]: Deploying daemon prometheus.a on smithi134 2024-09-21T09:39:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:53 smithi017 ceph-mon[29380]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:39:54.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:53 smithi134 ceph-mon[33541]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:39:54.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:53 smithi134 ceph-mon[33541]: Deploying daemon prometheus.a on smithi134 2024-09-21T09:39:54.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:53 smithi134 ceph-mon[33541]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:39:54.808 INFO:tasks.rados.rados.0.smithi017.stdout:478: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 195 (ObjNum 129 snap 0 seq_num 2399956496) dirty dne 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:477: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:479: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:477: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:479: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:477: finishing rollback tid 2 to smithi01785587-21 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 206 (ObjNum 170 snap 44 seq_num 170) dirty exists 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:479: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 170 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:476: done (4 left) 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:477: done (3 left) 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:478: done (2 left) 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:479: done (1 left) 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:480: done (0 left) 2024-09-21T09:39:54.809 INFO:tasks.rados.rados.0.smithi017.stdout:481: delete oid 22 current snap is 60 2024-09-21T09:39:54.812 INFO:tasks.rados.rados.0.smithi017.stdout:481: done (0 left) 2024-09-21T09:39:54.812 INFO:tasks.rados.rados.0.smithi017.stdout:482: read oid 37 snap -1 2024-09-21T09:39:54.812 INFO:tasks.rados.rados.0.smithi017.stdout:482: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:39:54.813 INFO:tasks.rados.rados.0.smithi017.stdout:483: read oid 47 snap 43 2024-09-21T09:39:54.813 INFO:tasks.rados.rados.0.smithi017.stdout:483: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:54.813 INFO:tasks.rados.rados.0.smithi017.stdout:484: snap_remove snap 52 2024-09-21T09:39:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:54 smithi017 ceph-mon[29380]: pgmap v45: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 357 KiB/s wr, 5 op/s 2024-09-21T09:39:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:54 smithi017 ceph-mon[40737]: pgmap v45: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 357 KiB/s wr, 5 op/s 2024-09-21T09:39:55.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:54 smithi134 ceph-mon[33541]: pgmap v45: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 357 KiB/s wr, 5 op/s 2024-09-21T09:39:55.814 INFO:tasks.rados.rados.0.smithi017.stdout:484: done (2 left) 2024-09-21T09:39:55.814 INFO:tasks.rados.rados.0.smithi017.stdout:485: rollback oid 17 current snap is 60 2024-09-21T09:39:55.814 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 54 2024-09-21T09:39:55.814 INFO:tasks.rados.rados.0.smithi017.stdout:482: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:39:55.900 INFO:tasks.rados.rados.0.smithi017.stdout:483: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:56.017 INFO:tasks.rados.rados.0.smithi017.stdout:485: finishing rollback tid 0 to smithi01785587-17 2024-09-21T09:39:56.017 INFO:tasks.rados.rados.0.smithi017.stdout:485: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:39:56.017 INFO:tasks.rados.rados.0.smithi017.stdout:485: finishing rollback tid 2 to smithi01785587-17 2024-09-21T09:39:56.017 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 133 (ObjNum 185 snap 49 seq_num 185) dirty exists 2024-09-21T09:39:56.017 INFO:tasks.rados.rados.0.smithi017.stdout:482: done (2 left) 2024-09-21T09:39:56.017 INFO:tasks.rados.rados.0.smithi017.stdout:483: done (1 left) 2024-09-21T09:39:56.017 INFO:tasks.rados.rados.0.smithi017.stdout:485: done (0 left) 2024-09-21T09:39:56.017 INFO:tasks.rados.rados.0.smithi017.stdout:486: write oid 50 current snap is 60 2024-09-21T09:39:56.017 INFO:tasks.rados.rados.0.smithi017.stdout:486: seq_num 212 ranges {763084=694319,1889359=541737,2955792=541902} 2024-09-21T09:39:56.024 INFO:tasks.rados.rados.0.smithi017.stdout:486: writing smithi01785587-50 from 763084 to 1457403 tid 1 2024-09-21T09:39:56.030 INFO:tasks.rados.rados.0.smithi017.stdout:486: writing smithi01785587-50 from 1889359 to 2431096 tid 2 2024-09-21T09:39:56.036 INFO:tasks.rados.rados.0.smithi017.stdout:486: writing smithi01785587-50 from 2955792 to 3497694 tid 3 2024-09-21T09:39:56.036 INFO:tasks.rados.rados.0.smithi017.stdout:487: delete oid 18 current snap is 60 2024-09-21T09:39:56.036 INFO:tasks.rados.rados.0.smithi017.stdout:486: finishing write tid 1 to smithi01785587-50 2024-09-21T09:39:56.037 INFO:tasks.rados.rados.0.smithi017.stdout:486: finishing write tid 2 to smithi01785587-50 2024-09-21T09:39:56.037 INFO:tasks.rados.rados.0.smithi017.stdout:487: done (1 left) 2024-09-21T09:39:56.037 INFO:tasks.rados.rados.0.smithi017.stdout:488: write oid 41 current snap is 60 2024-09-21T09:39:56.037 INFO:tasks.rados.rados.0.smithi017.stdout:488: seq_num 214 ranges {796698=561170,2006323=545406,3126991=469707} 2024-09-21T09:39:56.045 INFO:tasks.rados.rados.0.smithi017.stdout:488: writing smithi01785587-41 from 796698 to 1357868 tid 1 2024-09-21T09:39:56.051 INFO:tasks.rados.rados.0.smithi017.stdout:488: writing smithi01785587-41 from 2006323 to 2551729 tid 2 2024-09-21T09:39:56.058 INFO:tasks.rados.rados.0.smithi017.stdout:488: writing smithi01785587-41 from 3126991 to 3596698 tid 3 2024-09-21T09:39:56.058 INFO:tasks.rados.rados.0.smithi017.stdout:486: finishing write tid 3 to smithi01785587-50 2024-09-21T09:39:56.058 INFO:tasks.rados.rados.0.smithi017.stdout:486: finishing write tid 4 to smithi01785587-50 2024-09-21T09:39:56.058 INFO:tasks.rados.rados.0.smithi017.stdout:486: finishing write tid 5 to smithi01785587-50 2024-09-21T09:39:56.058 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 132 (ObjNum 212 snap 60 seq_num 212) dirty exists 2024-09-21T09:39:56.058 INFO:tasks.rados.rados.0.smithi017.stdout:486: left oid 50 (ObjNum 212 snap 60 seq_num 212) 2024-09-21T09:39:56.058 INFO:tasks.rados.rados.0.smithi017.stdout:488: finishing write tid 1 to smithi01785587-41 2024-09-21T09:39:56.058 INFO:tasks.rados.rados.0.smithi017.stdout:488: finishing write tid 2 to smithi01785587-41 2024-09-21T09:39:56.058 INFO:tasks.rados.rados.0.smithi017.stdout:489: read oid 6 snap 43 2024-09-21T09:39:56.059 INFO:tasks.rados.rados.0.smithi017.stdout:489: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:39:56.059 INFO:tasks.rados.rados.0.smithi017.stdout:486: done (2 left) 2024-09-21T09:39:56.059 INFO:tasks.rados.rados.0.smithi017.stdout:490: write oid 27 current snap is 60 2024-09-21T09:39:56.059 INFO:tasks.rados.rados.0.smithi017.stdout:490: seq_num 215 ranges {413505=636419,1664805=689064,3038061=492071} 2024-09-21T09:39:56.064 INFO:tasks.rados.rados.0.smithi017.stdout:490: writing smithi01785587-27 from 413505 to 1049924 tid 1 2024-09-21T09:39:56.071 INFO:tasks.rados.rados.0.smithi017.stdout:490: writing smithi01785587-27 from 1664805 to 2353869 tid 2 2024-09-21T09:39:56.078 INFO:tasks.rados.rados.0.smithi017.stdout:490: writing smithi01785587-27 from 3038061 to 3530132 tid 3 2024-09-21T09:39:56.078 INFO:tasks.rados.rados.0.smithi017.stdout:491: read oid 26 snap -1 2024-09-21T09:39:56.079 INFO:tasks.rados.rados.0.smithi017.stdout:491: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:56.079 INFO:tasks.rados.rados.0.smithi017.stdout:492: rollback oid 40 current snap is 60 2024-09-21T09:39:56.079 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 54 2024-09-21T09:39:56.079 INFO:tasks.rados.rados.0.smithi017.stdout:488: finishing write tid 3 to smithi01785587-41 2024-09-21T09:39:56.079 INFO:tasks.rados.rados.0.smithi017.stdout:488: finishing write tid 4 to smithi01785587-41 2024-09-21T09:39:56.079 INFO:tasks.rados.rados.0.smithi017.stdout:488: finishing write tid 5 to smithi01785587-41 2024-09-21T09:39:56.079 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 107 (ObjNum 214 snap 60 seq_num 214) dirty exists 2024-09-21T09:39:56.079 INFO:tasks.rados.rados.0.smithi017.stdout:488: left oid 41 (ObjNum 214 snap 60 seq_num 214) 2024-09-21T09:39:56.079 INFO:tasks.rados.rados.0.smithi017.stdout:489: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:490: finishing write tid 1 to smithi01785587-27 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:490: finishing write tid 2 to smithi01785587-27 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:492: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:492: finishing rollback tid 2 to smithi01785587-40 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 182 (ObjNum 158 snap 40 seq_num 158) dirty exists 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:490: finishing write tid 3 to smithi01785587-27 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:490: finishing write tid 4 to smithi01785587-27 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:488: done (4 left) 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:489: done (3 left) 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:492: done (2 left) 2024-09-21T09:39:56.151 INFO:tasks.rados.rados.0.smithi017.stdout:493: delete oid 28 current snap is 60 2024-09-21T09:39:56.152 INFO:tasks.rados.rados.0.smithi017.stdout:490: finishing write tid 5 to smithi01785587-27 2024-09-21T09:39:56.152 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 137 (ObjNum 215 snap 60 seq_num 215) dirty exists 2024-09-21T09:39:56.152 INFO:tasks.rados.rados.0.smithi017.stdout:490: left oid 27 (ObjNum 215 snap 60 seq_num 215) 2024-09-21T09:39:56.152 INFO:tasks.rados.rados.0.smithi017.stdout:491: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:56.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:55 smithi017 ceph-mon[29380]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:39:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:55 smithi017 ceph-mon[40737]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:39:56.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:55 smithi134 ceph-mon[33541]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:39:56.297 INFO:tasks.rados.rados.0.smithi017.stdout:490: done (2 left) 2024-09-21T09:39:56.297 INFO:tasks.rados.rados.0.smithi017.stdout:491: done (1 left) 2024-09-21T09:39:56.297 INFO:tasks.rados.rados.0.smithi017.stdout:493: done (0 left) 2024-09-21T09:39:56.297 INFO:tasks.rados.rados.0.smithi017.stdout:494: write oid 5 current snap is 60 2024-09-21T09:39:56.297 INFO:tasks.rados.rados.0.smithi017.stdout:494: seq_num 217 ranges {447119=786917,1665416=692733,2976554=514615} 2024-09-21T09:39:56.303 INFO:tasks.rados.rados.0.smithi017.stdout:494: writing smithi01785587-5 from 447119 to 1234036 tid 1 2024-09-21T09:39:56.310 INFO:tasks.rados.rados.0.smithi017.stdout:494: writing smithi01785587-5 from 1665416 to 2358149 tid 2 2024-09-21T09:39:56.316 INFO:tasks.rados.rados.0.smithi017.stdout:494: writing smithi01785587-5 from 2976554 to 3491169 tid 3 2024-09-21T09:39:56.316 INFO:tasks.rados.rados.0.smithi017.stdout:495: rollback oid 41 current snap is 60 2024-09-21T09:39:56.316 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 54 2024-09-21T09:39:56.316 INFO:tasks.rados.rados.0.smithi017.stdout:494: finishing write tid 1 to smithi01785587-5 2024-09-21T09:39:56.316 INFO:tasks.rados.rados.0.smithi017.stdout:496: write oid 9 current snap is 60 2024-09-21T09:39:56.316 INFO:tasks.rados.rados.0.smithi017.stdout:494: finishing write tid 2 to smithi01785587-5 2024-09-21T09:39:56.317 INFO:tasks.rados.rados.0.smithi017.stdout:496: seq_num 218 ranges {463926=462166,1723898=436391,2603977=725887} 2024-09-21T09:39:56.321 INFO:tasks.rados.rados.0.smithi017.stdout:496: writing smithi01785587-9 from 463926 to 926092 tid 1 2024-09-21T09:39:56.328 INFO:tasks.rados.rados.0.smithi017.stdout:496: writing smithi01785587-9 from 1723898 to 2160289 tid 2 2024-09-21T09:39:56.335 INFO:tasks.rados.rados.0.smithi017.stdout:496: writing smithi01785587-9 from 2603977 to 3329864 tid 3 2024-09-21T09:39:56.335 INFO:tasks.rados.rados.0.smithi017.stdout:497: read oid 43 snap -1 2024-09-21T09:39:56.335 INFO:tasks.rados.rados.0.smithi017.stdout:497: expect (ObjNum 199 snap 56 seq_num 199) 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:498: write oid 38 current snap is 60 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:495: finishing rollback tid 0 to smithi01785587-41 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:495: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:494: finishing write tid 3 to smithi01785587-5 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:495: finishing rollback tid 2 to smithi01785587-41 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 110 (ObjNum 175 snap 47 seq_num 175) dirty exists 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:494: finishing write tid 4 to smithi01785587-5 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:494: finishing write tid 5 to smithi01785587-5 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 267 (ObjNum 217 snap 60 seq_num 217) dirty exists 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:494: left oid 5 (ObjNum 217 snap 60 seq_num 217) 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:496: finishing write tid 1 to smithi01785587-9 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:496: finishing write tid 2 to smithi01785587-9 2024-09-21T09:39:56.336 INFO:tasks.rados.rados.0.smithi017.stdout:498: seq_num 219 ranges {480733=537415,1782380=696402,3031400=537159} 2024-09-21T09:39:56.340 INFO:tasks.rados.rados.0.smithi017.stdout:498: writing smithi01785587-38 from 480733 to 1018148 tid 1 2024-09-21T09:39:56.349 INFO:tasks.rados.rados.0.smithi017.stdout:498: writing smithi01785587-38 from 1782380 to 2478782 tid 2 2024-09-21T09:39:56.355 INFO:tasks.rados.rados.0.smithi017.stdout:498: writing smithi01785587-38 from 3031400 to 3568559 tid 3 2024-09-21T09:39:56.355 INFO:tasks.rados.rados.0.smithi017.stdout:494: done (4 left) 2024-09-21T09:39:56.355 INFO:tasks.rados.rados.0.smithi017.stdout:495: done (3 left) 2024-09-21T09:39:56.355 INFO:tasks.rados.rados.0.smithi017.stdout:499: snap_remove snap 31 2024-09-21T09:39:56.815 INFO:tasks.rados.rados.0.smithi017.stdout:499: done (3 left) 2024-09-21T09:39:56.815 INFO:tasks.rados.rados.0.smithi017.stdout:500: delete oid 12 current snap is 60 2024-09-21T09:39:56.815 INFO:tasks.rados.rados.0.smithi017.stdout:496: finishing write tid 3 to smithi01785587-9 2024-09-21T09:39:56.815 INFO:tasks.rados.rados.0.smithi017.stdout:496: finishing write tid 4 to smithi01785587-9 2024-09-21T09:39:56.815 INFO:tasks.rados.rados.0.smithi017.stdout:496: finishing write tid 5 to smithi01785587-9 2024-09-21T09:39:56.815 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 81 (ObjNum 218 snap 60 seq_num 218) dirty exists 2024-09-21T09:39:56.815 INFO:tasks.rados.rados.0.smithi017.stdout:496: left oid 9 (ObjNum 218 snap 60 seq_num 218) 2024-09-21T09:39:56.816 INFO:tasks.rados.rados.0.smithi017.stdout:498: finishing write tid 1 to smithi01785587-38 2024-09-21T09:39:56.816 INFO:tasks.rados.rados.0.smithi017.stdout:497: expect (ObjNum 199 snap 56 seq_num 199) 2024-09-21T09:39:56.816 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 09:39:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T09:39:56.502Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003338442s 2024-09-21T09:39:57.006 INFO:tasks.rados.rados.0.smithi017.stdout:498: finishing write tid 2 to smithi01785587-38 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:498: finishing write tid 3 to smithi01785587-38 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:498: finishing write tid 4 to smithi01785587-38 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:498: finishing write tid 5 to smithi01785587-38 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 136 (ObjNum 219 snap 60 seq_num 219) dirty exists 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:498: left oid 38 (ObjNum 219 snap 60 seq_num 219) 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:496: done (3 left) 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:497: done (2 left) 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:498: done (1 left) 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:500: done (0 left) 2024-09-21T09:39:57.007 INFO:tasks.rados.rados.0.smithi017.stdout:501: snap_create 2024-09-21T09:39:57.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:56 smithi017 ceph-mon[40737]: pgmap v48: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:57.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:56 smithi017 ceph-mon[40737]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:39:57.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:56 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:56 smithi017 ceph-mon[29380]: pgmap v48: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:56 smithi017 ceph-mon[29380]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:39:57.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:56 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:57.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:56 smithi134 ceph-mon[33541]: pgmap v48: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:39:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:56 smithi134 ceph-mon[33541]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:39:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:56 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:57.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:39:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:39:56] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.43.0" 2024-09-21T09:39:57.825 INFO:tasks.rados.rados.0.smithi017.stdout:501: done (0 left) 2024-09-21T09:39:57.825 INFO:tasks.rados.rados.0.smithi017.stdout:502: write oid 32 current snap is 61 2024-09-21T09:39:57.825 INFO:tasks.rados.rados.0.smithi017.stdout:502: seq_num 221 ranges {514347=404266,1499344=700071,2686246=676056} 2024-09-21T09:39:57.830 INFO:tasks.rados.rados.0.smithi017.stdout:502: writing smithi01785587-32 from 514347 to 918613 tid 1 2024-09-21T09:39:57.838 INFO:tasks.rados.rados.0.smithi017.stdout:502: writing smithi01785587-32 from 1499344 to 2199415 tid 2 2024-09-21T09:39:57.844 INFO:tasks.rados.rados.0.smithi017.stdout:502: writing smithi01785587-32 from 2686246 to 3362302 tid 3 2024-09-21T09:39:57.844 INFO:tasks.rados.rados.0.smithi017.stdout:503: read oid 47 snap -1 2024-09-21T09:39:57.845 INFO:tasks.rados.rados.0.smithi017.stdout:503: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:57.845 INFO:tasks.rados.rados.0.smithi017.stdout:504: write oid 21 current snap is 61 2024-09-21T09:39:57.845 INFO:tasks.rados.rados.0.smithi017.stdout:502: finishing write tid 1 to smithi01785587-32 2024-09-21T09:39:57.845 INFO:tasks.rados.rados.0.smithi017.stdout:504: seq_num 222 ranges {531154=479515,1557826=443729,2713669=487328,3679295=51859} 2024-09-21T09:39:57.850 INFO:tasks.rados.rados.0.smithi017.stdout:504: writing smithi01785587-21 from 531154 to 1010669 tid 1 2024-09-21T09:39:57.855 INFO:tasks.rados.rados.0.smithi017.stdout:504: writing smithi01785587-21 from 1557826 to 2001555 tid 2 2024-09-21T09:39:57.862 INFO:tasks.rados.rados.0.smithi017.stdout:504: writing smithi01785587-21 from 2713669 to 3200997 tid 3 2024-09-21T09:39:57.865 INFO:tasks.rados.rados.0.smithi017.stdout:504: writing smithi01785587-21 from 3679295 to 3731154 tid 4 2024-09-21T09:39:57.865 INFO:tasks.rados.rados.0.smithi017.stdout:505: rollback oid 11 current snap is 61 2024-09-21T09:39:57.866 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 58 2024-09-21T09:39:57.866 INFO:tasks.rados.rados.0.smithi017.stdout:502: finishing write tid 2 to smithi01785587-32 2024-09-21T09:39:57.866 INFO:tasks.rados.rados.0.smithi017.stdout:506: snap_create 2024-09-21T09:39:57.866 INFO:tasks.rados.rados.0.smithi017.stdout:503: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:57.982 INFO:tasks.rados.rados.0.smithi017.stdout:502: finishing write tid 3 to smithi01785587-32 2024-09-21T09:39:57.982 INFO:tasks.rados.rados.0.smithi017.stdout:502: finishing write tid 4 to smithi01785587-32 2024-09-21T09:39:57.982 INFO:tasks.rados.rados.0.smithi017.stdout:502: finishing write tid 5 to smithi01785587-32 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 248 (ObjNum 221 snap 61 seq_num 221) dirty exists 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:502: left oid 32 (ObjNum 221 snap 61 seq_num 221) 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:504: finishing write tid 1 to smithi01785587-21 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:504: finishing write tid 2 to smithi01785587-21 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:504: finishing write tid 3 to smithi01785587-21 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:504: finishing write tid 4 to smithi01785587-21 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:505: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 195 (ObjNum 129 snap 0 seq_num 2399878936) dirty dne 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:504: finishing write tid 5 to smithi01785587-21 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:504: finishing write tid 6 to smithi01785587-21 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 230 (ObjNum 222 snap 61 seq_num 222) dirty exists 2024-09-21T09:39:57.983 INFO:tasks.rados.rados.0.smithi017.stdout:504: left oid 21 (ObjNum 222 snap 61 seq_num 222) 2024-09-21T09:39:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:57 smithi017 ceph-mon[29380]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:39:58.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:57 smithi017 ceph-mon[40737]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:39:58.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:57 smithi134 ceph-mon[33541]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:39:58.837 INFO:tasks.rados.rados.0.smithi017.stdout:502: done (4 left) 2024-09-21T09:39:58.837 INFO:tasks.rados.rados.0.smithi017.stdout:503: done (3 left) 2024-09-21T09:39:58.837 INFO:tasks.rados.rados.0.smithi017.stdout:504: done (2 left) 2024-09-21T09:39:58.837 INFO:tasks.rados.rados.0.smithi017.stdout:505: done (1 left) 2024-09-21T09:39:58.837 INFO:tasks.rados.rados.0.smithi017.stdout:506: done (0 left) 2024-09-21T09:39:58.837 INFO:tasks.rados.rados.0.smithi017.stdout:507: delete oid 45 current snap is 62 2024-09-21T09:39:58.839 INFO:tasks.rados.rados.0.smithi017.stdout:507: done (0 left) 2024-09-21T09:39:58.839 INFO:tasks.rados.rados.0.smithi017.stdout:508: snap_create 2024-09-21T09:39:59.077 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:58 smithi134 ceph-mon[33541]: pgmap v51: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 363 KiB/s wr, 12 op/s 2024-09-21T09:39:59.077 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:58 smithi134 ceph-mon[33541]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:39:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:58 smithi017 ceph-mon[29380]: pgmap v51: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 363 KiB/s wr, 12 op/s 2024-09-21T09:39:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:58 smithi017 ceph-mon[29380]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:39:59.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:58 smithi017 ceph-mon[40737]: pgmap v51: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 363 KiB/s wr, 12 op/s 2024-09-21T09:39:59.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:58 smithi017 ceph-mon[40737]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:39:59.843 INFO:tasks.rados.rados.0.smithi017.stdout:508: done (0 left) 2024-09-21T09:39:59.843 INFO:tasks.rados.rados.0.smithi017.stdout:509: read oid 2 snap -1 2024-09-21T09:39:59.843 INFO:tasks.rados.rados.0.smithi017.stdout:509: expect deleted 2024-09-21T09:39:59.843 INFO:tasks.rados.rados.0.smithi017.stdout:510: rollback oid 25 current snap is 63 2024-09-21T09:39:59.843 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 59 2024-09-21T09:39:59.844 INFO:tasks.rados.rados.0.smithi017.stdout:511: write oid 50 current snap is 63 2024-09-21T09:39:59.844 INFO:tasks.rados.rados.0.smithi017.stdout:511: seq_num 224 ranges {564768=630013,1674790=447398,2768515=626225} 2024-09-21T09:39:59.849 INFO:tasks.rados.rados.0.smithi017.stdout:511: writing smithi01785587-50 from 564768 to 1194781 tid 1 2024-09-21T09:39:59.855 INFO:tasks.rados.rados.0.smithi017.stdout:511: writing smithi01785587-50 from 1674790 to 2122188 tid 2 2024-09-21T09:39:59.862 INFO:tasks.rados.rados.0.smithi017.stdout:511: writing smithi01785587-50 from 2768515 to 3394740 tid 3 2024-09-21T09:39:59.862 INFO:tasks.rados.rados.0.smithi017.stdout:512: read oid 34 snap -1 2024-09-21T09:39:59.862 INFO:tasks.rados.rados.0.smithi017.stdout:512: expect (ObjNum 112 snap 18 seq_num 112) 2024-09-21T09:39:59.863 INFO:tasks.rados.rados.0.smithi017.stdout:513: read oid 1 snap 62 2024-09-21T09:39:59.863 INFO:tasks.rados.rados.0.smithi017.stdout:513: expect deleted 2024-09-21T09:39:59.863 INFO:tasks.rados.rados.0.smithi017.stdout:514: read oid 39 snap -1 2024-09-21T09:39:59.863 INFO:tasks.rados.rados.0.smithi017.stdout:514: expect (ObjNum 190 snap 50 seq_num 190) 2024-09-21T09:39:59.863 INFO:tasks.rados.rados.0.smithi017.stdout:515: write oid 27 current snap is 63 2024-09-21T09:39:59.863 INFO:tasks.rados.rados.0.smithi017.stdout:515: seq_num 225 ranges {581575=705262,1733272=707409,3195938=437497} 2024-09-21T09:39:59.870 INFO:tasks.rados.rados.0.smithi017.stdout:515: writing smithi01785587-27 from 581575 to 1286837 tid 1 2024-09-21T09:39:59.877 INFO:tasks.rados.rados.0.smithi017.stdout:515: writing smithi01785587-27 from 1733272 to 2440681 tid 2 2024-09-21T09:39:59.884 INFO:tasks.rados.rados.0.smithi017.stdout:515: writing smithi01785587-27 from 3195938 to 3633435 tid 3 2024-09-21T09:39:59.884 INFO:tasks.rados.rados.0.smithi017.stdout:516: write oid 44 current snap is 63 2024-09-21T09:39:59.884 INFO:tasks.rados.rados.0.smithi017.stdout:516: seq_num 226 ranges {598382=780511,2075401=451067,3107008=648769} 2024-09-21T09:39:59.891 INFO:tasks.rados.rados.0.smithi017.stdout:516: writing smithi01785587-44 from 598382 to 1378893 tid 1 2024-09-21T09:39:59.898 INFO:tasks.rados.rados.0.smithi017.stdout:516: writing smithi01785587-44 from 2075401 to 2526468 tid 2 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:516: writing smithi01785587-44 from 3107008 to 3755777 tid 3 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:517: read oid 3 snap 62 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:517: expect deleted 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:518: rollback oid 9 current snap is 63 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 9 to 54 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:510: finishing rollback tid 0 to smithi01785587-25 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:510: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:510: finishing rollback tid 2 to smithi01785587-25 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 143 (ObjNum 194 snap 54 seq_num 194) dirty exists 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:511: finishing write tid 1 to smithi01785587-50 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:511: finishing write tid 2 to smithi01785587-50 2024-09-21T09:39:59.906 INFO:tasks.rados.rados.0.smithi017.stdout:509: done (9 left) 2024-09-21T09:39:59.907 INFO:tasks.rados.rados.0.smithi017.stdout:510: done (8 left) 2024-09-21T09:39:59.907 INFO:tasks.rados.rados.0.smithi017.stdout:519: snap_create 2024-09-21T09:39:59.907 INFO:tasks.rados.rados.0.smithi017.stdout:511: finishing write tid 3 to smithi01785587-50 2024-09-21T09:39:59.907 INFO:tasks.rados.rados.0.smithi017.stdout:511: finishing write tid 4 to smithi01785587-50 2024-09-21T09:39:59.907 INFO:tasks.rados.rados.0.smithi017.stdout:511: finishing write tid 5 to smithi01785587-50 2024-09-21T09:39:59.907 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 143 (ObjNum 224 snap 63 seq_num 224) dirty exists 2024-09-21T09:39:59.907 INFO:tasks.rados.rados.0.smithi017.stdout:511: left oid 50 (ObjNum 224 snap 63 seq_num 224) 2024-09-21T09:39:59.907 INFO:tasks.rados.rados.0.smithi017.stdout:514: expect (ObjNum 190 snap 50 seq_num 190) 2024-09-21T09:39:59.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:39:59 smithi134 ceph-mon[33541]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:40:00.055 INFO:tasks.rados.rados.0.smithi017.stdout:512: expect (ObjNum 112 snap 18 seq_num 112) 2024-09-21T09:40:00.187 INFO:tasks.rados.rados.0.smithi017.stdout:515: finishing write tid 1 to smithi01785587-27 2024-09-21T09:40:00.187 INFO:tasks.rados.rados.0.smithi017.stdout:515: finishing write tid 2 to smithi01785587-27 2024-09-21T09:40:00.187 INFO:tasks.rados.rados.0.smithi017.stdout:515: finishing write tid 3 to smithi01785587-27 2024-09-21T09:40:00.187 INFO:tasks.rados.rados.0.smithi017.stdout:515: finishing write tid 4 to smithi01785587-27 2024-09-21T09:40:00.187 INFO:tasks.rados.rados.0.smithi017.stdout:515: finishing write tid 5 to smithi01785587-27 2024-09-21T09:40:00.187 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 146 (ObjNum 225 snap 63 seq_num 225) dirty exists 2024-09-21T09:40:00.187 INFO:tasks.rados.rados.0.smithi017.stdout:515: left oid 27 (ObjNum 225 snap 63 seq_num 225) 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:516: finishing write tid 1 to smithi01785587-44 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:516: finishing write tid 2 to smithi01785587-44 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:518: finishing rollback tid 0 to smithi01785587-9 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:516: finishing write tid 3 to smithi01785587-44 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:518: finishing rollback tid 1 to smithi01785587-9 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:516: finishing write tid 4 to smithi01785587-44 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:518: finishing rollback tid 2 to smithi01785587-9 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 87 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:516: finishing write tid 5 to smithi01785587-44 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 279 (ObjNum 226 snap 63 seq_num 226) dirty exists 2024-09-21T09:40:00.188 INFO:tasks.rados.rados.0.smithi017.stdout:516: left oid 44 (ObjNum 226 snap 63 seq_num 226) 2024-09-21T09:40:00.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:39:59 smithi017 ceph-mon[29380]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:40:00.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:39:59 smithi017 ceph-mon[40737]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:40:00.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:00 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[29632]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:40:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:511: done (8 left) 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:512: done (7 left) 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:513: done (6 left) 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:514: done (5 left) 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:515: done (4 left) 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:516: done (3 left) 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:517: done (2 left) 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:518: done (1 left) 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:519: done (0 left) 2024-09-21T09:40:00.848 INFO:tasks.rados.rados.0.smithi017.stdout:520: delete oid 22 current snap is 64 2024-09-21T09:40:00.849 INFO:tasks.rados.rados.0.smithi017.stdout:520: done (0 left) 2024-09-21T09:40:00.849 INFO:tasks.rados.rados.0.smithi017.stdout:521: snap_create 2024-09-21T09:40:00.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:00 smithi134 ceph-mon[33541]: pgmap v54: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 363 KiB/s wr, 12 op/s 2024-09-21T09:40:00.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:00 smithi134 ceph-mon[33541]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:40:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:00 smithi134 ceph-mon[33541]: overall HEALTH_OK 2024-09-21T09:40:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:00 smithi017 ceph-mon[29380]: pgmap v54: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 363 KiB/s wr, 12 op/s 2024-09-21T09:40:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:00 smithi017 ceph-mon[29380]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:40:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:00 smithi017 ceph-mon[29380]: overall HEALTH_OK 2024-09-21T09:40:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:00 smithi017 ceph-mon[40737]: pgmap v54: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 363 KiB/s wr, 12 op/s 2024-09-21T09:40:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:00 smithi017 ceph-mon[40737]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:40:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:00 smithi017 ceph-mon[40737]: overall HEALTH_OK 2024-09-21T09:40:01.854 INFO:tasks.rados.rados.0.smithi017.stdout:521: done (0 left) 2024-09-21T09:40:01.855 INFO:tasks.rados.rados.0.smithi017.stdout:522: snap_remove snap 49 2024-09-21T09:40:02.159 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:01 smithi134 ceph-mon[33541]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:40:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:01 smithi017 ceph-mon[29380]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:40:02.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:01 smithi017 ceph-mon[40737]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:40:02.692 INFO:tasks.rados.rados.0.smithi017.stdout:522: done (0 left) 2024-09-21T09:40:02.692 INFO:tasks.rados.rados.0.smithi017.stdout:523: write oid 45 current snap is 65 2024-09-21T09:40:02.693 INFO:tasks.rados.rados.0.smithi017.stdout:523: seq_num 228 ranges {631996=531009,1792365=454736,2761854=671313} 2024-09-21T09:40:02.700 INFO:tasks.rados.rados.0.smithi017.stdout:523: writing smithi01785587-45 from 631996 to 1163005 tid 1 2024-09-21T09:40:02.706 INFO:tasks.rados.rados.0.smithi017.stdout:523: writing smithi01785587-45 from 1792365 to 2247101 tid 2 2024-09-21T09:40:02.713 INFO:tasks.rados.rados.0.smithi017.stdout:523: writing smithi01785587-45 from 2761854 to 3433167 tid 3 2024-09-21T09:40:02.713 INFO:tasks.rados.rados.0.smithi017.stdout:524: snap_remove snap 20 2024-09-21T09:40:02.737 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 systemd[1]: Stopping Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.429Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.429Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.429Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.429Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.429Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.429Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.429Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.429Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.429Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.430Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.430Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:40:02.738 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[65765]: ts=2024-09-21T09:40:02.430Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:40:02.739 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 podman[76865]: 2024-09-21 09:40:02.439881892 +0000 UTC m=+0.139288060 container died b50e22f97e5d14e90270738c523336c7e4c04085efd2cc6d39437673a2a224ab (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:02.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-mon[33541]: pgmap v57: 89 pgs: 89 active+clean; 243 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.6 MiB/s wr, 31 op/s 2024-09-21T09:40:02.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-mon[33541]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:40:02.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:02 smithi134 ceph-mon[33541]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:40:02.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 podman[76865]: 2024-09-21 09:40:02.944706485 +0000 UTC m=+0.644112647 container cleanup b50e22f97e5d14e90270738c523336c7e4c04085efd2cc6d39437673a2a224ab (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:02.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:02 smithi134 bash[76865]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:40:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:02 smithi017 ceph-mon[40737]: pgmap v57: 89 pgs: 89 active+clean; 243 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.6 MiB/s wr, 31 op/s 2024-09-21T09:40:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:02 smithi017 ceph-mon[40737]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:40:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:02 smithi017 ceph-mon[40737]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:40:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:02 smithi017 ceph-mon[29380]: pgmap v57: 89 pgs: 89 active+clean; 243 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.6 MiB/s wr, 31 op/s 2024-09-21T09:40:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:02 smithi017 ceph-mon[29380]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:40:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:02 smithi017 ceph-mon[29380]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:40:03.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:03 smithi134 podman[76882]: 2024-09-21 09:40:03.213562144 +0000 UTC m=+0.775433684 container remove b50e22f97e5d14e90270738c523336c7e4c04085efd2cc6d39437673a2a224ab (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:03.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:03 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:40:03.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:03 smithi134 systemd[1]: Stopped Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:03.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:03 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.699s CPU time. 2024-09-21T09:40:03.695 INFO:tasks.rados.rados.0.smithi017.stdout:524: done (1 left) 2024-09-21T09:40:03.695 INFO:tasks.rados.rados.0.smithi017.stdout:525: rollback oid 12 current snap is 65 2024-09-21T09:40:03.695 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 12 to 60 2024-09-21T09:40:03.696 INFO:tasks.rados.rados.0.smithi017.stdout:523: finishing write tid 1 to smithi01785587-45 2024-09-21T09:40:03.696 INFO:tasks.rados.rados.0.smithi017.stdout:523: finishing write tid 2 to smithi01785587-45 2024-09-21T09:40:03.696 INFO:tasks.rados.rados.0.smithi017.stdout:523: finishing write tid 3 to smithi01785587-45 2024-09-21T09:40:03.696 INFO:tasks.rados.rados.0.smithi017.stdout:523: finishing write tid 4 to smithi01785587-45 2024-09-21T09:40:03.696 INFO:tasks.rados.rados.0.smithi017.stdout:523: finishing write tid 5 to smithi01785587-45 2024-09-21T09:40:03.696 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 156 (ObjNum 228 snap 65 seq_num 228) dirty exists 2024-09-21T09:40:03.696 INFO:tasks.rados.rados.0.smithi017.stdout:523: left oid 45 (ObjNum 228 snap 65 seq_num 228) 2024-09-21T09:40:03.696 INFO:tasks.rados.rados.0.smithi017.stdout:523: done (1 left) 2024-09-21T09:40:03.696 INFO:tasks.rados.rados.0.smithi017.stdout:526: snap_create 2024-09-21T09:40:03.698 INFO:tasks.rados.rados.0.smithi017.stdout:525: finishing rollback tid 1 to smithi01785587-12 2024-09-21T09:40:03.698 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 170 (ObjNum 177 snap 0 seq_num 2399920736) dirty dne 2024-09-21T09:40:03.882 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:03 smithi134 systemd[1]: Starting Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:03.882 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:03 smithi134 podman[77012]: 2024-09-21 09:40:03.740007591 +0000 UTC m=+0.016754093 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-21T09:40:03.882 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:03 smithi134 podman[77012]: 2024-09-21 09:40:03.881336827 +0000 UTC m=+0.158083357 container create 9774662fd5df84505127f32125b1d520fb82cb80df489f49609ff382f88cad65 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:04.135 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 podman[77012]: 2024-09-21 09:40:04.076620022 +0000 UTC m=+0.353366525 container init 9774662fd5df84505127f32125b1d520fb82cb80df489f49609ff382f88cad65 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:04.135 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 podman[77012]: 2024-09-21 09:40:04.081425516 +0000 UTC m=+0.358172001 container start 9774662fd5df84505127f32125b1d520fb82cb80df489f49609ff382f88cad65 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:04.135 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.101Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-21T09:40:04.135 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.102Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-21T09:40:04.135 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.102Z caller=main.go:623 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi134 (none))" 2024-09-21T09:40:04.135 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.102Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.102Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.104Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.104Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.106Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.106Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.108Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.108Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.714µs 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.108Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.108Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.120Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.120Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.120Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.86µs wal_replay_duration=11.850598ms wbl_replay_duration=109ns total_replay_duration=11.905404ms 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.122Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.123Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-21T09:40:04.136 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.123Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:40:04.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.139Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.963937ms db_storage=903ns remote_storage=1.158µs web_handler=340ns query_engine=641ns scrape=466.469µs scrape_sd=115.8µs notify=15.981µs notify_sd=14.973µs rules=16.007866ms tracing=6.901µs 2024-09-21T09:40:04.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.140Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-21T09:40:04.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:04.140Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:40:04.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 bash[77012]: 9774662fd5df84505127f32125b1d520fb82cb80df489f49609ff382f88cad65 2024-09-21T09:40:04.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:04 smithi134 systemd[1]: Started Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:04.700 INFO:tasks.rados.rados.0.smithi017.stdout:525: done (1 left) 2024-09-21T09:40:04.701 INFO:tasks.rados.rados.0.smithi017.stdout:526: done (0 left) 2024-09-21T09:40:04.701 INFO:tasks.rados.rados.0.smithi017.stdout:527: rollback oid 7 current snap is 66 2024-09-21T09:40:04.701 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 42 2024-09-21T09:40:04.701 INFO:tasks.rados.rados.0.smithi017.stdout:528: read oid 4 snap -1 2024-09-21T09:40:04.701 INFO:tasks.rados.rados.0.smithi017.stdout:528: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:40:04.701 INFO:tasks.rados.rados.0.smithi017.stdout:529: write oid 34 current snap is 66 2024-09-21T09:40:04.701 INFO:tasks.rados.rados.0.smithi017.stdout:529: seq_num 229 ranges {648803=722611,1967200=714747,3421983=426820} 2024-09-21T09:40:04.708 INFO:tasks.rados.rados.0.smithi017.stdout:529: writing smithi01785587-34 from 648803 to 1371414 tid 1 2024-09-21T09:40:04.715 INFO:tasks.rados.rados.0.smithi017.stdout:529: writing smithi01785587-34 from 1967200 to 2681947 tid 2 2024-09-21T09:40:04.722 INFO:tasks.rados.rados.0.smithi017.stdout:529: writing smithi01785587-34 from 3421983 to 3848803 tid 3 2024-09-21T09:40:04.722 INFO:tasks.rados.rados.0.smithi017.stdout:530: snap_create 2024-09-21T09:40:04.722 INFO:tasks.rados.rados.0.smithi017.stdout:527: finishing rollback tid 0 to smithi01785587-7 2024-09-21T09:40:04.722 INFO:tasks.rados.rados.0.smithi017.stdout:527: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:40:04.722 INFO:tasks.rados.rados.0.smithi017.stdout:527: finishing rollback tid 2 to smithi01785587-7 2024-09-21T09:40:04.722 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 242 (ObjNum 156 snap 39 seq_num 156) dirty exists 2024-09-21T09:40:04.722 INFO:tasks.rados.rados.0.smithi017.stdout:529: finishing write tid 1 to smithi01785587-34 2024-09-21T09:40:04.722 INFO:tasks.rados.rados.0.smithi017.stdout:529: finishing write tid 2 to smithi01785587-34 2024-09-21T09:40:04.722 INFO:tasks.rados.rados.0.smithi017.stdout:528: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:40:04.796 INFO:tasks.rados.rados.0.smithi017.stdout:529: finishing write tid 3 to smithi01785587-34 2024-09-21T09:40:04.796 INFO:tasks.rados.rados.0.smithi017.stdout:529: finishing write tid 4 to smithi01785587-34 2024-09-21T09:40:04.796 INFO:tasks.rados.rados.0.smithi017.stdout:529: finishing write tid 5 to smithi01785587-34 2024-09-21T09:40:04.796 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 247 (ObjNum 229 snap 66 seq_num 229) dirty exists 2024-09-21T09:40:04.796 INFO:tasks.rados.rados.0.smithi017.stdout:529: left oid 34 (ObjNum 229 snap 66 seq_num 229) 2024-09-21T09:40:04.947 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-mon[33541]: pgmap v60: 89 pgs: 89 active+clean; 243 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.6 MiB/s wr, 31 op/s 2024-09-21T09:40:04.947 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-mon[33541]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:40:04.947 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:04.947 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:04 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:04 smithi017 ceph-mon[40737]: pgmap v60: 89 pgs: 89 active+clean; 243 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.6 MiB/s wr, 31 op/s 2024-09-21T09:40:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:04 smithi017 ceph-mon[40737]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:40:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:04 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:04 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:04 smithi017 ceph-mon[29380]: pgmap v60: 89 pgs: 89 active+clean; 243 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.6 MiB/s wr, 31 op/s 2024-09-21T09:40:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:04 smithi017 ceph-mon[29380]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:40:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:04 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:04 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:05.706 INFO:tasks.rados.rados.0.smithi017.stdout:527: done (3 left) 2024-09-21T09:40:05.706 INFO:tasks.rados.rados.0.smithi017.stdout:528: done (2 left) 2024-09-21T09:40:05.706 INFO:tasks.rados.rados.0.smithi017.stdout:529: done (1 left) 2024-09-21T09:40:05.706 INFO:tasks.rados.rados.0.smithi017.stdout:530: done (0 left) 2024-09-21T09:40:05.706 INFO:tasks.rados.rados.0.smithi017.stdout:531: delete oid 49 current snap is 67 2024-09-21T09:40:05.709 INFO:tasks.rados.rados.0.smithi017.stdout:531: done (0 left) 2024-09-21T09:40:05.709 INFO:tasks.rados.rados.0.smithi017.stdout:532: snap_remove snap 63 2024-09-21T09:40:05.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:05 smithi134 ceph-mon[33541]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T09:40:05.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:05 smithi134 ceph-mon[33541]: Reconfiguring daemon grafana.a on smithi134 2024-09-21T09:40:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:05 smithi134 ceph-mon[33541]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:40:06.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:05 smithi017 ceph-mon[40737]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T09:40:06.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:05 smithi017 ceph-mon[40737]: Reconfiguring daemon grafana.a on smithi134 2024-09-21T09:40:06.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:05 smithi017 ceph-mon[40737]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:40:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:05 smithi017 ceph-mon[29380]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T09:40:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:05 smithi017 ceph-mon[29380]: Reconfiguring daemon grafana.a on smithi134 2024-09-21T09:40:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:05 smithi017 ceph-mon[29380]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:40:06.708 INFO:tasks.rados.rados.0.smithi017.stdout:532: done (0 left) 2024-09-21T09:40:06.708 INFO:tasks.rados.rados.0.smithi017.stdout:533: write oid 45 current snap is 67 2024-09-21T09:40:06.708 INFO:tasks.rados.rados.0.smithi017.stdout:533: seq_num 231 ranges {682417=473109,1567811=602063,2844123=621482} 2024-09-21T09:40:06.714 INFO:tasks.rados.rados.0.smithi017.stdout:533: writing smithi01785587-45 from 682417 to 1155526 tid 1 2024-09-21T09:40:06.720 INFO:tasks.rados.rados.0.smithi017.stdout:533: writing smithi01785587-45 from 1567811 to 2169874 tid 2 2024-09-21T09:40:06.727 INFO:tasks.rados.rados.0.smithi017.stdout:533: writing smithi01785587-45 from 2844123 to 3465605 tid 3 2024-09-21T09:40:06.727 INFO:tasks.rados.rados.0.smithi017.stdout:534: delete oid 10 current snap is 67 2024-09-21T09:40:06.728 INFO:tasks.rados.rados.0.smithi017.stdout:533: finishing write tid 1 to smithi01785587-45 2024-09-21T09:40:06.728 INFO:tasks.rados.rados.0.smithi017.stdout:533: finishing write tid 2 to smithi01785587-45 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi017.stdout:533: finishing write tid 3 to smithi01785587-45 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi017.stdout:533: finishing write tid 4 to smithi01785587-45 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi017.stdout:533: finishing write tid 5 to smithi01785587-45 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 165 (ObjNum 231 snap 67 seq_num 231) dirty exists 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi017.stdout:533: left oid 45 (ObjNum 231 snap 67 seq_num 231) 2024-09-21T09:40:06.732 INFO:tasks.rados.rados.0.smithi017.stdout:533: done (1 left) 2024-09-21T09:40:06.732 INFO:tasks.rados.rados.0.smithi017.stdout:534: done (0 left) 2024-09-21T09:40:06.732 INFO:tasks.rados.rados.0.smithi017.stdout:535: write oid 28 current snap is 67 2024-09-21T09:40:06.732 INFO:tasks.rados.rados.0.smithi017.stdout:535: seq_num 233 ranges {716031=623607,2084775=605732,3298969=617062} 2024-09-21T09:40:06.739 INFO:tasks.rados.rados.0.smithi017.stdout:535: writing smithi01785587-28 from 716031 to 1339638 tid 1 2024-09-21T09:40:06.746 INFO:tasks.rados.rados.0.smithi017.stdout:535: writing smithi01785587-28 from 2084775 to 2690507 tid 2 2024-09-21T09:40:06.753 INFO:tasks.rados.rados.0.smithi017.stdout:535: writing smithi01785587-28 from 3298969 to 3916031 tid 3 2024-09-21T09:40:06.753 INFO:tasks.rados.rados.0.smithi017.stdout:536: snap_create 2024-09-21T09:40:06.754 INFO:tasks.rados.rados.0.smithi017.stdout:535: finishing write tid 1 to smithi01785587-28 2024-09-21T09:40:06.754 INFO:tasks.rados.rados.0.smithi017.stdout:535: finishing write tid 2 to smithi01785587-28 2024-09-21T09:40:06.756 INFO:tasks.rados.rados.0.smithi017.stdout:535: finishing write tid 3 to smithi01785587-28 2024-09-21T09:40:06.756 INFO:tasks.rados.rados.0.smithi017.stdout:535: finishing write tid 4 to smithi01785587-28 2024-09-21T09:40:06.756 INFO:tasks.rados.rados.0.smithi017.stdout:535: finishing write tid 5 to smithi01785587-28 2024-09-21T09:40:06.756 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 269 (ObjNum 233 snap 67 seq_num 233) dirty exists 2024-09-21T09:40:06.756 INFO:tasks.rados.rados.0.smithi017.stdout:535: left oid 28 (ObjNum 233 snap 67 seq_num 233) 2024-09-21T09:40:06.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:06 smithi134 ceph-mon[33541]: pgmap v63: 89 pgs: 89 active+clean; 243 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:40:06.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:06 smithi134 ceph-mon[33541]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:40:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:06 smithi017 ceph-mon[40737]: pgmap v63: 89 pgs: 89 active+clean; 243 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:40:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:06 smithi017 ceph-mon[40737]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:40:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:06 smithi017 ceph-mon[29380]: pgmap v63: 89 pgs: 89 active+clean; 243 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:40:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:06 smithi017 ceph-mon[29380]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:40:07.715 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:07 smithi134 systemd[1]: Stopping Ceph grafana.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:07.718 INFO:tasks.rados.rados.0.smithi017.stdout:535: done (1 left) 2024-09-21T09:40:07.718 INFO:tasks.rados.rados.0.smithi017.stdout:536: done (0 left) 2024-09-21T09:40:07.718 INFO:tasks.rados.rados.0.smithi017.stdout:537: read oid 42 snap -1 2024-09-21T09:40:07.718 INFO:tasks.rados.rados.0.smithi017.stdout:537: expect deleted 2024-09-21T09:40:07.718 INFO:tasks.rados.rados.0.smithi017.stdout:538: snap_create 2024-09-21T09:40:07.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:07 smithi134 ceph-mon[33541]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:40:07.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:07 smithi134 podman[77282]: 2024-09-21 09:40:07.729557627 +0000 UTC m=+0.118290455 container died f7936515cf4ed7a70e4d4d3fa60ecf6632e8b7e3abf494992adb14ae1060f9f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, distribution-scope=public, release=239.1651231664, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, version=8.5, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-09-21T09:40:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:07 smithi017 ceph-mon[29380]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:40:08.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:07 smithi017 ceph-mon[40737]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:40:08.452 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 podman[77282]: 2024-09-21 09:40:08.199401855 +0000 UTC m=+0.588134686 container cleanup f7936515cf4ed7a70e4d4d3fa60ecf6632e8b7e3abf494992adb14ae1060f9f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, io.openshift.tags=base rhel8, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, distribution-scope=public, version=8.5, com.redhat.component=ubi8-container, architecture=x86_64, io.buildah.version=1.32.0, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-09-21T09:40:08.452 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 bash[77282]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a 2024-09-21T09:40:08.716 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 podman[77300]: 2024-09-21 09:40:08.451080805 +0000 UTC m=+0.720170968 container remove f7936515cf4ed7a70e4d4d3fa60ecf6632e8b7e3abf494992adb14ae1060f9f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.expose-services=, maintainer=Paul Cuzner , version=8.5, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64) 2024-09-21T09:40:08.716 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:40:08.716 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@grafana.a.service: Failed with result 'exit-code'. 2024-09-21T09:40:08.716 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 systemd[1]: Stopped Ceph grafana.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:08.716 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@grafana.a.service: Consumed 5.431s CPU time. 2024-09-21T09:40:08.716 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 systemd[1]: Starting Ceph grafana.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:08.723 INFO:tasks.rados.rados.0.smithi017.stdout:537: done (1 left) 2024-09-21T09:40:08.723 INFO:tasks.rados.rados.0.smithi017.stdout:538: done (0 left) 2024-09-21T09:40:08.723 INFO:tasks.rados.rados.0.smithi017.stdout:539: delete oid 14 current snap is 69 2024-09-21T09:40:08.725 INFO:tasks.rados.rados.0.smithi017.stdout:539: done (0 left) 2024-09-21T09:40:08.725 INFO:tasks.rados.rados.0.smithi017.stdout:540: write oid 24 current snap is 69 2024-09-21T09:40:08.725 INFO:tasks.rados.rados.0.smithi017.stdout:540: seq_num 235 ranges {749645=774105,2085386=609401,3237462=712183} 2024-09-21T09:40:08.733 INFO:tasks.rados.rados.0.smithi017.stdout:540: writing smithi01785587-24 from 749645 to 1523750 tid 1 2024-09-21T09:40:08.740 INFO:tasks.rados.rados.0.smithi017.stdout:540: writing smithi01785587-24 from 2085386 to 2694787 tid 2 2024-09-21T09:40:08.748 INFO:tasks.rados.rados.0.smithi017.stdout:540: writing smithi01785587-24 from 3237462 to 3949645 tid 3 2024-09-21T09:40:08.748 INFO:tasks.rados.rados.0.smithi017.stdout:541: read oid 16 snap 43 2024-09-21T09:40:08.748 INFO:tasks.rados.rados.0.smithi017.stdout:541: expect (ObjNum 117 snap 20 seq_num 117) 2024-09-21T09:40:08.748 INFO:tasks.rados.rados.0.smithi017.stdout:542: snap_remove snap 68 2024-09-21T09:40:08.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:08 smithi134 ceph-mon[33541]: pgmap v66: 89 pgs: 89 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 860 KiB/s wr, 6 op/s 2024-09-21T09:40:08.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:08 smithi134 ceph-mon[33541]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:40:08.967 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 podman[77399]: 2024-09-21 09:40:08.732983735 +0000 UTC m=+0.017265783 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-21T09:40:08.967 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:08 smithi134 podman[77399]: 2024-09-21 09:40:08.852217526 +0000 UTC m=+0.136499604 container create 8d2c807b030864794a54fcbc34d81c4d1e0bb6e5dc2c9bc5295e10fd163e6d27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, io.openshift.expose-services=, version=8.5, description=Ceph Grafana Container, vendor=Red Hat, Inc., release=239.1651231664, maintainer=Paul Cuzner , architecture=x86_64, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8) 2024-09-21T09:40:09.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:08 smithi017 ceph-mon[29380]: pgmap v66: 89 pgs: 89 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 860 KiB/s wr, 6 op/s 2024-09-21T09:40:09.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:08 smithi017 ceph-mon[29380]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:40:09.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:08 smithi017 ceph-mon[40737]: pgmap v66: 89 pgs: 89 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 860 KiB/s wr, 6 op/s 2024-09-21T09:40:09.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:08 smithi017 ceph-mon[40737]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 podman[77399]: 2024-09-21 09:40:09.08776128 +0000 UTC m=+0.372043320 container init 8d2c807b030864794a54fcbc34d81c4d1e0bb6e5dc2c9bc5295e10fd163e6d27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, com.redhat.component=ubi8-container, version=8.5, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.openshift.expose-services=, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, distribution-scope=public, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc.) 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 podman[77399]: 2024-09-21 09:40:09.092510162 +0000 UTC m=+0.376792197 container start 8d2c807b030864794a54fcbc34d81c4d1e0bb6e5dc2c9bc5295e10fd163e6d27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vcs-type=git, io.openshift.tags=base rhel8, version=8.5, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, name=ubi8, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 bash[77399]: 8d2c807b030864794a54fcbc34d81c4d1e0bb6e5dc2c9bc5295e10fd163e6d27 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 systemd[1]: Started Ceph grafana.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.25132797Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251505494Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251524191Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251531843Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.25155041Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:40:09.357 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251558444Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251564452Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251570572Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251576824Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251583419Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251589296Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251594981Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.251600998Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=settings t=2024-09-21T09:40:09.25160701Z level=info msg="App mode production" 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=sqlstore t=2024-09-21T09:40:09.251652068Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=sqlstore t=2024-09-21T09:40:09.251668197Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=migrator t=2024-09-21T09:40:09.265325836Z level=info msg="Starting DB migrations" 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=migrator t=2024-09-21T09:40:09.269930274Z level=info msg="migrations completed" performed=0 skipped=481 duration=418.083µs 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=sqlstore t=2024-09-21T09:40:09.270279596Z level=info msg="Created default organization" 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=plugin.loader t=2024-09-21T09:40:09.326254131Z level=info msg="Plugin registered" pluginID=input 2024-09-21T09:40:09.358 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=plugin.loader t=2024-09-21T09:40:09.354825263Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-21T09:40:09.359 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=plugin.loader t=2024-09-21T09:40:09.354970044Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-21T09:40:09.359 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=secrets t=2024-09-21T09:40:09.355070714Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:40:09.726 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=query_data t=2024-09-21T09:40:09.357739223Z level=info msg="Query Service initialization" 2024-09-21T09:40:09.726 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=live.push_http t=2024-09-21T09:40:09.360065873Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:40:09.729 INFO:tasks.rados.rados.0.smithi017.stdout:542: done (2 left) 2024-09-21T09:40:09.729 INFO:tasks.rados.rados.0.smithi017.stdout:543: snap_create 2024-09-21T09:40:09.729 INFO:tasks.rados.rados.0.smithi017.stdout:540: finishing write tid 1 to smithi01785587-24 2024-09-21T09:40:09.729 INFO:tasks.rados.rados.0.smithi017.stdout:540: finishing write tid 2 to smithi01785587-24 2024-09-21T09:40:09.730 INFO:tasks.rados.rados.0.smithi017.stdout:540: finishing write tid 3 to smithi01785587-24 2024-09-21T09:40:09.730 INFO:tasks.rados.rados.0.smithi017.stdout:540: finishing write tid 4 to smithi01785587-24 2024-09-21T09:40:09.730 INFO:tasks.rados.rados.0.smithi017.stdout:540: finishing write tid 5 to smithi01785587-24 2024-09-21T09:40:09.730 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 153 (ObjNum 235 snap 69 seq_num 235) dirty exists 2024-09-21T09:40:09.730 INFO:tasks.rados.rados.0.smithi017.stdout:540: left oid 24 (ObjNum 235 snap 69 seq_num 235) 2024-09-21T09:40:09.730 INFO:tasks.rados.rados.0.smithi017.stdout:541: expect (ObjNum 117 snap 20 seq_num 117) 2024-09-21T09:40:09.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-mon[33541]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:40:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:09 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:09.994 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:09 smithi134 systemd[1]: Stopping Ceph node-exporter.b for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:10.120 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:09 smithi017 ceph-mon[29380]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:40:10.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:09 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:10.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:09 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:10.121 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:09 smithi017 ceph-mon[40737]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:40:10.121 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:09 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:10.121 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:09 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=infra.usagestats.collector t=2024-09-21T09:40:10.109270946Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:40:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=provisioning.datasources t=2024-09-21T09:40:10.195095367Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-21T09:40:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=provisioning.datasources t=2024-09-21T09:40:10.195121909Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-21T09:40:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=provisioning.datasources t=2024-09-21T09:40:10.205519071Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:40:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=provisioning.alerting t=2024-09-21T09:40:10.285172008Z level=info msg="starting to provision alerting" 2024-09-21T09:40:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=provisioning.alerting t=2024-09-21T09:40:10.285191901Z level=info msg="finished to provision alerting" 2024-09-21T09:40:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=grafanaStorageLogger t=2024-09-21T09:40:10.285283715Z level=info msg="storage starting" 2024-09-21T09:40:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=ngalert.state.manager t=2024-09-21T09:40:10.285491334Z level=info msg="Warming state cache for startup" 2024-09-21T09:40:10.495 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=http.server t=2024-09-21T09:40:10.287219841Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:40:10.495 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=ngalert.state.manager t=2024-09-21T09:40:10.296371354Z level=info msg="State cache has been initialized" states=0 duration=10.878939ms 2024-09-21T09:40:10.495 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=ticker t=2024-09-21T09:40:10.296444709Z level=info msg=starting first_tick=2024-09-21T09:40:20Z 2024-09-21T09:40:10.495 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:40:10.29643467Z level=info msg="starting MultiOrg Alertmanager" 2024-09-21T09:40:10.729 INFO:tasks.rados.rados.0.smithi017.stdout:540: done (2 left) 2024-09-21T09:40:10.729 INFO:tasks.rados.rados.0.smithi017.stdout:541: done (1 left) 2024-09-21T09:40:10.730 INFO:tasks.rados.rados.0.smithi017.stdout:543: done (0 left) 2024-09-21T09:40:10.730 INFO:tasks.rados.rados.0.smithi017.stdout:544: write oid 42 current snap is 70 2024-09-21T09:40:10.730 INFO:tasks.rados.rados.0.smithi017.stdout:544: seq_num 236 ranges {766452=565707,1860221=469412,2981238=594195} 2024-09-21T09:40:10.736 INFO:tasks.rados.rados.0.smithi017.stdout:544: writing smithi01785587-42 from 766452 to 1332159 tid 1 2024-09-21T09:40:10.742 INFO:tasks.rados.rados.0.smithi017.stdout:544: writing smithi01785587-42 from 1860221 to 2329633 tid 2 2024-09-21T09:40:10.749 INFO:tasks.rados.rados.0.smithi017.stdout:544: writing smithi01785587-42 from 2981238 to 3575433 tid 3 2024-09-21T09:40:10.749 INFO:tasks.rados.rados.0.smithi017.stdout:545: snap_remove snap 69 2024-09-21T09:40:10.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-mon[33541]: pgmap v69: 89 pgs: 89 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 860 KiB/s wr, 6 op/s 2024-09-21T09:40:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-mon[33541]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T09:40:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-mon[33541]: Deploying daemon node-exporter.b on smithi134 2024-09-21T09:40:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:10 smithi134 ceph-mon[33541]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:40:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:10 smithi017 ceph-mon[29380]: pgmap v69: 89 pgs: 89 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 860 KiB/s wr, 6 op/s 2024-09-21T09:40:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:10 smithi017 ceph-mon[29380]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T09:40:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:10 smithi017 ceph-mon[29380]: Deploying daemon node-exporter.b on smithi134 2024-09-21T09:40:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:10 smithi017 ceph-mon[29380]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:40:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:10 smithi017 ceph-mon[40737]: pgmap v69: 89 pgs: 89 active+clean; 188 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 860 KiB/s wr, 6 op/s 2024-09-21T09:40:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:10 smithi017 ceph-mon[40737]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T09:40:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:10 smithi017 ceph-mon[40737]: Deploying daemon node-exporter.b on smithi134 2024-09-21T09:40:11.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:10 smithi017 ceph-mon[40737]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:40:11.493 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:11 smithi134 podman[77560]: 2024-09-21 09:40:11.082570885 +0000 UTC m=+1.167003840 container died a405017b5317f8f31d672a2169654fc775b2c5e64868cc26525c3dfac44f785c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:11.740 INFO:tasks.rados.rados.0.smithi017.stdout:545: done (1 left) 2024-09-21T09:40:11.740 INFO:tasks.rados.rados.0.smithi017.stdout:546: snap_create 2024-09-21T09:40:11.740 INFO:tasks.rados.rados.0.smithi017.stdout:544: finishing write tid 1 to smithi01785587-42 2024-09-21T09:40:11.740 INFO:tasks.rados.rados.0.smithi017.stdout:544: finishing write tid 2 to smithi01785587-42 2024-09-21T09:40:11.740 INFO:tasks.rados.rados.0.smithi017.stdout:544: finishing write tid 3 to smithi01785587-42 2024-09-21T09:40:11.740 INFO:tasks.rados.rados.0.smithi017.stdout:544: finishing write tid 4 to smithi01785587-42 2024-09-21T09:40:11.740 INFO:tasks.rados.rados.0.smithi017.stdout:544: finishing write tid 5 to smithi01785587-42 2024-09-21T09:40:11.740 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 189 (ObjNum 236 snap 70 seq_num 236) dirty exists 2024-09-21T09:40:11.740 INFO:tasks.rados.rados.0.smithi017.stdout:544: left oid 42 (ObjNum 236 snap 70 seq_num 236) 2024-09-21T09:40:11.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:11 smithi134 ceph-mon[33541]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:40:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:11 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:11.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:40:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:40:11] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.51.0" 2024-09-21T09:40:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:11 smithi017 ceph-mon[29380]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:40:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:11 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:11 smithi017 ceph-mon[40737]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:40:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:11 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:12.493 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:12 smithi134 podman[77560]: 2024-09-21 09:40:12.111756254 +0000 UTC m=+2.196189254 container cleanup a405017b5317f8f31d672a2169654fc775b2c5e64868cc26525c3dfac44f785c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:12.494 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:12 smithi134 bash[77560]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T09:40:12.494 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:12 smithi134 podman[77579]: 2024-09-21 09:40:12.330020089 +0000 UTC m=+1.249707671 container remove a405017b5317f8f31d672a2169654fc775b2c5e64868cc26525c3dfac44f785c (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:12.494 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:12 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:40:12.694 INFO:tasks.rados.rados.0.smithi017.stdout:544: done (1 left) 2024-09-21T09:40:12.694 INFO:tasks.rados.rados.0.smithi017.stdout:546: done (0 left) 2024-09-21T09:40:12.694 INFO:tasks.rados.rados.0.smithi017.stdout:547: snap_remove snap 64 2024-09-21T09:40:12.945 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:12 smithi134 ceph-mon[33541]: pgmap v72: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-21T09:40:12.945 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:12 smithi134 ceph-mon[33541]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:40:12.945 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:12 smithi134 ceph-mon[33541]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:40:12.946 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:12 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T09:40:12.946 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:12 smithi134 systemd[1]: Stopped Ceph node-exporter.b for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:12.946 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:12 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 2.228s CPU time. 2024-09-21T09:40:12.946 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:12 smithi134 systemd[1]: Starting Ceph node-exporter.b for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:12 smithi017 ceph-mon[40737]: pgmap v72: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-21T09:40:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:12 smithi017 ceph-mon[40737]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:40:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:12 smithi017 ceph-mon[40737]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:40:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:12 smithi017 ceph-mon[29380]: pgmap v72: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-21T09:40:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:12 smithi017 ceph-mon[29380]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:40:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:12 smithi017 ceph-mon[29380]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:40:13.243 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:12 smithi134 bash[77711]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:40:13.696 INFO:tasks.rados.rados.0.smithi017.stdout:547: done (0 left) 2024-09-21T09:40:13.696 INFO:tasks.rados.rados.0.smithi017.stdout:548: rollback oid 39 current snap is 71 2024-09-21T09:40:13.696 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 70 2024-09-21T09:40:13.696 INFO:tasks.rados.rados.0.smithi017.stdout:549: read oid 9 snap -1 2024-09-21T09:40:13.696 INFO:tasks.rados.rados.0.smithi017.stdout:549: expect (ObjNum 151 snap 36 seq_num 151) 2024-09-21T09:40:13.696 INFO:tasks.rados.rados.0.smithi017.stdout:550: snap_create 2024-09-21T09:40:13.702 INFO:tasks.rados.rados.0.smithi017.stdout:549: expect (ObjNum 151 snap 36 seq_num 151) 2024-09-21T09:40:13.743 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:13 smithi134 bash[77711]: Getting image source signatures 2024-09-21T09:40:13.743 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:13 smithi134 bash[77711]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:40:13.744 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:13 smithi134 bash[77711]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:40:13.744 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:13 smithi134 bash[77711]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:40:13.834 INFO:tasks.rados.rados.0.smithi017.stdout:548: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:40:13.834 INFO:tasks.rados.rados.0.smithi017.stdout:548: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:40:13.834 INFO:tasks.rados.rados.0.smithi017.stdout:548: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:40:13.834 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 206 (ObjNum 190 snap 50 seq_num 190) dirty exists 2024-09-21T09:40:14.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:14.697 INFO:tasks.rados.rados.0.smithi017.stdout:548: done (2 left) 2024-09-21T09:40:14.697 INFO:tasks.rados.rados.0.smithi017.stdout:549: done (1 left) 2024-09-21T09:40:14.697 INFO:tasks.rados.rados.0.smithi017.stdout:550: done (0 left) 2024-09-21T09:40:14.697 INFO:tasks.rados.rados.0.smithi017.stdout:551: write oid 40 current snap is 72 2024-09-21T09:40:14.697 INFO:tasks.rados.rados.0.smithi017.stdout:551: seq_num 237 ranges {783259=640956,1918703=613070,3008661=405467,3821939=161320} 2024-09-21T09:40:14.704 INFO:tasks.rados.rados.0.smithi017.stdout:551: writing smithi01785587-40 from 783259 to 1424215 tid 1 2024-09-21T09:40:14.711 INFO:tasks.rados.rados.0.smithi017.stdout:551: writing smithi01785587-40 from 1918703 to 2531773 tid 2 2024-09-21T09:40:14.716 INFO:tasks.rados.rados.0.smithi017.stdout:551: writing smithi01785587-40 from 3008661 to 3414128 tid 3 2024-09-21T09:40:14.720 INFO:tasks.rados.rados.0.smithi017.stdout:551: writing smithi01785587-40 from 3821939 to 3983259 tid 4 2024-09-21T09:40:14.720 INFO:tasks.rados.rados.0.smithi017.stdout:552: read oid 4 snap 71 2024-09-21T09:40:14.720 INFO:tasks.rados.rados.0.smithi017.stdout:552: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:40:14.720 INFO:tasks.rados.rados.0.smithi017.stdout:553: write oid 24 current snap is 72 2024-09-21T09:40:14.720 INFO:tasks.rados.rados.0.smithi017.stdout:551: finishing write tid 1 to smithi01785587-40 2024-09-21T09:40:14.720 INFO:tasks.rados.rados.0.smithi017.stdout:553: seq_num 238 ranges {400066=716205,1860832=139234} 2024-09-21T09:40:14.726 INFO:tasks.rados.rados.0.smithi017.stdout:553: writing smithi01785587-24 from 400066 to 1116271 tid 1 2024-09-21T09:40:14.731 INFO:tasks.rados.rados.0.smithi017.stdout:553: writing smithi01785587-24 from 1860832 to 2000066 tid 2 2024-09-21T09:40:14.731 INFO:tasks.rados.rados.0.smithi017.stdout:554: rollback oid 23 current snap is 72 2024-09-21T09:40:14.732 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 23 to 70 2024-09-21T09:40:14.732 INFO:tasks.rados.rados.0.smithi017.stdout:551: finishing write tid 2 to smithi01785587-40 2024-09-21T09:40:14.732 INFO:tasks.rados.rados.0.smithi017.stdout:551: finishing write tid 3 to smithi01785587-40 2024-09-21T09:40:14.732 INFO:tasks.rados.rados.0.smithi017.stdout:551: finishing write tid 4 to smithi01785587-40 2024-09-21T09:40:14.732 INFO:tasks.rados.rados.0.smithi017.stdout:551: finishing write tid 5 to smithi01785587-40 2024-09-21T09:40:14.732 INFO:tasks.rados.rados.0.smithi017.stdout:551: finishing write tid 6 to smithi01785587-40 2024-09-21T09:40:14.732 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 212 (ObjNum 237 snap 72 seq_num 237) dirty exists 2024-09-21T09:40:14.732 INFO:tasks.rados.rados.0.smithi017.stdout:551: left oid 40 (ObjNum 237 snap 72 seq_num 237) 2024-09-21T09:40:14.732 INFO:tasks.rados.rados.0.smithi017.stdout:552: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:40:14.806 INFO:tasks.rados.rados.0.smithi017.stdout:553: finishing write tid 1 to smithi01785587-24 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:553: finishing write tid 2 to smithi01785587-24 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:553: finishing write tid 3 to smithi01785587-24 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:554: finishing rollback tid 0 to smithi01785587-23 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:553: finishing write tid 4 to smithi01785587-24 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 157 (ObjNum 238 snap 72 seq_num 238) dirty exists 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:553: left oid 24 (ObjNum 238 snap 72 seq_num 238) 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:554: finishing rollback tid 1 to smithi01785587-23 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:551: done (3 left) 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:552: done (2 left) 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:553: done (1 left) 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:555: read oid 17 snap -1 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:555: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:40:14.807 INFO:tasks.rados.rados.0.smithi017.stdout:556: write oid 16 current snap is 72 2024-09-21T09:40:14.808 INFO:tasks.rados.rados.0.smithi017.stdout:556: seq_num 239 ranges {416873=791454,1919314=97559} 2024-09-21T09:40:14.812 INFO:tasks.rados.rados.0.smithi017.stdout:556: writing smithi01785587-16 from 416873 to 1208327 tid 1 2024-09-21T09:40:14.818 INFO:tasks.rados.rados.0.smithi017.stdout:556: writing smithi01785587-16 from 1919314 to 2016873 tid 2 2024-09-21T09:40:14.818 INFO:tasks.rados.rados.0.smithi017.stdout:557: delete oid 45 current snap is 72 2024-09-21T09:40:14.818 INFO:tasks.rados.rados.0.smithi017.stdout:554: finishing rollback tid 2 to smithi01785587-23 2024-09-21T09:40:14.818 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 265 (ObjNum 186 snap 49 seq_num 186) dirty exists 2024-09-21T09:40:14.818 INFO:tasks.rados.rados.0.smithi017.stdout:555: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:40:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:14 smithi017 ceph-mon[29380]: pgmap v75: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-21T09:40:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:14 smithi017 ceph-mon[29380]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:40:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:14 smithi017 ceph-mon[40737]: pgmap v75: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-21T09:40:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:14 smithi017 ceph-mon[40737]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:40:14.962 INFO:tasks.rados.rados.0.smithi017.stdout:556: finishing write tid 1 to smithi01785587-16 2024-09-21T09:40:14.962 INFO:tasks.rados.rados.0.smithi017.stdout:556: finishing write tid 2 to smithi01785587-16 2024-09-21T09:40:14.962 INFO:tasks.rados.rados.0.smithi017.stdout:556: finishing write tid 3 to smithi01785587-16 2024-09-21T09:40:14.962 INFO:tasks.rados.rados.0.smithi017.stdout:556: finishing write tid 4 to smithi01785587-16 2024-09-21T09:40:14.962 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 272 (ObjNum 239 snap 72 seq_num 239) dirty exists 2024-09-21T09:40:14.962 INFO:tasks.rados.rados.0.smithi017.stdout:556: left oid 16 (ObjNum 239 snap 72 seq_num 239) 2024-09-21T09:40:14.962 INFO:tasks.rados.rados.0.smithi017.stdout:554: done (3 left) 2024-09-21T09:40:14.962 INFO:tasks.rados.rados.0.smithi017.stdout:555: done (2 left) 2024-09-21T09:40:14.963 INFO:tasks.rados.rados.0.smithi017.stdout:556: done (1 left) 2024-09-21T09:40:14.963 INFO:tasks.rados.rados.0.smithi017.stdout:557: done (0 left) 2024-09-21T09:40:14.963 INFO:tasks.rados.rados.0.smithi017.stdout:558: write oid 32 current snap is 72 2024-09-21T09:40:14.963 INFO:tasks.rados.rados.0.smithi017.stdout:558: seq_num 241 ranges {450487=541952,1636278=414209} 2024-09-21T09:40:14.967 INFO:tasks.rados.rados.0.smithi017.stdout:558: writing smithi01785587-32 from 450487 to 992439 tid 1 2024-09-21T09:40:14.973 INFO:tasks.rados.rados.0.smithi017.stdout:558: writing smithi01785587-32 from 1636278 to 2050487 tid 2 2024-09-21T09:40:14.973 INFO:tasks.rados.rados.0.smithi017.stdout:559: rollback oid 21 current snap is 72 2024-09-21T09:40:14.973 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 65 2024-09-21T09:40:14.973 INFO:tasks.rados.rados.0.smithi017.stdout:558: finishing write tid 1 to smithi01785587-32 2024-09-21T09:40:14.973 INFO:tasks.rados.rados.0.smithi017.stdout:560: delete oid 44 current snap is 72 2024-09-21T09:40:14.975 INFO:tasks.rados.rados.0.smithi017.stdout:560: done (2 left) 2024-09-21T09:40:14.975 INFO:tasks.rados.rados.0.smithi017.stdout:561: delete oid 10 current snap is 72 2024-09-21T09:40:14.975 INFO:tasks.rados.rados.0.smithi017.stdout:558: finishing write tid 2 to smithi01785587-32 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:558: finishing write tid 3 to smithi01785587-32 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:558: finishing write tid 4 to smithi01785587-32 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 269 (ObjNum 241 snap 72 seq_num 241) dirty exists 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:558: left oid 32 (ObjNum 241 snap 72 seq_num 241) 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:559: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:559: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:559: finishing rollback tid 2 to smithi01785587-21 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 265 (ObjNum 222 snap 61 seq_num 222) dirty exists 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:558: done (2 left) 2024-09-21T09:40:14.976 INFO:tasks.rados.rados.0.smithi017.stdout:559: done (1 left) 2024-09-21T09:40:14.977 INFO:tasks.rados.rados.0.smithi017.stdout:561: done (0 left) 2024-09-21T09:40:14.977 INFO:tasks.rados.rados.0.smithi017.stdout:562: delete oid 24 current snap is 72 2024-09-21T09:40:14.977 INFO:tasks.rados.rados.0.smithi017.stdout:562: done (0 left) 2024-09-21T09:40:14.977 INFO:tasks.rados.rados.0.smithi017.stdout:563: write oid 4 current snap is 72 2024-09-21T09:40:14.977 INFO:tasks.rados.rados.0.smithi017.stdout:563: seq_num 245 ranges {517715=559301,1870206=247509} 2024-09-21T09:40:14.983 INFO:tasks.rados.rados.0.smithi017.stdout:563: writing smithi01785587-4 from 517715 to 1077016 tid 1 2024-09-21T09:40:14.989 INFO:tasks.rados.rados.0.smithi017.stdout:563: writing smithi01785587-4 from 1870206 to 2117715 tid 2 2024-09-21T09:40:14.989 INFO:tasks.rados.rados.0.smithi017.stdout:564: read oid 31 snap -1 2024-09-21T09:40:14.989 INFO:tasks.rados.rados.0.smithi017.stdout:564: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:40:14.989 INFO:tasks.rados.rados.0.smithi017.stdout:565: snap_create 2024-09-21T09:40:14.990 INFO:tasks.rados.rados.0.smithi017.stdout:563: finishing write tid 1 to smithi01785587-4 2024-09-21T09:40:14.991 INFO:tasks.rados.rados.0.smithi017.stdout:563: finishing write tid 2 to smithi01785587-4 2024-09-21T09:40:14.991 INFO:tasks.rados.rados.0.smithi017.stdout:563: finishing write tid 3 to smithi01785587-4 2024-09-21T09:40:14.991 INFO:tasks.rados.rados.0.smithi017.stdout:563: finishing write tid 4 to smithi01785587-4 2024-09-21T09:40:14.991 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 276 (ObjNum 245 snap 72 seq_num 245) dirty exists 2024-09-21T09:40:14.991 INFO:tasks.rados.rados.0.smithi017.stdout:563: left oid 4 (ObjNum 245 snap 72 seq_num 245) 2024-09-21T09:40:14.991 INFO:tasks.rados.rados.0.smithi017.stdout:564: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:40:14.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:14 smithi134 ceph-mon[33541]: pgmap v75: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-21T09:40:14.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:14 smithi134 ceph-mon[33541]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:40:14.994 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:14 smithi134 bash[77711]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:40:15.374 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 bash[77711]: Writing manifest to image destination 2024-09-21T09:40:15.374 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 podman[77711]: 2024-09-21 09:40:15.266587032 +0000 UTC m=+2.419700297 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:40:15.374 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 podman[77711]: 2024-09-21 09:40:15.373469449 +0000 UTC m=+2.526582708 container create c6b046d44374f131cfaa2c1a1e02d202770926863880de056dd93546758311fd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:15.710 INFO:tasks.rados.rados.0.smithi017.stdout:563: done (2 left) 2024-09-21T09:40:15.710 INFO:tasks.rados.rados.0.smithi017.stdout:564: done (1 left) 2024-09-21T09:40:15.710 INFO:tasks.rados.rados.0.smithi017.stdout:565: done (0 left) 2024-09-21T09:40:15.710 INFO:tasks.rados.rados.0.smithi017.stdout:566: write oid 36 current snap is 73 2024-09-21T09:40:15.710 INFO:tasks.rados.rados.0.smithi017.stdout:566: seq_num 246 ranges {534522=634550,1812335=322187} 2024-09-21T09:40:15.716 INFO:tasks.rados.rados.0.smithi017.stdout:566: writing smithi01785587-36 from 534522 to 1169072 tid 1 2024-09-21T09:40:15.722 INFO:tasks.rados.rados.0.smithi017.stdout:566: writing smithi01785587-36 from 1812335 to 2134522 tid 2 2024-09-21T09:40:15.722 INFO:tasks.rados.rados.0.smithi017.stdout:567: rollback oid 12 current snap is 73 2024-09-21T09:40:15.722 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 12 to 65 2024-09-21T09:40:15.722 INFO:tasks.rados.rados.0.smithi017.stdout:566: finishing write tid 1 to smithi01785587-36 2024-09-21T09:40:15.722 INFO:tasks.rados.rados.0.smithi017.stdout:568: write oid 18 current snap is 73 2024-09-21T09:40:15.722 INFO:tasks.rados.rados.0.smithi017.stdout:568: seq_num 247 ranges {551329=709799,1870817=280512} 2024-09-21T09:40:15.728 INFO:tasks.rados.rados.0.smithi017.stdout:568: writing smithi01785587-18 from 551329 to 1261128 tid 1 2024-09-21T09:40:15.734 INFO:tasks.rados.rados.0.smithi017.stdout:568: writing smithi01785587-18 from 1870817 to 2151329 tid 2 2024-09-21T09:40:15.734 INFO:tasks.rados.rados.0.smithi017.stdout:569: rollback oid 21 current snap is 73 2024-09-21T09:40:15.734 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 59 2024-09-21T09:40:15.734 INFO:tasks.rados.rados.0.smithi017.stdout:567: finishing rollback tid 1 to smithi01785587-12 2024-09-21T09:40:15.734 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 170 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:40:15.734 INFO:tasks.rados.rados.0.smithi017.stdout:566: finishing write tid 2 to smithi01785587-36 2024-09-21T09:40:15.734 INFO:tasks.rados.rados.0.smithi017.stdout:566: finishing write tid 3 to smithi01785587-36 2024-09-21T09:40:15.734 INFO:tasks.rados.rados.0.smithi017.stdout:566: finishing write tid 4 to smithi01785587-36 2024-09-21T09:40:15.734 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 215 (ObjNum 246 snap 73 seq_num 246) dirty exists 2024-09-21T09:40:15.735 INFO:tasks.rados.rados.0.smithi017.stdout:566: left oid 36 (ObjNum 246 snap 73 seq_num 246) 2024-09-21T09:40:15.735 INFO:tasks.rados.rados.0.smithi017.stdout:568: finishing write tid 1 to smithi01785587-18 2024-09-21T09:40:15.735 INFO:tasks.rados.rados.0.smithi017.stdout:566: done (3 left) 2024-09-21T09:40:15.735 INFO:tasks.rados.rados.0.smithi017.stdout:567: done (2 left) 2024-09-21T09:40:15.735 INFO:tasks.rados.rados.0.smithi017.stdout:570: snap_remove snap 65 2024-09-21T09:40:15.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-mon[33541]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:40:15.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:15.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:15.994 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 podman[77711]: 2024-09-21 09:40:15.626212105 +0000 UTC m=+2.779325373 container init c6b046d44374f131cfaa2c1a1e02d202770926863880de056dd93546758311fd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:15.994 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 podman[77711]: 2024-09-21 09:40:15.631250047 +0000 UTC m=+2.784363307 container start c6b046d44374f131cfaa2c1a1e02d202770926863880de056dd93546758311fd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:15.994 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.634Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:40:15.994 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.634Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:40:15.994 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.634Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:40:15.994 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.634Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.634Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.634Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:40:15.995 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:40:15.996 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:40:15.997 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.635Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:40:15.998 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.636Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:40:15.999 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b[77786]: ts=2024-09-21T09:40:15.636Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:40:15.999 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 bash[77711]: c6b046d44374f131cfaa2c1a1e02d202770926863880de056dd93546758311fd 2024-09-21T09:40:15.999 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 09:40:15 smithi134 systemd[1]: Started Ceph node-exporter.b for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:16.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:15 smithi017 ceph-mon[29380]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:40:16.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:15 smithi017 ceph-mon[40737]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:40:16.714 INFO:tasks.rados.rados.0.smithi017.stdout:570: done (2 left) 2024-09-21T09:40:16.714 INFO:tasks.rados.rados.0.smithi017.stdout:571: read oid 15 snap -1 2024-09-21T09:40:16.714 INFO:tasks.rados.rados.0.smithi017.stdout:571: expect deleted 2024-09-21T09:40:16.714 INFO:tasks.rados.rados.0.smithi017.stdout:572: snap_remove snap 54 2024-09-21T09:40:16.951 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: pgmap v78: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:16.951 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:40:16.951 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:16.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:16.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:40:16.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:40:16.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:16.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:16.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:40:16.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:40:16.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: pgmap v78: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:40:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:40:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:40:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: pgmap v78: 89 pgs: 89 active+clean; 201 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:40:17.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:40:17.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:16 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:17.234 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:17.718 INFO:tasks.rados.rados.0.smithi017.stdout:572: done (3 left) 2024-09-21T09:40:17.718 INFO:tasks.rados.rados.0.smithi017.stdout:573: snap_create 2024-09-21T09:40:17.718 INFO:tasks.rados.rados.0.smithi017.stdout:569: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:40:17.718 INFO:tasks.rados.rados.0.smithi017.stdout:568: finishing write tid 2 to smithi01785587-18 2024-09-21T09:40:17.718 INFO:tasks.rados.rados.0.smithi017.stdout:569: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:40:17.719 INFO:tasks.rados.rados.0.smithi017.stdout:568: finishing write tid 3 to smithi01785587-18 2024-09-21T09:40:17.719 INFO:tasks.rados.rados.0.smithi017.stdout:569: finishing rollback tid 2 to smithi01785587-21 2024-09-21T09:40:17.719 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 269 (ObjNum 201 snap 56 seq_num 201) dirty exists 2024-09-21T09:40:17.719 INFO:tasks.rados.rados.0.smithi017.stdout:568: finishing write tid 4 to smithi01785587-18 2024-09-21T09:40:17.719 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 28 (ObjNum 247 snap 73 seq_num 247) dirty exists 2024-09-21T09:40:17.719 INFO:tasks.rados.rados.0.smithi017.stdout:568: left oid 18 (ObjNum 247 snap 73 seq_num 247) 2024-09-21T09:40:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:17 smithi017 ceph-mon[29380]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:40:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:17 smithi017 ceph-mon[40737]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:40:17.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:17 smithi134 ceph-mon[33541]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:40:18.719 INFO:tasks.rados.rados.0.smithi017.stdout:568: done (3 left) 2024-09-21T09:40:18.719 INFO:tasks.rados.rados.0.smithi017.stdout:569: done (2 left) 2024-09-21T09:40:18.719 INFO:tasks.rados.rados.0.smithi017.stdout:571: done (1 left) 2024-09-21T09:40:18.720 INFO:tasks.rados.rados.0.smithi017.stdout:573: done (0 left) 2024-09-21T09:40:18.720 INFO:tasks.rados.rados.0.smithi017.stdout:574: snap_create 2024-09-21T09:40:18.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:18 smithi134 ceph-mon[33541]: pgmap v81: 89 pgs: 89 active+clean; 214 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:40:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:18 smithi134 ceph-mon[33541]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:40:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:18 smithi017 ceph-mon[29380]: pgmap v81: 89 pgs: 89 active+clean; 214 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:40:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:18 smithi017 ceph-mon[29380]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:40:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:18 smithi017 ceph-mon[40737]: pgmap v81: 89 pgs: 89 active+clean; 214 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:40:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:18 smithi017 ceph-mon[40737]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:40:19.730 INFO:tasks.rados.rados.0.smithi017.stdout:574: done (0 left) 2024-09-21T09:40:19.731 INFO:tasks.rados.rados.0.smithi017.stdout:575: snap_remove snap 62 2024-09-21T09:40:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:19 smithi134 ceph-mon[33541]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:40:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:19 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:19 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:19 smithi017 ceph-mon[29380]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:40:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:19 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:19 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:20.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:19 smithi017 ceph-mon[40737]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:40:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:19 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:19 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:20.734 INFO:tasks.rados.rados.0.smithi017.stdout:575: done (0 left) 2024-09-21T09:40:20.734 INFO:tasks.rados.rados.0.smithi017.stdout:576: delete oid 39 current snap is 75 2024-09-21T09:40:20.742 INFO:tasks.rados.rados.0.smithi017.stdout:576: done (0 left) 2024-09-21T09:40:20.742 INFO:tasks.rados.rados.0.smithi017.stdout:577: delete oid 19 current snap is 75 2024-09-21T09:40:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:577: done (0 left) 2024-09-21T09:40:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:578: read oid 6 snap -1 2024-09-21T09:40:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:578: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:40:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:579: write oid 45 current snap is 75 2024-09-21T09:40:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:579: seq_num 250 ranges {601750=535546,1929910=271840} 2024-09-21T09:40:20.751 INFO:tasks.rados.rados.0.smithi017.stdout:579: writing smithi01785587-45 from 601750 to 1137296 tid 1 2024-09-21T09:40:20.757 INFO:tasks.rados.rados.0.smithi017.stdout:579: writing smithi01785587-45 from 1929910 to 2201750 tid 2 2024-09-21T09:40:20.757 INFO:tasks.rados.rados.0.smithi017.stdout:580: snap_create 2024-09-21T09:40:20.757 INFO:tasks.rados.rados.0.smithi017.stdout:578: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:40:20.829 INFO:tasks.rados.rados.0.smithi017.stdout:579: finishing write tid 1 to smithi01785587-45 2024-09-21T09:40:20.829 INFO:tasks.rados.rados.0.smithi017.stdout:579: finishing write tid 2 to smithi01785587-45 2024-09-21T09:40:20.829 INFO:tasks.rados.rados.0.smithi017.stdout:579: finishing write tid 3 to smithi01785587-45 2024-09-21T09:40:20.829 INFO:tasks.rados.rados.0.smithi017.stdout:579: finishing write tid 4 to smithi01785587-45 2024-09-21T09:40:20.829 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 180 (ObjNum 250 snap 75 seq_num 250) dirty exists 2024-09-21T09:40:20.829 INFO:tasks.rados.rados.0.smithi017.stdout:579: left oid 45 (ObjNum 250 snap 75 seq_num 250) 2024-09-21T09:40:20.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:20 smithi134 ceph-mon[33541]: pgmap v84: 89 pgs: 89 active+clean; 214 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:40:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:20 smithi134 ceph-mon[33541]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:40:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:20 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:20 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:20 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:20 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:20 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[40737]: pgmap v84: 89 pgs: 89 active+clean; 214 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:40:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[40737]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:40:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:21.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[29380]: pgmap v84: 89 pgs: 89 active+clean; 214 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:40:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[29380]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:40:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:20 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:578: done (2 left) 2024-09-21T09:40:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:579: done (1 left) 2024-09-21T09:40:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:580: done (0 left) 2024-09-21T09:40:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:581: write oid 2 current snap is 76 2024-09-21T09:40:21.747 INFO:tasks.rados.rados.0.smithi017.stdout:581: seq_num 251 ranges {618557=727148,2104745=113812} 2024-09-21T09:40:21.754 INFO:tasks.rados.rados.0.smithi017.stdout:581: writing smithi01785587-2 from 618557 to 1345705 tid 1 2024-09-21T09:40:21.759 INFO:tasks.rados.rados.0.smithi017.stdout:581: writing smithi01785587-2 from 2104745 to 2218557 tid 2 2024-09-21T09:40:21.759 INFO:tasks.rados.rados.0.smithi017.stdout:582: write oid 17 current snap is 76 2024-09-21T09:40:21.759 INFO:tasks.rados.rados.0.smithi017.stdout:582: seq_num 252 ranges {635364=402397,1763227=472137} 2024-09-21T09:40:21.765 INFO:tasks.rados.rados.0.smithi017.stdout:582: writing smithi01785587-17 from 635364 to 1037761 tid 1 2024-09-21T09:40:21.772 INFO:tasks.rados.rados.0.smithi017.stdout:582: writing smithi01785587-17 from 1763227 to 2235364 tid 2 2024-09-21T09:40:21.772 INFO:tasks.rados.rados.0.smithi017.stdout:583: write oid 30 current snap is 76 2024-09-21T09:40:21.772 INFO:tasks.rados.rados.0.smithi017.stdout:581: finishing write tid 1 to smithi01785587-2 2024-09-21T09:40:21.772 INFO:tasks.rados.rados.0.smithi017.stdout:583: seq_num 253 ranges {652171=477646,1821709=430462} 2024-09-21T09:40:21.778 INFO:tasks.rados.rados.0.smithi017.stdout:583: writing smithi01785587-30 from 652171 to 1129817 tid 1 2024-09-21T09:40:21.784 INFO:tasks.rados.rados.0.smithi017.stdout:583: writing smithi01785587-30 from 1821709 to 2252171 tid 2 2024-09-21T09:40:21.784 INFO:tasks.rados.rados.0.smithi017.stdout:584: delete oid 23 current snap is 76 2024-09-21T09:40:21.784 INFO:tasks.rados.rados.0.smithi017.stdout:581: finishing write tid 2 to smithi01785587-2 2024-09-21T09:40:21.784 INFO:tasks.rados.rados.0.smithi017.stdout:581: finishing write tid 3 to smithi01785587-2 2024-09-21T09:40:21.784 INFO:tasks.rados.rados.0.smithi017.stdout:581: finishing write tid 4 to smithi01785587-2 2024-09-21T09:40:21.784 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 98 (ObjNum 251 snap 76 seq_num 251) dirty exists 2024-09-21T09:40:21.785 INFO:tasks.rados.rados.0.smithi017.stdout:581: left oid 2 (ObjNum 251 snap 76 seq_num 251) 2024-09-21T09:40:21.785 INFO:tasks.rados.rados.0.smithi017.stdout:582: finishing write tid 1 to smithi01785587-17 2024-09-21T09:40:21.785 INFO:tasks.rados.rados.0.smithi017.stdout:582: finishing write tid 2 to smithi01785587-17 2024-09-21T09:40:21.785 INFO:tasks.rados.rados.0.smithi017.stdout:582: finishing write tid 3 to smithi01785587-17 2024-09-21T09:40:21.785 INFO:tasks.rados.rados.0.smithi017.stdout:582: finishing write tid 4 to smithi01785587-17 2024-09-21T09:40:21.785 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 184 (ObjNum 252 snap 76 seq_num 252) dirty exists 2024-09-21T09:40:21.785 INFO:tasks.rados.rados.0.smithi017.stdout:582: left oid 17 (ObjNum 252 snap 76 seq_num 252) 2024-09-21T09:40:21.785 INFO:tasks.rados.rados.0.smithi017.stdout:583: finishing write tid 1 to smithi01785587-30 2024-09-21T09:40:21.786 INFO:tasks.rados.rados.0.smithi017.stdout:581: done (3 left) 2024-09-21T09:40:21.786 INFO:tasks.rados.rados.0.smithi017.stdout:582: done (2 left) 2024-09-21T09:40:21.786 INFO:tasks.rados.rados.0.smithi017.stdout:584: done (1 left) 2024-09-21T09:40:21.786 INFO:tasks.rados.rados.0.smithi017.stdout:585: read oid 39 snap 43 2024-09-21T09:40:21.787 INFO:tasks.rados.rados.0.smithi017.stdout:585: expect (ObjNum 84 snap 10 seq_num 84) 2024-09-21T09:40:21.787 INFO:tasks.rados.rados.0.smithi017.stdout:586: snap_remove snap 72 2024-09-21T09:40:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:21 smithi017 ceph-mon[29380]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:40:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:21 smithi017 ceph-mon[40737]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:40:22.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:21 smithi134 ceph-mon[33541]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:40:22.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:40:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:40:21] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.51.0" 2024-09-21T09:40:22.698 INFO:tasks.rados.rados.0.smithi017.stdout:583: finishing write tid 2 to smithi01785587-30 2024-09-21T09:40:22.698 INFO:tasks.rados.rados.0.smithi017.stdout:583: finishing write tid 3 to smithi01785587-30 2024-09-21T09:40:22.698 INFO:tasks.rados.rados.0.smithi017.stdout:583: finishing write tid 4 to smithi01785587-30 2024-09-21T09:40:22.698 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 284 (ObjNum 253 snap 76 seq_num 253) dirty exists 2024-09-21T09:40:22.698 INFO:tasks.rados.rados.0.smithi017.stdout:583: left oid 30 (ObjNum 253 snap 76 seq_num 253) 2024-09-21T09:40:22.698 INFO:tasks.rados.rados.0.smithi017.stdout:585: expect (ObjNum 84 snap 10 seq_num 84) 2024-09-21T09:40:22.815 INFO:tasks.rados.rados.0.smithi017.stdout:583: done (2 left) 2024-09-21T09:40:22.815 INFO:tasks.rados.rados.0.smithi017.stdout:585: done (1 left) 2024-09-21T09:40:22.815 INFO:tasks.rados.rados.0.smithi017.stdout:586: done (0 left) 2024-09-21T09:40:22.815 INFO:tasks.rados.rados.0.smithi017.stdout:587: rollback oid 48 current snap is 76 2024-09-21T09:40:22.815 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 71 2024-09-21T09:40:22.815 INFO:tasks.rados.rados.0.smithi017.stdout:588: rollback oid 17 current snap is 76 2024-09-21T09:40:22.815 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 61 2024-09-21T09:40:22.815 INFO:tasks.rados.rados.0.smithi017.stdout:589: delete oid 29 current snap is 76 2024-09-21T09:40:22.817 INFO:tasks.rados.rados.0.smithi017.stdout:589: done (2 left) 2024-09-21T09:40:22.817 INFO:tasks.rados.rados.0.smithi017.stdout:590: read oid 5 snap 42 2024-09-21T09:40:22.818 INFO:tasks.rados.rados.0.smithi017.stdout:590: expect deleted 2024-09-21T09:40:22.818 INFO:tasks.rados.rados.0.smithi017.stdout:591: write oid 49 current snap is 76 2024-09-21T09:40:22.818 INFO:tasks.rados.rados.0.smithi017.stdout:591: seq_num 256 ranges {702592=703393,1880802=421790} 2024-09-21T09:40:22.825 INFO:tasks.rados.rados.0.smithi017.stdout:591: writing smithi01785587-49 from 702592 to 1405985 tid 1 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:591: writing smithi01785587-49 from 1880802 to 2302592 tid 2 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:592: snap_create 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:587: finishing rollback tid 0 to smithi01785587-48 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:587: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:587: finishing rollback tid 2 to smithi01785587-48 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 292 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:588: finishing rollback tid 0 to smithi01785587-17 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:588: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:588: finishing rollback tid 2 to smithi01785587-17 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 187 (ObjNum 185 snap 49 seq_num 185) dirty exists 2024-09-21T09:40:22.831 INFO:tasks.rados.rados.0.smithi017.stdout:591: finishing write tid 1 to smithi01785587-49 2024-09-21T09:40:22.833 INFO:tasks.rados.rados.0.smithi017.stdout:591: finishing write tid 2 to smithi01785587-49 2024-09-21T09:40:22.833 INFO:tasks.rados.rados.0.smithi017.stdout:591: finishing write tid 3 to smithi01785587-49 2024-09-21T09:40:22.833 INFO:tasks.rados.rados.0.smithi017.stdout:591: finishing write tid 4 to smithi01785587-49 2024-09-21T09:40:22.833 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 296 (ObjNum 256 snap 76 seq_num 256) dirty exists 2024-09-21T09:40:22.833 INFO:tasks.rados.rados.0.smithi017.stdout:591: left oid 49 (ObjNum 256 snap 76 seq_num 256) 2024-09-21T09:40:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:22 smithi017 ceph-mon[29380]: pgmap v87: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s rd, 197 KiB/s wr, 3 op/s 2024-09-21T09:40:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:22 smithi017 ceph-mon[29380]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:40:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:22 smithi017 ceph-mon[29380]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:40:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:22 smithi017 ceph-mon[40737]: pgmap v87: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s rd, 197 KiB/s wr, 3 op/s 2024-09-21T09:40:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:22 smithi017 ceph-mon[40737]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:40:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:22 smithi017 ceph-mon[40737]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:40:23.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:22 smithi134 ceph-mon[33541]: pgmap v87: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s rd, 197 KiB/s wr, 3 op/s 2024-09-21T09:40:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:22 smithi134 ceph-mon[33541]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:40:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:22 smithi134 ceph-mon[33541]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:40:23.752 INFO:tasks.rados.rados.0.smithi017.stdout:587: done (4 left) 2024-09-21T09:40:23.753 INFO:tasks.rados.rados.0.smithi017.stdout:588: done (3 left) 2024-09-21T09:40:23.753 INFO:tasks.rados.rados.0.smithi017.stdout:590: done (2 left) 2024-09-21T09:40:23.753 INFO:tasks.rados.rados.0.smithi017.stdout:591: done (1 left) 2024-09-21T09:40:23.753 INFO:tasks.rados.rados.0.smithi017.stdout:592: done (0 left) 2024-09-21T09:40:23.753 INFO:tasks.rados.rados.0.smithi017.stdout:593: snap_create 2024-09-21T09:40:24.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:24.755 INFO:tasks.rados.rados.0.smithi017.stdout:593: done (0 left) 2024-09-21T09:40:24.755 INFO:tasks.rados.rados.0.smithi017.stdout:594: read oid 13 snap -1 2024-09-21T09:40:24.755 INFO:tasks.rados.rados.0.smithi017.stdout:594: expect deleted 2024-09-21T09:40:24.755 INFO:tasks.rados.rados.0.smithi017.stdout:595: snap_remove snap 66 2024-09-21T09:40:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:24 smithi017 ceph-mon[29380]: pgmap v90: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s rd, 197 KiB/s wr, 3 op/s 2024-09-21T09:40:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:24 smithi017 ceph-mon[29380]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:40:25.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:24 smithi017 ceph-mon[40737]: pgmap v90: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s rd, 197 KiB/s wr, 3 op/s 2024-09-21T09:40:25.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:24 smithi017 ceph-mon[40737]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:40:25.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:24 smithi134 ceph-mon[33541]: pgmap v90: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s rd, 197 KiB/s wr, 3 op/s 2024-09-21T09:40:25.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:24 smithi134 ceph-mon[33541]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:40:25.757 INFO:tasks.rados.rados.0.smithi017.stdout:594: done (1 left) 2024-09-21T09:40:25.758 INFO:tasks.rados.rados.0.smithi017.stdout:595: done (0 left) 2024-09-21T09:40:25.758 INFO:tasks.rados.rados.0.smithi017.stdout:596: write oid 30 current snap is 78 2024-09-21T09:40:25.758 INFO:tasks.rados.rados.0.smithi017.stdout:596: seq_num 257 ranges {719399=778642,1939284=380115} 2024-09-21T09:40:25.767 INFO:tasks.rados.rados.0.smithi017.stdout:596: writing smithi01785587-30 from 719399 to 1498041 tid 1 2024-09-21T09:40:25.772 INFO:tasks.rados.rados.0.smithi017.stdout:596: writing smithi01785587-30 from 1939284 to 2319399 tid 2 2024-09-21T09:40:25.772 INFO:tasks.rados.rados.0.smithi017.stdout:597: read oid 23 snap 70 2024-09-21T09:40:25.773 INFO:tasks.rados.rados.0.smithi017.stdout:597: expect (ObjNum 186 snap 49 seq_num 186) 2024-09-21T09:40:25.773 INFO:tasks.rados.rados.0.smithi017.stdout:598: write oid 16 current snap is 78 2024-09-21T09:40:25.773 INFO:tasks.rados.rados.0.smithi017.stdout:596: finishing write tid 1 to smithi01785587-30 2024-09-21T09:40:25.773 INFO:tasks.rados.rados.0.smithi017.stdout:598: seq_num 258 ranges {736206=453891,1881413=454793} 2024-09-21T09:40:25.779 INFO:tasks.rados.rados.0.smithi017.stdout:598: writing smithi01785587-16 from 736206 to 1190097 tid 1 2024-09-21T09:40:25.785 INFO:tasks.rados.rados.0.smithi017.stdout:598: writing smithi01785587-16 from 1881413 to 2336206 tid 2 2024-09-21T09:40:25.786 INFO:tasks.rados.rados.0.smithi017.stdout:599: delete oid 41 current snap is 78 2024-09-21T09:40:25.786 INFO:tasks.rados.rados.0.smithi017.stdout:596: finishing write tid 2 to smithi01785587-30 2024-09-21T09:40:25.786 INFO:tasks.rados.rados.0.smithi017.stdout:596: finishing write tid 3 to smithi01785587-30 2024-09-21T09:40:25.786 INFO:tasks.rados.rados.0.smithi017.stdout:596: finishing write tid 4 to smithi01785587-30 2024-09-21T09:40:25.786 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 302 (ObjNum 257 snap 78 seq_num 257) dirty exists 2024-09-21T09:40:25.786 INFO:tasks.rados.rados.0.smithi017.stdout:596: left oid 30 (ObjNum 257 snap 78 seq_num 257) 2024-09-21T09:40:25.786 INFO:tasks.rados.rados.0.smithi017.stdout:597: expect (ObjNum 186 snap 49 seq_num 186) 2024-09-21T09:40:25.890 INFO:tasks.rados.rados.0.smithi017.stdout:598: finishing write tid 1 to smithi01785587-16 2024-09-21T09:40:25.890 INFO:tasks.rados.rados.0.smithi017.stdout:598: finishing write tid 2 to smithi01785587-16 2024-09-21T09:40:25.891 INFO:tasks.rados.rados.0.smithi017.stdout:598: finishing write tid 3 to smithi01785587-16 2024-09-21T09:40:25.891 INFO:tasks.rados.rados.0.smithi017.stdout:598: finishing write tid 4 to smithi01785587-16 2024-09-21T09:40:25.891 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 292 (ObjNum 258 snap 78 seq_num 258) dirty exists 2024-09-21T09:40:25.891 INFO:tasks.rados.rados.0.smithi017.stdout:598: left oid 16 (ObjNum 258 snap 78 seq_num 258) 2024-09-21T09:40:25.891 INFO:tasks.rados.rados.0.smithi017.stdout:596: done (3 left) 2024-09-21T09:40:25.891 INFO:tasks.rados.rados.0.smithi017.stdout:597: done (2 left) 2024-09-21T09:40:25.891 INFO:tasks.rados.rados.0.smithi017.stdout:598: done (1 left) 2024-09-21T09:40:25.891 INFO:tasks.rados.rados.0.smithi017.stdout:599: done (0 left) 2024-09-21T09:40:25.891 INFO:tasks.rados.rados.0.smithi017.stdout:600: snap_remove snap 77 2024-09-21T09:40:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:25 smithi134 ceph-mon[33541]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:40:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:25 smithi134 ceph-mon[33541]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:40:25.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:25.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:25 smithi017 ceph-mon[29380]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:40:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:25 smithi017 ceph-mon[29380]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:40:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:25 smithi017 ceph-mon[40737]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:40:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:25 smithi017 ceph-mon[40737]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:40:26.773 INFO:tasks.rados.rados.0.smithi017.stdout:600: done (0 left) 2024-09-21T09:40:26.773 INFO:tasks.rados.rados.0.smithi017.stdout:601: snap_create 2024-09-21T09:40:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:26 smithi017 ceph-mon[29380]: pgmap v93: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:26 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:27.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:26 smithi017 ceph-mon[40737]: pgmap v93: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:27.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:26 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:27.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:26 smithi134 ceph-mon[33541]: pgmap v93: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:26 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:27.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:27.775 INFO:tasks.rados.rados.0.smithi017.stdout:601: done (0 left) 2024-09-21T09:40:27.775 INFO:tasks.rados.rados.0.smithi017.stdout:602: write oid 46 current snap is 79 2024-09-21T09:40:27.775 INFO:tasks.rados.rados.0.smithi017.stdout:602: seq_num 260 ranges {769820=720742,2114730=255090} 2024-09-21T09:40:27.784 INFO:tasks.rados.rados.0.smithi017.stdout:602: writing smithi01785587-46 from 769820 to 1490562 tid 1 2024-09-21T09:40:27.789 INFO:tasks.rados.rados.0.smithi017.stdout:602: writing smithi01785587-46 from 2114730 to 2369820 tid 2 2024-09-21T09:40:27.789 INFO:tasks.rados.rados.0.smithi017.stdout:603: read oid 50 snap -1 2024-09-21T09:40:27.789 INFO:tasks.rados.rados.0.smithi017.stdout:603: expect (ObjNum 224 snap 63 seq_num 224) 2024-09-21T09:40:27.789 INFO:tasks.rados.rados.0.smithi017.stdout:604: delete oid 4 current snap is 79 2024-09-21T09:40:27.789 INFO:tasks.rados.rados.0.smithi017.stdout:602: finishing write tid 1 to smithi01785587-46 2024-09-21T09:40:27.791 INFO:tasks.rados.rados.0.smithi017.stdout:604: done (2 left) 2024-09-21T09:40:27.792 INFO:tasks.rados.rados.0.smithi017.stdout:605: write oid 27 current snap is 79 2024-09-21T09:40:27.792 INFO:tasks.rados.rados.0.smithi017.stdout:602: finishing write tid 2 to smithi01785587-46 2024-09-21T09:40:27.792 INFO:tasks.rados.rados.0.smithi017.stdout:605: seq_num 262 ranges {403434=471240,1315341=400756} 2024-09-21T09:40:27.796 INFO:tasks.rados.rados.0.smithi017.stdout:605: writing smithi01785587-27 from 403434 to 874674 tid 1 2024-09-21T09:40:27.801 INFO:tasks.rados.rados.0.smithi017.stdout:605: writing smithi01785587-27 from 1315341 to 1716097 tid 2 2024-09-21T09:40:27.802 INFO:tasks.rados.rados.0.smithi017.stdout:606: write oid 18 current snap is 79 2024-09-21T09:40:27.802 INFO:tasks.rados.rados.0.smithi017.stdout:602: finishing write tid 3 to smithi01785587-46 2024-09-21T09:40:27.802 INFO:tasks.rados.rados.0.smithi017.stdout:606: seq_num 263 ranges {420241=546489,1373823=544414} 2024-09-21T09:40:27.807 INFO:tasks.rados.rados.0.smithi017.stdout:606: writing smithi01785587-18 from 420241 to 966730 tid 1 2024-09-21T09:40:27.812 INFO:tasks.rados.rados.0.smithi017.stdout:606: writing smithi01785587-18 from 1373823 to 1918237 tid 2 2024-09-21T09:40:27.813 INFO:tasks.rados.rados.0.smithi017.stdout:607: write oid 44 current snap is 79 2024-09-21T09:40:27.813 INFO:tasks.rados.rados.0.smithi017.stdout:602: finishing write tid 4 to smithi01785587-46 2024-09-21T09:40:27.813 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 295 (ObjNum 260 snap 79 seq_num 260) dirty exists 2024-09-21T09:40:27.813 INFO:tasks.rados.rados.0.smithi017.stdout:602: left oid 46 (ObjNum 260 snap 79 seq_num 260) 2024-09-21T09:40:27.813 INFO:tasks.rados.rados.0.smithi017.stdout:603: expect (ObjNum 224 snap 63 seq_num 224) 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi017.stdout:605: finishing write tid 1 to smithi01785587-27 2024-09-21T09:40:28.081 INFO:tasks.rados.rados.0.smithi017.stdout:607: seq_num 264 ranges {437048=621738,1832305=404425} 2024-09-21T09:40:28.086 INFO:tasks.rados.rados.0.smithi017.stdout:607: writing smithi01785587-44 from 437048 to 1058786 tid 1 2024-09-21T09:40:28.093 INFO:tasks.rados.rados.0.smithi017.stdout:607: writing smithi01785587-44 from 1832305 to 2236730 tid 2 2024-09-21T09:40:28.093 INFO:tasks.rados.rados.0.smithi017.stdout:602: done (4 left) 2024-09-21T09:40:28.093 INFO:tasks.rados.rados.0.smithi017.stdout:603: done (3 left) 2024-09-21T09:40:28.093 INFO:tasks.rados.rados.0.smithi017.stdout:608: snap_create 2024-09-21T09:40:28.093 INFO:tasks.rados.rados.0.smithi017.stdout:605: finishing write tid 2 to smithi01785587-27 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:605: finishing write tid 3 to smithi01785587-27 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:605: finishing write tid 4 to smithi01785587-27 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 197 (ObjNum 262 snap 79 seq_num 262) dirty exists 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:605: left oid 27 (ObjNum 262 snap 79 seq_num 262) 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:606: finishing write tid 1 to smithi01785587-18 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:606: finishing write tid 2 to smithi01785587-18 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:606: finishing write tid 3 to smithi01785587-18 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:606: finishing write tid 4 to smithi01785587-18 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 32 (ObjNum 263 snap 79 seq_num 263) dirty exists 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:606: left oid 18 (ObjNum 263 snap 79 seq_num 263) 2024-09-21T09:40:28.094 INFO:tasks.rados.rados.0.smithi017.stdout:607: finishing write tid 1 to smithi01785587-44 2024-09-21T09:40:28.095 INFO:tasks.rados.rados.0.smithi017.stdout:607: finishing write tid 2 to smithi01785587-44 2024-09-21T09:40:28.095 INFO:tasks.rados.rados.0.smithi017.stdout:607: finishing write tid 3 to smithi01785587-44 2024-09-21T09:40:28.095 INFO:tasks.rados.rados.0.smithi017.stdout:607: finishing write tid 4 to smithi01785587-44 2024-09-21T09:40:28.096 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 318 (ObjNum 264 snap 79 seq_num 264) dirty exists 2024-09-21T09:40:28.096 INFO:tasks.rados.rados.0.smithi017.stdout:607: left oid 44 (ObjNum 264 snap 79 seq_num 264) 2024-09-21T09:40:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:27 smithi017 ceph-mon[29380]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:40:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:27 smithi017 ceph-mon[29380]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:40:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:27 smithi017 ceph-mon[40737]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:40:28.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:27 smithi017 ceph-mon[40737]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:40:28.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:27 smithi134 ceph-mon[33541]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:40:28.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:27 smithi134 ceph-mon[33541]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:40:28.786 INFO:tasks.rados.rados.0.smithi017.stdout:605: done (3 left) 2024-09-21T09:40:28.786 INFO:tasks.rados.rados.0.smithi017.stdout:606: done (2 left) 2024-09-21T09:40:28.786 INFO:tasks.rados.rados.0.smithi017.stdout:607: done (1 left) 2024-09-21T09:40:28.786 INFO:tasks.rados.rados.0.smithi017.stdout:608: done (0 left) 2024-09-21T09:40:28.787 INFO:tasks.rados.rados.0.smithi017.stdout:609: write oid 25 current snap is 80 2024-09-21T09:40:28.787 INFO:tasks.rados.rados.0.smithi017.stdout:609: seq_num 265 ranges {453855=696987,1890787=548083} 2024-09-21T09:40:28.792 INFO:tasks.rados.rados.0.smithi017.stdout:609: writing smithi01785587-25 from 453855 to 1150842 tid 1 2024-09-21T09:40:28.799 INFO:tasks.rados.rados.0.smithi017.stdout:609: writing smithi01785587-25 from 1890787 to 2438870 tid 2 2024-09-21T09:40:28.799 INFO:tasks.rados.rados.0.smithi017.stdout:610: snap_create 2024-09-21T09:40:28.800 INFO:tasks.rados.rados.0.smithi017.stdout:609: finishing write tid 1 to smithi01785587-25 2024-09-21T09:40:28.803 INFO:tasks.rados.rados.0.smithi017.stdout:609: finishing write tid 2 to smithi01785587-25 2024-09-21T09:40:28.803 INFO:tasks.rados.rados.0.smithi017.stdout:609: finishing write tid 3 to smithi01785587-25 2024-09-21T09:40:28.803 INFO:tasks.rados.rados.0.smithi017.stdout:609: finishing write tid 4 to smithi01785587-25 2024-09-21T09:40:28.803 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 158 (ObjNum 265 snap 80 seq_num 265) dirty exists 2024-09-21T09:40:28.803 INFO:tasks.rados.rados.0.smithi017.stdout:609: left oid 25 (ObjNum 265 snap 80 seq_num 265) 2024-09-21T09:40:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:28 smithi017 ceph-mon[29380]: pgmap v96: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:40:29.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:28 smithi017 ceph-mon[40737]: pgmap v96: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:40:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:28 smithi134 ceph-mon[33541]: pgmap v96: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:40:29.797 INFO:tasks.rados.rados.0.smithi017.stdout:609: done (1 left) 2024-09-21T09:40:29.797 INFO:tasks.rados.rados.0.smithi017.stdout:610: done (0 left) 2024-09-21T09:40:29.797 INFO:tasks.rados.rados.0.smithi017.stdout:611: delete oid 25 current snap is 81 2024-09-21T09:40:29.801 INFO:tasks.rados.rados.0.smithi017.stdout:611: done (0 left) 2024-09-21T09:40:29.801 INFO:tasks.rados.rados.0.smithi017.stdout:612: snap_remove snap 73 2024-09-21T09:40:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:29 smithi017 ceph-mon[29380]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:40:30.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:29 smithi017 ceph-mon[40737]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:40:30.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:29 smithi134 ceph-mon[33541]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:40:30.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:30.800 INFO:tasks.rados.rados.0.smithi017.stdout:612: done (0 left) 2024-09-21T09:40:30.800 INFO:tasks.rados.rados.0.smithi017.stdout:613: snap_create 2024-09-21T09:40:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:30 smithi017 ceph-mon[29380]: pgmap v99: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:40:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:30 smithi017 ceph-mon[29380]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:40:31.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:30 smithi017 ceph-mon[40737]: pgmap v99: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:40:31.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:30 smithi017 ceph-mon[40737]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:40:31.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:30 smithi134 ceph-mon[33541]: pgmap v99: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:40:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:30 smithi134 ceph-mon[33541]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:40:31.801 INFO:tasks.rados.rados.0.smithi017.stdout:613: done (0 left) 2024-09-21T09:40:31.802 INFO:tasks.rados.rados.0.smithi017.stdout:614: snap_create 2024-09-21T09:40:32.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:31 smithi017 ceph-mon[29380]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:40:32.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:31 smithi017 ceph-mon[29380]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:40:32.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:31 smithi017 ceph-mon[40737]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:40:32.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:31 smithi017 ceph-mon[40737]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:40:32.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:31 smithi134 ceph-mon[33541]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:40:32.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:31 smithi134 ceph-mon[33541]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:40:32.244 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:40:31 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:40:31] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.51.0" 2024-09-21T09:40:32.699 INFO:tasks.rados.rados.0.smithi017.stdout:614: done (0 left) 2024-09-21T09:40:32.699 INFO:tasks.rados.rados.0.smithi017.stdout:615: write oid 9 current snap is 83 2024-09-21T09:40:32.700 INFO:tasks.rados.rados.0.smithi017.stdout:615: seq_num 267 ranges {487469=563838,1607751=551752} 2024-09-21T09:40:32.705 INFO:tasks.rados.rados.0.smithi017.stdout:615: writing smithi01785587-9 from 487469 to 1051307 tid 1 2024-09-21T09:40:32.711 INFO:tasks.rados.rados.0.smithi017.stdout:615: writing smithi01785587-9 from 1607751 to 2159503 tid 2 2024-09-21T09:40:32.711 INFO:tasks.rados.rados.0.smithi017.stdout:616: write oid 15 current snap is 83 2024-09-21T09:40:32.711 INFO:tasks.rados.rados.0.smithi017.stdout:615: finishing write tid 1 to smithi01785587-9 2024-09-21T09:40:32.711 INFO:tasks.rados.rados.0.smithi017.stdout:616: seq_num 268 ranges {504276=639087,1666233=695410} 2024-09-21T09:40:32.717 INFO:tasks.rados.rados.0.smithi017.stdout:616: writing smithi01785587-15 from 504276 to 1143363 tid 1 2024-09-21T09:40:32.725 INFO:tasks.rados.rados.0.smithi017.stdout:616: writing smithi01785587-15 from 1666233 to 2361643 tid 2 2024-09-21T09:40:32.725 INFO:tasks.rados.rados.0.smithi017.stdout:617: read oid 50 snap -1 2024-09-21T09:40:32.725 INFO:tasks.rados.rados.0.smithi017.stdout:617: expect (ObjNum 224 snap 63 seq_num 224) 2024-09-21T09:40:32.725 INFO:tasks.rados.rados.0.smithi017.stdout:618: read oid 47 snap 81 2024-09-21T09:40:32.725 INFO:tasks.rados.rados.0.smithi017.stdout:618: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:40:32.725 INFO:tasks.rados.rados.0.smithi017.stdout:619: read oid 34 snap -1 2024-09-21T09:40:32.725 INFO:tasks.rados.rados.0.smithi017.stdout:619: expect (ObjNum 229 snap 66 seq_num 229) 2024-09-21T09:40:32.725 INFO:tasks.rados.rados.0.smithi017.stdout:620: write oid 13 current snap is 83 2024-09-21T09:40:32.725 INFO:tasks.rados.rados.0.smithi017.stdout:615: finishing write tid 2 to smithi01785587-9 2024-09-21T09:40:32.726 INFO:tasks.rados.rados.0.smithi017.stdout:615: finishing write tid 3 to smithi01785587-9 2024-09-21T09:40:32.726 INFO:tasks.rados.rados.0.smithi017.stdout:615: finishing write tid 4 to smithi01785587-9 2024-09-21T09:40:32.726 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 102 (ObjNum 267 snap 83 seq_num 267) dirty exists 2024-09-21T09:40:32.726 INFO:tasks.rados.rados.0.smithi017.stdout:615: left oid 9 (ObjNum 267 snap 83 seq_num 267) 2024-09-21T09:40:32.726 INFO:tasks.rados.rados.0.smithi017.stdout:616: finishing write tid 1 to smithi01785587-15 2024-09-21T09:40:32.726 INFO:tasks.rados.rados.0.smithi017.stdout:620: seq_num 269 ranges {521083=714336,1724715=555421} 2024-09-21T09:40:32.733 INFO:tasks.rados.rados.0.smithi017.stdout:620: writing smithi01785587-13 from 521083 to 1235419 tid 1 2024-09-21T09:40:32.739 INFO:tasks.rados.rados.0.smithi017.stdout:620: writing smithi01785587-13 from 1724715 to 2280136 tid 2 2024-09-21T09:40:32.739 INFO:tasks.rados.rados.0.smithi017.stdout:615: done (5 left) 2024-09-21T09:40:32.739 INFO:tasks.rados.rados.0.smithi017.stdout:621: read oid 12 snap 74 2024-09-21T09:40:32.739 INFO:tasks.rados.rados.0.smithi017.stdout:621: expect deleted 2024-09-21T09:40:32.739 INFO:tasks.rados.rados.0.smithi017.stdout:622: snap_remove snap 70 2024-09-21T09:40:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:32 smithi017 ceph-mon[29380]: pgmap v102: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:40:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:32 smithi017 ceph-mon[29380]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:40:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:32 smithi017 ceph-mon[40737]: pgmap v102: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:40:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:32 smithi017 ceph-mon[40737]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:40:33.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:32 smithi134 ceph-mon[33541]: pgmap v102: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:40:33.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:32 smithi134 ceph-mon[33541]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:40:33.701 INFO:tasks.rados.rados.0.smithi017.stdout:622: done (6 left) 2024-09-21T09:40:33.701 INFO:tasks.rados.rados.0.smithi017.stdout:623: rollback oid 32 current snap is 83 2024-09-21T09:40:33.701 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 78 2024-09-21T09:40:33.701 INFO:tasks.rados.rados.0.smithi017.stdout:616: finishing write tid 2 to smithi01785587-15 2024-09-21T09:40:33.701 INFO:tasks.rados.rados.0.smithi017.stdout:616: finishing write tid 3 to smithi01785587-15 2024-09-21T09:40:33.701 INFO:tasks.rados.rados.0.smithi017.stdout:616: finishing write tid 4 to smithi01785587-15 2024-09-21T09:40:33.702 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 131 (ObjNum 268 snap 83 seq_num 268) dirty exists 2024-09-21T09:40:33.702 INFO:tasks.rados.rados.0.smithi017.stdout:616: left oid 15 (ObjNum 268 snap 83 seq_num 268) 2024-09-21T09:40:33.702 INFO:tasks.rados.rados.0.smithi017.stdout:619: expect (ObjNum 229 snap 66 seq_num 229) 2024-09-21T09:40:33.874 INFO:tasks.rados.rados.0.smithi017.stdout:618: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:40:33.991 INFO:tasks.rados.rados.0.smithi017.stdout:617: expect (ObjNum 224 snap 63 seq_num 224) 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:620: finishing write tid 1 to smithi01785587-13 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:620: finishing write tid 2 to smithi01785587-13 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:616: done (6 left) 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:617: done (5 left) 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:618: done (4 left) 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:619: done (3 left) 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:621: done (2 left) 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:624: rollback oid 40 current snap is 83 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 78 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:620: finishing write tid 3 to smithi01785587-13 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:620: finishing write tid 4 to smithi01785587-13 2024-09-21T09:40:34.260 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 313 (ObjNum 269 snap 83 seq_num 269) dirty exists 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:620: left oid 13 (ObjNum 269 snap 83 seq_num 269) 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:623: finishing rollback tid 0 to smithi01785587-32 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:623: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:623: finishing rollback tid 2 to smithi01785587-32 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 303 (ObjNum 241 snap 72 seq_num 241) dirty exists 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:620: done (2 left) 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:623: done (1 left) 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:625: read oid 3 snap -1 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:625: expect deleted 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:626: write oid 13 current snap is 83 2024-09-21T09:40:34.261 INFO:tasks.rados.rados.0.smithi017.stdout:626: seq_num 270 ranges {537890=789585,1783197=699079} 2024-09-21T09:40:34.266 INFO:tasks.rados.rados.0.smithi017.stdout:626: writing smithi01785587-13 from 537890 to 1327475 tid 1 2024-09-21T09:40:34.273 INFO:tasks.rados.rados.0.smithi017.stdout:626: writing smithi01785587-13 from 1783197 to 2482276 tid 2 2024-09-21T09:40:34.273 INFO:tasks.rados.rados.0.smithi017.stdout:627: snap_remove snap 75 2024-09-21T09:40:34.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:34.816 INFO:tasks.rados.rados.0.smithi017.stdout:627: done (3 left) 2024-09-21T09:40:34.816 INFO:tasks.rados.rados.0.smithi017.stdout:628: write oid 49 current snap is 83 2024-09-21T09:40:34.816 INFO:tasks.rados.rados.0.smithi017.stdout:628: seq_num 271 ranges {554697=464834,1725326=559090} 2024-09-21T09:40:34.822 INFO:tasks.rados.rados.0.smithi017.stdout:628: writing smithi01785587-49 from 554697 to 1019531 tid 1 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:628: writing smithi01785587-49 from 1725326 to 2284416 tid 2 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:629: snap_create 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:624: finishing rollback tid 0 to smithi01785587-40 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:624: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:624: finishing rollback tid 2 to smithi01785587-40 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 247 (ObjNum 237 snap 72 seq_num 237) dirty exists 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:626: finishing write tid 1 to smithi01785587-13 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:626: finishing write tid 2 to smithi01785587-13 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:626: finishing write tid 3 to smithi01785587-13 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:626: finishing write tid 4 to smithi01785587-13 2024-09-21T09:40:34.830 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 322 (ObjNum 270 snap 83 seq_num 270) dirty exists 2024-09-21T09:40:34.831 INFO:tasks.rados.rados.0.smithi017.stdout:626: left oid 13 (ObjNum 270 snap 83 seq_num 270) 2024-09-21T09:40:34.831 INFO:tasks.rados.rados.0.smithi017.stdout:628: finishing write tid 1 to smithi01785587-49 2024-09-21T09:40:34.833 INFO:tasks.rados.rados.0.smithi017.stdout:628: finishing write tid 2 to smithi01785587-49 2024-09-21T09:40:34.833 INFO:tasks.rados.rados.0.smithi017.stdout:628: finishing write tid 3 to smithi01785587-49 2024-09-21T09:40:34.833 INFO:tasks.rados.rados.0.smithi017.stdout:628: finishing write tid 4 to smithi01785587-49 2024-09-21T09:40:34.833 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 306 (ObjNum 271 snap 83 seq_num 271) dirty exists 2024-09-21T09:40:34.833 INFO:tasks.rados.rados.0.smithi017.stdout:628: left oid 49 (ObjNum 271 snap 83 seq_num 271) 2024-09-21T09:40:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:34 smithi017 ceph-mon[29380]: pgmap v105: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:40:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:34 smithi017 ceph-mon[29380]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:40:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:34 smithi017 ceph-mon[40737]: pgmap v105: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:40:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:34 smithi017 ceph-mon[40737]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:40:34.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:34 smithi134 ceph-mon[33541]: pgmap v105: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:40:34.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:34 smithi134 ceph-mon[33541]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:40:35.770 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:35.770 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:624: done (4 left) 2024-09-21T09:40:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:625: done (3 left) 2024-09-21T09:40:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:626: done (2 left) 2024-09-21T09:40:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:628: done (1 left) 2024-09-21T09:40:35.821 INFO:tasks.rados.rados.0.smithi017.stdout:629: done (0 left) 2024-09-21T09:40:35.822 INFO:tasks.rados.rados.0.smithi017.stdout:630: write oid 50 current snap is 84 2024-09-21T09:40:35.822 INFO:tasks.rados.rados.0.smithi017.stdout:630: seq_num 272 ranges {571504=540083,1783808=702748} 2024-09-21T09:40:35.827 INFO:tasks.rados.rados.0.smithi017.stdout:630: writing smithi01785587-50 from 571504 to 1111587 tid 1 2024-09-21T09:40:35.835 INFO:tasks.rados.rados.0.smithi017.stdout:630: writing smithi01785587-50 from 1783808 to 2486556 tid 2 2024-09-21T09:40:35.835 INFO:tasks.rados.rados.0.smithi017.stdout:631: rollback oid 33 current snap is 84 2024-09-21T09:40:35.835 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 74 2024-09-21T09:40:35.835 INFO:tasks.rados.rados.0.smithi017.stdout:630: finishing write tid 1 to smithi01785587-50 2024-09-21T09:40:35.835 INFO:tasks.rados.rados.0.smithi017.stdout:632: rollback oid 10 current snap is 84 2024-09-21T09:40:35.835 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 81 2024-09-21T09:40:35.835 INFO:tasks.rados.rados.0.smithi017.stdout:633: snap_remove snap 71 2024-09-21T09:40:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:35 smithi017 ceph-mon[29380]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:40:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:35 smithi017 ceph-mon[40737]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:40:36.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:35 smithi134 ceph-mon[33541]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:40:36.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade resume' 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:631: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:632: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 0 (ObjNum 892680247 snap 875378488 seq_num 304) dirty dne 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:630: finishing write tid 2 to smithi01785587-50 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:630: finishing write tid 3 to smithi01785587-50 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:630: finishing write tid 4 to smithi01785587-50 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 168 (ObjNum 272 snap 84 seq_num 272) dirty exists 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:630: left oid 50 (ObjNum 272 snap 84 seq_num 272) 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:630: done (3 left) 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:631: done (2 left) 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:632: done (1 left) 2024-09-21T09:40:36.827 INFO:tasks.rados.rados.0.smithi017.stdout:633: done (0 left) 2024-09-21T09:40:36.828 INFO:tasks.rados.rados.0.smithi017.stdout:634: read oid 30 snap -1 2024-09-21T09:40:36.828 INFO:tasks.rados.rados.0.smithi017.stdout:634: expect (ObjNum 257 snap 78 seq_num 257) 2024-09-21T09:40:36.828 INFO:tasks.rados.rados.0.smithi017.stdout:635: rollback oid 3 current snap is 84 2024-09-21T09:40:36.828 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 81 2024-09-21T09:40:36.828 INFO:tasks.rados.rados.0.smithi017.stdout:636: snap_create 2024-09-21T09:40:36.833 INFO:tasks.rados.rados.0.smithi017.stdout:635: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:40:36.833 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 184 (ObjNum 129 snap 0 seq_num 2399678600) dirty dne 2024-09-21T09:40:36.841 INFO:tasks.rados.rados.0.smithi017.stdout:634: expect (ObjNum 257 snap 78 seq_num 257) 2024-09-21T09:40:36.920 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:40:36.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:36 smithi017 ceph-mon[29380]: pgmap v108: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:36 smithi017 ceph-mon[29380]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:40:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:36 smithi017 ceph-mon[40737]: pgmap v108: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:36 smithi017 ceph-mon[40737]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:40:36.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:36 smithi134 ceph-mon[33541]: pgmap v108: 89 pgs: 89 active+clean; 219 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:36 smithi134 ceph-mon[33541]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:40:36.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:37.831 INFO:tasks.rados.rados.0.smithi017.stdout:634: done (2 left) 2024-09-21T09:40:37.831 INFO:tasks.rados.rados.0.smithi017.stdout:635: done (1 left) 2024-09-21T09:40:37.831 INFO:tasks.rados.rados.0.smithi017.stdout:636: done (0 left) 2024-09-21T09:40:37.831 INFO:tasks.rados.rados.0.smithi017.stdout:637: read oid 18 snap -1 2024-09-21T09:40:37.831 INFO:tasks.rados.rados.0.smithi017.stdout:637: expect (ObjNum 263 snap 79 seq_num 263) 2024-09-21T09:40:37.831 INFO:tasks.rados.rados.0.smithi017.stdout:638: snap_remove snap 84 2024-09-21T09:40:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:37 smithi017 ceph-mon[29380]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:40:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:37 smithi017 ceph-mon[40737]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:40:38.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:37 smithi134 ceph-mon[33541]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:40:38.832 INFO:tasks.rados.rados.0.smithi017.stdout:638: done (1 left) 2024-09-21T09:40:38.832 INFO:tasks.rados.rados.0.smithi017.stdout:639: read oid 13 snap -1 2024-09-21T09:40:38.832 INFO:tasks.rados.rados.0.smithi017.stdout:639: expect (ObjNum 270 snap 83 seq_num 270) 2024-09-21T09:40:38.832 INFO:tasks.rados.rados.0.smithi017.stdout:640: delete oid 39 current snap is 85 2024-09-21T09:40:38.832 INFO:tasks.rados.rados.0.smithi017.stdout:637: expect (ObjNum 263 snap 79 seq_num 263) 2024-09-21T09:40:38.941 INFO:tasks.rados.rados.0.smithi017.stdout:639: expect (ObjNum 270 snap 83 seq_num 270) 2024-09-21T09:40:39.040 INFO:teuthology.orchestra.run.smithi017.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:40:39.064 INFO:tasks.rados.rados.0.smithi017.stdout:637: done (2 left) 2024-09-21T09:40:39.064 INFO:tasks.rados.rados.0.smithi017.stdout:639: done (1 left) 2024-09-21T09:40:39.064 INFO:tasks.rados.rados.0.smithi017.stdout:640: done (0 left) 2024-09-21T09:40:39.064 INFO:tasks.rados.rados.0.smithi017.stdout:641: read oid 40 snap -1 2024-09-21T09:40:39.064 INFO:tasks.rados.rados.0.smithi017.stdout:641: expect (ObjNum 237 snap 72 seq_num 237) 2024-09-21T09:40:39.064 INFO:tasks.rados.rados.0.smithi017.stdout:642: read oid 11 snap -1 2024-09-21T09:40:39.065 INFO:tasks.rados.rados.0.smithi017.stdout:642: expect deleted 2024-09-21T09:40:39.065 INFO:tasks.rados.rados.0.smithi017.stdout:643: snap_remove snap 74 2024-09-21T09:40:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:38 smithi017 ceph-mon[29380]: pgmap v111: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-09-21T09:40:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:38 smithi017 ceph-mon[29380]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:40:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:38 smithi017 ceph-mon[40737]: pgmap v111: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-09-21T09:40:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:38 smithi017 ceph-mon[40737]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:40:39.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:38 smithi134 ceph-mon[33541]: pgmap v111: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-09-21T09:40:39.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:38 smithi134 ceph-mon[33541]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:40:39.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-21T09:40:39.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:39 smithi134 ceph-mon[33541]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:40:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:39 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:39 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:39 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:39 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:40.048 INFO:tasks.rados.rados.0.smithi017.stdout:643: done (2 left) 2024-09-21T09:40:40.048 INFO:tasks.rados.rados.0.smithi017.stdout:644: read oid 18 snap -1 2024-09-21T09:40:40.048 INFO:tasks.rados.rados.0.smithi017.stdout:644: expect (ObjNum 263 snap 79 seq_num 263) 2024-09-21T09:40:40.048 INFO:tasks.rados.rados.0.smithi017.stdout:645: read oid 25 snap -1 2024-09-21T09:40:40.048 INFO:tasks.rados.rados.0.smithi017.stdout:645: expect deleted 2024-09-21T09:40:40.049 INFO:tasks.rados.rados.0.smithi017.stdout:646: delete oid 14 current snap is 85 2024-09-21T09:40:40.049 INFO:tasks.rados.rados.0.smithi017.stdout:641: expect (ObjNum 237 snap 72 seq_num 237) 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[40737]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[29380]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:39 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:40.315 INFO:tasks.rados.rados.0.smithi017.stdout:644: expect (ObjNum 263 snap 79 seq_num 263) 2024-09-21T09:40:40.425 INFO:tasks.rados.rados.0.smithi017.stdout:641: done (4 left) 2024-09-21T09:40:40.425 INFO:tasks.rados.rados.0.smithi017.stdout:642: done (3 left) 2024-09-21T09:40:40.425 INFO:tasks.rados.rados.0.smithi017.stdout:644: done (2 left) 2024-09-21T09:40:40.425 INFO:tasks.rados.rados.0.smithi017.stdout:645: done (1 left) 2024-09-21T09:40:40.425 INFO:tasks.rados.rados.0.smithi017.stdout:646: done (0 left) 2024-09-21T09:40:40.425 INFO:tasks.rados.rados.0.smithi017.stdout:647: snap_remove snap 59 2024-09-21T09:40:40.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:40.855 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:40:41.044 INFO:tasks.rados.rados.0.smithi017.stdout:647: done (0 left) 2024-09-21T09:40:41.045 INFO:tasks.rados.rados.0.smithi017.stdout:648: write oid 29 current snap is 85 2024-09-21T09:40:41.045 INFO:tasks.rados.rados.0.smithi017.stdout:648: seq_num 275 ranges {621925=482183,1559254=566428} 2024-09-21T09:40:41.051 INFO:tasks.rados.rados.0.smithi017.stdout:648: writing smithi01785587-29 from 621925 to 1104108 tid 1 2024-09-21T09:40:41.057 INFO:tasks.rados.rados.0.smithi017.stdout:648: writing smithi01785587-29 from 1559254 to 2125682 tid 2 2024-09-21T09:40:41.057 INFO:tasks.rados.rados.0.smithi017.stdout:649: delete oid 28 current snap is 85 2024-09-21T09:40:41.057 INFO:tasks.rados.rados.0.smithi017.stdout:648: finishing write tid 1 to smithi01785587-29 2024-09-21T09:40:41.059 INFO:tasks.rados.rados.0.smithi017.stdout:649: done (1 left) 2024-09-21T09:40:41.059 INFO:tasks.rados.rados.0.smithi017.stdout:650: delete oid 7 current snap is 85 2024-09-21T09:40:41.060 INFO:tasks.rados.rados.0.smithi017.stdout:648: finishing write tid 2 to smithi01785587-29 2024-09-21T09:40:41.060 INFO:tasks.rados.rados.0.smithi017.stdout:648: finishing write tid 3 to smithi01785587-29 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:648: finishing write tid 4 to smithi01785587-29 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 175 (ObjNum 275 snap 85 seq_num 275) dirty exists 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:648: left oid 29 (ObjNum 275 snap 85 seq_num 275) 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:648: done (1 left) 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:650: done (0 left) 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:651: read oid 4 snap -1 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:651: expect deleted 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:652: rollback oid 16 current snap is 85 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 16 to 80 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:653: write oid 45 current snap is 85 2024-09-21T09:40:41.061 INFO:tasks.rados.rados.0.smithi017.stdout:653: seq_num 278 ranges {672346=707930,2134700=537646} 2024-09-21T09:40:41.068 INFO:tasks.rados.rados.0.smithi017.stdout:653: writing smithi01785587-45 from 672346 to 1380276 tid 1 2024-09-21T09:40:41.075 INFO:tasks.rados.rados.0.smithi017.stdout:653: writing smithi01785587-45 from 2134700 to 2672346 tid 2 2024-09-21T09:40:41.075 INFO:tasks.rados.rados.0.smithi017.stdout:654: delete oid 11 current snap is 85 2024-09-21T09:40:41.075 INFO:tasks.rados.rados.0.smithi017.stdout:652: finishing rollback tid 0 to smithi01785587-16 2024-09-21T09:40:41.076 INFO:tasks.rados.rados.0.smithi017.stdout:652: finishing rollback tid 1 to smithi01785587-16 2024-09-21T09:40:41.076 INFO:tasks.rados.rados.0.smithi017.stdout:652: finishing rollback tid 2 to smithi01785587-16 2024-09-21T09:40:41.076 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 322 (ObjNum 258 snap 78 seq_num 258) dirty exists 2024-09-21T09:40:41.076 INFO:tasks.rados.rados.0.smithi017.stdout:653: finishing write tid 1 to smithi01785587-45 2024-09-21T09:40:41.077 INFO:tasks.rados.rados.0.smithi017.stdout:651: done (3 left) 2024-09-21T09:40:41.077 INFO:tasks.rados.rados.0.smithi017.stdout:652: done (2 left) 2024-09-21T09:40:41.077 INFO:tasks.rados.rados.0.smithi017.stdout:654: done (1 left) 2024-09-21T09:40:41.077 INFO:tasks.rados.rados.0.smithi017.stdout:655: read oid 41 snap -1 2024-09-21T09:40:41.077 INFO:tasks.rados.rados.0.smithi017.stdout:655: expect deleted 2024-09-21T09:40:41.077 INFO:tasks.rados.rados.0.smithi017.stdout:656: read oid 8 snap 42 2024-09-21T09:40:41.078 INFO:tasks.rados.rados.0.smithi017.stdout:656: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-21T09:40:41.078 INFO:tasks.rados.rados.0.smithi017.stdout:657: rollback oid 10 current snap is 85 2024-09-21T09:40:41.078 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 78 2024-09-21T09:40:41.078 INFO:tasks.rados.rados.0.smithi017.stdout:658: snap_remove snap 67 2024-09-21T09:40:41.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:40 smithi017 ceph-mon[29380]: pgmap v114: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-09-21T09:40:41.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:40 smithi017 ceph-mon[29380]: from='client.25244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:41.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:40 smithi017 ceph-mon[29380]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:40:41.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:40 smithi017 ceph-mon[29380]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:40:41.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:40 smithi017 ceph-mon[40737]: pgmap v114: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-09-21T09:40:41.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:40 smithi017 ceph-mon[40737]: from='client.25244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:41.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:40 smithi017 ceph-mon[40737]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:40:41.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:40 smithi017 ceph-mon[40737]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:40:41.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:40 smithi134 ceph-mon[33541]: pgmap v114: 89 pgs: 89 active+clean; 227 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-09-21T09:40:41.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:40 smithi134 ceph-mon[33541]: from='client.25244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:41.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:40 smithi134 ceph-mon[33541]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:40:41.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:40 smithi134 ceph-mon[33541]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:40:41.893 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:40:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:40:41] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.51.0" 2024-09-21T09:40:42.050 INFO:tasks.rados.rados.0.smithi017.stdout:658: done (4 left) 2024-09-21T09:40:42.050 INFO:tasks.rados.rados.0.smithi017.stdout:659: read oid 11 snap -1 2024-09-21T09:40:42.051 INFO:tasks.rados.rados.0.smithi017.stdout:659: expect deleted 2024-09-21T09:40:42.051 INFO:tasks.rados.rados.0.smithi017.stdout:660: snap_remove snap 79 2024-09-21T09:40:42.226 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:42 smithi134 ceph-mon[33541]: pgmap v116: 89 pgs: 89 active+clean; 223 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 10 op/s 2024-09-21T09:40:42.226 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:42 smithi134 ceph-mon[33541]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:40:42.227 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:42 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:42.337 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:42 smithi017 ceph-mon[29380]: pgmap v116: 89 pgs: 89 active+clean; 223 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 10 op/s 2024-09-21T09:40:42.337 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:42 smithi017 ceph-mon[29380]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:40:42.337 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:42 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:42.337 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:42 smithi017 ceph-mon[40737]: pgmap v116: 89 pgs: 89 active+clean; 223 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 10 op/s 2024-09-21T09:40:42.337 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:42 smithi017 ceph-mon[40737]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:40:42.337 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:42 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:42.703 INFO:tasks.rados.rados.0.smithi017.stdout:660: done (5 left) 2024-09-21T09:40:42.704 INFO:tasks.rados.rados.0.smithi017.stdout:661: delete oid 16 current snap is 85 2024-09-21T09:40:42.704 INFO:tasks.rados.rados.0.smithi017.stdout:653: finishing write tid 2 to smithi01785587-45 2024-09-21T09:40:42.704 INFO:tasks.rados.rados.0.smithi017.stdout:653: finishing write tid 3 to smithi01785587-45 2024-09-21T09:40:42.704 INFO:tasks.rados.rados.0.smithi017.stdout:657: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:40:42.704 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:40:42.704 INFO:tasks.rados.rados.0.smithi017.stdout:653: finishing write tid 4 to smithi01785587-45 2024-09-21T09:40:42.704 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 229 (ObjNum 278 snap 85 seq_num 278) dirty exists 2024-09-21T09:40:42.704 INFO:tasks.rados.rados.0.smithi017.stdout:653: left oid 45 (ObjNum 278 snap 85 seq_num 278) 2024-09-21T09:40:42.704 INFO:tasks.rados.rados.0.smithi017.stdout:656: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-21T09:40:42.825 INFO:tasks.rados.rados.0.smithi017.stdout:653: done (5 left) 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:655: done (4 left) 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:656: done (3 left) 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:657: done (2 left) 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:659: done (1 left) 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:661: done (0 left) 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:662: rollback oid 37 current snap is 85 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 42 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:663: rollback oid 21 current snap is 85 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 83 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:664: write oid 40 current snap is 85 2024-09-21T09:40:42.826 INFO:tasks.rados.rados.0.smithi017.stdout:664: seq_num 281 ranges {722767=533677,1793793=461082} 2024-09-21T09:40:42.832 INFO:tasks.rados.rados.0.smithi017.stdout:664: writing smithi01785587-40 from 722767 to 1256444 tid 1 2024-09-21T09:40:42.838 INFO:tasks.rados.rados.0.smithi017.stdout:664: writing smithi01785587-40 from 1793793 to 2254875 tid 2 2024-09-21T09:40:42.838 INFO:tasks.rados.rados.0.smithi017.stdout:665: write oid 42 current snap is 85 2024-09-21T09:40:42.838 INFO:tasks.rados.rados.0.smithi017.stdout:662: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:40:42.839 INFO:tasks.rados.rados.0.smithi017.stdout:662: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:40:42.839 INFO:tasks.rados.rados.0.smithi017.stdout:665: seq_num 282 ranges {739574=725279,1968628=721093} 2024-09-21T09:40:42.846 INFO:tasks.rados.rados.0.smithi017.stdout:665: writing smithi01785587-42 from 739574 to 1464853 tid 1 2024-09-21T09:40:42.853 INFO:tasks.rados.rados.0.smithi017.stdout:665: writing smithi01785587-42 from 1968628 to 2689721 tid 2 2024-09-21T09:40:42.853 INFO:tasks.rados.rados.0.smithi017.stdout:666: snap_create 2024-09-21T09:40:42.853 INFO:tasks.rados.rados.0.smithi017.stdout:663: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:40:42.853 INFO:tasks.rados.rados.0.smithi017.stdout:663: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:40:42.853 INFO:tasks.rados.rados.0.smithi017.stdout:662: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:40:42.853 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 270 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:40:42.854 INFO:tasks.rados.rados.0.smithi017.stdout:663: finishing rollback tid 2 to smithi01785587-21 2024-09-21T09:40:42.854 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 330 (ObjNum 201 snap 56 seq_num 201) dirty exists 2024-09-21T09:40:42.854 INFO:tasks.rados.rados.0.smithi017.stdout:664: finishing write tid 1 to smithi01785587-40 2024-09-21T09:40:42.854 INFO:tasks.rados.rados.0.smithi017.stdout:664: finishing write tid 2 to smithi01785587-40 2024-09-21T09:40:42.854 INFO:tasks.rados.rados.0.smithi017.stdout:664: finishing write tid 3 to smithi01785587-40 2024-09-21T09:40:42.854 INFO:tasks.rados.rados.0.smithi017.stdout:664: finishing write tid 4 to smithi01785587-40 2024-09-21T09:40:42.854 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 274 (ObjNum 281 snap 85 seq_num 281) dirty exists 2024-09-21T09:40:42.854 INFO:tasks.rados.rados.0.smithi017.stdout:664: left oid 40 (ObjNum 281 snap 85 seq_num 281) 2024-09-21T09:40:42.854 INFO:tasks.rados.rados.0.smithi017.stdout:665: finishing write tid 1 to smithi01785587-42 2024-09-21T09:40:42.856 INFO:tasks.rados.rados.0.smithi017.stdout:665: finishing write tid 2 to smithi01785587-42 2024-09-21T09:40:42.857 INFO:tasks.rados.rados.0.smithi017.stdout:665: finishing write tid 3 to smithi01785587-42 2024-09-21T09:40:42.857 INFO:tasks.rados.rados.0.smithi017.stdout:665: finishing write tid 4 to smithi01785587-42 2024-09-21T09:40:42.857 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 198 (ObjNum 282 snap 85 seq_num 282) dirty exists 2024-09-21T09:40:42.857 INFO:tasks.rados.rados.0.smithi017.stdout:665: left oid 42 (ObjNum 282 snap 85 seq_num 282) 2024-09-21T09:40:42.895 INFO:teuthology.orchestra.run.smithi017.stdout:wait for majority of mons to upgrade 2024-09-21T09:40:43.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:43 smithi134 ceph-mon[33541]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:40:43.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:43 smithi134 ceph-mon[33541]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:40:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:43 smithi017 ceph-mon[29380]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:40:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:43 smithi017 ceph-mon[29380]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:40:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:43 smithi017 ceph-mon[40737]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:40:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:43 smithi017 ceph-mon[40737]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:40:43.707 INFO:tasks.rados.rados.0.smithi017.stdout:662: done (4 left) 2024-09-21T09:40:43.707 INFO:tasks.rados.rados.0.smithi017.stdout:663: done (3 left) 2024-09-21T09:40:43.707 INFO:tasks.rados.rados.0.smithi017.stdout:664: done (2 left) 2024-09-21T09:40:43.708 INFO:tasks.rados.rados.0.smithi017.stdout:665: done (1 left) 2024-09-21T09:40:43.708 INFO:tasks.rados.rados.0.smithi017.stdout:666: done (0 left) 2024-09-21T09:40:43.708 INFO:tasks.rados.rados.0.smithi017.stdout:667: rollback oid 28 current snap is 86 2024-09-21T09:40:43.708 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 28 to 83 2024-09-21T09:40:43.708 INFO:tasks.rados.rados.0.smithi017.stdout:668: snap_remove snap 58 2024-09-21T09:40:44.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-21T09:40:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:44 smithi134 ceph-mon[33541]: pgmap v120: 89 pgs: 89 active+clean; 223 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 10 op/s 2024-09-21T09:40:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:44 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:44 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:44 smithi134 ceph-mon[33541]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:40:44.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:44.546 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:44 smithi017 ceph-mon[29380]: pgmap v120: 89 pgs: 89 active+clean; 223 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 10 op/s 2024-09-21T09:40:44.546 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:44 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:44.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:44 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:44.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:44 smithi017 ceph-mon[29380]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:40:44.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:44 smithi017 ceph-mon[40737]: pgmap v120: 89 pgs: 89 active+clean; 223 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 10 op/s 2024-09-21T09:40:44.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:44 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:44.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:44 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:44.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:44 smithi017 ceph-mon[40737]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:40:44.710 INFO:tasks.rados.rados.0.smithi017.stdout:668: done (1 left) 2024-09-21T09:40:44.710 INFO:tasks.rados.rados.0.smithi017.stdout:669: delete oid 29 current snap is 86 2024-09-21T09:40:44.710 INFO:tasks.rados.rados.0.smithi017.stdout:667: finishing rollback tid 1 to smithi01785587-28 2024-09-21T09:40:44.710 INFO:tasks.rados.rados.0.smithi017.stdout:667: finishing rollback tid 2 to smithi01785587-28 2024-09-21T09:40:44.710 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 333 (ObjNum 233 snap 67 seq_num 233) dirty exists 2024-09-21T09:40:44.717 INFO:tasks.rados.rados.0.smithi017.stdout:667: done (1 left) 2024-09-21T09:40:44.717 INFO:tasks.rados.rados.0.smithi017.stdout:669: done (0 left) 2024-09-21T09:40:44.717 INFO:tasks.rados.rados.0.smithi017.stdout:670: read oid 5 snap -1 2024-09-21T09:40:44.717 INFO:tasks.rados.rados.0.smithi017.stdout:670: expect (ObjNum 217 snap 60 seq_num 217) 2024-09-21T09:40:44.718 INFO:tasks.rados.rados.0.smithi017.stdout:671: snap_remove snap 43 2024-09-21T09:40:44.888 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:40:45.714 INFO:tasks.rados.rados.0.smithi017.stdout:671: done (1 left) 2024-09-21T09:40:45.715 INFO:tasks.rados.rados.0.smithi017.stdout:672: write oid 24 current snap is 86 2024-09-21T09:40:45.715 INFO:tasks.rados.rados.0.smithi017.stdout:672: seq_num 284 ranges {773188=475777,1969239=724762} 2024-09-21T09:40:45.722 INFO:tasks.rados.rados.0.smithi017.stdout:672: writing smithi01785587-24 from 773188 to 1248965 tid 1 2024-09-21T09:40:45.731 INFO:tasks.rados.rados.0.smithi017.stdout:672: writing smithi01785587-24 from 1969239 to 2694001 tid 2 2024-09-21T09:40:45.731 INFO:tasks.rados.rados.0.smithi017.stdout:673: snap_remove snap 61 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[40737]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[29380]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:45.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:45.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:45.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:45.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:45 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:45.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:45.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:45 smithi134 ceph-mon[33541]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:40:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:45 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:45 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:45 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:45 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:45 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:45 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:46.716 INFO:tasks.rados.rados.0.smithi017.stdout:673: done (2 left) 2024-09-21T09:40:46.716 INFO:tasks.rados.rados.0.smithi017.stdout:674: write oid 30 current snap is 86 2024-09-21T09:40:46.716 INFO:tasks.rados.rados.0.smithi017.stdout:674: seq_num 285 ranges {789995=551026,2027721=468420} 2024-09-21T09:40:46.724 INFO:tasks.rados.rados.0.smithi017.stdout:674: writing smithi01785587-30 from 789995 to 1341021 tid 1 2024-09-21T09:40:46.731 INFO:tasks.rados.rados.0.smithi017.stdout:674: writing smithi01785587-30 from 2027721 to 2496141 tid 2 2024-09-21T09:40:46.731 INFO:tasks.rados.rados.0.smithi017.stdout:675: read oid 13 snap 42 2024-09-21T09:40:46.731 INFO:tasks.rados.rados.0.smithi017.stdout:675: expect (ObjNum 142 snap 34 seq_num 142) 2024-09-21T09:40:46.731 INFO:tasks.rados.rados.0.smithi017.stdout:676: read oid 19 snap 83 2024-09-21T09:40:46.731 INFO:tasks.rados.rados.0.smithi017.stdout:676: expect deleted 2024-09-21T09:40:46.731 INFO:tasks.rados.rados.0.smithi017.stdout:677: write oid 22 current snap is 86 2024-09-21T09:40:46.731 INFO:tasks.rados.rados.0.smithi017.stdout:670: expect (ObjNum 217 snap 60 seq_num 217) 2024-09-21T09:40:46.902 INFO:tasks.rados.rados.0.smithi017.stdout:672: finishing write tid 1 to smithi01785587-24 2024-09-21T09:40:46.902 INFO:tasks.rados.rados.0.smithi017.stdout:672: finishing write tid 2 to smithi01785587-24 2024-09-21T09:40:46.902 INFO:tasks.rados.rados.0.smithi017.stdout:672: finishing write tid 3 to smithi01785587-24 2024-09-21T09:40:46.902 INFO:tasks.rados.rados.0.smithi017.stdout:672: finishing write tid 4 to smithi01785587-24 2024-09-21T09:40:46.902 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 187 (ObjNum 284 snap 86 seq_num 284) dirty exists 2024-09-21T09:40:46.902 INFO:tasks.rados.rados.0.smithi017.stdout:672: left oid 24 (ObjNum 284 snap 86 seq_num 284) 2024-09-21T09:40:46.902 INFO:tasks.rados.rados.0.smithi017.stdout:674: finishing write tid 1 to smithi01785587-30 2024-09-21T09:40:46.902 INFO:tasks.rados.rados.0.smithi017.stdout:677: seq_num 286 ranges {406802=626275,1686203=728431} 2024-09-21T09:40:46.907 INFO:tasks.rados.rados.0.smithi017.stdout:677: writing smithi01785587-22 from 406802 to 1033077 tid 1 2024-09-21T09:40:46.915 INFO:tasks.rados.rados.0.smithi017.stdout:677: writing smithi01785587-22 from 1686203 to 2414634 tid 2 2024-09-21T09:40:46.915 INFO:tasks.rados.rados.0.smithi017.stdout:670: done (5 left) 2024-09-21T09:40:46.915 INFO:tasks.rados.rados.0.smithi017.stdout:672: done (4 left) 2024-09-21T09:40:46.915 INFO:tasks.rados.rados.0.smithi017.stdout:678: read oid 44 snap 78 2024-09-21T09:40:46.915 INFO:tasks.rados.rados.0.smithi017.stdout:678: expect deleted 2024-09-21T09:40:46.915 INFO:tasks.rados.rados.0.smithi017.stdout:679: snap_create 2024-09-21T09:40:46.915 INFO:tasks.rados.rados.0.smithi017.stdout:674: finishing write tid 2 to smithi01785587-30 2024-09-21T09:40:46.915 INFO:tasks.rados.rados.0.smithi017.stdout:674: finishing write tid 3 to smithi01785587-30 2024-09-21T09:40:46.915 INFO:tasks.rados.rados.0.smithi017.stdout:674: finishing write tid 4 to smithi01785587-30 2024-09-21T09:40:46.916 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 376 (ObjNum 285 snap 86 seq_num 285) dirty exists 2024-09-21T09:40:46.916 INFO:tasks.rados.rados.0.smithi017.stdout:674: left oid 30 (ObjNum 285 snap 86 seq_num 285) 2024-09-21T09:40:46.916 INFO:tasks.rados.rados.0.smithi017.stdout:675: expect (ObjNum 142 snap 34 seq_num 142) 2024-09-21T09:40:46.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:46 smithi134 ceph-mon[33541]: pgmap v123: 89 pgs: 89 active+clean; 223 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:46.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:46 smithi134 ceph-mon[33541]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:40:46.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:46 smithi134 ceph-mon[33541]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:40:46.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:46 smithi134 ceph-mon[33541]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:40:46.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:46.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:47.021 INFO:tasks.rados.rados.0.smithi017.stdout:677: finishing write tid 1 to smithi01785587-22 2024-09-21T09:40:47.021 INFO:tasks.rados.rados.0.smithi017.stdout:677: finishing write tid 2 to smithi01785587-22 2024-09-21T09:40:47.021 INFO:tasks.rados.rados.0.smithi017.stdout:677: finishing write tid 3 to smithi01785587-22 2024-09-21T09:40:47.022 INFO:tasks.rados.rados.0.smithi017.stdout:677: finishing write tid 4 to smithi01785587-22 2024-09-21T09:40:47.022 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 348 (ObjNum 286 snap 86 seq_num 286) dirty exists 2024-09-21T09:40:47.022 INFO:tasks.rados.rados.0.smithi017.stdout:677: left oid 22 (ObjNum 286 snap 86 seq_num 286) 2024-09-21T09:40:47.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:46 smithi017 ceph-mon[29380]: pgmap v123: 89 pgs: 89 active+clean; 223 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:47.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:46 smithi017 ceph-mon[29380]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:40:47.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:46 smithi017 ceph-mon[29380]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:40:47.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:46 smithi017 ceph-mon[29380]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:40:47.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:46 smithi017 ceph-mon[40737]: pgmap v123: 89 pgs: 89 active+clean; 223 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:47.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:46 smithi017 ceph-mon[40737]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:40:47.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:46 smithi017 ceph-mon[40737]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:40:47.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:46 smithi017 ceph-mon[40737]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:40:47.730 INFO:tasks.rados.rados.0.smithi017.stdout:674: done (5 left) 2024-09-21T09:40:47.730 INFO:tasks.rados.rados.0.smithi017.stdout:675: done (4 left) 2024-09-21T09:40:47.730 INFO:tasks.rados.rados.0.smithi017.stdout:676: done (3 left) 2024-09-21T09:40:47.730 INFO:tasks.rados.rados.0.smithi017.stdout:677: done (2 left) 2024-09-21T09:40:47.730 INFO:tasks.rados.rados.0.smithi017.stdout:678: done (1 left) 2024-09-21T09:40:47.730 INFO:tasks.rados.rados.0.smithi017.stdout:679: done (0 left) 2024-09-21T09:40:47.730 INFO:tasks.rados.rados.0.smithi017.stdout:680: write oid 15 current snap is 87 2024-09-21T09:40:47.730 INFO:tasks.rados.rados.0.smithi017.stdout:680: seq_num 287 ranges {423609=701524,1628332=472089,2648987=174622} 2024-09-21T09:40:47.735 INFO:tasks.rados.rados.0.smithi017.stdout:680: writing smithi01785587-15 from 423609 to 1125133 tid 1 2024-09-21T09:40:47.742 INFO:tasks.rados.rados.0.smithi017.stdout:680: writing smithi01785587-15 from 1628332 to 2100421 tid 2 2024-09-21T09:40:47.747 INFO:tasks.rados.rados.0.smithi017.stdout:680: writing smithi01785587-15 from 2648987 to 2823609 tid 3 2024-09-21T09:40:47.747 INFO:tasks.rados.rados.0.smithi017.stdout:681: rollback oid 5 current snap is 87 2024-09-21T09:40:47.747 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 85 2024-09-21T09:40:47.747 INFO:tasks.rados.rados.0.smithi017.stdout:680: finishing write tid 1 to smithi01785587-15 2024-09-21T09:40:47.747 INFO:tasks.rados.rados.0.smithi017.stdout:680: finishing write tid 2 to smithi01785587-15 2024-09-21T09:40:47.747 INFO:tasks.rados.rados.0.smithi017.stdout:682: read oid 36 snap -1 2024-09-21T09:40:47.747 INFO:tasks.rados.rados.0.smithi017.stdout:682: expect (ObjNum 246 snap 73 seq_num 246) 2024-09-21T09:40:47.747 INFO:tasks.rados.rados.0.smithi017.stdout:683: snap_remove snap 83 2024-09-21T09:40:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:47 smithi134 ceph-mon[33541]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:40:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:47 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:47 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:47 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:47 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:40:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:47 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:48.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[29380]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:40:48.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[40737]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:40:48.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:47 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:48.255 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:40:48.735 INFO:tasks.rados.rados.0.smithi017.stdout:683: done (3 left) 2024-09-21T09:40:48.735 INFO:tasks.rados.rados.0.smithi017.stdout:684: read oid 2 snap -1 2024-09-21T09:40:48.735 INFO:tasks.rados.rados.0.smithi017.stdout:684: expect (ObjNum 251 snap 76 seq_num 251) 2024-09-21T09:40:48.735 INFO:tasks.rados.rados.0.smithi017.stdout:685: snap_create 2024-09-21T09:40:48.735 INFO:tasks.rados.rados.0.smithi017.stdout:680: finishing write tid 3 to smithi01785587-15 2024-09-21T09:40:48.736 INFO:tasks.rados.rados.0.smithi017.stdout:680: finishing write tid 4 to smithi01785587-15 2024-09-21T09:40:48.736 INFO:tasks.rados.rados.0.smithi017.stdout:680: finishing write tid 5 to smithi01785587-15 2024-09-21T09:40:48.736 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 156 (ObjNum 287 snap 87 seq_num 287) dirty exists 2024-09-21T09:40:48.736 INFO:tasks.rados.rados.0.smithi017.stdout:680: left oid 15 (ObjNum 287 snap 87 seq_num 287) 2024-09-21T09:40:48.736 INFO:tasks.rados.rados.0.smithi017.stdout:681: finishing rollback tid 0 to smithi01785587-5 2024-09-21T09:40:48.736 INFO:tasks.rados.rados.0.smithi017.stdout:681: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:40:48.736 INFO:tasks.rados.rados.0.smithi017.stdout:681: finishing rollback tid 2 to smithi01785587-5 2024-09-21T09:40:48.736 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 348 (ObjNum 217 snap 60 seq_num 217) dirty exists 2024-09-21T09:40:48.736 INFO:tasks.rados.rados.0.smithi017.stdout:682: expect (ObjNum 246 snap 73 seq_num 246) 2024-09-21T09:40:48.808 INFO:tasks.rados.rados.0.smithi017.stdout:684: expect (ObjNum 251 snap 76 seq_num 251) 2024-09-21T09:40:48.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[29380]: pgmap v126: 89 pgs: 89 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:40:48.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[29380]: Upgrade: Updating mgr.y 2024-09-21T09:40:48.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[29380]: Deploying daemon mgr.y on smithi017 2024-09-21T09:40:48.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[29380]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:40:48.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/406849061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:48.883 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[40737]: pgmap v126: 89 pgs: 89 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:40:48.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[40737]: Upgrade: Updating mgr.y 2024-09-21T09:40:48.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[40737]: Deploying daemon mgr.y on smithi017 2024-09-21T09:40:48.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[40737]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:40:48.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:48 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/406849061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:48.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:48 smithi134 ceph-mon[33541]: pgmap v126: 89 pgs: 89 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:40:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:48 smithi134 ceph-mon[33541]: Upgrade: Updating mgr.y 2024-09-21T09:40:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:48 smithi134 ceph-mon[33541]: Deploying daemon mgr.y on smithi017 2024-09-21T09:40:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:48 smithi134 ceph-mon[33541]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:40:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:48 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/406849061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:49.438 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:49 smithi017 systemd[1]: Stopping Ceph mgr.y for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:49.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:49 smithi017 podman[93678]: 2024-09-21 09:40:49.436703907 +0000 UTC m=+0.131978566 container died cd2cad7817e108cbab465fb29c70427eb1d5bafcc35760c39bf13079b8e1673d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T09:40:49.742 INFO:tasks.rados.rados.0.smithi017.stdout:680: done (4 left) 2024-09-21T09:40:49.742 INFO:tasks.rados.rados.0.smithi017.stdout:681: done (3 left) 2024-09-21T09:40:49.742 INFO:tasks.rados.rados.0.smithi017.stdout:682: done (2 left) 2024-09-21T09:40:49.743 INFO:tasks.rados.rados.0.smithi017.stdout:684: done (1 left) 2024-09-21T09:40:49.743 INFO:tasks.rados.rados.0.smithi017.stdout:685: done (0 left) 2024-09-21T09:40:49.743 INFO:tasks.rados.rados.0.smithi017.stdout:686: write oid 27 current snap is 88 2024-09-21T09:40:49.743 INFO:tasks.rados.rados.0.smithi017.stdout:686: seq_num 288 ranges {440416=776773,1686814=732100} 2024-09-21T09:40:49.749 INFO:tasks.rados.rados.0.smithi017.stdout:686: writing smithi01785587-27 from 440416 to 1217189 tid 1 2024-09-21T09:40:49.756 INFO:tasks.rados.rados.0.smithi017.stdout:686: writing smithi01785587-27 from 1686814 to 2418914 tid 2 2024-09-21T09:40:49.757 INFO:tasks.rados.rados.0.smithi017.stdout:686: finishing write tid 1 to smithi01785587-27 2024-09-21T09:40:49.757 INFO:tasks.rados.rados.0.smithi017.stdout:687: read oid 16 snap -1 2024-09-21T09:40:49.757 INFO:tasks.rados.rados.0.smithi017.stdout:687: expect deleted 2024-09-21T09:40:49.757 INFO:tasks.rados.rados.0.smithi017.stdout:688: snap_remove snap 80 2024-09-21T09:40:49.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:49 smithi134 ceph-mon[33541]: from='client.25256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:49 smithi134 ceph-mon[33541]: from='client.25262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:49 smithi134 ceph-mon[33541]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:40:50.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:49 smithi017 ceph-mon[29380]: from='client.25256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:50.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:49 smithi017 ceph-mon[29380]: from='client.25262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:50.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:49 smithi017 ceph-mon[29380]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:40:50.040 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:49 smithi017 ceph-mon[40737]: from='client.25256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:50.040 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:49 smithi017 ceph-mon[40737]: from='client.25262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:50.040 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:49 smithi017 ceph-mon[40737]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:40:50.040 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:49 smithi017 podman[93678]: 2024-09-21 09:40:49.84515847 +0000 UTC m=+0.540433159 container cleanup cd2cad7817e108cbab465fb29c70427eb1d5bafcc35760c39bf13079b8e1673d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T09:40:50.040 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:49 smithi017 bash[93678]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y 2024-09-21T09:40:50.382 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 podman[93696]: 2024-09-21 09:40:50.03830977 +0000 UTC m=+0.602134030 container remove cd2cad7817e108cbab465fb29c70427eb1d5bafcc35760c39bf13079b8e1673d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:40:50.382 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:40:50.382 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y.service: Failed with result 'exit-code'. 2024-09-21T09:40:50.382 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 systemd[1]: Stopped Ceph mgr.y for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:50.382 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y.service: Consumed 44.814s CPU time. 2024-09-21T09:40:50.382 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 systemd[1]: Starting Ceph mgr.y for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:50.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:50.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 podman[93835]: 2024-09-21 09:40:50.493668925 +0000 UTC m=+0.016428223 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:40:50.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 podman[93835]: 2024-09-21 09:40:50.614697188 +0000 UTC m=+0.137456474 container create 14c61243c535dfa35de3c16d7ee0023566ab65b7e2800bf0a5f95d597cfb0f88 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T09:40:50.745 INFO:tasks.rados.rados.0.smithi017.stdout:688: done (2 left) 2024-09-21T09:40:50.745 INFO:tasks.rados.rados.0.smithi017.stdout:689: rollback oid 3 current snap is 88 2024-09-21T09:40:50.745 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 85 2024-09-21T09:40:50.745 INFO:tasks.rados.rados.0.smithi017.stdout:690: snap_create 2024-09-21T09:40:50.746 INFO:tasks.rados.rados.0.smithi017.stdout:686: finishing write tid 2 to smithi01785587-27 2024-09-21T09:40:50.746 INFO:tasks.rados.rados.0.smithi017.stdout:686: finishing write tid 3 to smithi01785587-27 2024-09-21T09:40:50.746 INFO:tasks.rados.rados.0.smithi017.stdout:686: finishing write tid 4 to smithi01785587-27 2024-09-21T09:40:50.746 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 257 (ObjNum 288 snap 88 seq_num 288) dirty exists 2024-09-21T09:40:50.746 INFO:tasks.rados.rados.0.smithi017.stdout:686: left oid 27 (ObjNum 288 snap 88 seq_num 288) 2024-09-21T09:40:50.749 INFO:tasks.rados.rados.0.smithi017.stdout:689: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:40:50.749 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 184 (ObjNum 81 snap 0 seq_num 1540943998) dirty dne 2024-09-21T09:40:50.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:50 smithi134 ceph-mon[33541]: pgmap v129: 89 pgs: 89 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:40:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:50 smithi134 ceph-mon[33541]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:40:51.001 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:50 smithi017 ceph-mon[29380]: pgmap v129: 89 pgs: 89 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:40:51.001 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:50 smithi017 ceph-mon[29380]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:40:51.001 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:50 smithi017 ceph-mon[40737]: pgmap v129: 89 pgs: 89 active+clean; 215 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:40:51.002 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:50 smithi017 ceph-mon[40737]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:40:51.002 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 podman[93835]: 2024-09-21 09:40:50.892552658 +0000 UTC m=+0.415311960 container init 14c61243c535dfa35de3c16d7ee0023566ab65b7e2800bf0a5f95d597cfb0f88 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T09:40:51.002 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 podman[93835]: 2024-09-21 09:40:50.899331374 +0000 UTC m=+0.422090678 container start 14c61243c535dfa35de3c16d7ee0023566ab65b7e2800bf0a5f95d597cfb0f88 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:40:51.002 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:50 smithi017 ceph-mgr[93899]: -- 172.21.15.17:0/1827719951 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563bbfa230e0 con 0x563bbfa01000 2024-09-21T09:40:51.254 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:51 smithi017 bash[93835]: 14c61243c535dfa35de3c16d7ee0023566ab65b7e2800bf0a5f95d597cfb0f88 2024-09-21T09:40:51.254 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:51 smithi017 systemd[1]: Started Ceph mgr.y for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:51.254 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:40:51.751 INFO:tasks.rados.rados.0.smithi017.stdout:686: done (3 left) 2024-09-21T09:40:51.751 INFO:tasks.rados.rados.0.smithi017.stdout:687: done (2 left) 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:689: done (1 left) 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:690: done (0 left) 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:691: read oid 17 snap 78 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:691: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:692: rollback oid 3 current snap is 89 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 42 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:693: rollback oid 39 current snap is 89 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 82 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:694: write oid 37 current snap is 89 2024-09-21T09:40:51.752 INFO:tasks.rados.rados.0.smithi017.stdout:694: seq_num 289 ranges {457223=568375,1461649=475758,2420186=437037} 2024-09-21T09:40:51.758 INFO:tasks.rados.rados.0.smithi017.stdout:694: writing smithi01785587-37 from 457223 to 1025598 tid 1 2024-09-21T09:40:51.763 INFO:tasks.rados.rados.0.smithi017.stdout:694: writing smithi01785587-37 from 1461649 to 1937407 tid 2 2024-09-21T09:40:51.768 INFO:tasks.rados.rados.0.smithi017.stdout:694: writing smithi01785587-37 from 2420186 to 2857223 tid 3 2024-09-21T09:40:51.769 INFO:tasks.rados.rados.0.smithi017.stdout:695: snap_create 2024-09-21T09:40:51.769 INFO:tasks.rados.rados.0.smithi017.stdout:692: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:40:51.769 INFO:tasks.rados.rados.0.smithi017.stdout:692: finishing rollback tid 2 to smithi01785587-3 2024-09-21T09:40:51.769 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 355 (ObjNum 150 snap 35 seq_num 150) dirty exists 2024-09-21T09:40:51.769 INFO:tasks.rados.rados.0.smithi017.stdout:693: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:40:51.769 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 227 (ObjNum 145 snap 0 seq_num 2399927816) dirty dne 2024-09-21T09:40:51.769 INFO:tasks.rados.rados.0.smithi017.stdout:691: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:40:51.916 INFO:tasks.rados.rados.0.smithi017.stdout:694: finishing write tid 1 to smithi01785587-37 2024-09-21T09:40:51.916 INFO:tasks.rados.rados.0.smithi017.stdout:694: finishing write tid 2 to smithi01785587-37 2024-09-21T09:40:51.916 INFO:tasks.rados.rados.0.smithi017.stdout:694: finishing write tid 3 to smithi01785587-37 2024-09-21T09:40:51.916 INFO:tasks.rados.rados.0.smithi017.stdout:694: finishing write tid 4 to smithi01785587-37 2024-09-21T09:40:51.916 INFO:tasks.rados.rados.0.smithi017.stdout:694: finishing write tid 5 to smithi01785587-37 2024-09-21T09:40:51.916 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 306 (ObjNum 289 snap 89 seq_num 289) dirty exists 2024-09-21T09:40:51.916 INFO:tasks.rados.rados.0.smithi017.stdout:694: left oid 37 (ObjNum 289 snap 89 seq_num 289) 2024-09-21T09:40:51.970 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:40:51 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:40:51] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.51.0" 2024-09-21T09:40:51.971 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:51 smithi134 ceph-mon[33541]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:40:51.971 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:51 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1988245909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:51.971 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:51 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:51.971 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:51 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:51.971 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:51 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[40737]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1988245909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[29380]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1988245909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:52.010 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:51 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:691: done (4 left) 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:692: done (3 left) 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:693: done (2 left) 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:694: done (1 left) 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:695: done (0 left) 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:696: rollback oid 45 current snap is 90 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 45 to 85 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:697: read oid 9 snap -1 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:697: expect (ObjNum 267 snap 83 seq_num 267) 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:698: write oid 10 current snap is 90 2024-09-21T09:40:52.704 INFO:tasks.rados.rados.0.smithi017.stdout:698: seq_num 290 ranges {474030=643624,1520131=735769} 2024-09-21T09:40:52.710 INFO:tasks.rados.rados.0.smithi017.stdout:698: writing smithi01785587-10 from 474030 to 1117654 tid 1 2024-09-21T09:40:52.717 INFO:tasks.rados.rados.0.smithi017.stdout:698: writing smithi01785587-10 from 1520131 to 2255900 tid 2 2024-09-21T09:40:52.717 INFO:tasks.rados.rados.0.smithi017.stdout:699: write oid 46 current snap is 90 2024-09-21T09:40:52.717 INFO:tasks.rados.rados.0.smithi017.stdout:699: seq_num 291 ranges {490837=718873,1862260=479427,2875032=15805} 2024-09-21T09:40:52.723 INFO:tasks.rados.rados.0.smithi017.stdout:699: writing smithi01785587-46 from 490837 to 1209710 tid 1 2024-09-21T09:40:52.730 INFO:tasks.rados.rados.0.smithi017.stdout:699: writing smithi01785587-46 from 1862260 to 2341687 tid 2 2024-09-21T09:40:52.733 INFO:tasks.rados.rados.0.smithi017.stdout:699: writing smithi01785587-46 from 2875032 to 2890837 tid 3 2024-09-21T09:40:52.733 INFO:tasks.rados.rados.0.smithi017.stdout:700: write oid 42 current snap is 90 2024-09-21T09:40:52.733 INFO:tasks.rados.rados.0.smithi017.stdout:700: seq_num 292 ranges {507644=794122,1920742=623085} 2024-09-21T09:40:52.740 INFO:tasks.rados.rados.0.smithi017.stdout:700: writing smithi01785587-42 from 507644 to 1301766 tid 1 2024-09-21T09:40:52.747 INFO:tasks.rados.rados.0.smithi017.stdout:700: writing smithi01785587-42 from 1920742 to 2543827 tid 2 2024-09-21T09:40:52.747 INFO:tasks.rados.rados.0.smithi017.stdout:701: write oid 28 current snap is 90 2024-09-21T09:40:52.747 INFO:tasks.rados.rados.0.smithi017.stdout:701: seq_num 293 ranges {524451=469371,1579224=483096,2529878=394573} 2024-09-21T09:40:52.753 INFO:tasks.rados.rados.0.smithi017.stdout:701: writing smithi01785587-28 from 524451 to 993822 tid 1 2024-09-21T09:40:52.759 INFO:tasks.rados.rados.0.smithi017.stdout:701: writing smithi01785587-28 from 1579224 to 2062320 tid 2 2024-09-21T09:40:52.764 INFO:tasks.rados.rados.0.smithi017.stdout:701: writing smithi01785587-28 from 2529878 to 2924451 tid 3 2024-09-21T09:40:52.764 INFO:tasks.rados.rados.0.smithi017.stdout:702: read oid 47 snap -1 2024-09-21T09:40:52.764 INFO:tasks.rados.rados.0.smithi017.stdout:702: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:40:52.764 INFO:tasks.rados.rados.0.smithi017.stdout:703: write oid 49 current snap is 90 2024-09-21T09:40:52.764 INFO:tasks.rados.rados.0.smithi017.stdout:696: finishing rollback tid 0 to smithi01785587-45 2024-09-21T09:40:52.764 INFO:tasks.rados.rados.0.smithi017.stdout:703: seq_num 294 ranges {541258=544620,1637706=626754,2840948=100310} 2024-09-21T09:40:52.769 INFO:tasks.rados.rados.0.smithi017.stdout:703: writing smithi01785587-49 from 541258 to 1085878 tid 1 2024-09-21T09:40:52.776 INFO:tasks.rados.rados.0.smithi017.stdout:703: writing smithi01785587-49 from 1637706 to 2264460 tid 2 2024-09-21T09:40:52.780 INFO:tasks.rados.rados.0.smithi017.stdout:703: writing smithi01785587-49 from 2840948 to 2941258 tid 3 2024-09-21T09:40:52.780 INFO:tasks.rados.rados.0.smithi017.stdout:696: finishing rollback tid 1 to smithi01785587-45 2024-09-21T09:40:52.780 INFO:tasks.rados.rados.0.smithi017.stdout:696: finishing rollback tid 2 to smithi01785587-45 2024-09-21T09:40:52.780 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 265 (ObjNum 278 snap 85 seq_num 278) dirty exists 2024-09-21T09:40:52.780 INFO:tasks.rados.rados.0.smithi017.stdout:697: expect (ObjNum 267 snap 83 seq_num 267) 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[29380]: pgmap v132: 89 pgs: 89 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[29380]: from='client.15387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[29380]: from='client.25286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[29380]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[29380]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[40737]: pgmap v132: 89 pgs: 89 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[40737]: from='client.15387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[40737]: from='client.25286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[40737]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:40:52.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:52 smithi017 ceph-mon[40737]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:40:52.937 INFO:tasks.rados.rados.0.smithi017.stdout:698: finishing write tid 1 to smithi01785587-10 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:698: finishing write tid 2 to smithi01785587-10 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:704: read oid 1 snap -1 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:704: expect deleted 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:696: done (8 left) 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:697: done (7 left) 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:705: write oid 15 current snap is 90 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:698: finishing write tid 3 to smithi01785587-10 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:698: finishing write tid 4 to smithi01785587-10 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 35 (ObjNum 290 snap 90 seq_num 290) dirty exists 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:698: left oid 10 (ObjNum 290 snap 90 seq_num 290) 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:699: finishing write tid 1 to smithi01785587-46 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:699: finishing write tid 2 to smithi01785587-46 2024-09-21T09:40:52.938 INFO:tasks.rados.rados.0.smithi017.stdout:699: finishing write tid 3 to smithi01785587-46 2024-09-21T09:40:52.939 INFO:tasks.rados.rados.0.smithi017.stdout:705: seq_num 295 ranges {558065=619869,1579835=486765,2468371=489694} 2024-09-21T09:40:52.943 INFO:tasks.rados.rados.0.smithi017.stdout:705: writing smithi01785587-15 from 558065 to 1177934 tid 1 2024-09-21T09:40:52.948 INFO:tasks.rados.rados.0.smithi017.stdout:705: writing smithi01785587-15 from 1579835 to 2066600 tid 2 2024-09-21T09:40:52.954 INFO:tasks.rados.rados.0.smithi017.stdout:705: writing smithi01785587-15 from 2468371 to 2958065 tid 3 2024-09-21T09:40:52.954 INFO:tasks.rados.rados.0.smithi017.stdout:698: done (7 left) 2024-09-21T09:40:52.954 INFO:tasks.rados.rados.0.smithi017.stdout:706: write oid 21 current snap is 90 2024-09-21T09:40:52.954 INFO:tasks.rados.rados.0.smithi017.stdout:706: seq_num 296 ranges {574872=695118,2038317=630423} 2024-09-21T09:40:52.960 INFO:tasks.rados.rados.0.smithi017.stdout:706: writing smithi01785587-21 from 574872 to 1269990 tid 1 2024-09-21T09:40:52.969 INFO:tasks.rados.rados.0.smithi017.stdout:706: writing smithi01785587-21 from 2038317 to 2668740 tid 2 2024-09-21T09:40:52.969 INFO:tasks.rados.rados.0.smithi017.stdout:707: read oid 9 snap -1 2024-09-21T09:40:52.969 INFO:tasks.rados.rados.0.smithi017.stdout:707: expect (ObjNum 267 snap 83 seq_num 267) 2024-09-21T09:40:52.969 INFO:tasks.rados.rados.0.smithi017.stdout:708: delete oid 23 current snap is 90 2024-09-21T09:40:52.969 INFO:tasks.rados.rados.0.smithi017.stdout:699: finishing write tid 4 to smithi01785587-46 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:699: finishing write tid 5 to smithi01785587-46 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 361 (ObjNum 291 snap 90 seq_num 291) dirty exists 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:699: left oid 46 (ObjNum 291 snap 90 seq_num 291) 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:700: finishing write tid 1 to smithi01785587-42 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:700: finishing write tid 2 to smithi01785587-42 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:700: finishing write tid 3 to smithi01785587-42 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:700: finishing write tid 4 to smithi01785587-42 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 210 (ObjNum 292 snap 90 seq_num 292) dirty exists 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:700: left oid 42 (ObjNum 292 snap 90 seq_num 292) 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:701: finishing write tid 1 to smithi01785587-28 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:701: finishing write tid 2 to smithi01785587-28 2024-09-21T09:40:52.970 INFO:tasks.rados.rados.0.smithi017.stdout:702: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:40:53.093 INFO:tasks.rados.rados.0.smithi017.stdout:701: finishing write tid 3 to smithi01785587-28 2024-09-21T09:40:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:699: done (9 left) 2024-09-21T09:40:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:700: done (8 left) 2024-09-21T09:40:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:702: done (7 left) 2024-09-21T09:40:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:708: done (6 left) 2024-09-21T09:40:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:709: snap_remove snap 42 2024-09-21T09:40:53.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:52 smithi134 ceph-mon[33541]: pgmap v132: 89 pgs: 89 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-09-21T09:40:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:52 smithi134 ceph-mon[33541]: from='client.15387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:52 smithi134 ceph-mon[33541]: from='client.25286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:52 smithi134 ceph-mon[33541]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:40:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:52 smithi134 ceph-mon[33541]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:40:53.762 INFO:tasks.rados.rados.0.smithi017.stdout:709: done (6 left) 2024-09-21T09:40:53.762 INFO:tasks.rados.rados.0.smithi017.stdout:710: read oid 44 snap -1 2024-09-21T09:40:53.762 INFO:tasks.rados.rados.0.smithi017.stdout:710: expect (ObjNum 264 snap 79 seq_num 264) 2024-09-21T09:40:53.762 INFO:tasks.rados.rados.0.smithi017.stdout:711: read oid 4 snap -1 2024-09-21T09:40:53.762 INFO:tasks.rados.rados.0.smithi017.stdout:711: expect deleted 2024-09-21T09:40:53.762 INFO:tasks.rados.rados.0.smithi017.stdout:712: rollback oid 35 current snap is 90 2024-09-21T09:40:53.762 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 82 2024-09-21T09:40:53.763 INFO:tasks.rados.rados.0.smithi017.stdout:713: write oid 38 current snap is 90 2024-09-21T09:40:53.763 INFO:tasks.rados.rados.0.smithi017.stdout:713: seq_num 298 ranges {608486=561969,1871634=634092,2950640=57846} 2024-09-21T09:40:53.770 INFO:tasks.rados.rados.0.smithi017.stdout:713: writing smithi01785587-38 from 608486 to 1170455 tid 1 2024-09-21T09:40:53.779 INFO:tasks.rados.rados.0.smithi017.stdout:713: writing smithi01785587-38 from 1871634 to 2505726 tid 2 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:713: writing smithi01785587-38 from 2950640 to 3008486 tid 3 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:714: write oid 26 current snap is 90 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:701: finishing write tid 4 to smithi01785587-28 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:701: finishing write tid 5 to smithi01785587-28 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 361 (ObjNum 293 snap 90 seq_num 293) dirty exists 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:701: left oid 28 (ObjNum 293 snap 90 seq_num 293) 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:703: finishing write tid 1 to smithi01785587-49 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:703: finishing write tid 2 to smithi01785587-49 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:703: finishing write tid 3 to smithi01785587-49 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:703: finishing write tid 4 to smithi01785587-49 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:703: finishing write tid 5 to smithi01785587-49 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 369 (ObjNum 294 snap 90 seq_num 294) dirty exists 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:703: left oid 49 (ObjNum 294 snap 90 seq_num 294) 2024-09-21T09:40:53.783 INFO:tasks.rados.rados.0.smithi017.stdout:714: seq_num 299 ranges {625293=637218,1813763=494103,2978063=47230} 2024-09-21T09:40:53.789 INFO:tasks.rados.rados.0.smithi017.stdout:714: writing smithi01785587-26 from 625293 to 1262511 tid 1 2024-09-21T09:40:53.795 INFO:tasks.rados.rados.0.smithi017.stdout:714: writing smithi01785587-26 from 1813763 to 2307866 tid 2 2024-09-21T09:40:53.799 INFO:tasks.rados.rados.0.smithi017.stdout:714: writing smithi01785587-26 from 2978063 to 3025293 tid 3 2024-09-21T09:40:53.799 INFO:tasks.rados.rados.0.smithi017.stdout:701: done (10 left) 2024-09-21T09:40:53.799 INFO:tasks.rados.rados.0.smithi017.stdout:703: done (9 left) 2024-09-21T09:40:53.799 INFO:tasks.rados.rados.0.smithi017.stdout:715: write oid 22 current snap is 90 2024-09-21T09:40:53.800 INFO:tasks.rados.rados.0.smithi017.stdout:715: seq_num 300 ranges {642100=712467,1872245=637761} 2024-09-21T09:40:53.806 INFO:tasks.rados.rados.0.smithi017.stdout:715: writing smithi01785587-22 from 642100 to 1354567 tid 1 2024-09-21T09:40:53.813 INFO:tasks.rados.rados.0.smithi017.stdout:715: writing smithi01785587-22 from 1872245 to 2510006 tid 2 2024-09-21T09:40:53.813 INFO:tasks.rados.rados.0.smithi017.stdout:716: write oid 10 current snap is 90 2024-09-21T09:40:53.813 INFO:tasks.rados.rados.0.smithi017.stdout:716: seq_num 301 ranges {658907=787716,1930727=497772,3032909=25998} 2024-09-21T09:40:53.821 INFO:tasks.rados.rados.0.smithi017.stdout:716: writing smithi01785587-10 from 658907 to 1446623 tid 1 2024-09-21T09:40:53.826 INFO:tasks.rados.rados.0.smithi017.stdout:716: writing smithi01785587-10 from 1930727 to 2428499 tid 2 2024-09-21T09:40:53.830 INFO:tasks.rados.rados.0.smithi017.stdout:716: writing smithi01785587-10 from 3032909 to 3058907 tid 3 2024-09-21T09:40:53.830 INFO:tasks.rados.rados.0.smithi017.stdout:717: snap_create 2024-09-21T09:40:53.830 INFO:tasks.rados.rados.0.smithi017.stdout:705: finishing write tid 1 to smithi01785587-15 2024-09-21T09:40:53.830 INFO:tasks.rados.rados.0.smithi017.stdout:705: finishing write tid 2 to smithi01785587-15 2024-09-21T09:40:53.830 INFO:tasks.rados.rados.0.smithi017.stdout:705: finishing write tid 3 to smithi01785587-15 2024-09-21T09:40:53.830 INFO:tasks.rados.rados.0.smithi017.stdout:705: finishing write tid 4 to smithi01785587-15 2024-09-21T09:40:53.830 INFO:tasks.rados.rados.0.smithi017.stdout:705: finishing write tid 5 to smithi01785587-15 2024-09-21T09:40:53.831 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 163 (ObjNum 295 snap 90 seq_num 295) dirty exists 2024-09-21T09:40:53.831 INFO:tasks.rados.rados.0.smithi017.stdout:705: left oid 15 (ObjNum 295 snap 90 seq_num 295) 2024-09-21T09:40:53.831 INFO:tasks.rados.rados.0.smithi017.stdout:706: finishing write tid 1 to smithi01785587-21 2024-09-21T09:40:53.831 INFO:tasks.rados.rados.0.smithi017.stdout:706: finishing write tid 2 to smithi01785587-21 2024-09-21T09:40:53.831 INFO:tasks.rados.rados.0.smithi017.stdout:707: expect (ObjNum 267 snap 83 seq_num 267) 2024-09-21T09:40:53.988 INFO:tasks.rados.rados.0.smithi017.stdout:706: finishing write tid 3 to smithi01785587-21 2024-09-21T09:40:53.988 INFO:tasks.rados.rados.0.smithi017.stdout:706: finishing write tid 4 to smithi01785587-21 2024-09-21T09:40:53.988 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 373 (ObjNum 296 snap 90 seq_num 296) dirty exists 2024-09-21T09:40:53.988 INFO:tasks.rados.rados.0.smithi017.stdout:706: left oid 21 (ObjNum 296 snap 90 seq_num 296) 2024-09-21T09:40:53.988 INFO:tasks.rados.rados.0.smithi017.stdout:710: expect (ObjNum 264 snap 79 seq_num 264) 2024-09-21T09:40:54.085 INFO:tasks.rados.rados.0.smithi017.stdout:712: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:40:54.085 INFO:tasks.rados.rados.0.smithi017.stdout:712: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:40:54.085 INFO:tasks.rados.rados.0.smithi017.stdout:712: finishing rollback tid 2 to smithi01785587-35 2024-09-21T09:40:54.085 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 75 (ObjNum 196 snap 54 seq_num 196) dirty exists 2024-09-21T09:40:54.085 INFO:tasks.rados.rados.0.smithi017.stdout:713: finishing write tid 1 to smithi01785587-38 2024-09-21T09:40:54.085 INFO:tasks.rados.rados.0.smithi017.stdout:713: finishing write tid 2 to smithi01785587-38 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:713: finishing write tid 3 to smithi01785587-38 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:713: finishing write tid 4 to smithi01785587-38 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:713: finishing write tid 5 to smithi01785587-38 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 200 (ObjNum 298 snap 90 seq_num 298) dirty exists 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:713: left oid 38 (ObjNum 298 snap 90 seq_num 298) 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:714: finishing write tid 1 to smithi01785587-26 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:714: finishing write tid 2 to smithi01785587-26 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:714: finishing write tid 3 to smithi01785587-26 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:714: finishing write tid 4 to smithi01785587-26 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:714: finishing write tid 5 to smithi01785587-26 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 372 (ObjNum 299 snap 90 seq_num 299) dirty exists 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:714: left oid 26 (ObjNum 299 snap 90 seq_num 299) 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:715: finishing write tid 1 to smithi01785587-22 2024-09-21T09:40:54.086 INFO:tasks.rados.rados.0.smithi017.stdout:715: finishing write tid 2 to smithi01785587-22 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:715: finishing write tid 3 to smithi01785587-22 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:715: finishing write tid 4 to smithi01785587-22 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 376 (ObjNum 300 snap 90 seq_num 300) dirty exists 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:715: left oid 22 (ObjNum 300 snap 90 seq_num 300) 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:716: finishing write tid 1 to smithi01785587-10 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:716: finishing write tid 2 to smithi01785587-10 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:716: finishing write tid 3 to smithi01785587-10 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:716: finishing write tid 4 to smithi01785587-10 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:716: finishing write tid 5 to smithi01785587-10 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 39 (ObjNum 301 snap 90 seq_num 301) dirty exists 2024-09-21T09:40:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:716: left oid 10 (ObjNum 301 snap 90 seq_num 301) 2024-09-21T09:40:54.291 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:40:54.455 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:54 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:704: done (11 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:705: done (10 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:706: done (9 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:707: done (8 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:710: done (7 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:711: done (6 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:712: done (5 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:713: done (4 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:714: done (3 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:715: done (2 left) 2024-09-21T09:40:54.762 INFO:tasks.rados.rados.0.smithi017.stdout:716: done (1 left) 2024-09-21T09:40:54.763 INFO:tasks.rados.rados.0.smithi017.stdout:717: done (0 left) 2024-09-21T09:40:54.763 INFO:tasks.rados.rados.0.smithi017.stdout:718: snap_remove snap 85 2024-09-21T09:40:55.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:54 smithi134 ceph-mon[33541]: pgmap v135: 89 pgs: 89 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-09-21T09:40:55.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:54 smithi134 ceph-mon[33541]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:40:55.060 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:54 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2614528432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:54 smithi017 ceph-mon[29380]: pgmap v135: 89 pgs: 89 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-09-21T09:40:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:54 smithi017 ceph-mon[29380]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:40:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:54 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2614528432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:54 smithi017 ceph-mon[40737]: pgmap v135: 89 pgs: 89 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-09-21T09:40:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:54 smithi017 ceph-mon[40737]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:40:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:54 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2614528432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:55.764 INFO:tasks.rados.rados.0.smithi017.stdout:718: done (0 left) 2024-09-21T09:40:55.764 INFO:tasks.rados.rados.0.smithi017.stdout:719: read oid 6 snap -1 2024-09-21T09:40:55.764 INFO:tasks.rados.rados.0.smithi017.stdout:719: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:40:55.765 INFO:tasks.rados.rados.0.smithi017.stdout:720: delete oid 29 current snap is 91 2024-09-21T09:40:55.768 INFO:tasks.rados.rados.0.smithi017.stdout:720: done (1 left) 2024-09-21T09:40:55.768 INFO:tasks.rados.rados.0.smithi017.stdout:721: write oid 46 current snap is 91 2024-09-21T09:40:55.768 INFO:tasks.rados.rados.0.smithi017.stdout:721: seq_num 303 ranges {692521=538214,1931338=785088} 2024-09-21T09:40:55.775 INFO:tasks.rados.rados.0.smithi017.stdout:721: writing smithi01785587-46 from 692521 to 1230735 tid 1 2024-09-21T09:40:55.784 INFO:tasks.rados.rados.0.smithi017.stdout:721: writing smithi01785587-46 from 1931338 to 2716426 tid 2 2024-09-21T09:40:55.784 INFO:tasks.rados.rados.0.smithi017.stdout:722: read oid 18 snap -1 2024-09-21T09:40:55.785 INFO:tasks.rados.rados.0.smithi017.stdout:722: expect (ObjNum 263 snap 79 seq_num 263) 2024-09-21T09:40:55.785 INFO:tasks.rados.rados.0.smithi017.stdout:723: rollback oid 35 current snap is 91 2024-09-21T09:40:55.785 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 86 2024-09-21T09:40:55.785 INFO:tasks.rados.rados.0.smithi017.stdout:719: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:40:55.857 INFO:tasks.rados.rados.0.smithi017.stdout:721: finishing write tid 1 to smithi01785587-46 2024-09-21T09:40:55.857 INFO:tasks.rados.rados.0.smithi017.stdout:719: done (3 left) 2024-09-21T09:40:55.857 INFO:tasks.rados.rados.0.smithi017.stdout:724: write oid 31 current snap is 91 2024-09-21T09:40:55.857 INFO:tasks.rados.rados.0.smithi017.stdout:723: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:40:55.857 INFO:tasks.rados.rados.0.smithi017.stdout:724: seq_num 304 ranges {709328=613463,1989820=645099} 2024-09-21T09:40:55.864 INFO:tasks.rados.rados.0.smithi017.stdout:724: writing smithi01785587-31 from 709328 to 1322791 tid 1 2024-09-21T09:40:55.871 INFO:tasks.rados.rados.0.smithi017.stdout:724: writing smithi01785587-31 from 1989820 to 2634919 tid 2 2024-09-21T09:40:55.872 INFO:tasks.rados.rados.0.smithi017.stdout:725: write oid 32 current snap is 91 2024-09-21T09:40:55.872 INFO:tasks.rados.rados.0.smithi017.stdout:725: seq_num 305 ranges {726135=405065,1764655=788757,3026248=99887} 2024-09-21T09:40:55.878 INFO:tasks.rados.rados.0.smithi017.stdout:725: writing smithi01785587-32 from 726135 to 1131200 tid 1 2024-09-21T09:40:55.885 INFO:tasks.rados.rados.0.smithi017.stdout:725: writing smithi01785587-32 from 1764655 to 2553412 tid 2 2024-09-21T09:40:55.889 INFO:tasks.rados.rados.0.smithi017.stdout:725: writing smithi01785587-32 from 3026248 to 3126135 tid 3 2024-09-21T09:40:55.889 INFO:tasks.rados.rados.0.smithi017.stdout:726: delete oid 7 current snap is 91 2024-09-21T09:40:55.889 INFO:tasks.rados.rados.0.smithi017.stdout:723: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:40:55.889 INFO:tasks.rados.rados.0.smithi017.stdout:723: finishing rollback tid 2 to smithi01785587-35 2024-09-21T09:40:55.889 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 81 (ObjNum 196 snap 54 seq_num 196) dirty exists 2024-09-21T09:40:55.889 INFO:tasks.rados.rados.0.smithi017.stdout:721: finishing write tid 2 to smithi01785587-46 2024-09-21T09:40:55.890 INFO:tasks.rados.rados.0.smithi017.stdout:721: finishing write tid 3 to smithi01785587-46 2024-09-21T09:40:55.890 INFO:tasks.rados.rados.0.smithi017.stdout:721: finishing write tid 4 to smithi01785587-46 2024-09-21T09:40:55.890 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 372 (ObjNum 303 snap 91 seq_num 303) dirty exists 2024-09-21T09:40:55.890 INFO:tasks.rados.rados.0.smithi017.stdout:721: left oid 46 (ObjNum 303 snap 91 seq_num 303) 2024-09-21T09:40:55.890 INFO:tasks.rados.rados.0.smithi017.stdout:722: expect (ObjNum 263 snap 79 seq_num 263) 2024-09-21T09:40:55.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:55.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-mon[33541]: from='client.25081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-mon[33541]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-mon[33541]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:40:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:55.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:55 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.002 INFO:tasks.rados.rados.0.smithi017.stdout:724: finishing write tid 1 to smithi01785587-31 2024-09-21T09:40:56.002 INFO:tasks.rados.rados.0.smithi017.stdout:724: finishing write tid 2 to smithi01785587-31 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:724: finishing write tid 3 to smithi01785587-31 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:724: finishing write tid 4 to smithi01785587-31 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 367 (ObjNum 304 snap 91 seq_num 304) dirty exists 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:724: left oid 31 (ObjNum 304 snap 91 seq_num 304) 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:725: finishing write tid 1 to smithi01785587-32 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:721: done (5 left) 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:722: done (4 left) 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:723: done (3 left) 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:724: done (2 left) 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:726: done (1 left) 2024-09-21T09:40:56.003 INFO:tasks.rados.rados.0.smithi017.stdout:727: snap_remove snap 90 2024-09-21T09:40:56.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[29380]: from='client.25081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:56.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[29380]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:56.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[29380]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:40:56.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[40737]: from='client.25081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:56.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[40737]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:56.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[40737]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:40:56.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:55 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:40:56.769 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:40:56] ENGINE Bus STARTING 2024-09-21T09:40:56.769 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: CherryPy Checker: 2024-09-21T09:40:56.769 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: The Application mounted at '' has an empty config. 2024-09-21T09:40:56.769 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:56.769 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:40:56] ENGINE Serving on http://:::9283 2024-09-21T09:40:56.769 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:40:56] ENGINE Bus STARTED 2024-09-21T09:40:56.839 INFO:tasks.rados.rados.0.smithi017.stdout:727: done (1 left) 2024-09-21T09:40:56.839 INFO:tasks.rados.rados.0.smithi017.stdout:728: write oid 47 current snap is 91 2024-09-21T09:40:56.839 INFO:tasks.rados.rados.0.smithi017.stdout:725: finishing write tid 2 to smithi01785587-32 2024-09-21T09:40:56.839 INFO:tasks.rados.rados.0.smithi017.stdout:728: seq_num 307 ranges {759749=555563,1765266=792426,2964741=195008} 2024-09-21T09:40:56.846 INFO:tasks.rados.rados.0.smithi017.stdout:728: writing smithi01785587-47 from 759749 to 1315312 tid 1 2024-09-21T09:40:56.854 INFO:tasks.rados.rados.0.smithi017.stdout:728: writing smithi01785587-47 from 1765266 to 2557692 tid 2 2024-09-21T09:40:56.858 INFO:tasks.rados.rados.0.smithi017.stdout:728: writing smithi01785587-47 from 2964741 to 3159749 tid 3 2024-09-21T09:40:56.858 INFO:tasks.rados.rados.0.smithi017.stdout:729: delete oid 13 current snap is 91 2024-09-21T09:40:56.858 INFO:tasks.rados.rados.0.smithi017.stdout:725: finishing write tid 3 to smithi01785587-32 2024-09-21T09:40:56.858 INFO:tasks.rados.rados.0.smithi017.stdout:725: finishing write tid 4 to smithi01785587-32 2024-09-21T09:40:56.858 INFO:tasks.rados.rados.0.smithi017.stdout:725: finishing write tid 5 to smithi01785587-32 2024-09-21T09:40:56.858 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 377 (ObjNum 305 snap 91 seq_num 305) dirty exists 2024-09-21T09:40:56.858 INFO:tasks.rados.rados.0.smithi017.stdout:725: left oid 32 (ObjNum 305 snap 91 seq_num 305) 2024-09-21T09:40:56.858 INFO:tasks.rados.rados.0.smithi017.stdout:728: finishing write tid 1 to smithi01785587-47 2024-09-21T09:40:56.858 INFO:tasks.rados.rados.0.smithi017.stdout:728: finishing write tid 2 to smithi01785587-47 2024-09-21T09:40:56.860 INFO:tasks.rados.rados.0.smithi017.stdout:728: finishing write tid 3 to smithi01785587-47 2024-09-21T09:40:56.860 INFO:tasks.rados.rados.0.smithi017.stdout:728: finishing write tid 4 to smithi01785587-47 2024-09-21T09:40:56.860 INFO:tasks.rados.rados.0.smithi017.stdout:725: done (2 left) 2024-09-21T09:40:56.860 INFO:tasks.rados.rados.0.smithi017.stdout:729: done (1 left) 2024-09-21T09:40:56.860 INFO:tasks.rados.rados.0.smithi017.stdout:730: write oid 34 current snap is 91 2024-09-21T09:40:56.860 INFO:tasks.rados.rados.0.smithi017.stdout:730: seq_num 309 ranges {793363=706061,2282230=796095} 2024-09-21T09:40:56.868 INFO:tasks.rados.rados.0.smithi017.stdout:730: writing smithi01785587-34 from 793363 to 1499424 tid 1 2024-09-21T09:40:56.877 INFO:tasks.rados.rados.0.smithi017.stdout:730: writing smithi01785587-34 from 2282230 to 3078325 tid 2 2024-09-21T09:40:56.877 INFO:tasks.rados.rados.0.smithi017.stdout:731: write oid 33 current snap is 91 2024-09-21T09:40:56.877 INFO:tasks.rados.rados.0.smithi017.stdout:731: seq_num 310 ranges {410170=781310,1940712=656106,3163363=46807} 2024-09-21T09:40:56.883 INFO:tasks.rados.rados.0.smithi017.stdout:731: writing smithi01785587-33 from 410170 to 1191480 tid 1 2024-09-21T09:40:56.891 INFO:tasks.rados.rados.0.smithi017.stdout:731: writing smithi01785587-33 from 1940712 to 2596818 tid 2 2024-09-21T09:40:56.896 INFO:tasks.rados.rados.0.smithi017.stdout:731: writing smithi01785587-33 from 3163363 to 3210170 tid 3 2024-09-21T09:40:56.896 INFO:tasks.rados.rados.0.smithi017.stdout:732: rollback oid 37 current snap is 91 2024-09-21T09:40:56.896 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 81 2024-09-21T09:40:56.896 INFO:tasks.rados.rados.0.smithi017.stdout:728: finishing write tid 5 to smithi01785587-47 2024-09-21T09:40:56.896 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 221 (ObjNum 307 snap 91 seq_num 307) dirty exists 2024-09-21T09:40:56.896 INFO:tasks.rados.rados.0.smithi017.stdout:728: left oid 47 (ObjNum 307 snap 91 seq_num 307) 2024-09-21T09:40:56.896 INFO:tasks.rados.rados.0.smithi017.stdout:728: done (3 left) 2024-09-21T09:40:56.896 INFO:tasks.rados.rados.0.smithi017.stdout:733: rollback oid 46 current snap is 91 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 60 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:730: finishing write tid 1 to smithi01785587-34 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:730: finishing write tid 2 to smithi01785587-34 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:730: finishing write tid 3 to smithi01785587-34 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:730: finishing write tid 4 to smithi01785587-34 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 390 (ObjNum 309 snap 91 seq_num 309) dirty exists 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:730: left oid 34 (ObjNum 309 snap 91 seq_num 309) 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:731: finishing write tid 1 to smithi01785587-33 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:731: finishing write tid 2 to smithi01785587-33 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:730: done (3 left) 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:734: read oid 17 snap 60 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:734: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:735: read oid 22 snap -1 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:735: expect (ObjNum 300 snap 90 seq_num 300) 2024-09-21T09:40:56.897 INFO:tasks.rados.rados.0.smithi017.stdout:736: snap_create 2024-09-21T09:40:56.899 INFO:tasks.rados.rados.0.smithi017.stdout:732: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:40:56.899 INFO:tasks.rados.rados.0.smithi017.stdout:733: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:40:56.899 INFO:tasks.rados.rados.0.smithi017.stdout:733: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:40:56.899 INFO:tasks.rados.rados.0.smithi017.stdout:733: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:40:56.900 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 384 (ObjNum 210 snap 60 seq_num 210) dirty exists 2024-09-21T09:40:56.900 INFO:tasks.rados.rados.0.smithi017.stdout:734: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:40:57.044 INFO:tasks.rados.rados.0.smithi017.stdout:732: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:40:57.044 INFO:tasks.rados.rados.0.smithi017.stdout:732: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:40:57.044 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 318 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:40:57.044 INFO:tasks.rados.rados.0.smithi017.stdout:731: finishing write tid 3 to smithi01785587-33 2024-09-21T09:40:57.044 INFO:tasks.rados.rados.0.smithi017.stdout:731: finishing write tid 4 to smithi01785587-33 2024-09-21T09:40:57.045 INFO:tasks.rados.rados.0.smithi017.stdout:731: finishing write tid 5 to smithi01785587-33 2024-09-21T09:40:57.045 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 382 (ObjNum 310 snap 91 seq_num 310) dirty exists 2024-09-21T09:40:57.045 INFO:tasks.rados.rados.0.smithi017.stdout:731: left oid 33 (ObjNum 310 snap 91 seq_num 310) 2024-09-21T09:40:57.045 INFO:tasks.rados.rados.0.smithi017.stdout:735: expect (ObjNum 300 snap 90 seq_num 300) 2024-09-21T09:40:57.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[29380]: pgmap v138: 89 pgs: 89 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:57.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[29380]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:40:57.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:57.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[29380]: Standby manager daemon y restarted 2024-09-21T09:40:57.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[29380]: Standby manager daemon y started 2024-09-21T09:40:57.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:40:57.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:57.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:40:57.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:57.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[40737]: pgmap v138: 89 pgs: 89 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:57.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[40737]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:40:57.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:57.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[40737]: Standby manager daemon y restarted 2024-09-21T09:40:57.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[40737]: Standby manager daemon y started 2024-09-21T09:40:57.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:40:57.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:57.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:40:57.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:56 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:57.243 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:40:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:40:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-mon[33541]: pgmap v138: 89 pgs: 89 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:40:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-mon[33541]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:40:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-mon[33541]: Standby manager daemon y restarted 2024-09-21T09:40:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-mon[33541]: Standby manager daemon y started 2024-09-21T09:40:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:40:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:40:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:56 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:57.304 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:40:57.847 INFO:tasks.rados.rados.0.smithi017.stdout:731: done (5 left) 2024-09-21T09:40:57.847 INFO:tasks.rados.rados.0.smithi017.stdout:732: done (4 left) 2024-09-21T09:40:57.848 INFO:tasks.rados.rados.0.smithi017.stdout:733: done (3 left) 2024-09-21T09:40:57.848 INFO:tasks.rados.rados.0.smithi017.stdout:734: done (2 left) 2024-09-21T09:40:57.848 INFO:tasks.rados.rados.0.smithi017.stdout:735: done (1 left) 2024-09-21T09:40:57.848 INFO:tasks.rados.rados.0.smithi017.stdout:736: done (0 left) 2024-09-21T09:40:57.848 INFO:tasks.rados.rados.0.smithi017.stdout:737: snap_create 2024-09-21T09:40:58.144 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:57 smithi017 ceph-mon[29380]: mgrmap e27: x(active, since 91s), standbys: y 2024-09-21T09:40:58.144 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:57 smithi017 ceph-mon[29380]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:40:58.144 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:57 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1346304386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:58.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:57 smithi017 ceph-mon[40737]: mgrmap e27: x(active, since 91s), standbys: y 2024-09-21T09:40:58.145 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:57 smithi017 ceph-mon[40737]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:40:58.145 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:57 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1346304386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:58.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:57 smithi134 ceph-mon[33541]: mgrmap e27: x(active, since 91s), standbys: y 2024-09-21T09:40:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:57 smithi134 ceph-mon[33541]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:40:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:57 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1346304386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:40:58.845 INFO:tasks.rados.rados.0.smithi017.stdout:737: done (0 left) 2024-09-21T09:40:58.845 INFO:tasks.rados.rados.0.smithi017.stdout:738: rollback oid 17 current snap is 93 2024-09-21T09:40:58.846 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 91 2024-09-21T09:40:58.846 INFO:tasks.rados.rados.0.smithi017.stdout:739: write oid 42 current snap is 93 2024-09-21T09:40:58.846 INFO:tasks.rados.rados.0.smithi017.stdout:739: seq_num 311 ranges {426977=456559,1482841=799764,2958080=268897} 2024-09-21T09:40:58.851 INFO:tasks.rados.rados.0.smithi017.stdout:739: writing smithi01785587-42 from 426977 to 883536 tid 1 2024-09-21T09:40:58.859 INFO:tasks.rados.rados.0.smithi017.stdout:739: writing smithi01785587-42 from 1482841 to 2282605 tid 2 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:739: writing smithi01785587-42 from 2958080 to 3226977 tid 3 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:740: rollback oid 47 current snap is 93 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 47 to 92 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:738: finishing rollback tid 0 to smithi01785587-17 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:738: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:738: finishing rollback tid 2 to smithi01785587-17 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 277 (ObjNum 185 snap 49 seq_num 185) dirty exists 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:739: finishing write tid 1 to smithi01785587-42 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:739: finishing write tid 2 to smithi01785587-42 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:738: done (2 left) 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:741: write oid 4 current snap is 93 2024-09-21T09:40:58.865 INFO:tasks.rados.rados.0.smithi017.stdout:741: seq_num 312 ranges {443784=648161,1657676=659775,2818209=425575} 2024-09-21T09:40:58.871 INFO:tasks.rados.rados.0.smithi017.stdout:741: writing smithi01785587-4 from 443784 to 1091945 tid 1 2024-09-21T09:40:58.878 INFO:tasks.rados.rados.0.smithi017.stdout:741: writing smithi01785587-4 from 1657676 to 2317451 tid 2 2024-09-21T09:40:58.883 INFO:tasks.rados.rados.0.smithi017.stdout:741: writing smithi01785587-4 from 2818209 to 3243784 tid 3 2024-09-21T09:40:58.883 INFO:tasks.rados.rados.0.smithi017.stdout:742: write oid 36 current snap is 93 2024-09-21T09:40:58.883 INFO:tasks.rados.rados.0.smithi017.stdout:739: finishing write tid 3 to smithi01785587-42 2024-09-21T09:40:58.883 INFO:tasks.rados.rados.0.smithi017.stdout:739: finishing write tid 4 to smithi01785587-42 2024-09-21T09:40:58.883 INFO:tasks.rados.rados.0.smithi017.stdout:739: finishing write tid 5 to smithi01785587-42 2024-09-21T09:40:58.883 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 226 (ObjNum 311 snap 93 seq_num 311) dirty exists 2024-09-21T09:40:58.884 INFO:tasks.rados.rados.0.smithi017.stdout:739: left oid 42 (ObjNum 311 snap 93 seq_num 311) 2024-09-21T09:40:58.884 INFO:tasks.rados.rados.0.smithi017.stdout:740: finishing rollback tid 0 to smithi01785587-47 2024-09-21T09:40:58.884 INFO:tasks.rados.rados.0.smithi017.stdout:740: finishing rollback tid 1 to smithi01785587-47 2024-09-21T09:40:58.884 INFO:tasks.rados.rados.0.smithi017.stdout:740: finishing rollback tid 2 to smithi01785587-47 2024-09-21T09:40:58.884 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 230 (ObjNum 307 snap 91 seq_num 307) dirty exists 2024-09-21T09:40:58.884 INFO:tasks.rados.rados.0.smithi017.stdout:741: finishing write tid 1 to smithi01785587-4 2024-09-21T09:40:58.884 INFO:tasks.rados.rados.0.smithi017.stdout:741: finishing write tid 2 to smithi01785587-4 2024-09-21T09:40:58.884 INFO:tasks.rados.rados.0.smithi017.stdout:742: seq_num 313 ranges {460591=723410,1716158=403433,2729279=440140} 2024-09-21T09:40:58.890 INFO:tasks.rados.rados.0.smithi017.stdout:742: writing smithi01785587-36 from 460591 to 1184001 tid 1 2024-09-21T09:40:58.895 INFO:tasks.rados.rados.0.smithi017.stdout:742: writing smithi01785587-36 from 1716158 to 2119591 tid 2 2024-09-21T09:40:58.901 INFO:tasks.rados.rados.0.smithi017.stdout:742: writing smithi01785587-36 from 2729279 to 3169419 tid 3 2024-09-21T09:40:58.901 INFO:tasks.rados.rados.0.smithi017.stdout:739: done (3 left) 2024-09-21T09:40:58.901 INFO:tasks.rados.rados.0.smithi017.stdout:740: done (2 left) 2024-09-21T09:40:58.901 INFO:tasks.rados.rados.0.smithi017.stdout:743: read oid 17 snap -1 2024-09-21T09:40:58.901 INFO:tasks.rados.rados.0.smithi017.stdout:743: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:40:58.901 INFO:tasks.rados.rados.0.smithi017.stdout:744: delete oid 18 current snap is 93 2024-09-21T09:40:58.901 INFO:tasks.rados.rados.0.smithi017.stdout:741: finishing write tid 3 to smithi01785587-4 2024-09-21T09:40:58.901 INFO:tasks.rados.rados.0.smithi017.stdout:741: finishing write tid 4 to smithi01785587-4 2024-09-21T09:40:58.902 INFO:tasks.rados.rados.0.smithi017.stdout:741: finishing write tid 5 to smithi01785587-4 2024-09-21T09:40:58.902 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 386 (ObjNum 312 snap 93 seq_num 312) dirty exists 2024-09-21T09:40:58.902 INFO:tasks.rados.rados.0.smithi017.stdout:741: left oid 4 (ObjNum 312 snap 93 seq_num 312) 2024-09-21T09:40:58.902 INFO:tasks.rados.rados.0.smithi017.stdout:742: finishing write tid 1 to smithi01785587-36 2024-09-21T09:40:58.902 INFO:tasks.rados.rados.0.smithi017.stdout:742: finishing write tid 2 to smithi01785587-36 2024-09-21T09:40:58.903 INFO:tasks.rados.rados.0.smithi017.stdout:741: done (3 left) 2024-09-21T09:40:58.903 INFO:tasks.rados.rados.0.smithi017.stdout:744: done (2 left) 2024-09-21T09:40:58.903 INFO:tasks.rados.rados.0.smithi017.stdout:745: snap_create 2024-09-21T09:40:58.905 INFO:tasks.rados.rados.0.smithi017.stdout:742: finishing write tid 3 to smithi01785587-36 2024-09-21T09:40:58.905 INFO:tasks.rados.rados.0.smithi017.stdout:742: finishing write tid 4 to smithi01785587-36 2024-09-21T09:40:58.905 INFO:tasks.rados.rados.0.smithi017.stdout:742: finishing write tid 5 to smithi01785587-36 2024-09-21T09:40:58.905 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 323 (ObjNum 313 snap 93 seq_num 313) dirty exists 2024-09-21T09:40:58.905 INFO:tasks.rados.rados.0.smithi017.stdout:742: left oid 36 (ObjNum 313 snap 93 seq_num 313) 2024-09-21T09:40:58.905 INFO:tasks.rados.rados.0.smithi017.stdout:743: expect (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:40:59.087 INFO:tasks.rados.rados.0.smithi017.stdout:742: done (2 left) 2024-09-21T09:40:59.087 INFO:tasks.rados.rados.0.smithi017.stdout:743: done (1 left) 2024-09-21T09:40:59.087 INFO:tasks.rados.rados.0.smithi017.stdout:745: done (0 left) 2024-09-21T09:40:59.087 INFO:tasks.rados.rados.0.smithi017.stdout:746: rollback oid 17 current snap is 94 2024-09-21T09:40:59.087 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 76 2024-09-21T09:40:59.087 INFO:tasks.rados.rados.0.smithi017.stdout:747: snap_remove snap 89 2024-09-21T09:40:59.115 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:40:59.084+0000 7f676caf0640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-21T09:40:59.116 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:40:59] ENGINE Bus STOPPING 2024-09-21T09:40:59.116 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:58 smithi134 ceph-mon[33541]: pgmap v141: 89 pgs: 89 active+clean; 187 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-21T09:40:59.116 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:58 smithi134 ceph-mon[33541]: from='client.25316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:59.116 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:58 smithi134 ceph-mon[33541]: from='client.25322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:59.116 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:58 smithi134 ceph-mon[33541]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:40:59.116 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:58 smithi134 ceph-mon[33541]: mgrmap e28: x(active, since 92s), standbys: y 2024-09-21T09:40:59.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[29380]: pgmap v141: 89 pgs: 89 active+clean; 187 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-21T09:40:59.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[29380]: from='client.25316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:59.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[29380]: from='client.25322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:59.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[29380]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:40:59.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[29380]: mgrmap e28: x(active, since 92s), standbys: y 2024-09-21T09:40:59.117 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[40737]: pgmap v141: 89 pgs: 89 active+clean; 187 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-21T09:40:59.117 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[40737]: from='client.25316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:59.117 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[40737]: from='client.25322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:59.117 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[40737]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:40:59.117 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:58 smithi017 ceph-mon[40737]: mgrmap e28: x(active, since 92s), standbys: y 2024-09-21T09:40:59.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:40:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:40:59.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:40:59] ENGINE Bus STOPPED 2024-09-21T09:40:59.493 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ignoring --setuser ceph since I am not root 2024-09-21T09:40:59.493 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ignoring --setgroup ceph since I am not root 2024-09-21T09:40:59.493 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mgr[68823]: -- 172.21.15.134:0/4070756833 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ff714cf0e0 con 0x55ff714ac800 2024-09-21T09:41:00.060 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:41:00.060 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:41:00.060 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: Activating manager daemon y 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.24904 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: mgrmap e29: y(active, starting, since 0.00743313s) 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:41:00.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.coodep"}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: Manager daemon y is now available 2024-09-21T09:41:00.062 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.704+0000 7f5528a67640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.704+0000 7f5528a67640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.704+0000 7f5528a67640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.704+0000 7f5528a67640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.704+0000 7f5528a67640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.063 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.706+0000 7f55258e5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.706+0000 7f55258e5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.706+0000 7f55258e5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.706+0000 7f55258e5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:40:59.706+0000 7f55258e5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:00.064 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: Activating manager daemon y 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.24904 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: mgrmap e29: y(active, starting, since 0.00743313s) 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.coodep"}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:41:00.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: Manager daemon y is now available 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:41:00.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:40:59 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:41:00.087 INFO:tasks.rados.rados.0.smithi017.stdout:747: done (1 left) 2024-09-21T09:41:00.087 INFO:tasks.rados.rados.0.smithi017.stdout:748: rollback oid 3 current snap is 94 2024-09-21T09:41:00.087 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 88 2024-09-21T09:41:00.088 INFO:tasks.rados.rados.0.smithi017.stdout:746: finishing rollback tid 0 to smithi01785587-17 2024-09-21T09:41:00.088 INFO:tasks.rados.rados.0.smithi017.stdout:746: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:41:00.088 INFO:tasks.rados.rados.0.smithi017.stdout:746: finishing rollback tid 2 to smithi01785587-17 2024-09-21T09:41:00.088 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 281 (ObjNum 185 snap 49 seq_num 185) dirty exists 2024-09-21T09:41:00.088 INFO:tasks.rados.rados.0.smithi017.stdout:746: done (1 left) 2024-09-21T09:41:00.088 INFO:tasks.rados.rados.0.smithi017.stdout:749: delete oid 13 current snap is 94 2024-09-21T09:41:00.094 INFO:tasks.rados.rados.0.smithi017.stdout:748: finishing rollback tid 0 to smithi01785587-3 2024-09-21T09:41:00.095 INFO:tasks.rados.rados.0.smithi017.stdout:748: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:41:00.095 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 372 (ObjNum 129 snap 0 seq_num 2399868688) dirty dne 2024-09-21T09:41:00.097 INFO:tasks.rados.rados.0.smithi017.stdout:748: done (1 left) 2024-09-21T09:41:00.097 INFO:tasks.rados.rados.0.smithi017.stdout:749: done (0 left) 2024-09-21T09:41:00.097 INFO:tasks.rados.rados.0.smithi017.stdout:750: delete oid 17 current snap is 94 2024-09-21T09:41:00.101 INFO:tasks.rados.rados.0.smithi017.stdout:750: done (0 left) 2024-09-21T09:41:00.101 INFO:tasks.rados.rados.0.smithi017.stdout:751: write oid 27 current snap is 94 2024-09-21T09:41:00.101 INFO:tasks.rados.rados.0.smithi017.stdout:751: seq_num 317 ranges {527819=624406,1833733=410771,2722618=601581} 2024-09-21T09:41:00.108 INFO:tasks.rados.rados.0.smithi017.stdout:751: writing smithi01785587-27 from 527819 to 1152225 tid 1 2024-09-21T09:41:00.114 INFO:tasks.rados.rados.0.smithi017.stdout:751: writing smithi01785587-27 from 1833733 to 2244504 tid 2 2024-09-21T09:41:00.121 INFO:tasks.rados.rados.0.smithi017.stdout:751: writing smithi01785587-27 from 2722618 to 3324199 tid 3 2024-09-21T09:41:00.121 INFO:tasks.rados.rados.0.smithi017.stdout:752: delete oid 39 current snap is 94 2024-09-21T09:41:00.121 INFO:tasks.rados.rados.0.smithi017.stdout:751: finishing write tid 1 to smithi01785587-27 2024-09-21T09:41:00.121 INFO:tasks.rados.rados.0.smithi017.stdout:751: finishing write tid 2 to smithi01785587-27 2024-09-21T09:41:00.123 INFO:tasks.rados.rados.0.smithi017.stdout:752: done (1 left) 2024-09-21T09:41:00.123 INFO:tasks.rados.rados.0.smithi017.stdout:753: snap_create 2024-09-21T09:41:00.124 INFO:tasks.rados.rados.0.smithi017.stdout:751: finishing write tid 3 to smithi01785587-27 2024-09-21T09:41:00.124 INFO:tasks.rados.rados.0.smithi017.stdout:751: finishing write tid 4 to smithi01785587-27 2024-09-21T09:41:00.124 INFO:tasks.rados.rados.0.smithi017.stdout:751: finishing write tid 5 to smithi01785587-27 2024-09-21T09:41:00.124 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 291 (ObjNum 317 snap 94 seq_num 317) dirty exists 2024-09-21T09:41:00.124 INFO:tasks.rados.rados.0.smithi017.stdout:751: left oid 27 (ObjNum 317 snap 94 seq_num 317) 2024-09-21T09:41:00.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.24904 172.21.15.134:0/3313156565' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: Activating manager daemon y 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.24904 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: mgrmap e29: y(active, starting, since 0.00743313s) 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:41:00.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.coodep"}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:41:00.216 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: Manager daemon y is now available 2024-09-21T09:41:00.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:00.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:00.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:00.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:41:00.217 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:40:59 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:41:00.337 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:00.338 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:00 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:00] ENGINE Bus STARTING 2024-09-21T09:41:00.338 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:00 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:00] ENGINE Serving on http://:::9283 2024-09-21T09:41:00.338 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:00 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:00] ENGINE Bus STARTED 2024-09-21T09:41:00.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:00.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:00 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/4275860173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:00 smithi134 ceph-mon[33541]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:41:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:00 smithi134 ceph-mon[33541]: mgrmap e30: y(active, since 1.01714s) 2024-09-21T09:41:01.099 INFO:tasks.rados.rados.0.smithi017.stdout:751: done (1 left) 2024-09-21T09:41:01.100 INFO:tasks.rados.rados.0.smithi017.stdout:753: done (0 left) 2024-09-21T09:41:01.100 INFO:tasks.rados.rados.0.smithi017.stdout:754: delete oid 21 current snap is 95 2024-09-21T09:41:01.105 INFO:tasks.rados.rados.0.smithi017.stdout:754: done (0 left) 2024-09-21T09:41:01.105 INFO:tasks.rados.rados.0.smithi017.stdout:755: snap_remove snap 91 2024-09-21T09:41:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:00 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/4275860173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:00 smithi017 ceph-mon[29380]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:41:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:00 smithi017 ceph-mon[29380]: mgrmap e30: y(active, since 1.01714s) 2024-09-21T09:41:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:00 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/4275860173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:00 smithi017 ceph-mon[40737]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:41:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:00 smithi017 ceph-mon[40737]: mgrmap e30: y(active, since 1.01714s) 2024-09-21T09:41:02.100 INFO:tasks.rados.rados.0.smithi017.stdout:755: done (0 left) 2024-09-21T09:41:02.101 INFO:tasks.rados.rados.0.smithi017.stdout:756: read oid 41 snap 76 2024-09-21T09:41:02.101 INFO:tasks.rados.rados.0.smithi017.stdout:756: expect (ObjNum 175 snap 47 seq_num 175) 2024-09-21T09:41:02.101 INFO:tasks.rados.rados.0.smithi017.stdout:757: write oid 14 current snap is 95 2024-09-21T09:41:02.101 INFO:tasks.rados.rados.0.smithi017.stdout:757: seq_num 320 ranges {578240=566506,1609179=558098,2804887=551750} 2024-09-21T09:41:02.107 INFO:tasks.rados.rados.0.smithi017.stdout:757: writing smithi01785587-14 from 578240 to 1144746 tid 1 2024-09-21T09:41:02.113 INFO:tasks.rados.rados.0.smithi017.stdout:757: writing smithi01785587-14 from 1609179 to 2167277 tid 2 2024-09-21T09:41:02.121 INFO:tasks.rados.rados.0.smithi017.stdout:757: writing smithi01785587-14 from 2804887 to 3356637 tid 3 2024-09-21T09:41:02.121 INFO:tasks.rados.rados.0.smithi017.stdout:758: read oid 10 snap 86 2024-09-21T09:41:02.121 INFO:tasks.rados.rados.0.smithi017.stdout:758: expect deleted 2024-09-21T09:41:02.121 INFO:tasks.rados.rados.0.smithi017.stdout:759: read oid 6 snap -1 2024-09-21T09:41:02.121 INFO:tasks.rados.rados.0.smithi017.stdout:759: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:41:02.121 INFO:tasks.rados.rados.0.smithi017.stdout:760: snap_create 2024-09-21T09:41:02.121 INFO:tasks.rados.rados.0.smithi017.stdout:757: finishing write tid 1 to smithi01785587-14 2024-09-21T09:41:02.121 INFO:tasks.rados.rados.0.smithi017.stdout:756: expect (ObjNum 175 snap 47 seq_num 175) 2024-09-21T09:41:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[29380]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[29380]: pgmap v4: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[29380]: from='client.25364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[29380]: from='client.25370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[29380]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:41:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[29380]: mgrmap e31: y(active, since 2s) 2024-09-21T09:41:02.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[40737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:02.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[40737]: pgmap v4: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:02.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[40737]: from='client.25364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[40737]: from='client.25370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[40737]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-mon[40737]: mgrmap e31: y(active, since 2s) 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:01.784+0000 7f55097ae640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:41:01] ENGINE Error in HTTPServer.serve 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: Traceback (most recent call last): 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: self._connections.run(self.expiration_interval) 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: self._run(expiration_interval) 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:41:02.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:41:02.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: s = self.context.wrap_socket( 2024-09-21T09:41:02.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:41:02.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: return self.sslsocket_class._create( 2024-09-21T09:41:02.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:41:02.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: self.do_handshake() 2024-09-21T09:41:02.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:41:02.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: self._sslobj.do_handshake() 2024-09-21T09:41:02.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:41:02.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:01 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:02.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:01 smithi134 ceph-mon[33541]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:02.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:01 smithi134 ceph-mon[33541]: pgmap v4: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:02.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:01 smithi134 ceph-mon[33541]: from='client.25364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:02.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:01 smithi134 ceph-mon[33541]: from='client.25370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:01 smithi134 ceph-mon[33541]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:41:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:01 smithi134 ceph-mon[33541]: mgrmap e31: y(active, since 2s) 2024-09-21T09:41:02.289 INFO:tasks.rados.rados.0.smithi017.stdout:757: finishing write tid 2 to smithi01785587-14 2024-09-21T09:41:02.289 INFO:tasks.rados.rados.0.smithi017.stdout:757: finishing write tid 3 to smithi01785587-14 2024-09-21T09:41:02.290 INFO:tasks.rados.rados.0.smithi017.stdout:757: finishing write tid 4 to smithi01785587-14 2024-09-21T09:41:02.290 INFO:tasks.rados.rados.0.smithi017.stdout:757: finishing write tid 5 to smithi01785587-14 2024-09-21T09:41:02.290 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 380 (ObjNum 320 snap 95 seq_num 320) dirty exists 2024-09-21T09:41:02.290 INFO:tasks.rados.rados.0.smithi017.stdout:757: left oid 14 (ObjNum 320 snap 95 seq_num 320) 2024-09-21T09:41:02.290 INFO:tasks.rados.rados.0.smithi017.stdout:759: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:41:02.705 INFO:tasks.rados.rados.0.smithi017.stdout:756: done (4 left) 2024-09-21T09:41:02.705 INFO:tasks.rados.rados.0.smithi017.stdout:757: done (3 left) 2024-09-21T09:41:02.705 INFO:tasks.rados.rados.0.smithi017.stdout:758: done (2 left) 2024-09-21T09:41:02.706 INFO:tasks.rados.rados.0.smithi017.stdout:759: done (1 left) 2024-09-21T09:41:02.706 INFO:tasks.rados.rados.0.smithi017.stdout:760: done (0 left) 2024-09-21T09:41:02.706 INFO:tasks.rados.rados.0.smithi017.stdout:761: snap_create 2024-09-21T09:41:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:01] ENGINE Bus STARTING 2024-09-21T09:41:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:01] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:41:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:01] ENGINE Error in HTTPServer.serve 2024-09-21T09:41:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:41:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:41:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:41:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: self.do_handshake() 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:01] ENGINE Serving on http://172.21.15.17:8765 2024-09-21T09:41:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:01] ENGINE Bus STARTED 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[40737]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[40737]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[29380]: [21/Sep/2024:09:41:01] ENGINE Bus STARTING 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[29380]: [21/Sep/2024:09:41:01] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[29380]: [21/Sep/2024:09:41:01] ENGINE Error in HTTPServer.serve 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:41:03.202 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[29380]: [21/Sep/2024:09:41:01] ENGINE Serving on http://172.21.15.17:8765 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[29380]: [21/Sep/2024:09:41:01] ENGINE Bus STARTED 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[29380]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:41:03.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:02 smithi017 ceph-mon[29380]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:41:03.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:02 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:01] ENGINE Bus STARTING 2024-09-21T09:41:03.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:02 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:01] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:41:03.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:02 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:01] ENGINE Error in HTTPServer.serve 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: Traceback (most recent call last): 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._run(expiration_interval) 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: s = self.context.wrap_socket( 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: return self.sslsocket_class._create( 2024-09-21T09:41:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:41:03.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: self.do_handshake() 2024-09-21T09:41:03.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:41:03.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._sslobj.do_handshake() 2024-09-21T09:41:03.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:41:03.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:02 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:01] ENGINE Serving on http://172.21.15.17:8765 2024-09-21T09:41:03.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:02 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:01] ENGINE Bus STARTED 2024-09-21T09:41:03.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:02 smithi134 ceph-mon[33541]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:41:03.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:02 smithi134 ceph-mon[33541]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:41:03.332 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:03.706 INFO:tasks.rados.rados.0.smithi017.stdout:761: done (0 left) 2024-09-21T09:41:03.706 INFO:tasks.rados.rados.0.smithi017.stdout:762: write oid 42 current snap is 97 2024-09-21T09:41:03.706 INFO:tasks.rados.rados.0.smithi017.stdout:762: seq_num 321 ranges {595047=641755,1667661=418109,2548663=763022} 2024-09-21T09:41:03.713 INFO:tasks.rados.rados.0.smithi017.stdout:762: writing smithi01785587-42 from 595047 to 1236802 tid 1 2024-09-21T09:41:03.718 INFO:tasks.rados.rados.0.smithi017.stdout:762: writing smithi01785587-42 from 1667661 to 2085770 tid 2 2024-09-21T09:41:03.726 INFO:tasks.rados.rados.0.smithi017.stdout:762: writing smithi01785587-42 from 2548663 to 3311685 tid 3 2024-09-21T09:41:03.726 INFO:tasks.rados.rados.0.smithi017.stdout:763: read oid 4 snap 76 2024-09-21T09:41:03.726 INFO:tasks.rados.rados.0.smithi017.stdout:763: expect (ObjNum 245 snap 72 seq_num 245) 2024-09-21T09:41:03.726 INFO:tasks.rados.rados.0.smithi017.stdout:764: rollback oid 35 current snap is 97 2024-09-21T09:41:03.726 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 82 2024-09-21T09:41:03.726 INFO:tasks.rados.rados.0.smithi017.stdout:762: finishing write tid 1 to smithi01785587-42 2024-09-21T09:41:03.726 INFO:tasks.rados.rados.0.smithi017.stdout:762: finishing write tid 2 to smithi01785587-42 2024-09-21T09:41:03.727 INFO:tasks.rados.rados.0.smithi017.stdout:765: snap_create 2024-09-21T09:41:03.727 INFO:tasks.rados.rados.0.smithi017.stdout:764: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:41:03.727 INFO:tasks.rados.rados.0.smithi017.stdout:764: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:41:03.728 INFO:tasks.rados.rados.0.smithi017.stdout:764: finishing rollback tid 2 to smithi01785587-35 2024-09-21T09:41:03.728 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 89 (ObjNum 196 snap 54 seq_num 196) dirty exists 2024-09-21T09:41:03.729 INFO:tasks.rados.rados.0.smithi017.stdout:762: finishing write tid 3 to smithi01785587-42 2024-09-21T09:41:03.729 INFO:tasks.rados.rados.0.smithi017.stdout:762: finishing write tid 4 to smithi01785587-42 2024-09-21T09:41:03.729 INFO:tasks.rados.rados.0.smithi017.stdout:762: finishing write tid 5 to smithi01785587-42 2024-09-21T09:41:03.729 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 243 (ObjNum 321 snap 97 seq_num 321) dirty exists 2024-09-21T09:41:03.729 INFO:tasks.rados.rados.0.smithi017.stdout:762: left oid 42 (ObjNum 321 snap 97 seq_num 321) 2024-09-21T09:41:03.732 INFO:tasks.rados.rados.0.smithi017.stdout:763: expect (ObjNum 245 snap 72 seq_num 245) 2024-09-21T09:41:04.128 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:03 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3762993579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:04.128 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:03 smithi134 ceph-mon[33541]: mgrmap e32: y(active, since 4s) 2024-09-21T09:41:04.128 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:03 smithi134 ceph-mon[33541]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:41:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:03 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3762993579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:03 smithi017 ceph-mon[29380]: mgrmap e32: y(active, since 4s) 2024-09-21T09:41:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:03 smithi017 ceph-mon[29380]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:41:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:03 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3762993579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:03 smithi017 ceph-mon[40737]: mgrmap e32: y(active, since 4s) 2024-09-21T09:41:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:03 smithi017 ceph-mon[40737]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:41:04.382 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:04.127Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.134:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.134:8765: connect: connection refused" 2024-09-21T09:41:04.382 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:04.127Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.134:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.134:8765: connect: connection refused" 2024-09-21T09:41:04.382 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:04.130Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.134:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.134:8765: connect: connection refused" 2024-09-21T09:41:04.382 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:04.130Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.134:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.134:8765: connect: connection refused" 2024-09-21T09:41:04.382 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:04.130Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.134:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.134:8765: connect: connection refused" 2024-09-21T09:41:04.382 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:04.130Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.134:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.134:8765: connect: connection refused" 2024-09-21T09:41:04.707 INFO:tasks.rados.rados.0.smithi017.stdout:762: done (3 left) 2024-09-21T09:41:04.708 INFO:tasks.rados.rados.0.smithi017.stdout:763: done (2 left) 2024-09-21T09:41:04.708 INFO:tasks.rados.rados.0.smithi017.stdout:764: done (1 left) 2024-09-21T09:41:04.708 INFO:tasks.rados.rados.0.smithi017.stdout:765: done (0 left) 2024-09-21T09:41:04.708 INFO:tasks.rados.rados.0.smithi017.stdout:766: write oid 19 current snap is 98 2024-09-21T09:41:04.708 INFO:tasks.rados.rados.0.smithi017.stdout:766: seq_num 322 ranges {611854=717004,2126143=561767,3259733=152121} 2024-09-21T09:41:04.715 INFO:tasks.rados.rados.0.smithi017.stdout:766: writing smithi01785587-19 from 611854 to 1328858 tid 1 2024-09-21T09:41:04.723 INFO:tasks.rados.rados.0.smithi017.stdout:766: writing smithi01785587-19 from 2126143 to 2687910 tid 2 2024-09-21T09:41:04.727 INFO:tasks.rados.rados.0.smithi017.stdout:766: writing smithi01785587-19 from 3259733 to 3411854 tid 3 2024-09-21T09:41:04.727 INFO:tasks.rados.rados.0.smithi017.stdout:767: write oid 44 current snap is 98 2024-09-21T09:41:04.727 INFO:tasks.rados.rados.0.smithi017.stdout:767: seq_num 323 ranges {628661=792253,2184625=421778,3403509=25152} 2024-09-21T09:41:04.735 INFO:tasks.rados.rados.0.smithi017.stdout:767: writing smithi01785587-44 from 628661 to 1420914 tid 1 2024-09-21T09:41:04.741 INFO:tasks.rados.rados.0.smithi017.stdout:767: writing smithi01785587-44 from 2184625 to 2606403 tid 2 2024-09-21T09:41:04.743 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:41:04] ENGINE Bus STARTING 2024-09-21T09:41:04.743 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: CherryPy Checker: 2024-09-21T09:41:04.743 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: The Application mounted at '' has an empty config. 2024-09-21T09:41:04.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: 2024-09-21T09:41:04.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:41:04] ENGINE Serving on http://:::9283 2024-09-21T09:41:04.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: [21/Sep/2024:09:41:04] ENGINE Bus STARTED 2024-09-21T09:41:04.746 INFO:tasks.rados.rados.0.smithi017.stdout:767: writing smithi01785587-44 from 3403509 to 3428661 tid 3 2024-09-21T09:41:04.746 INFO:tasks.rados.rados.0.smithi017.stdout:768: snap_remove snap 81 2024-09-21T09:41:05.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: pgmap v9: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:05.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='client.25388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:05.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:05.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: Standby manager daemon x started 2024-09-21T09:41:05.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:41:05.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[29380]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: pgmap v9: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='client.25388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: Standby manager daemon x started 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:04 smithi017 ceph-mon[40737]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: pgmap v9: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='client.25388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: Standby manager daemon x started 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/2383238918' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:41:05.181 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.182 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.182 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.182 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:05.182 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:04 smithi134 ceph-mon[33541]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:41:05.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:05 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:05.179Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.134:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.134:8765: connect: connection refused" 2024-09-21T09:41:05.710 INFO:tasks.rados.rados.0.smithi017.stdout:768: done (2 left) 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:769: delete oid 41 current snap is 98 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:766: finishing write tid 1 to smithi01785587-19 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:766: finishing write tid 2 to smithi01785587-19 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:766: finishing write tid 3 to smithi01785587-19 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:766: finishing write tid 4 to smithi01785587-19 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:766: finishing write tid 5 to smithi01785587-19 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 399 (ObjNum 322 snap 98 seq_num 322) dirty exists 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:766: left oid 19 (ObjNum 322 snap 98 seq_num 322) 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:767: finishing write tid 1 to smithi01785587-44 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:767: finishing write tid 2 to smithi01785587-44 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:767: finishing write tid 3 to smithi01785587-44 2024-09-21T09:41:05.711 INFO:tasks.rados.rados.0.smithi017.stdout:767: finishing write tid 4 to smithi01785587-44 2024-09-21T09:41:05.712 INFO:tasks.rados.rados.0.smithi017.stdout:767: finishing write tid 5 to smithi01785587-44 2024-09-21T09:41:05.712 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 385 (ObjNum 323 snap 98 seq_num 323) dirty exists 2024-09-21T09:41:05.712 INFO:tasks.rados.rados.0.smithi017.stdout:767: left oid 44 (ObjNum 323 snap 98 seq_num 323) 2024-09-21T09:41:05.713 INFO:tasks.rados.rados.0.smithi017.stdout:766: done (2 left) 2024-09-21T09:41:05.713 INFO:tasks.rados.rados.0.smithi017.stdout:767: done (1 left) 2024-09-21T09:41:05.713 INFO:tasks.rados.rados.0.smithi017.stdout:769: done (0 left) 2024-09-21T09:41:05.713 INFO:tasks.rados.rados.0.smithi017.stdout:770: snap_remove snap 78 2024-09-21T09:41:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[29380]: mgrmap e33: y(active, since 5s), standbys: x 2024-09-21T09:41:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:41:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[29380]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:41:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[40737]: mgrmap e33: y(active, since 5s), standbys: x 2024-09-21T09:41:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:41:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:06.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:05 smithi017 ceph-mon[40737]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:41:06.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:05 smithi134 ceph-mon[33541]: mgrmap e33: y(active, since 5s), standbys: x 2024-09-21T09:41:06.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:05 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:41:06.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:05 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:05 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:05 smithi134 ceph-mon[33541]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:41:06.311 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:06.712 INFO:tasks.rados.rados.0.smithi017.stdout:770: done (0 left) 2024-09-21T09:41:06.712 INFO:tasks.rados.rados.0.smithi017.stdout:771: write oid 24 current snap is 98 2024-09-21T09:41:06.712 INFO:tasks.rados.rados.0.smithi017.stdout:771: seq_num 325 ranges {662275=542751,1785236=425447,2942002=520273} 2024-09-21T09:41:06.719 INFO:tasks.rados.rados.0.smithi017.stdout:771: writing smithi01785587-24 from 662275 to 1205026 tid 1 2024-09-21T09:41:06.725 INFO:tasks.rados.rados.0.smithi017.stdout:771: writing smithi01785587-24 from 1785236 to 2210683 tid 2 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:771: writing smithi01785587-24 from 2942002 to 3462275 tid 3 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:772: read oid 19 snap 86 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:772: expect deleted 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:773: read oid 3 snap 82 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:773: expect deleted 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:774: read oid 43 snap -1 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:774: expect (ObjNum 199 snap 56 seq_num 199) 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:775: read oid 46 snap -1 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:775: expect (ObjNum 210 snap 60 seq_num 210) 2024-09-21T09:41:06.733 INFO:tasks.rados.rados.0.smithi017.stdout:776: rollback oid 4 current snap is 98 2024-09-21T09:41:06.734 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 4 to 95 2024-09-21T09:41:06.734 INFO:tasks.rados.rados.0.smithi017.stdout:771: finishing write tid 1 to smithi01785587-24 2024-09-21T09:41:06.734 INFO:tasks.rados.rados.0.smithi017.stdout:771: finishing write tid 2 to smithi01785587-24 2024-09-21T09:41:06.734 INFO:tasks.rados.rados.0.smithi017.stdout:777: write oid 18 current snap is 98 2024-09-21T09:41:06.734 INFO:tasks.rados.rados.0.smithi017.stdout:777: seq_num 326 ranges {679082=618000,1843718=569105,2853072=626010} 2024-09-21T09:41:06.740 INFO:tasks.rados.rados.0.smithi017.stdout:777: writing smithi01785587-18 from 679082 to 1297082 tid 1 2024-09-21T09:41:06.746 INFO:tasks.rados.rados.0.smithi017.stdout:777: writing smithi01785587-18 from 1843718 to 2412823 tid 2 2024-09-21T09:41:06.752 INFO:tasks.rados.rados.0.smithi017.stdout:777: writing smithi01785587-18 from 2853072 to 3479082 tid 3 2024-09-21T09:41:06.752 INFO:tasks.rados.rados.0.smithi017.stdout:778: snap_create 2024-09-21T09:41:06.752 INFO:tasks.rados.rados.0.smithi017.stdout:775: expect (ObjNum 210 snap 60 seq_num 210) 2024-09-21T09:41:06.873 INFO:tasks.rados.rados.0.smithi017.stdout:771: finishing write tid 3 to smithi01785587-24 2024-09-21T09:41:06.873 INFO:tasks.rados.rados.0.smithi017.stdout:771: finishing write tid 4 to smithi01785587-24 2024-09-21T09:41:06.873 INFO:tasks.rados.rados.0.smithi017.stdout:771: finishing write tid 5 to smithi01785587-24 2024-09-21T09:41:06.873 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 217 (ObjNum 325 snap 98 seq_num 325) dirty exists 2024-09-21T09:41:06.873 INFO:tasks.rados.rados.0.smithi017.stdout:771: left oid 24 (ObjNum 325 snap 98 seq_num 325) 2024-09-21T09:41:06.873 INFO:tasks.rados.rados.0.smithi017.stdout:774: expect (ObjNum 199 snap 56 seq_num 199) 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:776: finishing rollback tid 0 to smithi01785587-4 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:776: finishing rollback tid 1 to smithi01785587-4 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:776: finishing rollback tid 2 to smithi01785587-4 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 410 (ObjNum 312 snap 93 seq_num 312) dirty exists 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:777: finishing write tid 1 to smithi01785587-18 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:777: finishing write tid 2 to smithi01785587-18 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:777: finishing write tid 3 to smithi01785587-18 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:777: finishing write tid 4 to smithi01785587-18 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:777: finishing write tid 5 to smithi01785587-18 2024-09-21T09:41:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 52 (ObjNum 326 snap 98 seq_num 326) dirty exists 2024-09-21T09:41:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:777: left oid 18 (ObjNum 326 snap 98 seq_num 326) 2024-09-21T09:41:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:06 smithi017 ceph-mon[40737]: pgmap v12: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:06 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/58554158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:06 smithi017 ceph-mon[40737]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:41:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:06 smithi017 ceph-mon[29380]: pgmap v12: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:06 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/58554158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:06 smithi017 ceph-mon[29380]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:41:07.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:06 smithi134 ceph-mon[33541]: pgmap v12: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:06 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/58554158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:06 smithi134 ceph-mon[33541]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:41:07.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:06 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:07.712 INFO:tasks.rados.rados.0.smithi017.stdout:771: done (7 left) 2024-09-21T09:41:07.713 INFO:tasks.rados.rados.0.smithi017.stdout:772: done (6 left) 2024-09-21T09:41:07.713 INFO:tasks.rados.rados.0.smithi017.stdout:773: done (5 left) 2024-09-21T09:41:07.713 INFO:tasks.rados.rados.0.smithi017.stdout:774: done (4 left) 2024-09-21T09:41:07.713 INFO:tasks.rados.rados.0.smithi017.stdout:775: done (3 left) 2024-09-21T09:41:07.713 INFO:tasks.rados.rados.0.smithi017.stdout:776: done (2 left) 2024-09-21T09:41:07.713 INFO:tasks.rados.rados.0.smithi017.stdout:777: done (1 left) 2024-09-21T09:41:07.713 INFO:tasks.rados.rados.0.smithi017.stdout:778: done (0 left) 2024-09-21T09:41:07.713 INFO:tasks.rados.rados.0.smithi017.stdout:779: snap_remove snap 94 2024-09-21T09:41:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[29380]: from='client.25406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[29380]: from='client.15519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[29380]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:41:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:08.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[40737]: from='client.25406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[40737]: from='client.15519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[40737]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:41:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:07 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:08.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:07 smithi134 ceph-mon[33541]: from='client.25406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:08.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:07 smithi134 ceph-mon[33541]: from='client.15519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:08.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:07 smithi134 ceph-mon[33541]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:41:08.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:07 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:07 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:07 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:07 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:08.715 INFO:tasks.rados.rados.0.smithi017.stdout:779: done (0 left) 2024-09-21T09:41:08.716 INFO:tasks.rados.rados.0.smithi017.stdout:780: read oid 47 snap 98 2024-09-21T09:41:08.716 INFO:tasks.rados.rados.0.smithi017.stdout:780: expect (ObjNum 307 snap 91 seq_num 307) 2024-09-21T09:41:08.716 INFO:tasks.rados.rados.0.smithi017.stdout:781: read oid 28 snap 96 2024-09-21T09:41:08.716 INFO:tasks.rados.rados.0.smithi017.stdout:781: expect (ObjNum 293 snap 90 seq_num 293) 2024-09-21T09:41:08.716 INFO:tasks.rados.rados.0.smithi017.stdout:782: snap_remove snap 60 2024-09-21T09:41:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: pgmap v15: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.9 MiB/s wr, 33 op/s 2024-09-21T09:41:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:41:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:41:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:09.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: Reconfiguring daemon prometheus.a on smithi134 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[40737]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: pgmap v15: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.9 MiB/s wr, 33 op/s 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:09.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: Reconfiguring daemon prometheus.a on smithi134 2024-09-21T09:41:09.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:08 smithi017 ceph-mon[29380]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: pgmap v15: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.9 MiB/s wr, 33 op/s 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:09.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:41:09.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: Reconfiguring daemon prometheus.a on smithi134 2024-09-21T09:41:09.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:08 smithi134 ceph-mon[33541]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:41:09.275 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:09.716 INFO:tasks.rados.rados.0.smithi017.stdout:782: done (2 left) 2024-09-21T09:41:09.716 INFO:tasks.rados.rados.0.smithi017.stdout:783: write oid 36 current snap is 99 2024-09-21T09:41:09.716 INFO:tasks.rados.rados.0.smithi017.stdout:783: seq_num 327 ranges {695889=409602,1618553=712763,2996848=499041} 2024-09-21T09:41:09.722 INFO:tasks.rados.rados.0.smithi017.stdout:783: writing smithi01785587-36 from 695889 to 1105491 tid 1 2024-09-21T09:41:09.730 INFO:tasks.rados.rados.0.smithi017.stdout:783: writing smithi01785587-36 from 1618553 to 2331316 tid 2 2024-09-21T09:41:09.737 INFO:tasks.rados.rados.0.smithi017.stdout:783: writing smithi01785587-36 from 2996848 to 3495889 tid 3 2024-09-21T09:41:09.737 INFO:tasks.rados.rados.0.smithi017.stdout:784: snap_create 2024-09-21T09:41:09.737 INFO:tasks.rados.rados.0.smithi017.stdout:780: expect (ObjNum 307 snap 91 seq_num 307) 2024-09-21T09:41:09.889 INFO:tasks.rados.rados.0.smithi017.stdout:781: expect (ObjNum 293 snap 90 seq_num 293) 2024-09-21T09:41:10.017 INFO:tasks.rados.rados.0.smithi017.stdout:783: finishing write tid 1 to smithi01785587-36 2024-09-21T09:41:10.017 INFO:tasks.rados.rados.0.smithi017.stdout:783: finishing write tid 2 to smithi01785587-36 2024-09-21T09:41:10.018 INFO:tasks.rados.rados.0.smithi017.stdout:783: finishing write tid 3 to smithi01785587-36 2024-09-21T09:41:10.018 INFO:tasks.rados.rados.0.smithi017.stdout:783: finishing write tid 4 to smithi01785587-36 2024-09-21T09:41:10.018 INFO:tasks.rados.rados.0.smithi017.stdout:783: finishing write tid 5 to smithi01785587-36 2024-09-21T09:41:10.018 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 353 (ObjNum 327 snap 99 seq_num 327) dirty exists 2024-09-21T09:41:10.018 INFO:tasks.rados.rados.0.smithi017.stdout:783: left oid 36 (ObjNum 327 snap 99 seq_num 327) 2024-09-21T09:41:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1942004792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[29380]: pgmap v18: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.9 MiB/s wr, 33 op/s 2024-09-21T09:41:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[29380]: from='client.15531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[29380]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[29380]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:41:10.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1942004792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:10.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[40737]: pgmap v18: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.9 MiB/s wr, 33 op/s 2024-09-21T09:41:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[40737]: from='client.15531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[40737]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:09 smithi017 ceph-mon[40737]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:41:10.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:09 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1942004792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:10.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:09 smithi134 ceph-mon[33541]: pgmap v18: 89 pgs: 89 active+clean; 212 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.9 MiB/s wr, 33 op/s 2024-09-21T09:41:10.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:09 smithi134 ceph-mon[33541]: from='client.15531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:10.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:09 smithi134 ceph-mon[33541]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:10.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:09 smithi134 ceph-mon[33541]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:41:10.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:10.717 INFO:tasks.rados.rados.0.smithi017.stdout:780: done (3 left) 2024-09-21T09:41:10.717 INFO:tasks.rados.rados.0.smithi017.stdout:781: done (2 left) 2024-09-21T09:41:10.717 INFO:tasks.rados.rados.0.smithi017.stdout:783: done (1 left) 2024-09-21T09:41:10.717 INFO:tasks.rados.rados.0.smithi017.stdout:784: done (0 left) 2024-09-21T09:41:10.717 INFO:tasks.rados.rados.0.smithi017.stdout:785: delete oid 3 current snap is 100 2024-09-21T09:41:10.718 INFO:tasks.rados.rados.0.smithi017.stdout:785: done (0 left) 2024-09-21T09:41:10.718 INFO:tasks.rados.rados.0.smithi017.stdout:786: delete oid 4 current snap is 100 2024-09-21T09:41:10.720 INFO:tasks.rados.rados.0.smithi017.stdout:786: done (0 left) 2024-09-21T09:41:10.720 INFO:tasks.rados.rados.0.smithi017.stdout:787: read oid 46 snap 99 2024-09-21T09:41:10.720 INFO:tasks.rados.rados.0.smithi017.stdout:787: expect (ObjNum 210 snap 60 seq_num 210) 2024-09-21T09:41:10.721 INFO:tasks.rados.rados.0.smithi017.stdout:788: delete oid 9 current snap is 100 2024-09-21T09:41:10.722 INFO:tasks.rados.rados.0.smithi017.stdout:788: done (1 left) 2024-09-21T09:41:10.722 INFO:tasks.rados.rados.0.smithi017.stdout:789: read oid 3 snap 88 2024-09-21T09:41:10.722 INFO:tasks.rados.rados.0.smithi017.stdout:789: expect deleted 2024-09-21T09:41:10.723 INFO:tasks.rados.rados.0.smithi017.stdout:790: read oid 39 snap -1 2024-09-21T09:41:10.723 INFO:tasks.rados.rados.0.smithi017.stdout:790: expect deleted 2024-09-21T09:41:10.723 INFO:tasks.rados.rados.0.smithi017.stdout:791: snap_remove snap 95 2024-09-21T09:41:11.723 INFO:tasks.rados.rados.0.smithi017.stdout:791: done (3 left) 2024-09-21T09:41:11.723 INFO:tasks.rados.rados.0.smithi017.stdout:792: delete oid 15 current snap is 100 2024-09-21T09:41:11.724 INFO:tasks.rados.rados.0.smithi017.stdout:787: expect (ObjNum 210 snap 60 seq_num 210) 2024-09-21T09:41:11.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:11 smithi134 ceph-mon[33541]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:41:11.843 INFO:tasks.rados.rados.0.smithi017.stdout:787: done (3 left) 2024-09-21T09:41:11.843 INFO:tasks.rados.rados.0.smithi017.stdout:789: done (2 left) 2024-09-21T09:41:11.843 INFO:tasks.rados.rados.0.smithi017.stdout:790: done (1 left) 2024-09-21T09:41:11.843 INFO:tasks.rados.rados.0.smithi017.stdout:792: done (0 left) 2024-09-21T09:41:11.843 INFO:tasks.rados.rados.0.smithi017.stdout:793: write oid 6 current snap is 100 2024-09-21T09:41:11.843 INFO:tasks.rados.rados.0.smithi017.stdout:793: seq_num 332 ranges {779924=785847,2078257=580112,3301257=278667} 2024-09-21T09:41:11.851 INFO:tasks.rados.rados.0.smithi017.stdout:793: writing smithi01785587-6 from 779924 to 1565771 tid 1 2024-09-21T09:41:11.857 INFO:tasks.rados.rados.0.smithi017.stdout:793: writing smithi01785587-6 from 2078257 to 2658369 tid 2 2024-09-21T09:41:11.863 INFO:tasks.rados.rados.0.smithi017.stdout:793: writing smithi01785587-6 from 3301257 to 3579924 tid 3 2024-09-21T09:41:11.863 INFO:tasks.rados.rados.0.smithi017.stdout:794: write oid 11 current snap is 100 2024-09-21T09:41:11.863 INFO:tasks.rados.rados.0.smithi017.stdout:794: seq_num 333 ranges {796731=461096,1736739=723770,2928680=668051} 2024-09-21T09:41:11.869 INFO:tasks.rados.rados.0.smithi017.stdout:794: writing smithi01785587-11 from 796731 to 1257827 tid 1 2024-09-21T09:41:11.876 INFO:tasks.rados.rados.0.smithi017.stdout:794: writing smithi01785587-11 from 1736739 to 2460509 tid 2 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:794: writing smithi01785587-11 from 2928680 to 3596731 tid 3 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:795: rollback oid 37 current snap is 100 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 99 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:793: finishing write tid 1 to smithi01785587-6 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:793: finishing write tid 2 to smithi01785587-6 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:793: finishing write tid 3 to smithi01785587-6 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:793: finishing write tid 4 to smithi01785587-6 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:793: finishing write tid 5 to smithi01785587-6 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 221 (ObjNum 332 snap 100 seq_num 332) dirty exists 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:793: left oid 6 (ObjNum 332 snap 100 seq_num 332) 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:794: finishing write tid 1 to smithi01785587-11 2024-09-21T09:41:11.883 INFO:tasks.rados.rados.0.smithi017.stdout:794: finishing write tid 2 to smithi01785587-11 2024-09-21T09:41:11.884 INFO:tasks.rados.rados.0.smithi017.stdout:793: done (2 left) 2024-09-21T09:41:11.884 INFO:tasks.rados.rados.0.smithi017.stdout:796: snap_create 2024-09-21T09:41:11.885 INFO:tasks.rados.rados.0.smithi017.stdout:795: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:41:11.887 INFO:tasks.rados.rados.0.smithi017.stdout:794: finishing write tid 3 to smithi01785587-11 2024-09-21T09:41:11.888 INFO:tasks.rados.rados.0.smithi017.stdout:794: finishing write tid 4 to smithi01785587-11 2024-09-21T09:41:11.888 INFO:tasks.rados.rados.0.smithi017.stdout:795: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:41:11.888 INFO:tasks.rados.rados.0.smithi017.stdout:795: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:41:11.888 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 359 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:41:11.888 INFO:tasks.rados.rados.0.smithi017.stdout:794: finishing write tid 5 to smithi01785587-11 2024-09-21T09:41:11.888 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 406 (ObjNum 333 snap 100 seq_num 333) dirty exists 2024-09-21T09:41:11.888 INFO:tasks.rados.rados.0.smithi017.stdout:794: left oid 11 (ObjNum 333 snap 100 seq_num 333) 2024-09-21T09:41:12.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:11 smithi017 ceph-mon[29380]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:41:12.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:11 smithi017 ceph-mon[40737]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:41:12.202 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:11 smithi134 systemd[1]: Stopping Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:12.202 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[68800]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:41:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-09-21T09:41:12.238 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.201Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.201Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.201Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.201Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.201Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.201Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.201Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.201Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.201Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-21T09:41:12.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.203Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:41:12.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.203Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-21T09:41:12.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[77027]: ts=2024-09-21T09:41:12.203Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-21T09:41:12.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 podman[81072]: 2024-09-21 09:41:12.211731663 +0000 UTC m=+0.133853525 container died 9774662fd5df84505127f32125b1d520fb82cb80df489f49609ff382f88cad65 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:12.707 INFO:tasks.rados.rados.0.smithi017.stdout:794: done (2 left) 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi017.stdout:795: done (1 left) 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi017.stdout:796: done (0 left) 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi017.stdout:797: snap_remove snap 86 2024-09-21T09:41:12.867 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 podman[81072]: 2024-09-21 09:41:12.590299227 +0000 UTC m=+0.512421076 container cleanup 9774662fd5df84505127f32125b1d520fb82cb80df489f49609ff382f88cad65 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:12.868 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 bash[81072]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:41:12.868 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-mon[33541]: pgmap v21: 89 pgs: 89 active+clean; 184 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:41:12.868 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-mon[33541]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:41:12.868 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/1336058200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:12.868 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:12 smithi134 ceph-mon[33541]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:41:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:12 smithi017 ceph-mon[40737]: pgmap v21: 89 pgs: 89 active+clean; 184 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:41:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:12 smithi017 ceph-mon[40737]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:41:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:12 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/1336058200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:13.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:12 smithi017 ceph-mon[40737]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:41:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:12 smithi017 ceph-mon[29380]: pgmap v21: 89 pgs: 89 active+clean; 184 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:41:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:12 smithi017 ceph-mon[29380]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:41:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:12 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/1336058200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:12 smithi017 ceph-mon[29380]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:41:13.229 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:12 smithi134 podman[81089]: 2024-09-21 09:41:12.86668171 +0000 UTC m=+0.656064239 container remove 9774662fd5df84505127f32125b1d520fb82cb80df489f49609ff382f88cad65 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:13.229 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:41:13.229 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 systemd[1]: Stopped Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:13.229 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.301s CPU time. 2024-09-21T09:41:13.229 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 systemd[1]: Starting Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:13.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 podman[81188]: 2024-09-21 09:41:13.131631079 +0000 UTC m=+0.016743033 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-21T09:41:13.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 podman[81188]: 2024-09-21 09:41:13.410720888 +0000 UTC m=+0.295832828 container create 061a3294723d9bba519864e40145c126bd16d472e0cd54dc709160e191fbfd2e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:13.711 INFO:tasks.rados.rados.0.smithi017.stdout:797: done (0 left) 2024-09-21T09:41:13.711 INFO:tasks.rados.rados.0.smithi017.stdout:798: read oid 14 snap -1 2024-09-21T09:41:13.711 INFO:tasks.rados.rados.0.smithi017.stdout:798: expect (ObjNum 320 snap 95 seq_num 320) 2024-09-21T09:41:13.711 INFO:tasks.rados.rados.0.smithi017.stdout:799: write oid 30 current snap is 101 2024-09-21T09:41:13.711 INFO:tasks.rados.rados.0.smithi017.stdout:799: seq_num 334 ranges {413538=536345,1395221=583781,2556103=658617} 2024-09-21T09:41:13.717 INFO:tasks.rados.rados.0.smithi017.stdout:799: writing smithi01785587-30 from 413538 to 949883 tid 1 2024-09-21T09:41:13.725 INFO:tasks.rados.rados.0.smithi017.stdout:799: writing smithi01785587-30 from 1395221 to 1979002 tid 2 2024-09-21T09:41:13.733 INFO:tasks.rados.rados.0.smithi017.stdout:799: writing smithi01785587-30 from 2556103 to 3214720 tid 3 2024-09-21T09:41:13.733 INFO:tasks.rados.rados.0.smithi017.stdout:800: snap_create 2024-09-21T09:41:13.733 INFO:tasks.rados.rados.0.smithi017.stdout:799: finishing write tid 1 to smithi01785587-30 2024-09-21T09:41:13.733 INFO:tasks.rados.rados.0.smithi017.stdout:798: expect (ObjNum 320 snap 95 seq_num 320) 2024-09-21T09:41:13.845 INFO:tasks.rados.rados.0.smithi017.stdout:799: finishing write tid 2 to smithi01785587-30 2024-09-21T09:41:13.846 INFO:tasks.rados.rados.0.smithi017.stdout:799: finishing write tid 3 to smithi01785587-30 2024-09-21T09:41:13.846 INFO:tasks.rados.rados.0.smithi017.stdout:799: finishing write tid 4 to smithi01785587-30 2024-09-21T09:41:13.846 INFO:tasks.rados.rados.0.smithi017.stdout:799: finishing write tid 5 to smithi01785587-30 2024-09-21T09:41:13.846 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 416 (ObjNum 334 snap 101 seq_num 334) dirty exists 2024-09-21T09:41:13.846 INFO:tasks.rados.rados.0.smithi017.stdout:799: left oid 30 (ObjNum 334 snap 101 seq_num 334) 2024-09-21T09:41:13.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-mon[33541]: from='client.25442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-mon[33541]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-mon[33541]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:41:13.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 podman[81188]: 2024-09-21 09:41:13.662410055 +0000 UTC m=+0.547522003 container init 061a3294723d9bba519864e40145c126bd16d472e0cd54dc709160e191fbfd2e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:13.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 podman[81188]: 2024-09-21 09:41:13.667476351 +0000 UTC m=+0.552588295 container start 061a3294723d9bba519864e40145c126bd16d472e0cd54dc709160e191fbfd2e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:13.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.686Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-21T09:41:13.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.686Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-21T09:41:13.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.686Z caller=main.go:623 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi134 (none))" 2024-09-21T09:41:13.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.686Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.686Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.688Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.688Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.690Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.690Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.692Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.692Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.594µs 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.692Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.692Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.703Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.713Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.713Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-21T09:41:13.995 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.713Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.032µs wal_replay_duration=21.885824ms wbl_replay_duration=126ns total_replay_duration=21.929282ms 2024-09-21T09:41:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.720Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:41:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.720Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-21T09:41:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.720Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:41:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.738Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.762896ms db_storage=1.257µs remote_storage=1.544µs web_handler=516ns query_engine=1.004µs scrape=691.087µs scrape_sd=174.517µs notify=17.682µs notify_sd=15.893µs rules=16.235264ms tracing=6.131µs 2024-09-21T09:41:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.738Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-21T09:41:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:13.738Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:41:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 bash[81188]: 061a3294723d9bba519864e40145c126bd16d472e0cd54dc709160e191fbfd2e 2024-09-21T09:41:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:13 smithi134 systemd[1]: Started Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:13 smithi017 ceph-mon[29380]: from='client.25442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:13 smithi017 ceph-mon[29380]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:13 smithi017 ceph-mon[29380]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:41:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:13 smithi017 ceph-mon[40737]: from='client.25442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:13 smithi017 ceph-mon[40737]: from='client.15555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:13 smithi017 ceph-mon[40737]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:41:14.733 INFO:tasks.rados.rados.0.smithi017.stdout:798: done (2 left) 2024-09-21T09:41:14.733 INFO:tasks.rados.rados.0.smithi017.stdout:799: done (1 left) 2024-09-21T09:41:14.733 INFO:tasks.rados.rados.0.smithi017.stdout:800: done (0 left) 2024-09-21T09:41:14.733 INFO:tasks.rados.rados.0.smithi017.stdout:801: delete oid 35 current snap is 102 2024-09-21T09:41:14.736 INFO:tasks.rados.rados.0.smithi017.stdout:801: done (0 left) 2024-09-21T09:41:14.737 INFO:tasks.rados.rados.0.smithi017.stdout:802: write oid 17 current snap is 102 2024-09-21T09:41:14.737 INFO:tasks.rados.rados.0.smithi017.stdout:802: seq_num 336 ranges {447152=403196,1512185=587450,2727302=681161} 2024-09-21T09:41:14.741 INFO:tasks.rados.rados.0.smithi017.stdout:802: writing smithi01785587-17 from 447152 to 850348 tid 1 2024-09-21T09:41:14.748 INFO:tasks.rados.rados.0.smithi017.stdout:802: writing smithi01785587-17 from 1512185 to 2099635 tid 2 2024-09-21T09:41:14.755 INFO:tasks.rados.rados.0.smithi017.stdout:802: writing smithi01785587-17 from 2727302 to 3408463 tid 3 2024-09-21T09:41:14.755 INFO:tasks.rados.rados.0.smithi017.stdout:803: read oid 50 snap -1 2024-09-21T09:41:14.755 INFO:tasks.rados.rados.0.smithi017.stdout:803: expect (ObjNum 272 snap 84 seq_num 272) 2024-09-21T09:41:14.756 INFO:tasks.rados.rados.0.smithi017.stdout:804: write oid 1 current snap is 102 2024-09-21T09:41:14.756 INFO:tasks.rados.rados.0.smithi017.stdout:802: finishing write tid 1 to smithi01785587-17 2024-09-21T09:41:14.756 INFO:tasks.rados.rados.0.smithi017.stdout:804: seq_num 337 ranges {463959=478445,1570667=731108,3038372=492433} 2024-09-21T09:41:14.761 INFO:tasks.rados.rados.0.smithi017.stdout:804: writing smithi01785587-1 from 463959 to 942404 tid 1 2024-09-21T09:41:14.768 INFO:tasks.rados.rados.0.smithi017.stdout:804: writing smithi01785587-1 from 1570667 to 2301775 tid 2 2024-09-21T09:41:14.775 INFO:tasks.rados.rados.0.smithi017.stdout:804: writing smithi01785587-1 from 3038372 to 3530805 tid 3 2024-09-21T09:41:14.775 INFO:tasks.rados.rados.0.smithi017.stdout:805: snap_remove snap 76 2024-09-21T09:41:15.196 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:14 smithi134 ceph-mon[33541]: pgmap v24: 89 pgs: 89 active+clean; 184 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:41:15.196 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:14 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:15.196 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:14 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:15.196 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:14 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:15.196 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:14 smithi134 ceph-mon[33541]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:15.196 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:14 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:15.196 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:14 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:15.196 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:14 smithi134 ceph-mon[33541]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[29380]: pgmap v24: 89 pgs: 89 active+clean; 184 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[29380]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[29380]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[40737]: pgmap v24: 89 pgs: 89 active+clean; 184 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:15.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:15.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:15.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[40737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:15.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:15.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:15.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:14 smithi017 ceph-mon[40737]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:41:15.207 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:15.736 INFO:tasks.rados.rados.0.smithi017.stdout:805: done (3 left) 2024-09-21T09:41:15.736 INFO:tasks.rados.rados.0.smithi017.stdout:806: read oid 14 snap -1 2024-09-21T09:41:15.736 INFO:tasks.rados.rados.0.smithi017.stdout:806: expect (ObjNum 320 snap 95 seq_num 320) 2024-09-21T09:41:15.736 INFO:tasks.rados.rados.0.smithi017.stdout:807: snap_remove snap 82 2024-09-21T09:41:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/620817830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:41:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:41:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[29380]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:41:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/620817830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:41:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:41:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:15 smithi017 ceph-mon[40737]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:41:16.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:15 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/620817830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:16.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:15 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:41:16.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:15 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:41:16.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:15 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:16.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:15 smithi134 ceph-mon[33541]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:41:16.740 INFO:tasks.rados.rados.0.smithi017.stdout:807: done (4 left) 2024-09-21T09:41:16.740 INFO:tasks.rados.rados.0.smithi017.stdout:808: rollback oid 7 current snap is 102 2024-09-21T09:41:16.740 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 88 2024-09-21T09:41:16.740 INFO:tasks.rados.rados.0.smithi017.stdout:802: finishing write tid 2 to smithi01785587-17 2024-09-21T09:41:16.741 INFO:tasks.rados.rados.0.smithi017.stdout:803: expect (ObjNum 272 snap 84 seq_num 272) 2024-09-21T09:41:16.930 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:16 smithi134 ceph-mon[33541]: pgmap v27: 89 pgs: 89 active+clean; 184 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:16.930 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:16 smithi134 ceph-mon[33541]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:16.930 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:16 smithi134 ceph-mon[33541]: Upgrade: Updating mgr.x 2024-09-21T09:41:16.930 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:16 smithi134 ceph-mon[33541]: Deploying daemon mgr.x on smithi134 2024-09-21T09:41:16.930 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:16 smithi134 ceph-mon[33541]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:16.930 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:16 smithi134 ceph-mon[33541]: osdmap e265: 8 total, 8 up, 8 in 2024-09-21T09:41:16.931 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:16 smithi134 systemd[1]: Stopping Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:16.939 INFO:tasks.rados.rados.0.smithi017.stdout:802: finishing write tid 3 to smithi01785587-17 2024-09-21T09:41:16.939 INFO:tasks.rados.rados.0.smithi017.stdout:802: finishing write tid 4 to smithi01785587-17 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:802: finishing write tid 5 to smithi01785587-17 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 323 (ObjNum 336 snap 102 seq_num 336) dirty exists 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:802: left oid 17 (ObjNum 336 snap 102 seq_num 336) 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:804: finishing write tid 1 to smithi01785587-1 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:804: finishing write tid 2 to smithi01785587-1 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:804: finishing write tid 3 to smithi01785587-1 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:804: finishing write tid 4 to smithi01785587-1 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:804: finishing write tid 5 to smithi01785587-1 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 227 (ObjNum 337 snap 102 seq_num 337) dirty exists 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:804: left oid 1 (ObjNum 337 snap 102 seq_num 337) 2024-09-21T09:41:16.940 INFO:tasks.rados.rados.0.smithi017.stdout:806: expect (ObjNum 320 snap 95 seq_num 320) 2024-09-21T09:41:17.050 INFO:tasks.rados.rados.0.smithi017.stdout:808: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:41:17.050 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 318 (ObjNum 353 snap 0 seq_num 2399702832) dirty dne 2024-09-21T09:41:17.050 INFO:tasks.rados.rados.0.smithi017.stdout:802: done (4 left) 2024-09-21T09:41:17.050 INFO:tasks.rados.rados.0.smithi017.stdout:803: done (3 left) 2024-09-21T09:41:17.050 INFO:tasks.rados.rados.0.smithi017.stdout:804: done (2 left) 2024-09-21T09:41:17.050 INFO:tasks.rados.rados.0.smithi017.stdout:806: done (1 left) 2024-09-21T09:41:17.050 INFO:tasks.rados.rados.0.smithi017.stdout:808: done (0 left) 2024-09-21T09:41:17.050 INFO:tasks.rados.rados.0.smithi017.stdout:809: read oid 19 snap 92 2024-09-21T09:41:17.051 INFO:tasks.rados.rados.0.smithi017.stdout:809: expect deleted 2024-09-21T09:41:17.051 INFO:tasks.rados.rados.0.smithi017.stdout:810: rollback oid 47 current snap is 102 2024-09-21T09:41:17.051 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 47 to 88 2024-09-21T09:41:17.051 INFO:tasks.rados.rados.0.smithi017.stdout:811: rollback oid 9 current snap is 102 2024-09-21T09:41:17.051 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 9 to 100 2024-09-21T09:41:17.051 INFO:tasks.rados.rados.0.smithi017.stdout:812: snap_remove snap 93 2024-09-21T09:41:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[29380]: pgmap v27: 89 pgs: 89 active+clean; 184 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[29380]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[29380]: Upgrade: Updating mgr.x 2024-09-21T09:41:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[29380]: Deploying daemon mgr.x on smithi134 2024-09-21T09:41:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[29380]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[29380]: osdmap e265: 8 total, 8 up, 8 in 2024-09-21T09:41:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[40737]: pgmap v27: 89 pgs: 89 active+clean; 184 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[40737]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:17.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[40737]: Upgrade: Updating mgr.x 2024-09-21T09:41:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[40737]: Deploying daemon mgr.x on smithi134 2024-09-21T09:41:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[40737]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:16 smithi017 ceph-mon[40737]: osdmap e265: 8 total, 8 up, 8 in 2024-09-21T09:41:17.218 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:17.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:17 smithi134 podman[81530]: 2024-09-21 09:41:17.216878427 +0000 UTC m=+0.144286159 container died 7b270a3ec585c2953579e3b64c75c8001885ca7cc5a5631f11eda2c4df20102d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, ceph=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T09:41:17.800 INFO:tasks.rados.rados.0.smithi017.stdout:811: finishing rollback tid 1 to smithi01785587-9 2024-09-21T09:41:17.800 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 134 (ObjNum 177 snap 0 seq_num 2399901504) dirty dne 2024-09-21T09:41:17.800 INFO:tasks.rados.rados.0.smithi017.stdout:810: finishing rollback tid 0 to smithi01785587-47 2024-09-21T09:41:17.800 INFO:tasks.rados.rados.0.smithi017.stdout:810: finishing rollback tid 1 to smithi01785587-47 2024-09-21T09:41:17.800 INFO:tasks.rados.rados.0.smithi017.stdout:810: finishing rollback tid 2 to smithi01785587-47 2024-09-21T09:41:17.800 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 273 (ObjNum 155 snap 38 seq_num 155) dirty exists 2024-09-21T09:41:17.800 INFO:tasks.rados.rados.0.smithi017.stdout:809: done (3 left) 2024-09-21T09:41:17.800 INFO:tasks.rados.rados.0.smithi017.stdout:810: done (2 left) 2024-09-21T09:41:17.801 INFO:tasks.rados.rados.0.smithi017.stdout:811: done (1 left) 2024-09-21T09:41:17.801 INFO:tasks.rados.rados.0.smithi017.stdout:812: done (0 left) 2024-09-21T09:41:17.801 INFO:tasks.rados.rados.0.smithi017.stdout:813: snap_remove snap 88 2024-09-21T09:41:17.857 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:17 smithi134 podman[81530]: 2024-09-21 09:41:17.628417064 +0000 UTC m=+0.555824794 container cleanup 7b270a3ec585c2953579e3b64c75c8001885ca7cc5a5631f11eda2c4df20102d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T09:41:17.857 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:17 smithi134 bash[81530]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:41:18.122 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:17 smithi134 podman[81548]: 2024-09-21 09:41:17.856103805 +0000 UTC m=+0.640794138 container remove 7b270a3ec585c2953579e3b64c75c8001885ca7cc5a5631f11eda2c4df20102d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-62da54b) 2024-09-21T09:41:18.123 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:17 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:41:18.123 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:18 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:41:18.123 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:18 smithi134 systemd[1]: Stopped Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:18.123 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:18 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.x.service: Consumed 18.982s CPU time. 2024-09-21T09:41:18.168 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:18.493 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:18 smithi134 systemd[1]: Starting Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:18.780 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:18 smithi134 podman[81685]: 2024-09-21 09:41:18.411185435 +0000 UTC m=+0.016159315 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:41:18.781 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:18 smithi134 podman[81685]: 2024-09-21 09:41:18.532244215 +0000 UTC m=+0.137218130 container create 4feb5ade269b0d0fcb1733c61dcc9fec3a695df503a649ddaf1a5dab4357b473 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-21T09:41:18.781 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:18 smithi134 podman[81685]: 2024-09-21 09:41:18.779502695 +0000 UTC m=+0.384476563 container init 4feb5ade269b0d0fcb1733c61dcc9fec3a695df503a649ddaf1a5dab4357b473 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T09:41:18.801 INFO:tasks.rados.rados.0.smithi017.stdout:813: done (0 left) 2024-09-21T09:41:18.801 INFO:tasks.rados.rados.0.smithi017.stdout:814: delete oid 50 current snap is 102 2024-09-21T09:41:18.811 INFO:tasks.rados.rados.0.smithi017.stdout:814: done (0 left) 2024-09-21T09:41:18.811 INFO:tasks.rados.rados.0.smithi017.stdout:815: read oid 39 snap 101 2024-09-21T09:41:18.811 INFO:tasks.rados.rados.0.smithi017.stdout:815: expect deleted 2024-09-21T09:41:18.811 INFO:tasks.rados.rados.0.smithi017.stdout:816: read oid 43 snap 92 2024-09-21T09:41:18.811 INFO:tasks.rados.rados.0.smithi017.stdout:816: expect (ObjNum 199 snap 56 seq_num 199) 2024-09-21T09:41:18.811 INFO:tasks.rados.rados.0.smithi017.stdout:817: snap_create 2024-09-21T09:41:18.819 INFO:tasks.rados.rados.0.smithi017.stdout:816: expect (ObjNum 199 snap 56 seq_num 199) 2024-09-21T09:41:19.122 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:18 smithi134 ceph-mon[33541]: pgmap v30: 89 pgs: 89 active+clean; 198 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:41:19.123 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:18 smithi134 ceph-mon[33541]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:41:19.123 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:18 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2285816042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:19.123 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:18 smithi134 podman[81685]: 2024-09-21 09:41:18.784116711 +0000 UTC m=+0.389090580 container start 4feb5ade269b0d0fcb1733c61dcc9fec3a695df503a649ddaf1a5dab4357b473 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-21T09:41:19.123 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:18 smithi134 ceph-mgr[81723]: -- 172.21.15.134:0/2857452879 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560dfaef30e0 con 0x560dfaed1400 2024-09-21T09:41:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:18 smithi017 ceph-mon[40737]: pgmap v30: 89 pgs: 89 active+clean; 198 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:41:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:18 smithi017 ceph-mon[40737]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:41:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:18 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2285816042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:18 smithi017 ceph-mon[29380]: pgmap v30: 89 pgs: 89 active+clean; 198 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:41:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:18 smithi017 ceph-mon[29380]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:41:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:18 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2285816042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:19.493 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:19 smithi134 bash[81685]: 4feb5ade269b0d0fcb1733c61dcc9fec3a695df503a649ddaf1a5dab4357b473 2024-09-21T09:41:19.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:19 smithi134 systemd[1]: Started Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:19.816 INFO:tasks.rados.rados.0.smithi017.stdout:815: done (2 left) 2024-09-21T09:41:19.816 INFO:tasks.rados.rados.0.smithi017.stdout:816: done (1 left) 2024-09-21T09:41:19.816 INFO:tasks.rados.rados.0.smithi017.stdout:817: done (0 left) 2024-09-21T09:41:19.816 INFO:tasks.rados.rados.0.smithi017.stdout:818: write oid 35 current snap is 103 2024-09-21T09:41:19.816 INFO:tasks.rados.rados.0.smithi017.stdout:818: seq_num 339 ranges {497573=628943,1687631=734777,3093218=604355} 2024-09-21T09:41:19.821 INFO:tasks.rados.rados.0.smithi017.stdout:818: writing smithi01785587-35 from 497573 to 1126516 tid 1 2024-09-21T09:41:19.829 INFO:tasks.rados.rados.0.smithi017.stdout:818: writing smithi01785587-35 from 1687631 to 2422408 tid 2 2024-09-21T09:41:19.836 INFO:tasks.rados.rados.0.smithi017.stdout:818: writing smithi01785587-35 from 3093218 to 3697573 tid 3 2024-09-21T09:41:19.836 INFO:tasks.rados.rados.0.smithi017.stdout:819: write oid 48 current snap is 103 2024-09-21T09:41:19.836 INFO:tasks.rados.rados.0.smithi017.stdout:818: finishing write tid 1 to smithi01785587-35 2024-09-21T09:41:19.836 INFO:tasks.rados.rados.0.smithi017.stdout:818: finishing write tid 2 to smithi01785587-35 2024-09-21T09:41:19.837 INFO:tasks.rados.rados.0.smithi017.stdout:819: seq_num 340 ranges {514380=704192,1629760=478435,2604288=442602} 2024-09-21T09:41:19.843 INFO:tasks.rados.rados.0.smithi017.stdout:819: writing smithi01785587-48 from 514380 to 1218572 tid 1 2024-09-21T09:41:19.848 INFO:tasks.rados.rados.0.smithi017.stdout:819: writing smithi01785587-48 from 1629760 to 2108195 tid 2 2024-09-21T09:41:19.853 INFO:tasks.rados.rados.0.smithi017.stdout:819: writing smithi01785587-48 from 2604288 to 3046890 tid 3 2024-09-21T09:41:19.853 INFO:tasks.rados.rados.0.smithi017.stdout:820: write oid 49 current snap is 103 2024-09-21T09:41:19.853 INFO:tasks.rados.rados.0.smithi017.stdout:820: seq_num 341 ranges {531187=779441,2088242=738446,3431711=299476} 2024-09-21T09:41:19.860 INFO:tasks.rados.rados.0.smithi017.stdout:820: writing smithi01785587-49 from 531187 to 1310628 tid 1 2024-09-21T09:41:19.869 INFO:tasks.rados.rados.0.smithi017.stdout:820: writing smithi01785587-49 from 2088242 to 2826688 tid 2 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:820: writing smithi01785587-49 from 3431711 to 3731187 tid 3 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:821: read oid 32 snap -1 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:821: expect (ObjNum 305 snap 91 seq_num 305) 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:822: delete oid 27 current snap is 103 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:818: finishing write tid 3 to smithi01785587-35 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:818: finishing write tid 4 to smithi01785587-35 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:818: finishing write tid 5 to smithi01785587-35 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 115 (ObjNum 339 snap 103 seq_num 339) dirty exists 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:818: left oid 35 (ObjNum 339 snap 103 seq_num 339) 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:819: finishing write tid 1 to smithi01785587-48 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:819: finishing write tid 2 to smithi01785587-48 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:819: finishing write tid 3 to smithi01785587-48 2024-09-21T09:41:19.875 INFO:tasks.rados.rados.0.smithi017.stdout:819: finishing write tid 4 to smithi01785587-48 2024-09-21T09:41:19.876 INFO:tasks.rados.rados.0.smithi017.stdout:819: finishing write tid 5 to smithi01785587-48 2024-09-21T09:41:19.876 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 431 (ObjNum 340 snap 103 seq_num 340) dirty exists 2024-09-21T09:41:19.876 INFO:tasks.rados.rados.0.smithi017.stdout:819: left oid 48 (ObjNum 340 snap 103 seq_num 340) 2024-09-21T09:41:19.876 INFO:tasks.rados.rados.0.smithi017.stdout:820: finishing write tid 1 to smithi01785587-49 2024-09-21T09:41:19.876 INFO:tasks.rados.rados.0.smithi017.stdout:820: finishing write tid 2 to smithi01785587-49 2024-09-21T09:41:19.877 INFO:tasks.rados.rados.0.smithi017.stdout:818: done (4 left) 2024-09-21T09:41:19.877 INFO:tasks.rados.rados.0.smithi017.stdout:819: done (3 left) 2024-09-21T09:41:19.877 INFO:tasks.rados.rados.0.smithi017.stdout:822: done (2 left) 2024-09-21T09:41:19.877 INFO:tasks.rados.rados.0.smithi017.stdout:823: write oid 21 current snap is 103 2024-09-21T09:41:19.877 INFO:tasks.rados.rados.0.smithi017.stdout:823: seq_num 343 ranges {564801=646292,1921559=742115,3202910=561891} 2024-09-21T09:41:19.884 INFO:tasks.rados.rados.0.smithi017.stdout:823: writing smithi01785587-21 from 564801 to 1211093 tid 1 2024-09-21T09:41:19.892 INFO:tasks.rados.rados.0.smithi017.stdout:823: writing smithi01785587-21 from 1921559 to 2663674 tid 2 2024-09-21T09:41:19.898 INFO:tasks.rados.rados.0.smithi017.stdout:823: writing smithi01785587-21 from 3202910 to 3764801 tid 3 2024-09-21T09:41:19.898 INFO:tasks.rados.rados.0.smithi017.stdout:824: write oid 39 current snap is 103 2024-09-21T09:41:19.898 INFO:tasks.rados.rados.0.smithi017.stdout:824: seq_num 344 ranges {581608=721541,1863688=485773,3113980=604043} 2024-09-21T09:41:19.905 INFO:tasks.rados.rados.0.smithi017.stdout:824: writing smithi01785587-39 from 581608 to 1303149 tid 1 2024-09-21T09:41:19.911 INFO:tasks.rados.rados.0.smithi017.stdout:824: writing smithi01785587-39 from 1863688 to 2349461 tid 2 2024-09-21T09:41:19.919 INFO:tasks.rados.rados.0.smithi017.stdout:824: writing smithi01785587-39 from 3113980 to 3718023 tid 3 2024-09-21T09:41:19.919 INFO:tasks.rados.rados.0.smithi017.stdout:825: read oid 37 snap 102 2024-09-21T09:41:19.919 INFO:tasks.rados.rados.0.smithi017.stdout:825: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:41:19.919 INFO:tasks.rados.rados.0.smithi017.stdout:826: rollback oid 1 current snap is 103 2024-09-21T09:41:19.919 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 1 to 97 2024-09-21T09:41:19.919 INFO:tasks.rados.rados.0.smithi017.stdout:820: finishing write tid 3 to smithi01785587-49 2024-09-21T09:41:19.919 INFO:tasks.rados.rados.0.smithi017.stdout:821: expect (ObjNum 305 snap 91 seq_num 305) 2024-09-21T09:41:20.125 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:19 smithi134 ceph-mon[33541]: from='client.25478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:20.125 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:19 smithi134 ceph-mon[33541]: from='client.25484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:20.126 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:19 smithi134 ceph-mon[33541]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:41:20.126 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:19 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.126 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:19 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.126 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:19 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:20.126 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:19 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.126 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:19 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:820: finishing write tid 4 to smithi01785587-49 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:820: finishing write tid 5 to smithi01785587-49 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 462 (ObjNum 341 snap 103 seq_num 341) dirty exists 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:820: left oid 49 (ObjNum 341 snap 103 seq_num 341) 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:820: done (5 left) 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:821: done (4 left) 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:827: delete oid 24 current snap is 103 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:823: finishing write tid 1 to smithi01785587-21 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:823: finishing write tid 2 to smithi01785587-21 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:823: finishing write tid 3 to smithi01785587-21 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:823: finishing write tid 4 to smithi01785587-21 2024-09-21T09:41:20.127 INFO:tasks.rados.rados.0.smithi017.stdout:823: finishing write tid 5 to smithi01785587-21 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 466 (ObjNum 343 snap 103 seq_num 343) dirty exists 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:823: left oid 21 (ObjNum 343 snap 103 seq_num 343) 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:824: finishing write tid 1 to smithi01785587-39 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:824: finishing write tid 2 to smithi01785587-39 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:826: finishing rollback tid 0 to smithi01785587-1 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:826: finishing rollback tid 1 to smithi01785587-1 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 238 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:824: finishing write tid 3 to smithi01785587-39 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:824: finishing write tid 4 to smithi01785587-39 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:824: finishing write tid 5 to smithi01785587-39 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 387 (ObjNum 344 snap 103 seq_num 344) dirty exists 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:824: left oid 39 (ObjNum 344 snap 103 seq_num 344) 2024-09-21T09:41:20.128 INFO:tasks.rados.rados.0.smithi017.stdout:825: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:41:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[29380]: from='client.25478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[29380]: from='client.25484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[29380]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:41:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[40737]: from='client.25478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[40737]: from='client.25484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[40737]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:41:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:19 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:20.210 INFO:tasks.rados.rados.0.smithi017.stdout:823: done (4 left) 2024-09-21T09:41:20.210 INFO:tasks.rados.rados.0.smithi017.stdout:824: done (3 left) 2024-09-21T09:41:20.210 INFO:tasks.rados.rados.0.smithi017.stdout:825: done (2 left) 2024-09-21T09:41:20.211 INFO:tasks.rados.rados.0.smithi017.stdout:826: done (1 left) 2024-09-21T09:41:20.211 INFO:tasks.rados.rados.0.smithi017.stdout:827: done (0 left) 2024-09-21T09:41:20.211 INFO:tasks.rados.rados.0.smithi017.stdout:828: read oid 43 snap -1 2024-09-21T09:41:20.211 INFO:tasks.rados.rados.0.smithi017.stdout:828: expect (ObjNum 199 snap 56 seq_num 199) 2024-09-21T09:41:20.211 INFO:tasks.rados.rados.0.smithi017.stdout:829: snap_create 2024-09-21T09:41:20.221 INFO:tasks.rados.rados.0.smithi017.stdout:828: expect (ObjNum 199 snap 56 seq_num 199) 2024-09-21T09:41:20.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:20.820 INFO:tasks.rados.rados.0.smithi017.stdout:828: done (1 left) 2024-09-21T09:41:20.820 INFO:tasks.rados.rados.0.smithi017.stdout:829: done (0 left) 2024-09-21T09:41:20.820 INFO:tasks.rados.rados.0.smithi017.stdout:830: rollback oid 18 current snap is 104 2024-09-21T09:41:20.820 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 18 to 102 2024-09-21T09:41:20.820 INFO:tasks.rados.rados.0.smithi017.stdout:831: snap_create 2024-09-21T09:41:20.824 INFO:tasks.rados.rados.0.smithi017.stdout:830: finishing rollback tid 0 to smithi01785587-18 2024-09-21T09:41:20.824 INFO:tasks.rados.rados.0.smithi017.stdout:830: finishing rollback tid 1 to smithi01785587-18 2024-09-21T09:41:20.824 INFO:tasks.rados.rados.0.smithi017.stdout:830: finishing rollback tid 2 to smithi01785587-18 2024-09-21T09:41:20.824 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 64 (ObjNum 326 snap 98 seq_num 326) dirty exists 2024-09-21T09:41:21.131 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:20 smithi017 ceph-mon[29380]: pgmap v33: 89 pgs: 89 active+clean; 198 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:41:21.131 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:20 smithi017 ceph-mon[29380]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:41:21.131 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:21.131 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:20 smithi017 ceph-mon[40737]: pgmap v33: 89 pgs: 89 active+clean; 198 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:41:21.132 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:20 smithi017 ceph-mon[40737]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:41:21.132 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:20 smithi134 ceph-mon[33541]: pgmap v33: 89 pgs: 89 active+clean; 198 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:41:21.132 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:20 smithi134 ceph-mon[33541]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:41:21.824 INFO:tasks.rados.rados.0.smithi017.stdout:830: done (1 left) 2024-09-21T09:41:21.825 INFO:tasks.rados.rados.0.smithi017.stdout:831: done (0 left) 2024-09-21T09:41:21.825 INFO:tasks.rados.rados.0.smithi017.stdout:832: rollback oid 38 current snap is 105 2024-09-21T09:41:21.825 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 101 2024-09-21T09:41:21.825 INFO:tasks.rados.rados.0.smithi017.stdout:833: read oid 40 snap 102 2024-09-21T09:41:21.825 INFO:tasks.rados.rados.0.smithi017.stdout:833: expect (ObjNum 281 snap 85 seq_num 281) 2024-09-21T09:41:21.825 INFO:tasks.rados.rados.0.smithi017.stdout:834: write oid 39 current snap is 105 2024-09-21T09:41:21.825 INFO:tasks.rados.rados.0.smithi017.stdout:834: seq_num 346 ranges {615222=472039,1580652=489442,2768826=626587} 2024-09-21T09:41:21.830 INFO:tasks.rados.rados.0.smithi017.stdout:834: writing smithi01785587-39 from 615222 to 1087261 tid 1 2024-09-21T09:41:21.836 INFO:tasks.rados.rados.0.smithi017.stdout:834: writing smithi01785587-39 from 1580652 to 2070094 tid 2 2024-09-21T09:41:21.844 INFO:tasks.rados.rados.0.smithi017.stdout:834: writing smithi01785587-39 from 2768826 to 3395413 tid 3 2024-09-21T09:41:21.844 INFO:tasks.rados.rados.0.smithi017.stdout:835: write oid 36 current snap is 105 2024-09-21T09:41:21.844 INFO:tasks.rados.rados.0.smithi017.stdout:835: seq_num 347 ranges {632029=547288,1639134=749453,2796249=437859,3653524=178505} 2024-09-21T09:41:21.851 INFO:tasks.rados.rados.0.smithi017.stdout:835: writing smithi01785587-36 from 632029 to 1179317 tid 1 2024-09-21T09:41:21.858 INFO:tasks.rados.rados.0.smithi017.stdout:835: writing smithi01785587-36 from 1639134 to 2388587 tid 2 2024-09-21T09:41:21.863 INFO:tasks.rados.rados.0.smithi017.stdout:835: writing smithi01785587-36 from 2796249 to 3234108 tid 3 2024-09-21T09:41:21.866 INFO:tasks.rados.rados.0.smithi017.stdout:835: writing smithi01785587-36 from 3653524 to 3832029 tid 4 2024-09-21T09:41:21.867 INFO:tasks.rados.rados.0.smithi017.stdout:836: snap_remove snap 92 2024-09-21T09:41:22.069 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:21 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/19226335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:22.069 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:21 smithi134 ceph-mon[33541]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:41:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:21 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/19226335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:21 smithi017 ceph-mon[29380]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:41:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:21 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/19226335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:22.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:21 smithi017 ceph-mon[40737]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:41:22.712 INFO:tasks.rados.rados.0.smithi017.stdout:836: done (4 left) 2024-09-21T09:41:22.712 INFO:tasks.rados.rados.0.smithi017.stdout:837: write oid 47 current snap is 105 2024-09-21T09:41:22.712 INFO:tasks.rados.rados.0.smithi017.stdout:837: seq_num 348 ranges {648836=622537,1981263=493111,3107319=741517} 2024-09-21T09:41:22.720 INFO:tasks.rados.rados.0.smithi017.stdout:837: writing smithi01785587-47 from 648836 to 1271373 tid 1 2024-09-21T09:41:22.727 INFO:tasks.rados.rados.0.smithi017.stdout:837: writing smithi01785587-47 from 1981263 to 2474374 tid 2 2024-09-21T09:41:22.734 INFO:tasks.rados.rados.0.smithi017.stdout:837: writing smithi01785587-47 from 3107319 to 3848836 tid 3 2024-09-21T09:41:22.734 INFO:tasks.rados.rados.0.smithi017.stdout:838: write oid 16 current snap is 105 2024-09-21T09:41:22.734 INFO:tasks.rados.rados.0.smithi017.stdout:832: finishing rollback tid 0 to smithi01785587-38 2024-09-21T09:41:22.735 INFO:tasks.rados.rados.0.smithi017.stdout:832: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:41:22.735 INFO:tasks.rados.rados.0.smithi017.stdout:832: finishing rollback tid 2 to smithi01785587-38 2024-09-21T09:41:22.735 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 251 (ObjNum 298 snap 90 seq_num 298) dirty exists 2024-09-21T09:41:22.735 INFO:tasks.rados.rados.0.smithi017.stdout:838: seq_num 349 ranges {665643=697786,2039745=753122,3534742=330901} 2024-09-21T09:41:22.742 INFO:tasks.rados.rados.0.smithi017.stdout:838: writing smithi01785587-16 from 665643 to 1363429 tid 1 2024-09-21T09:41:22.750 INFO:tasks.rados.rados.0.smithi017.stdout:838: writing smithi01785587-16 from 2039745 to 2792867 tid 2 2024-09-21T09:41:22.756 INFO:tasks.rados.rados.0.smithi017.stdout:838: writing smithi01785587-16 from 3534742 to 3865643 tid 3 2024-09-21T09:41:22.756 INFO:tasks.rados.rados.0.smithi017.stdout:832: done (5 left) 2024-09-21T09:41:22.756 INFO:tasks.rados.rados.0.smithi017.stdout:839: write oid 30 current snap is 105 2024-09-21T09:41:22.756 INFO:tasks.rados.rados.0.smithi017.stdout:833: expect (ObjNum 281 snap 85 seq_num 281) 2024-09-21T09:41:22.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:22 smithi134 ceph-mon[33541]: pgmap v36: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:41:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:22 smithi134 ceph-mon[33541]: from='client.25499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:22 smithi134 ceph-mon[33541]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:22 smithi134 ceph-mon[33541]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:41:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:22 smithi134 ceph-mon[33541]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:41:22.998 INFO:tasks.rados.rados.0.smithi017.stdout:834: finishing write tid 1 to smithi01785587-39 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:834: finishing write tid 2 to smithi01785587-39 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:834: finishing write tid 3 to smithi01785587-39 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:834: finishing write tid 4 to smithi01785587-39 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:834: finishing write tid 5 to smithi01785587-39 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 392 (ObjNum 346 snap 105 seq_num 346) dirty exists 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:834: left oid 39 (ObjNum 346 snap 105 seq_num 346) 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:835: finishing write tid 1 to smithi01785587-36 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:835: finishing write tid 2 to smithi01785587-36 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:835: finishing write tid 3 to smithi01785587-36 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:835: finishing write tid 4 to smithi01785587-36 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:835: finishing write tid 5 to smithi01785587-36 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:835: finishing write tid 6 to smithi01785587-36 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 398 (ObjNum 347 snap 105 seq_num 347) dirty exists 2024-09-21T09:41:22.999 INFO:tasks.rados.rados.0.smithi017.stdout:835: left oid 36 (ObjNum 347 snap 105 seq_num 347) 2024-09-21T09:41:23.000 INFO:tasks.rados.rados.0.smithi017.stdout:837: finishing write tid 1 to smithi01785587-47 2024-09-21T09:41:23.000 INFO:tasks.rados.rados.0.smithi017.stdout:839: seq_num 350 ranges {682450=489388,1814580=496780,2878518=788028} 2024-09-21T09:41:23.005 INFO:tasks.rados.rados.0.smithi017.stdout:839: writing smithi01785587-30 from 682450 to 1171838 tid 1 2024-09-21T09:41:23.011 INFO:tasks.rados.rados.0.smithi017.stdout:839: writing smithi01785587-30 from 1814580 to 2311360 tid 2 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:839: writing smithi01785587-30 from 2878518 to 3666546 tid 3 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:833: done (5 left) 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:834: done (4 left) 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:835: done (3 left) 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:840: snap_create 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:837: finishing write tid 2 to smithi01785587-47 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:837: finishing write tid 3 to smithi01785587-47 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:837: finishing write tid 4 to smithi01785587-47 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:837: finishing write tid 5 to smithi01785587-47 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 290 (ObjNum 348 snap 105 seq_num 348) dirty exists 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:837: left oid 47 (ObjNum 348 snap 105 seq_num 348) 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:838: finishing write tid 1 to smithi01785587-16 2024-09-21T09:41:23.019 INFO:tasks.rados.rados.0.smithi017.stdout:838: finishing write tid 2 to smithi01785587-16 2024-09-21T09:41:23.020 INFO:tasks.rados.rados.0.smithi017.stdout:838: finishing write tid 3 to smithi01785587-16 2024-09-21T09:41:23.020 INFO:tasks.rados.rados.0.smithi017.stdout:838: finishing write tid 4 to smithi01785587-16 2024-09-21T09:41:23.020 INFO:tasks.rados.rados.0.smithi017.stdout:838: finishing write tid 5 to smithi01785587-16 2024-09-21T09:41:23.020 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 452 (ObjNum 349 snap 105 seq_num 349) dirty exists 2024-09-21T09:41:23.020 INFO:tasks.rados.rados.0.smithi017.stdout:838: left oid 16 (ObjNum 349 snap 105 seq_num 349) 2024-09-21T09:41:23.020 INFO:tasks.rados.rados.0.smithi017.stdout:839: finishing write tid 1 to smithi01785587-30 2024-09-21T09:41:23.020 INFO:tasks.rados.rados.0.smithi017.stdout:839: finishing write tid 2 to smithi01785587-30 2024-09-21T09:41:23.022 INFO:tasks.rados.rados.0.smithi017.stdout:839: finishing write tid 3 to smithi01785587-30 2024-09-21T09:41:23.022 INFO:tasks.rados.rados.0.smithi017.stdout:839: finishing write tid 4 to smithi01785587-30 2024-09-21T09:41:23.022 INFO:tasks.rados.rados.0.smithi017.stdout:839: finishing write tid 5 to smithi01785587-30 2024-09-21T09:41:23.022 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 440 (ObjNum 350 snap 105 seq_num 350) dirty exists 2024-09-21T09:41:23.022 INFO:tasks.rados.rados.0.smithi017.stdout:839: left oid 30 (ObjNum 350 snap 105 seq_num 350) 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[40737]: pgmap v36: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[40737]: from='client.25499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[40737]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[40737]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[40737]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[29380]: pgmap v36: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[29380]: from='client.25499 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[29380]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[29380]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:41:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:22 smithi017 ceph-mon[29380]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:41:23.831 INFO:tasks.rados.rados.0.smithi017.stdout:837: done (3 left) 2024-09-21T09:41:23.831 INFO:tasks.rados.rados.0.smithi017.stdout:838: done (2 left) 2024-09-21T09:41:23.831 INFO:tasks.rados.rados.0.smithi017.stdout:839: done (1 left) 2024-09-21T09:41:23.831 INFO:tasks.rados.rados.0.smithi017.stdout:840: done (0 left) 2024-09-21T09:41:23.831 INFO:tasks.rados.rados.0.smithi017.stdout:841: rollback oid 4 current snap is 106 2024-09-21T09:41:23.831 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 4 to 98 2024-09-21T09:41:23.831 INFO:tasks.rados.rados.0.smithi017.stdout:842: write oid 5 current snap is 106 2024-09-21T09:41:23.831 INFO:tasks.rados.rados.0.smithi017.stdout:842: seq_num 351 ranges {699257=564637,1873062=640438,3305941=593316} 2024-09-21T09:41:23.837 INFO:tasks.rados.rados.0.smithi017.stdout:842: writing smithi01785587-5 from 699257 to 1263894 tid 1 2024-09-21T09:41:23.845 INFO:tasks.rados.rados.0.smithi017.stdout:842: writing smithi01785587-5 from 1873062 to 2513500 tid 2 2024-09-21T09:41:23.852 INFO:tasks.rados.rados.0.smithi017.stdout:842: writing smithi01785587-5 from 3305941 to 3899257 tid 3 2024-09-21T09:41:23.853 INFO:tasks.rados.rados.0.smithi017.stdout:843: snap_create 2024-09-21T09:41:23.853 INFO:tasks.rados.rados.0.smithi017.stdout:841: finishing rollback tid 1 to smithi01785587-4 2024-09-21T09:41:23.853 INFO:tasks.rados.rados.0.smithi017.stdout:841: finishing rollback tid 2 to smithi01785587-4 2024-09-21T09:41:23.853 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 454 (ObjNum 312 snap 93 seq_num 312) dirty exists 2024-09-21T09:41:23.853 INFO:tasks.rados.rados.0.smithi017.stdout:842: finishing write tid 1 to smithi01785587-5 2024-09-21T09:41:23.853 INFO:tasks.rados.rados.0.smithi017.stdout:842: finishing write tid 2 to smithi01785587-5 2024-09-21T09:41:23.855 INFO:tasks.rados.rados.0.smithi017.stdout:842: finishing write tid 3 to smithi01785587-5 2024-09-21T09:41:23.855 INFO:tasks.rados.rados.0.smithi017.stdout:842: finishing write tid 4 to smithi01785587-5 2024-09-21T09:41:23.855 INFO:tasks.rados.rados.0.smithi017.stdout:842: finishing write tid 5 to smithi01785587-5 2024-09-21T09:41:23.855 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 439 (ObjNum 351 snap 106 seq_num 351) dirty exists 2024-09-21T09:41:23.856 INFO:tasks.rados.rados.0.smithi017.stdout:842: left oid 5 (ObjNum 351 snap 106 seq_num 351) 2024-09-21T09:41:24.104 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:24.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:23 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/563825755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:24.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:23 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/563825755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:24.140 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:23 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/563825755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:24.493 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: [21/Sep/2024:09:41:24] ENGINE Bus STARTING 2024-09-21T09:41:24.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: CherryPy Checker: 2024-09-21T09:41:24.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: The Application mounted at '' has an empty config. 2024-09-21T09:41:24.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: 2024-09-21T09:41:24.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: [21/Sep/2024:09:41:24] ENGINE Serving on http://:::9283 2024-09-21T09:41:24.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: [21/Sep/2024:09:41:24] ENGINE Bus STARTED 2024-09-21T09:41:24.837 INFO:tasks.rados.rados.0.smithi017.stdout:841: done (2 left) 2024-09-21T09:41:24.837 INFO:tasks.rados.rados.0.smithi017.stdout:842: done (1 left) 2024-09-21T09:41:24.837 INFO:tasks.rados.rados.0.smithi017.stdout:843: done (0 left) 2024-09-21T09:41:24.837 INFO:tasks.rados.rados.0.smithi017.stdout:844: read oid 12 snap -1 2024-09-21T09:41:24.837 INFO:tasks.rados.rados.0.smithi017.stdout:844: expect deleted 2024-09-21T09:41:24.837 INFO:tasks.rados.rados.0.smithi017.stdout:845: rollback oid 9 current snap is 107 2024-09-21T09:41:24.837 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 9 to 102 2024-09-21T09:41:24.837 INFO:tasks.rados.rados.0.smithi017.stdout:846: snap_remove snap 106 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: pgmap v39: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: Standby manager daemon x restarted 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: Standby manager daemon x started 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.168 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:24 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:41:24] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.51.0" 2024-09-21T09:41:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: pgmap v39: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:41:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:41:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: Standby manager daemon x restarted 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: Standby manager daemon x started 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: pgmap v39: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: Standby manager daemon x restarted 2024-09-21T09:41:25.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: Standby manager daemon x started 2024-09-21T09:41:25.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:41:25.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:41:25.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:41:25.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/58206102' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:41:25.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:24 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:25.840 INFO:tasks.rados.rados.0.smithi017.stdout:846: done (2 left) 2024-09-21T09:41:25.840 INFO:tasks.rados.rados.0.smithi017.stdout:847: write oid 25 current snap is 107 2024-09-21T09:41:25.840 INFO:tasks.rados.rados.0.smithi017.stdout:847: seq_num 352 ranges {716064=639886,1815191=500449,2817011=526925} 2024-09-21T09:41:25.847 INFO:tasks.rados.rados.0.smithi017.stdout:847: writing smithi01785587-25 from 716064 to 1355950 tid 1 2024-09-21T09:41:25.853 INFO:tasks.rados.rados.0.smithi017.stdout:847: writing smithi01785587-25 from 1815191 to 2315640 tid 2 2024-09-21T09:41:25.859 INFO:tasks.rados.rados.0.smithi017.stdout:847: writing smithi01785587-25 from 2817011 to 3343936 tid 3 2024-09-21T09:41:25.859 INFO:tasks.rados.rados.0.smithi017.stdout:848: delete oid 34 current snap is 107 2024-09-21T09:41:25.859 INFO:tasks.rados.rados.0.smithi017.stdout:845: finishing rollback tid 1 to smithi01785587-9 2024-09-21T09:41:25.859 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 134 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:41:25.859 INFO:tasks.rados.rados.0.smithi017.stdout:847: finishing write tid 1 to smithi01785587-25 2024-09-21T09:41:25.859 INFO:tasks.rados.rados.0.smithi017.stdout:847: finishing write tid 2 to smithi01785587-25 2024-09-21T09:41:25.862 INFO:tasks.rados.rados.0.smithi017.stdout:844: done (3 left) 2024-09-21T09:41:25.862 INFO:tasks.rados.rados.0.smithi017.stdout:845: done (2 left) 2024-09-21T09:41:25.862 INFO:tasks.rados.rados.0.smithi017.stdout:848: done (1 left) 2024-09-21T09:41:25.862 INFO:tasks.rados.rados.0.smithi017.stdout:849: read oid 5 snap -1 2024-09-21T09:41:25.863 INFO:tasks.rados.rados.0.smithi017.stdout:849: expect (ObjNum 351 snap 106 seq_num 351) 2024-09-21T09:41:25.863 INFO:tasks.rados.rados.0.smithi017.stdout:850: rollback oid 27 current snap is 107 2024-09-21T09:41:25.863 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 105 2024-09-21T09:41:25.863 INFO:tasks.rados.rados.0.smithi017.stdout:851: snap_create 2024-09-21T09:41:25.863 INFO:tasks.rados.rados.0.smithi017.stdout:847: finishing write tid 3 to smithi01785587-25 2024-09-21T09:41:25.863 INFO:tasks.rados.rados.0.smithi017.stdout:847: finishing write tid 4 to smithi01785587-25 2024-09-21T09:41:25.863 INFO:tasks.rados.rados.0.smithi017.stdout:847: finishing write tid 5 to smithi01785587-25 2024-09-21T09:41:25.863 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 202 (ObjNum 352 snap 107 seq_num 352) dirty exists 2024-09-21T09:41:25.863 INFO:tasks.rados.rados.0.smithi017.stdout:847: left oid 25 (ObjNum 352 snap 107 seq_num 352) 2024-09-21T09:41:25.866 INFO:tasks.rados.rados.0.smithi017.stdout:850: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:41:25.866 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 347 (ObjNum 33 snap 0 seq_num 3570511944) dirty dne 2024-09-21T09:41:25.872 INFO:tasks.rados.rados.0.smithi017.stdout:849: expect (ObjNum 351 snap 106 seq_num 351) 2024-09-21T09:41:25.880 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:25 smithi134 ceph-mon[33541]: from='client.25517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:25.881 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:25 smithi134 ceph-mon[33541]: from='client.25523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:25.881 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:25 smithi134 ceph-mon[33541]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:41:25.881 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:25 smithi134 ceph-mon[33541]: mgrmap e34: y(active, since 25s), standbys: x 2024-09-21T09:41:25.881 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:25.881 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:25 smithi017 ceph-mon[29380]: from='client.25517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:25 smithi017 ceph-mon[29380]: from='client.25523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:25 smithi017 ceph-mon[29380]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:41:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:25 smithi017 ceph-mon[29380]: mgrmap e34: y(active, since 25s), standbys: x 2024-09-21T09:41:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:25 smithi017 ceph-mon[40737]: from='client.25517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:25 smithi017 ceph-mon[40737]: from='client.25523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:25 smithi017 ceph-mon[40737]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:41:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:25 smithi017 ceph-mon[40737]: mgrmap e34: y(active, since 25s), standbys: x 2024-09-21T09:41:26.850 INFO:tasks.rados.rados.0.smithi017.stdout:847: done (3 left) 2024-09-21T09:41:26.850 INFO:tasks.rados.rados.0.smithi017.stdout:849: done (2 left) 2024-09-21T09:41:26.850 INFO:tasks.rados.rados.0.smithi017.stdout:850: done (1 left) 2024-09-21T09:41:26.850 INFO:tasks.rados.rados.0.smithi017.stdout:851: done (0 left) 2024-09-21T09:41:26.850 INFO:tasks.rados.rados.0.smithi017.stdout:852: read oid 25 snap -1 2024-09-21T09:41:26.850 INFO:tasks.rados.rados.0.smithi017.stdout:852: expect (ObjNum 352 snap 107 seq_num 352) 2024-09-21T09:41:26.850 INFO:tasks.rados.rados.0.smithi017.stdout:853: snap_remove snap 96 2024-09-21T09:41:27.063 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:26 smithi017 ceph-mon[29380]: pgmap v42: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:26 smithi017 ceph-mon[29380]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:41:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:26 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/2664924403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:27.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:26 smithi017 ceph-mon[40737]: pgmap v42: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:27.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:26 smithi017 ceph-mon[40737]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:41:27.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:26 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2664924403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:26 smithi134 ceph-mon[33541]: pgmap v42: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:26 smithi134 ceph-mon[33541]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:41:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:26 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2664924403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:27.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:27.854 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:41:27.855 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:41:27.856 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:27 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:41:27.859 INFO:tasks.rados.rados.0.smithi017.stdout:853: done (1 left) 2024-09-21T09:41:27.859 INFO:tasks.rados.rados.0.smithi017.stdout:854: write oid 12 current snap is 108 2024-09-21T09:41:27.859 INFO:tasks.rados.rados.0.smithi017.stdout:854: seq_num 354 ranges {749678=790384,2332155=504118,3271857=549469} 2024-09-21T09:41:27.867 INFO:tasks.rados.rados.0.smithi017.stdout:854: writing smithi01785587-12 from 749678 to 1540062 tid 1 2024-09-21T09:41:27.874 INFO:tasks.rados.rados.0.smithi017.stdout:854: writing smithi01785587-12 from 2332155 to 2836273 tid 2 2024-09-21T09:41:27.880 INFO:tasks.rados.rados.0.smithi017.stdout:854: writing smithi01785587-12 from 3271857 to 3821326 tid 3 2024-09-21T09:41:27.880 INFO:tasks.rados.rados.0.smithi017.stdout:855: delete oid 11 current snap is 108 2024-09-21T09:41:27.880 INFO:tasks.rados.rados.0.smithi017.stdout:852: expect (ObjNum 352 snap 107 seq_num 352) 2024-09-21T09:41:28.025 INFO:tasks.rados.rados.0.smithi017.stdout:854: finishing write tid 1 to smithi01785587-12 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:854: finishing write tid 2 to smithi01785587-12 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:854: finishing write tid 3 to smithi01785587-12 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:854: finishing write tid 4 to smithi01785587-12 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:854: finishing write tid 5 to smithi01785587-12 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 298 (ObjNum 354 snap 108 seq_num 354) dirty exists 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:854: left oid 12 (ObjNum 354 snap 108 seq_num 354) 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:852: done (2 left) 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:854: done (1 left) 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:855: done (0 left) 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:856: read oid 37 snap -1 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:856: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:857: read oid 31 snap -1 2024-09-21T09:41:28.026 INFO:tasks.rados.rados.0.smithi017.stdout:857: expect (ObjNum 304 snap 91 seq_num 304) 2024-09-21T09:41:28.027 INFO:tasks.rados.rados.0.smithi017.stdout:858: snap_remove snap 101 2024-09-21T09:41:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:41:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:28.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:41:28.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:41:28.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:27 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:41:28.862 INFO:tasks.rados.rados.0.smithi017.stdout:858: done (2 left) 2024-09-21T09:41:28.862 INFO:tasks.rados.rados.0.smithi017.stdout:859: delete oid 36 current snap is 108 2024-09-21T09:41:28.865 INFO:tasks.rados.rados.0.smithi017.stdout:856: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:41:28.948 INFO:tasks.rados.rados.0.smithi017.stdout:857: expect (ObjNum 304 snap 91 seq_num 304) 2024-09-21T09:41:29.119 INFO:tasks.rados.rados.0.smithi017.stdout:856: done (2 left) 2024-09-21T09:41:29.119 INFO:tasks.rados.rados.0.smithi017.stdout:857: done (1 left) 2024-09-21T09:41:29.119 INFO:tasks.rados.rados.0.smithi017.stdout:859: done (0 left) 2024-09-21T09:41:29.119 INFO:tasks.rados.rados.0.smithi017.stdout:860: read oid 11 snap 103 2024-09-21T09:41:29.119 INFO:tasks.rados.rados.0.smithi017.stdout:860: expect (ObjNum 333 snap 100 seq_num 333) 2024-09-21T09:41:29.119 INFO:tasks.rados.rados.0.smithi017.stdout:861: rollback oid 3 current snap is 108 2024-09-21T09:41:29.119 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 103 2024-09-21T09:41:29.119 INFO:tasks.rados.rados.0.smithi017.stdout:862: rollback oid 41 current snap is 108 2024-09-21T09:41:29.120 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 87 2024-09-21T09:41:29.120 INFO:tasks.rados.rados.0.smithi017.stdout:863: snap_remove snap 97 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[29380]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[29380]: pgmap v45: 89 pgs: 89 active+clean; 191 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[29380]: from='client.15648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[29380]: Upgrade: Setting container_image for all mgr 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[29380]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[29380]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[40737]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[40737]: pgmap v45: 89 pgs: 89 active+clean; 191 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[40737]: from='client.15648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[40737]: Upgrade: Setting container_image for all mgr 2024-09-21T09:41:29.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[40737]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:41:29.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:28 smithi017 ceph-mon[40737]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:41:29.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:28 smithi134 ceph-mon[33541]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:28 smithi134 ceph-mon[33541]: pgmap v45: 89 pgs: 89 active+clean; 191 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-21T09:41:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:28 smithi134 ceph-mon[33541]: from='client.15648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:28 smithi134 ceph-mon[33541]: Upgrade: Setting container_image for all mgr 2024-09-21T09:41:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:28 smithi134 ceph-mon[33541]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:41:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:28 smithi134 ceph-mon[33541]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:41:29.870 INFO:tasks.rados.rados.0.smithi017.stdout:863: done (3 left) 2024-09-21T09:41:29.870 INFO:tasks.rados.rados.0.smithi017.stdout:864: snap_remove snap 99 2024-09-21T09:41:30.032 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[29380]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:41:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:41:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:41:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[29380]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[29380]: from='client.? 172.21.15.17:0/3367376279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[40737]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:41:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:41:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:41:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:29 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3367376279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:30.212 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:29 smithi134 ceph-mon[33541]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:41:30.212 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:29 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' 2024-09-21T09:41:30.212 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:29 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:41:30.212 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:29 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:41:30.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:29 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:30.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:29 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:30.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:29 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3367376279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:30.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:30.876 INFO:tasks.rados.rados.0.smithi017.stdout:864: done (3 left) 2024-09-21T09:41:30.876 INFO:tasks.rados.rados.0.smithi017.stdout:865: delete oid 8 current snap is 108 2024-09-21T09:41:30.876 INFO:tasks.rados.rados.0.smithi017.stdout:861: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:41:30.876 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 0 (ObjNum 129 snap 0 seq_num 2399928688) dirty dne 2024-09-21T09:41:30.876 INFO:tasks.rados.rados.0.smithi017.stdout:862: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:41:30.876 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 0 (ObjNum 97 snap 0 seq_num 544760173) dirty dne 2024-09-21T09:41:30.877 INFO:tasks.rados.rados.0.smithi017.stdout:860: expect (ObjNum 333 snap 100 seq_num 333) 2024-09-21T09:41:31.001 INFO:tasks.rados.rados.0.smithi017.stdout:860: done (3 left) 2024-09-21T09:41:31.001 INFO:tasks.rados.rados.0.smithi017.stdout:861: done (2 left) 2024-09-21T09:41:31.001 INFO:tasks.rados.rados.0.smithi017.stdout:862: done (1 left) 2024-09-21T09:41:31.001 INFO:tasks.rados.rados.0.smithi017.stdout:865: done (0 left) 2024-09-21T09:41:31.001 INFO:tasks.rados.rados.0.smithi017.stdout:866: read oid 1 snap -1 2024-09-21T09:41:31.001 INFO:tasks.rados.rados.0.smithi017.stdout:866: expect deleted 2024-09-21T09:41:31.002 INFO:tasks.rados.rados.0.smithi017.stdout:867: read oid 7 snap 103 2024-09-21T09:41:31.002 INFO:tasks.rados.rados.0.smithi017.stdout:867: expect deleted 2024-09-21T09:41:31.002 INFO:tasks.rados.rados.0.smithi017.stdout:868: write oid 28 current snap is 108 2024-09-21T09:41:31.002 INFO:tasks.rados.rados.0.smithi017.stdout:868: seq_num 358 ranges {416906=407733,1366083=511456} 2024-09-21T09:41:31.005 INFO:tasks.rados.rados.0.smithi017.stdout:868: writing smithi01785587-28 from 416906 to 824639 tid 1 2024-09-21T09:41:31.011 INFO:tasks.rados.rados.0.smithi017.stdout:868: writing smithi01785587-28 from 1366083 to 1877539 tid 2 2024-09-21T09:41:31.011 INFO:tasks.rados.rados.0.smithi017.stdout:869: read oid 4 snap -1 2024-09-21T09:41:31.012 INFO:tasks.rados.rados.0.smithi017.stdout:869: expect (ObjNum 312 snap 93 seq_num 312) 2024-09-21T09:41:31.012 INFO:tasks.rados.rados.0.smithi017.stdout:870: read oid 18 snap 102 2024-09-21T09:41:31.012 INFO:tasks.rados.rados.0.smithi017.stdout:870: expect (ObjNum 326 snap 98 seq_num 326) 2024-09-21T09:41:31.012 INFO:tasks.rados.rados.0.smithi017.stdout:871: write oid 39 current snap is 108 2024-09-21T09:41:31.012 INFO:tasks.rados.rados.0.smithi017.stdout:871: seq_num 359 ranges {433713=482982,1424565=609148} 2024-09-21T09:41:31.016 INFO:tasks.rados.rados.0.smithi017.stdout:871: writing smithi01785587-39 from 433713 to 916695 tid 1 2024-09-21T09:41:31.023 INFO:tasks.rados.rados.0.smithi017.stdout:871: writing smithi01785587-39 from 1424565 to 2033713 tid 2 2024-09-21T09:41:31.023 INFO:tasks.rados.rados.0.smithi017.stdout:872: snap_create 2024-09-21T09:41:31.023 INFO:tasks.rados.rados.0.smithi017.stdout:868: finishing write tid 1 to smithi01785587-28 2024-09-21T09:41:31.023 INFO:tasks.rados.rados.0.smithi017.stdout:871: finishing write tid 1 to smithi01785587-39 2024-09-21T09:41:31.025 INFO:tasks.rados.rados.0.smithi017.stdout:868: finishing write tid 2 to smithi01785587-28 2024-09-21T09:41:31.025 INFO:tasks.rados.rados.0.smithi017.stdout:868: finishing write tid 3 to smithi01785587-28 2024-09-21T09:41:31.025 INFO:tasks.rados.rados.0.smithi017.stdout:868: finishing write tid 4 to smithi01785587-28 2024-09-21T09:41:31.025 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 464 (ObjNum 358 snap 108 seq_num 358) dirty exists 2024-09-21T09:41:31.025 INFO:tasks.rados.rados.0.smithi017.stdout:868: left oid 28 (ObjNum 358 snap 108 seq_num 358) 2024-09-21T09:41:31.027 INFO:tasks.rados.rados.0.smithi017.stdout:871: finishing write tid 2 to smithi01785587-39 2024-09-21T09:41:31.027 INFO:tasks.rados.rados.0.smithi017.stdout:871: finishing write tid 3 to smithi01785587-39 2024-09-21T09:41:31.027 INFO:tasks.rados.rados.0.smithi017.stdout:871: finishing write tid 4 to smithi01785587-39 2024-09-21T09:41:31.027 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 422 (ObjNum 359 snap 108 seq_num 359) dirty exists 2024-09-21T09:41:31.027 INFO:tasks.rados.rados.0.smithi017.stdout:871: left oid 39 (ObjNum 359 snap 108 seq_num 359) 2024-09-21T09:41:31.028 INFO:tasks.rados.rados.0.smithi017.stdout:869: expect (ObjNum 312 snap 93 seq_num 312) 2024-09-21T09:41:31.051 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:30 smithi017 ceph-mon[29380]: pgmap v48: 89 pgs: 89 active+clean; 191 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-21T09:41:31.051 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:30 smithi017 ceph-mon[29380]: Upgrade: Updating mon.a 2024-09-21T09:41:31.052 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:30 smithi017 ceph-mon[29380]: Deploying daemon mon.a on smithi017 2024-09-21T09:41:31.052 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:30 smithi017 ceph-mon[29380]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:41:31.052 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:30 smithi017 ceph-mon[40737]: pgmap v48: 89 pgs: 89 active+clean; 191 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-21T09:41:31.052 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:30 smithi017 ceph-mon[40737]: Upgrade: Updating mon.a 2024-09-21T09:41:31.052 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:30 smithi017 ceph-mon[40737]: Deploying daemon mon.a on smithi017 2024-09-21T09:41:31.052 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:30 smithi017 ceph-mon[40737]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:41:31.137 INFO:tasks.rados.rados.0.smithi017.stdout:870: expect (ObjNum 326 snap 98 seq_num 326) 2024-09-21T09:41:31.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:30 smithi134 ceph-mon[33541]: pgmap v48: 89 pgs: 89 active+clean; 191 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-21T09:41:31.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:30 smithi134 ceph-mon[33541]: Upgrade: Updating mon.a 2024-09-21T09:41:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:30 smithi134 ceph-mon[33541]: Deploying daemon mon.a on smithi017 2024-09-21T09:41:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:30 smithi134 ceph-mon[33541]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:41:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:31 smithi017 systemd[1]: Stopping Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:31.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:31 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:31] ENGINE Bus STOPPING 2024-09-21T09:41:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:31 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[29357]: 2024-09-21T09:41:31.304+0000 7f6052941640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:41:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:31 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[29357]: 2024-09-21T09:41:31.304+0000 7f6052941640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T09:41:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:31 smithi017 podman[102665]: 2024-09-21 09:41:31.469522314 +0000 UTC m=+0.268002950 container died 52c5c6bf49345abeb6b6dbf2e5b057bc5a7aa49d47bc9f160c510c122d9f15e9 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T09:41:32.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:31 smithi017 podman[102665]: 2024-09-21 09:41:31.828893684 +0000 UTC m=+0.627374319 container cleanup 52c5c6bf49345abeb6b6dbf2e5b057bc5a7aa49d47bc9f160c510c122d9f15e9 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, GIT_BRANCH=HEAD, ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:41:32.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:31 smithi017 bash[102665]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:41:32.098 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:31 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:41:32.098 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:31 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:31] ENGINE Bus STOPPED 2024-09-21T09:41:32.098 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:31 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:31] ENGINE Bus STARTING 2024-09-21T09:41:32.098 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:31 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:31] ENGINE Serving on http://:::9283 2024-09-21T09:41:32.098 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:31 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:31] ENGINE Bus STARTED 2024-09-21T09:41:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 podman[102682]: 2024-09-21 09:41:32.096699374 +0000 UTC m=+0.622190187 container remove 52c5c6bf49345abeb6b6dbf2e5b057bc5a7aa49d47bc9f160c510c122d9f15e9 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T09:41:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:41:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 systemd[1]: Stopped Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:32.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service: Consumed 15.620s CPU time. 2024-09-21T09:41:32.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 systemd[1]: Starting Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:32.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 podman[102852]: 2024-09-21 09:41:32.599209422 +0000 UTC m=+0.017792608 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:41:32.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 podman[102852]: 2024-09-21 09:41:32.714940614 +0000 UTC m=+0.133523785 container create 0590143b06cf4e29de8e56942b7a39ee42652dc198eb59c58ac4ea25b7d618e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T09:41:33.012 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 podman[102852]: 2024-09-21 09:41:32.951430794 +0000 UTC m=+0.370013988 container init 0590143b06cf4e29de8e56942b7a39ee42652dc198eb59c58ac4ea25b7d618e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 podman[102852]: 2024-09-21 09:41:32.956885222 +0000 UTC m=+0.375468401 container start 0590143b06cf4e29de8e56942b7a39ee42652dc198eb59c58ac4ea25b7d618e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: pidfile_write: ignore empty --pid-file 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: load: jerasure load: lrc 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Git sha 0 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: DB SUMMARY 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: DB Session ID: S3XZF0P8BHXPC3U4OD7V 2024-09-21T09:41:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1166 Bytes 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 4849784 ; 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.env: 0x561d57868120 2024-09-21T09:41:33.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:41:33.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.info_log: 0x561d597a3700 2024-09-21T09:41:33.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:41:33.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.statistics: (nil) 2024-09-21T09:41:33.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.use_fsync: 0 2024-09-21T09:41:33.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:41:33.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:41:33.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:41:33.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:41:33.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:41:33.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.db_log_dir: 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.wal_dir: 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:41:33.455 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.write_buffer_manager: 0x561d597a61e0 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.unordered_write: 0 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.row_cache: None 2024-09-21T09:41:33.456 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.wal_filter: None 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.wal_compression: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:41:33.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_open_files: -1 2024-09-21T09:41:33.458 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Compression algorithms supported: 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: kZSTD supported: 0 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:41:33.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.merge_operator: 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_filter: None 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561d597a3320) 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:41:33.460 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_type: 0 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_index_type: 0 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_shortening: 1 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: checksum: 4 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: no_block_cache: 0 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache: 0x561d59797610 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_options: 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: capacity : 536870912 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: num_shard_bits : 4 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: strict_capacity_limit : 0 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:41:33.461 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_compressed: (nil) 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: persistent_cache: (nil) 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size: 4096 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size_deviation: 10 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_restart_interval: 16 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_block_restart_interval: 1 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: metadata_block_size: 4096 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: partition_filters: 0 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: use_delta_encoding: 1 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: filter_policy: bloomfilter 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: whole_key_filtering: 1 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: verify_compression: 0 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:41:33.462 INFO:journalctl@ceph.mon.a.smithi017.stdout: format_version: 5 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_index_compression: 1 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_align: 0 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_auto_readahead_size: 262144 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout: prepopulate_block_cache: 0 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression: NoCompression 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:41:33.463 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.num_levels: 7 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:41:33.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:41:33.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:41:33.466 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:41:33.467 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.ttl: 2592000 2024-09-21T09:41:33.468 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12800, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:41:33.469 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 1f7def57-f1d6-4669-a7c2-40690f0a21ac 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911692984539, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911692998478, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 4529199, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12794, "largest_seqno": 15148, "table_properties": {"data_size": 4517120, "index_size": 8473, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2565, "raw_key_size": 24452, "raw_average_key_size": 24, "raw_value_size": 4494174, "raw_average_value_size": 4440, "num_data_blocks": 371, "num_entries": 1012, "num_filter_entries": 1012, "num_deletions": 48, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911692, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "1f7def57-f1d6-4669-a7c2-40690f0a21ac", "db_session_id": "S3XZF0P8BHXPC3U4OD7V", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911692998585, "job": 1, "event": "recovery_finished"} 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:32 smithi017 ceph-mon[102913]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561d597c6e00 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: rocksdb: DB pointer 0x561d598c8000 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** DB Stats ** 2024-09-21T09:41:33.470 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: L0 1/0 4.32 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 636.2 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: L6 1/0 7.47 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: Sum 2/0 11.79 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 636.2 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:41:33.471 INFO:journalctl@ceph.mon.a.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 636.2 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 636.2 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative compaction: 0.00 GB write, 259.16 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:41:33.472 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval compaction: 0.00 GB write, 259.16 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: Block cache BinnedLRUCache@0x561d59797610#7 capacity: 512.00 MB usage: 11.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.78 KB,0.000530481%) IndexBlock(1,9.08 KB,0.00173151%) Misc(1,0.00 KB,0%) 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: starting mon.a rank 0 at public addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] at bind addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: mon.a@-1(???) e3 preinit fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: mon.a@-1(???).mds e8 new map 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: mon.a@-1(???).mds e8 print_map 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: e8 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: legacy client fscid: 1 2024-09-21T09:41:33.473 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: Filesystem 'foo' (1) 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: fs_name foo 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: epoch 7 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: created 2024-09-21T09:36:43.123303+0000 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: modified 2024-09-21T09:36:59.247371+0000 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: tableserver 0 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: root 0 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: session_timeout 60 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: session_autoclose 300 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_file_size 1099511627776 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_xattr_size 65536 2024-09-21T09:41:33.474 INFO:journalctl@ceph.mon.a.smithi017.stdout: required_client_features {} 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: last_failure 0 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: last_failure_osd_epoch 0 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_mds 1 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: in 0 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: up {0=14655} 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: failed 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: damaged 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: stopped 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_pools [4] 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: metadata_pool 3 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: inline_data disabled 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: balancer 2024-09-21T09:41:33.475 INFO:journalctl@ceph.mon.a.smithi017.stdout: bal_rank_mask -1 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout: standby_count_wanted 1 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout: [mds.foo.smithi017.coodep{0:14655} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout: Standby daemons: 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout: [mds.foo.smithi134.bwovvb{-1:24493} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.134:6832/3208820776,v1:172.21.15.134:6833/3208820776] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: mon.a@-1(???).osd e279 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: mon.a@-1(???).osd e279 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: mon.a@-1(???).osd e279 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: mon.a@-1(???).osd e279 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 ceph-mon[102913]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-09-21T09:41:33.476 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 bash[102852]: 0590143b06cf4e29de8e56942b7a39ee42652dc198eb59c58ac4ea25b7d618e4 2024-09-21T09:41:33.477 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:33 smithi017 systemd[1]: Started Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:34.475 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:34.486 INFO:tasks.rados.rados.0.smithi017.stdout:866: done (6 left) 2024-09-21T09:41:34.486 INFO:tasks.rados.rados.0.smithi017.stdout:867: done (5 left) 2024-09-21T09:41:34.486 INFO:tasks.rados.rados.0.smithi017.stdout:868: done (4 left) 2024-09-21T09:41:34.486 INFO:tasks.rados.rados.0.smithi017.stdout:869: done (3 left) 2024-09-21T09:41:34.486 INFO:tasks.rados.rados.0.smithi017.stdout:870: done (2 left) 2024-09-21T09:41:34.486 INFO:tasks.rados.rados.0.smithi017.stdout:871: done (1 left) 2024-09-21T09:41:34.486 INFO:tasks.rados.rados.0.smithi017.stdout:872: done (0 left) 2024-09-21T09:41:34.486 INFO:tasks.rados.rados.0.smithi017.stdout:873: write oid 45 current snap is 109 2024-09-21T09:41:34.486 INFO:tasks.rados.rados.0.smithi017.stdout:873: seq_num 360 ranges {450520=558231,1483047=515125} 2024-09-21T09:41:34.491 INFO:tasks.rados.rados.0.smithi017.stdout:873: writing smithi01785587-45 from 450520 to 1008751 tid 1 2024-09-21T09:41:34.497 INFO:tasks.rados.rados.0.smithi017.stdout:873: writing smithi01785587-45 from 1483047 to 1998172 tid 2 2024-09-21T09:41:34.497 INFO:tasks.rados.rados.0.smithi017.stdout:874: rollback oid 30 current snap is 109 2024-09-21T09:41:34.497 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 104 2024-09-21T09:41:34.497 INFO:tasks.rados.rados.0.smithi017.stdout:873: finishing write tid 1 to smithi01785587-45 2024-09-21T09:41:34.498 INFO:tasks.rados.rados.0.smithi017.stdout:875: read oid 50 snap -1 2024-09-21T09:41:34.498 INFO:tasks.rados.rados.0.smithi017.stdout:875: expect deleted 2024-09-21T09:41:34.498 INFO:tasks.rados.rados.0.smithi017.stdout:876: delete oid 22 current snap is 109 2024-09-21T09:41:34.503 INFO:tasks.rados.rados.0.smithi017.stdout:875: done (3 left) 2024-09-21T09:41:34.503 INFO:tasks.rados.rados.0.smithi017.stdout:876: done (2 left) 2024-09-21T09:41:34.503 INFO:tasks.rados.rados.0.smithi017.stdout:877: snap_create 2024-09-21T09:41:34.503 INFO:tasks.rados.rados.0.smithi017.stdout:873: finishing write tid 2 to smithi01785587-45 2024-09-21T09:41:34.503 INFO:tasks.rados.rados.0.smithi017.stdout:873: finishing write tid 3 to smithi01785587-45 2024-09-21T09:41:34.503 INFO:tasks.rados.rados.0.smithi017.stdout:873: finishing write tid 4 to smithi01785587-45 2024-09-21T09:41:34.503 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 364 (ObjNum 360 snap 109 seq_num 360) dirty exists 2024-09-21T09:41:34.504 INFO:tasks.rados.rados.0.smithi017.stdout:873: left oid 45 (ObjNum 360 snap 109 seq_num 360) 2024-09-21T09:41:34.504 INFO:tasks.rados.rados.0.smithi017.stdout:874: finishing rollback tid 0 to smithi01785587-30 2024-09-21T09:41:34.504 INFO:tasks.rados.rados.0.smithi017.stdout:874: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:41:34.504 INFO:tasks.rados.rados.0.smithi017.stdout:874: finishing rollback tid 2 to smithi01785587-30 2024-09-21T09:41:34.504 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 460 (ObjNum 334 snap 101 seq_num 334) dirty exists 2024-09-21T09:41:34.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: ignoring --setuser ceph since I am not root 2024-09-21T09:41:34.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: ignoring --setgroup ceph since I am not root 2024-09-21T09:41:34.744 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mgr[81723]: -- 172.21.15.134:0/3843686568 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b16383f0e0 con 0x55b16381d400 2024-09-21T09:41:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:41:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:41:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:41:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/4121811425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: from='client.25571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: pgmap v52: 89 pgs: 89 active+clean; 197 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 654 KiB/s wr, 11 op/s 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: from='client.25577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: mon.a calling monitor election 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: monmap epoch 3 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: last_changed 2024-09-21T09:31:25.701063+0000 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: created 2024-09-21T09:29:44.848123+0000 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: min_mon_release 17 (quincy) 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: election_strategy: 1 2024-09-21T09:41:34.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-21T09:41:34.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: 1: [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon.b 2024-09-21T09:41:34.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-21T09:41:34.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:41:34.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:41:34.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: mgrmap e34: y(active, since 35s), standbys: x 2024-09-21T09:41:34.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: overall HEALTH_OK 2024-09-21T09:41:34.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: from='mgr.25274 ' entity='' 2024-09-21T09:41:34.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:34 smithi134 ceph-mon[33541]: mgrmap e35: y(active, since 35s), standbys: x 2024-09-21T09:41:34.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ignoring --setuser ceph since I am not root 2024-09-21T09:41:34.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ignoring --setgroup ceph since I am not root 2024-09-21T09:41:34.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mgr[93899]: -- 172.21.15.17:0/974119554 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5623921d10e0 con 0x5623921ae800 2024-09-21T09:41:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:41:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/4121811425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: from='client.25571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: pgmap v52: 89 pgs: 89 active+clean; 197 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 654 KiB/s wr, 11 op/s 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: from='client.25577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: mon.a calling monitor election 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: monmap epoch 3 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: last_changed 2024-09-21T09:31:25.701063+0000 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: created 2024-09-21T09:29:44.848123+0000 2024-09-21T09:41:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: min_mon_release 17 (quincy) 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: election_strategy: 1 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: 1: [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon.b 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: mgrmap e34: y(active, since 35s), standbys: x 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: overall HEALTH_OK 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: from='mgr.25274 ' entity='' 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[40737]: mgrmap e35: y(active, since 35s), standbys: x 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:41:34.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: from='mgr.25274 172.21.15.17:0/1888004991' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4121811425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: from='client.25571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: pgmap v52: 89 pgs: 89 active+clean; 197 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 654 KiB/s wr, 11 op/s 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: from='client.25577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: mon.a calling monitor election 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: monmap epoch 3 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: last_changed 2024-09-21T09:31:25.701063+0000 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: created 2024-09-21T09:29:44.848123+0000 2024-09-21T09:41:34.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: min_mon_release 17 (quincy) 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: election_strategy: 1 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: 1: [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon.b 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: mgrmap e34: y(active, since 35s), standbys: x 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: overall HEALTH_OK 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: from='mgr.25274 ' entity='' 2024-09-21T09:41:34.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:34 smithi017 ceph-mon[102913]: mgrmap e35: y(active, since 35s), standbys: x 2024-09-21T09:41:35.497 INFO:tasks.rados.rados.0.smithi017.stdout:873: done (2 left) 2024-09-21T09:41:35.498 INFO:tasks.rados.rados.0.smithi017.stdout:874: done (1 left) 2024-09-21T09:41:35.498 INFO:tasks.rados.rados.0.smithi017.stdout:877: done (0 left) 2024-09-21T09:41:35.498 INFO:tasks.rados.rados.0.smithi017.stdout:878: snap_create 2024-09-21T09:41:35.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:35 smithi134 ceph-mon[33541]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:41:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:35 smithi017 ceph-mon[102913]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:41:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:35 smithi017 ceph-mon[40737]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:41:36.496 INFO:tasks.rados.rados.0.smithi017.stdout:878: done (0 left) 2024-09-21T09:41:36.496 INFO:tasks.rados.rados.0.smithi017.stdout:879: snap_create 2024-09-21T09:41:36.769 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:36 smithi134 ceph-mon[33541]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:41:36.769 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:36 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3543408471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:36 smithi017 ceph-mon[40737]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:41:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:36 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/3543408471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:36 smithi017 ceph-mon[102913]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:41:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:36 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3543408471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:37.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:37.508 INFO:tasks.rados.rados.0.smithi017.stdout:879: done (0 left) 2024-09-21T09:41:37.508 INFO:tasks.rados.rados.0.smithi017.stdout:880: delete oid 27 current snap is 112 2024-09-21T09:41:37.511 INFO:tasks.rados.rados.0.smithi017.stdout:880: done (0 left) 2024-09-21T09:41:37.511 INFO:tasks.rados.rados.0.smithi017.stdout:881: read oid 34 snap -1 2024-09-21T09:41:37.511 INFO:tasks.rados.rados.0.smithi017.stdout:881: expect deleted 2024-09-21T09:41:37.511 INFO:tasks.rados.rados.0.smithi017.stdout:882: delete oid 17 current snap is 112 2024-09-21T09:41:37.513 INFO:tasks.rados.rados.0.smithi017.stdout:882: done (1 left) 2024-09-21T09:41:37.513 INFO:tasks.rados.rados.0.smithi017.stdout:883: snap_remove snap 108 2024-09-21T09:41:37.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:37 smithi017 ceph-mon[40737]: osdmap e282: 8 total, 8 up, 8 in 2024-09-21T09:41:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:37 smithi017 ceph-mon[102913]: osdmap e282: 8 total, 8 up, 8 in 2024-09-21T09:41:37.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:37 smithi134 ceph-mon[33541]: osdmap e282: 8 total, 8 up, 8 in 2024-09-21T09:41:38.012 INFO:tasks.rados.rados.0.smithi017.stdout:881: done (1 left) 2024-09-21T09:41:38.012 INFO:tasks.rados.rados.0.smithi017.stdout:883: done (0 left) 2024-09-21T09:41:38.012 INFO:tasks.rados.rados.0.smithi017.stdout:884: read oid 21 snap 105 2024-09-21T09:41:38.012 INFO:tasks.rados.rados.0.smithi017.stdout:884: expect (ObjNum 343 snap 103 seq_num 343) 2024-09-21T09:41:38.012 INFO:tasks.rados.rados.0.smithi017.stdout:885: snap_create 2024-09-21T09:41:38.020 INFO:tasks.rados.rados.0.smithi017.stdout:884: expect (ObjNum 343 snap 103 seq_num 343) 2024-09-21T09:41:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:38 smithi017 ceph-mon[102913]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:41:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:38 smithi017 ceph-mon[102913]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:41:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:38 smithi017 ceph-mon[40737]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:41:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:38 smithi017 ceph-mon[40737]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:41:38.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:38 smithi134 ceph-mon[33541]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:41:38.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:38 smithi134 ceph-mon[33541]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:41:39.013 INFO:tasks.rados.rados.0.smithi017.stdout:884: done (1 left) 2024-09-21T09:41:39.013 INFO:tasks.rados.rados.0.smithi017.stdout:885: done (0 left) 2024-09-21T09:41:39.013 INFO:tasks.rados.rados.0.smithi017.stdout:886: read oid 5 snap -1 2024-09-21T09:41:39.013 INFO:tasks.rados.rados.0.smithi017.stdout:886: expect (ObjNum 351 snap 106 seq_num 351) 2024-09-21T09:41:39.013 INFO:tasks.rados.rados.0.smithi017.stdout:887: read oid 36 snap -1 2024-09-21T09:41:39.013 INFO:tasks.rados.rados.0.smithi017.stdout:887: expect deleted 2024-09-21T09:41:39.013 INFO:tasks.rados.rados.0.smithi017.stdout:888: snap_remove snap 109 2024-09-21T09:41:39.772 INFO:tasks.rados.rados.0.smithi017.stdout:888: done (2 left) 2024-09-21T09:41:39.772 INFO:tasks.rados.rados.0.smithi017.stdout:889: snap_create 2024-09-21T09:41:39.772 INFO:tasks.rados.rados.0.smithi017.stdout:886: expect (ObjNum 351 snap 106 seq_num 351) 2024-09-21T09:41:39.993 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: [21/Sep/2024:09:41:39] ENGINE Bus STARTING 2024-09-21T09:41:39.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: CherryPy Checker: 2024-09-21T09:41:39.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: The Application mounted at '' has an empty config. 2024-09-21T09:41:39.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: 2024-09-21T09:41:39.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: [21/Sep/2024:09:41:39] ENGINE Serving on http://:::9283 2024-09-21T09:41:39.994 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 09:41:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x[81700]: [21/Sep/2024:09:41:39] ENGINE Bus STARTED 2024-09-21T09:41:40.277 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:41:40.277 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: Standby manager daemon x restarted 2024-09-21T09:41:40.277 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: Standby manager daemon x started 2024-09-21T09:41:40.277 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:41:40.277 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: Active manager daemon y restarted 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: Activating manager daemon y 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: mgrmap e36: y(active, starting, since 0.00905946s), standbys: x 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi017.coodep"} : dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"} : dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-21T09:41:40.278 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T09:41:40.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[102913]: Manager daemon y is now available 2024-09-21T09:41:40.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:41:40.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: Standby manager daemon x restarted 2024-09-21T09:41:40.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: Standby manager daemon x started 2024-09-21T09:41:40.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:41:40.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:41:40.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:41:40.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:41:40.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: Active manager daemon y restarted 2024-09-21T09:41:40.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: Activating manager daemon y 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: mgrmap e36: y(active, starting, since 0.00905946s), standbys: x 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi017.coodep"} : dispatch 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"} : dispatch 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-21T09:41:40.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T09:41:40.282 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-mon[40737]: Manager daemon y is now available 2024-09-21T09:41:40.442 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:40.443 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:41:40.443 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: Standby manager daemon x restarted 2024-09-21T09:41:40.443 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: Standby manager daemon x started 2024-09-21T09:41:40.443 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:41:40.443 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:41:40.443 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:41:40.443 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.? 172.21.15.134:0/732558729' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:41:40.443 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: Active manager daemon y restarted 2024-09-21T09:41:40.443 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: Activating manager daemon y 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: mgrmap e36: y(active, starting, since 0.00905946s), standbys: x 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi017.coodep"} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi134.bwovvb"} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:41:40.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:41:40.445 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:41:40.445 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:41:40.445 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T09:41:40.445 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T09:41:40.445 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T09:41:40.445 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:40 smithi134 ceph-mon[33541]: Manager daemon y is now available 2024-09-21T09:41:40.643 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.274+0000 7f310281e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.643 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.274+0000 7f310281e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.643 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.274+0000 7f310281e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.643 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.274+0000 7f310281e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.643 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.274+0000 7f310281e640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.644 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.275+0000 7f30ff017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.644 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.275+0000 7f30ff017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.644 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.275+0000 7f30ff017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.644 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.275+0000 7f30ff017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.644 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.275+0000 7f30ff017640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:41:40.768 INFO:tasks.rados.rados.0.smithi017.stdout:886: done (2 left) 2024-09-21T09:41:40.768 INFO:tasks.rados.rados.0.smithi017.stdout:887: done (1 left) 2024-09-21T09:41:40.768 INFO:tasks.rados.rados.0.smithi017.stdout:889: done (0 left) 2024-09-21T09:41:40.768 INFO:tasks.rados.rados.0.smithi017.stdout:890: write oid 35 current snap is 114 2024-09-21T09:41:40.768 INFO:tasks.rados.rados.0.smithi017.stdout:890: seq_num 364 ranges {517748=459227,1600622=406110} 2024-09-21T09:41:40.775 INFO:tasks.rados.rados.0.smithi017.stdout:890: writing smithi01785587-35 from 517748 to 976975 tid 1 2024-09-21T09:41:40.782 INFO:tasks.rados.rados.0.smithi017.stdout:890: writing smithi01785587-35 from 1600622 to 2006732 tid 2 2024-09-21T09:41:40.782 INFO:tasks.rados.rados.0.smithi017.stdout:891: read oid 45 snap -1 2024-09-21T09:41:40.782 INFO:tasks.rados.rados.0.smithi017.stdout:891: expect (ObjNum 360 snap 109 seq_num 360) 2024-09-21T09:41:40.782 INFO:tasks.rados.rados.0.smithi017.stdout:892: snap_remove snap 100 2024-09-21T09:41:40.806 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:40.901 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.785+0000 7f3131349640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-21T09:41:40.901 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:40] ENGINE Bus STARTING 2024-09-21T09:41:40.901 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: CherryPy Checker: 2024-09-21T09:41:40.901 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: The Application mounted at '' has an empty config. 2024-09-21T09:41:40.901 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:40.901 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:40] ENGINE Serving on http://:::9283 2024-09-21T09:41:40.902 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:09:41:40] ENGINE Bus STARTED 2024-09-21T09:41:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[102913]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[102913]: mgrmap e37: y(active, since 1.01595s), standbys: x 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:41:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[40737]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:41:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:41 smithi017 ceph-mon[40737]: mgrmap e37: y(active, since 1.01595s), standbys: x 2024-09-21T09:41:41.321 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:41 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:41.321 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:41 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:41:41.321 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:41 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T09:41:41.321 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:41 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:41:41.321 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:41 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:41:41.321 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:41 smithi134 ceph-mon[33541]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:41:41.321 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:41 smithi134 ceph-mon[33541]: mgrmap e37: y(active, since 1.01595s), standbys: x 2024-09-21T09:41:41.775 INFO:tasks.rados.rados.0.smithi017.stdout:892: done (2 left) 2024-09-21T09:41:41.776 INFO:tasks.rados.rados.0.smithi017.stdout:893: write oid 18 current snap is 114 2024-09-21T09:41:41.776 INFO:tasks.rados.rados.0.smithi017.stdout:890: finishing write tid 1 to smithi01785587-35 2024-09-21T09:41:41.776 INFO:tasks.rados.rados.0.smithi017.stdout:893: seq_num 365 ranges {534555=650829,1659104=475451} 2024-09-21T09:41:41.782 INFO:tasks.rados.rados.0.smithi017.stdout:893: writing smithi01785587-18 from 534555 to 1185384 tid 1 2024-09-21T09:41:41.787 INFO:tasks.rados.rados.0.smithi017.stdout:893: writing smithi01785587-18 from 1659104 to 2134555 tid 2 2024-09-21T09:41:41.788 INFO:tasks.rados.rados.0.smithi017.stdout:894: snap_remove snap 107 2024-09-21T09:41:42.108 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[102913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:42.108 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[102913]: from='client.25601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:42.108 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[102913]: pgmap v4: 89 pgs: 89 active+clean; 202 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:42.108 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[102913]: from='client.25375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:42.108 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[102913]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:41:42.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[40737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:42.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[40737]: from='client.25601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:42.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[40737]: pgmap v4: 89 pgs: 89 active+clean; 202 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:42.108 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[40737]: from='client.25375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:42.109 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-mon[40737]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:41:42.267 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:42 smithi134 ceph-mon[33541]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:41:42.267 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:42 smithi134 ceph-mon[33541]: from='client.25601 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:42.267 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:42 smithi134 ceph-mon[33541]: pgmap v4: 89 pgs: 89 active+clean; 202 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:42.267 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:42 smithi134 ceph-mon[33541]: from='client.25375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:42.267 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:42 smithi134 ceph-mon[33541]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:41:42.426 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:42.104+0000 7f30e4e29640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:41:42] ENGINE Error in HTTPServer.serve 2024-09-21T09:41:42.426 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: Traceback (most recent call last): 2024-09-21T09:41:42.426 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:41:42.426 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: self._connections.run(self.expiration_interval) 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: self._run(expiration_interval) 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: s = self.context.wrap_socket( 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: return self.sslsocket_class._create( 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: self.do_handshake() 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:41:42.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: self._sslobj.do_handshake() 2024-09-21T09:41:42.428 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:41:42.428 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:41:42.795 INFO:tasks.rados.rados.0.smithi017.stdout:894: done (3 left) 2024-09-21T09:41:42.795 INFO:tasks.rados.rados.0.smithi017.stdout:895: rollback oid 15 current snap is 114 2024-09-21T09:41:42.795 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 113 2024-09-21T09:41:42.795 INFO:tasks.rados.rados.0.smithi017.stdout:890: finishing write tid 2 to smithi01785587-35 2024-09-21T09:41:42.795 INFO:tasks.rados.rados.0.smithi017.stdout:890: finishing write tid 3 to smithi01785587-35 2024-09-21T09:41:42.795 INFO:tasks.rados.rados.0.smithi017.stdout:890: finishing write tid 4 to smithi01785587-35 2024-09-21T09:41:42.795 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 129 (ObjNum 364 snap 114 seq_num 364) dirty exists 2024-09-21T09:41:42.795 INFO:tasks.rados.rados.0.smithi017.stdout:890: left oid 35 (ObjNum 364 snap 114 seq_num 364) 2024-09-21T09:41:42.795 INFO:tasks.rados.rados.0.smithi017.stdout:891: expect (ObjNum 360 snap 109 seq_num 360) 2024-09-21T09:41:42.920 INFO:tasks.rados.rados.0.smithi017.stdout:893: finishing write tid 1 to smithi01785587-18 2024-09-21T09:41:42.920 INFO:tasks.rados.rados.0.smithi017.stdout:893: finishing write tid 2 to smithi01785587-18 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:890: done (3 left) 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:891: done (2 left) 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:896: read oid 4 snap -1 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:896: expect (ObjNum 312 snap 93 seq_num 312) 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:897: snap_create 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:893: finishing write tid 3 to smithi01785587-18 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:893: finishing write tid 4 to smithi01785587-18 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 76 (ObjNum 365 snap 114 seq_num 365) dirty exists 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:893: left oid 18 (ObjNum 365 snap 114 seq_num 365) 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:895: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:41:42.921 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 176 (ObjNum 129 snap 0 seq_num 2399667440) dirty dne 2024-09-21T09:41:42.928 INFO:tasks.rados.rados.0.smithi017.stdout:896: expect (ObjNum 312 snap 93 seq_num 312) 2024-09-21T09:41:43.040 INFO:tasks.rados.rados.0.smithi017.stdout:893: done (3 left) 2024-09-21T09:41:43.040 INFO:tasks.rados.rados.0.smithi017.stdout:895: done (2 left) 2024-09-21T09:41:43.040 INFO:tasks.rados.rados.0.smithi017.stdout:896: done (1 left) 2024-09-21T09:41:43.040 INFO:tasks.rados.rados.0.smithi017.stdout:897: done (0 left) 2024-09-21T09:41:43.040 INFO:tasks.rados.rados.0.smithi017.stdout:898: write oid 12 current snap is 115 2024-09-21T09:41:43.040 INFO:tasks.rados.rados.0.smithi017.stdout:898: seq_num 366 ranges {551362=726078,1717586=409779} 2024-09-21T09:41:43.047 INFO:tasks.rados.rados.0.smithi017.stdout:898: writing smithi01785587-12 from 551362 to 1277440 tid 1 2024-09-21T09:41:43.052 INFO:tasks.rados.rados.0.smithi017.stdout:898: writing smithi01785587-12 from 1717586 to 2127365 tid 2 2024-09-21T09:41:43.052 INFO:tasks.rados.rados.0.smithi017.stdout:899: delete oid 36 current snap is 115 2024-09-21T09:41:43.054 INFO:tasks.rados.rados.0.smithi017.stdout:898: finishing write tid 1 to smithi01785587-12 2024-09-21T09:41:43.055 INFO:tasks.rados.rados.0.smithi017.stdout:898: finishing write tid 2 to smithi01785587-12 2024-09-21T09:41:43.055 INFO:tasks.rados.rados.0.smithi017.stdout:898: finishing write tid 3 to smithi01785587-12 2024-09-21T09:41:43.055 INFO:tasks.rados.rados.0.smithi017.stdout:898: finishing write tid 4 to smithi01785587-12 2024-09-21T09:41:43.055 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 310 (ObjNum 366 snap 115 seq_num 366) dirty exists 2024-09-21T09:41:43.055 INFO:tasks.rados.rados.0.smithi017.stdout:898: left oid 12 (ObjNum 366 snap 115 seq_num 366) 2024-09-21T09:41:43.055 INFO:tasks.rados.rados.0.smithi017.stdout:898: done (1 left) 2024-09-21T09:41:43.055 INFO:tasks.rados.rados.0.smithi017.stdout:899: done (0 left) 2024-09-21T09:41:43.056 INFO:tasks.rados.rados.0.smithi017.stdout:900: read oid 33 snap -1 2024-09-21T09:41:43.056 INFO:tasks.rados.rados.0.smithi017.stdout:900: expect (ObjNum 310 snap 91 seq_num 310) 2024-09-21T09:41:43.056 INFO:tasks.rados.rados.0.smithi017.stdout:901: delete oid 34 current snap is 115 2024-09-21T09:41:43.056 INFO:tasks.rados.rados.0.smithi017.stdout:901: done (1 left) 2024-09-21T09:41:43.056 INFO:tasks.rados.rados.0.smithi017.stdout:902: snap_create 2024-09-21T09:41:43.063 INFO:tasks.rados.rados.0.smithi017.stdout:900: expect (ObjNum 310 snap 91 seq_num 310) 2024-09-21T09:41:43.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:43 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:41] ENGINE Bus STARTING 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:43 smithi134 ceph-mon[33541]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:43 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:42] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:43 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:42] ENGINE Error in HTTPServer.serve 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: Traceback (most recent call last): 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._run(expiration_interval) 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:41:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: s = self.context.wrap_socket( 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: return self.sslsocket_class._create( 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: self.do_handshake() 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: self._sslobj.do_handshake() 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:43 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:42] ENGINE Serving on http://172.21.15.17:8765 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:43 smithi134 ceph-mon[33541]: [21/Sep/2024:09:41:42] ENGINE Bus STARTED 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:43 smithi134 ceph-mon[33541]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:41:43.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:43 smithi134 ceph-mon[33541]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[102913]: [21/Sep/2024:09:41:41] ENGINE Bus STARTING 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[102913]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[102913]: [21/Sep/2024:09:41:42] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[102913]: [21/Sep/2024:09:41:42] ENGINE Error in HTTPServer.serve 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:41:43.372 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: self.do_handshake() 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[102913]: [21/Sep/2024:09:41:42] ENGINE Serving on http://172.21.15.17:8765 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[102913]: [21/Sep/2024:09:41:42] ENGINE Bus STARTED 2024-09-21T09:41:43.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[102913]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[102913]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:41] ENGINE Bus STARTING 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[40737]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:42] ENGINE Serving on https://172.21.15.17:7150 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:42] ENGINE Error in HTTPServer.serve 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout: Traceback (most recent call last): 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._run(expiration_interval) 2024-09-21T09:41:43.374 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: s = self.context.wrap_socket( 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: return self.sslsocket_class._create( 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: self.do_handshake() 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: self._sslobj.do_handshake() 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:42] ENGINE Serving on http://172.21.15.17:8765 2024-09-21T09:41:43.375 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[40737]: [21/Sep/2024:09:41:42] ENGINE Bus STARTED 2024-09-21T09:41:43.376 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[40737]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:41:43.376 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:43 smithi017 ceph-mon[40737]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:41:43.824 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:44.032 INFO:tasks.rados.rados.0.smithi017.stdout:900: done (1 left) 2024-09-21T09:41:44.033 INFO:tasks.rados.rados.0.smithi017.stdout:902: done (0 left) 2024-09-21T09:41:44.033 INFO:tasks.rados.rados.0.smithi017.stdout:903: delete oid 23 current snap is 116 2024-09-21T09:41:44.035 INFO:tasks.rados.rados.0.smithi017.stdout:903: done (0 left) 2024-09-21T09:41:44.035 INFO:tasks.rados.rados.0.smithi017.stdout:904: delete oid 30 current snap is 116 2024-09-21T09:41:44.038 INFO:tasks.rados.rados.0.smithi017.stdout:904: done (0 left) 2024-09-21T09:41:44.038 INFO:tasks.rados.rados.0.smithi017.stdout:905: write oid 39 current snap is 116 2024-09-21T09:41:44.038 INFO:tasks.rados.rados.0.smithi017.stdout:905: seq_num 371 ranges {635397=702323,1893643=341754} 2024-09-21T09:41:44.045 INFO:tasks.rados.rados.0.smithi017.stdout:905: writing smithi01785587-39 from 635397 to 1337720 tid 1 2024-09-21T09:41:44.051 INFO:tasks.rados.rados.0.smithi017.stdout:905: writing smithi01785587-39 from 1893643 to 2235397 tid 2 2024-09-21T09:41:44.051 INFO:tasks.rados.rados.0.smithi017.stdout:906: snap_create 2024-09-21T09:41:44.053 INFO:tasks.rados.rados.0.smithi017.stdout:905: finishing write tid 1 to smithi01785587-39 2024-09-21T09:41:44.054 INFO:tasks.rados.rados.0.smithi017.stdout:905: finishing write tid 2 to smithi01785587-39 2024-09-21T09:41:44.054 INFO:tasks.rados.rados.0.smithi017.stdout:905: finishing write tid 3 to smithi01785587-39 2024-09-21T09:41:44.054 INFO:tasks.rados.rados.0.smithi017.stdout:905: finishing write tid 4 to smithi01785587-39 2024-09-21T09:41:44.054 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 433 (ObjNum 371 snap 116 seq_num 371) dirty exists 2024-09-21T09:41:44.054 INFO:tasks.rados.rados.0.smithi017.stdout:905: left oid 39 (ObjNum 371 snap 116 seq_num 371) 2024-09-21T09:41:44.177 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2979347697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:44.178 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:44 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2979347697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:44.453 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:44 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2979347697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:45.026 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:41:44] "GET /metrics HTTP/1.1" 200 33569 "" "Prometheus/2.51.0" 2024-09-21T09:41:45.035 INFO:tasks.rados.rados.0.smithi017.stdout:905: done (1 left) 2024-09-21T09:41:45.036 INFO:tasks.rados.rados.0.smithi017.stdout:906: done (0 left) 2024-09-21T09:41:45.036 INFO:tasks.rados.rados.0.smithi017.stdout:907: delete oid 43 current snap is 117 2024-09-21T09:41:45.038 INFO:tasks.rados.rados.0.smithi017.stdout:907: done (0 left) 2024-09-21T09:41:45.038 INFO:tasks.rados.rados.0.smithi017.stdout:908: delete oid 5 current snap is 117 2024-09-21T09:41:45.039 INFO:tasks.rados.rados.0.smithi017.stdout:908: done (0 left) 2024-09-21T09:41:45.039 INFO:tasks.rados.rados.0.smithi017.stdout:909: read oid 3 snap -1 2024-09-21T09:41:45.039 INFO:tasks.rados.rados.0.smithi017.stdout:909: expect deleted 2024-09-21T09:41:45.040 INFO:tasks.rados.rados.0.smithi017.stdout:910: rollback oid 37 current snap is 117 2024-09-21T09:41:45.040 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 113 2024-09-21T09:41:45.040 INFO:tasks.rados.rados.0.smithi017.stdout:911: snap_remove snap 102 2024-09-21T09:41:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[102913]: pgmap v9: 89 pgs: 89 active+clean; 202 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[102913]: from='client.25652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[102913]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:41:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[102913]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:41:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[102913]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[40737]: pgmap v9: 89 pgs: 89 active+clean; 202 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:45.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[40737]: from='client.25652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:45.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[40737]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:41:45.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[40737]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:41:45.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[40737]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:45.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:45 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-mon[33541]: pgmap v9: 89 pgs: 89 active+clean; 202 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-mon[33541]: from='client.25652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-mon[33541]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:41:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-mon[33541]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:41:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-mon[33541]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:45.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:45.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:46.043 INFO:tasks.rados.rados.0.smithi017.stdout:911: done (2 left) 2024-09-21T09:41:46.043 INFO:tasks.rados.rados.0.smithi017.stdout:912: rollback oid 40 current snap is 117 2024-09-21T09:41:46.043 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 113 2024-09-21T09:41:46.043 INFO:tasks.rados.rados.0.smithi017.stdout:913: write oid 2 current snap is 117 2024-09-21T09:41:46.043 INFO:tasks.rados.rados.0.smithi017.stdout:913: seq_num 374 ranges {685818=644423,2069089=216729} 2024-09-21T09:41:46.050 INFO:tasks.rados.rados.0.smithi017.stdout:913: writing smithi01785587-2 from 685818 to 1330241 tid 1 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:913: writing smithi01785587-2 from 2069089 to 2285818 tid 2 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:914: delete oid 25 current snap is 117 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:910: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:910: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:910: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 437 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:912: finishing rollback tid 0 to smithi01785587-40 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:912: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:912: finishing rollback tid 2 to smithi01785587-40 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 445 (ObjNum 281 snap 85 seq_num 281) dirty exists 2024-09-21T09:41:46.056 INFO:tasks.rados.rados.0.smithi017.stdout:913: finishing write tid 1 to smithi01785587-2 2024-09-21T09:41:46.058 INFO:tasks.rados.rados.0.smithi017.stdout:909: done (4 left) 2024-09-21T09:41:46.058 INFO:tasks.rados.rados.0.smithi017.stdout:910: done (3 left) 2024-09-21T09:41:46.058 INFO:tasks.rados.rados.0.smithi017.stdout:912: done (2 left) 2024-09-21T09:41:46.058 INFO:tasks.rados.rados.0.smithi017.stdout:914: done (1 left) 2024-09-21T09:41:46.058 INFO:tasks.rados.rados.0.smithi017.stdout:915: rollback oid 27 current snap is 117 2024-09-21T09:41:46.058 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 103 2024-09-21T09:41:46.058 INFO:tasks.rados.rados.0.smithi017.stdout:916: read oid 38 snap -1 2024-09-21T09:41:46.058 INFO:tasks.rados.rados.0.smithi017.stdout:916: expect (ObjNum 298 snap 90 seq_num 298) 2024-09-21T09:41:46.058 INFO:tasks.rados.rados.0.smithi017.stdout:917: read oid 33 snap -1 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi017.stdout:917: expect (ObjNum 310 snap 91 seq_num 310) 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi017.stdout:918: rollback oid 7 current snap is 117 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 116 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi017.stdout:913: finishing write tid 2 to smithi01785587-2 2024-09-21T09:41:46.059 INFO:tasks.rados.rados.0.smithi017.stdout:919: snap_remove snap 110 2024-09-21T09:41:46.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:46 smithi134 ceph-mon[33541]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:41:46.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:46 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:46 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:46 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.309 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:46 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[102913]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:41:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[40737]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:41:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:46 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:46.807 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:47.045 INFO:tasks.rados.rados.0.smithi017.stdout:913: finishing write tid 3 to smithi01785587-2 2024-09-21T09:41:47.045 INFO:tasks.rados.rados.0.smithi017.stdout:913: finishing write tid 4 to smithi01785587-2 2024-09-21T09:41:47.045 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 160 (ObjNum 374 snap 117 seq_num 374) dirty exists 2024-09-21T09:41:47.045 INFO:tasks.rados.rados.0.smithi017.stdout:913: left oid 2 (ObjNum 374 snap 117 seq_num 374) 2024-09-21T09:41:47.045 INFO:tasks.rados.rados.0.smithi017.stdout:918: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:41:47.045 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 0 (ObjNum 129 snap 0 seq_num 2399828008) dirty dne 2024-09-21T09:41:47.046 INFO:tasks.rados.rados.0.smithi017.stdout:915: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:41:47.046 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 347 (ObjNum 81 snap 0 seq_num 1) dirty dne 2024-09-21T09:41:47.046 INFO:tasks.rados.rados.0.smithi017.stdout:916: expect (ObjNum 298 snap 90 seq_num 298) 2024-09-21T09:41:47.178 INFO:tasks.rados.rados.0.smithi017.stdout:917: expect (ObjNum 310 snap 91 seq_num 310) 2024-09-21T09:41:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:47 smithi017 ceph-mon[102913]: pgmap v12: 89 pgs: 89 active+clean; 202 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:47 smithi017 ceph-mon[102913]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:41:47.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:47 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/906399728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:47.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:47 smithi017 ceph-mon[40737]: pgmap v12: 89 pgs: 89 active+clean; 202 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:47.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:47 smithi017 ceph-mon[40737]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:41:47.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:47 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/906399728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:47.234 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:47 smithi134 ceph-mon[33541]: pgmap v12: 89 pgs: 89 active+clean; 202 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:41:47.234 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:47 smithi134 ceph-mon[33541]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:41:47.234 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:47 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/906399728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-21T09:41:47.234 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:47.285 INFO:tasks.rados.rados.0.smithi017.stdout:913: done (5 left) 2024-09-21T09:41:47.285 INFO:tasks.rados.rados.0.smithi017.stdout:915: done (4 left) 2024-09-21T09:41:47.285 INFO:tasks.rados.rados.0.smithi017.stdout:916: done (3 left) 2024-09-21T09:41:47.285 INFO:tasks.rados.rados.0.smithi017.stdout:917: done (2 left) 2024-09-21T09:41:47.285 INFO:tasks.rados.rados.0.smithi017.stdout:918: done (1 left) 2024-09-21T09:41:47.285 INFO:tasks.rados.rados.0.smithi017.stdout:919: done (0 left) 2024-09-21T09:41:47.286 INFO:tasks.rados.rados.0.smithi017.stdout:920: rollback oid 41 current snap is 117 2024-09-21T09:41:47.286 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 87 2024-09-21T09:41:47.286 INFO:tasks.rados.rados.0.smithi017.stdout:921: snap_remove snap 113 2024-09-21T09:41:48.052 INFO:tasks.rados.rados.0.smithi017.stdout:921: done (1 left) 2024-09-21T09:41:48.053 INFO:tasks.rados.rados.0.smithi017.stdout:920: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:41:48.053 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 0 (ObjNum 129 snap 0 seq_num 3570419963) dirty dne 2024-09-21T09:41:48.053 INFO:tasks.rados.rados.0.smithi017.stdout:922: read oid 27 snap -1 2024-09-21T09:41:48.053 INFO:tasks.rados.rados.0.smithi017.stdout:922: expect deleted 2024-09-21T09:41:48.053 INFO:tasks.rados.rados.0.smithi017.stdout:920: done (1 left) 2024-09-21T09:41:48.053 INFO:tasks.rados.rados.0.smithi017.stdout:923: write oid 35 current snap is 117 2024-09-21T09:41:48.053 INFO:tasks.rados.rados.0.smithi017.stdout:923: seq_num 376 ranges {719432=794921,2186053=133379} 2024-09-21T09:41:48.061 INFO:tasks.rados.rados.0.smithi017.stdout:923: writing smithi01785587-35 from 719432 to 1514353 tid 1 2024-09-21T09:41:48.066 INFO:tasks.rados.rados.0.smithi017.stdout:923: writing smithi01785587-35 from 2186053 to 2319432 tid 2 2024-09-21T09:41:48.066 INFO:tasks.rados.rados.0.smithi017.stdout:924: write oid 17 current snap is 117 2024-09-21T09:41:48.066 INFO:tasks.rados.rados.0.smithi017.stdout:924: seq_num 377 ranges {736239=470170,1728182=571782} 2024-09-21T09:41:48.072 INFO:tasks.rados.rados.0.smithi017.stdout:924: writing smithi01785587-17 from 736239 to 1206409 tid 1 2024-09-21T09:41:48.079 INFO:tasks.rados.rados.0.smithi017.stdout:924: writing smithi01785587-17 from 1728182 to 2299964 tid 2 2024-09-21T09:41:48.079 INFO:tasks.rados.rados.0.smithi017.stdout:925: read oid 31 snap -1 2024-09-21T09:41:48.079 INFO:tasks.rados.rados.0.smithi017.stdout:925: expect (ObjNum 304 snap 91 seq_num 304) 2024-09-21T09:41:48.079 INFO:tasks.rados.rados.0.smithi017.stdout:926: snap_remove snap 114 2024-09-21T09:41:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:48 smithi017 ceph-mon[40737]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:48 smithi017 ceph-mon[40737]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:41:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:48 smithi017 ceph-mon[40737]: from='client.25676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:48 smithi017 ceph-mon[102913]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:48 smithi017 ceph-mon[102913]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:41:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:48 smithi017 ceph-mon[102913]: from='client.25676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:48.453 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:48 smithi134 ceph-mon[33541]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:48.454 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:48 smithi134 ceph-mon[33541]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:41:48.454 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:48 smithi134 ceph-mon[33541]: from='client.25676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:49.056 INFO:tasks.rados.rados.0.smithi017.stdout:926: done (4 left) 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:927: read oid 34 snap -1 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:927: expect deleted 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:928: snap_create 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:923: finishing write tid 1 to smithi01785587-35 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:923: finishing write tid 2 to smithi01785587-35 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:923: finishing write tid 3 to smithi01785587-35 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:923: finishing write tid 4 to smithi01785587-35 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 141 (ObjNum 376 snap 117 seq_num 376) dirty exists 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:923: left oid 35 (ObjNum 376 snap 117 seq_num 376) 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:924: finishing write tid 1 to smithi01785587-17 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:924: finishing write tid 2 to smithi01785587-17 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:924: finishing write tid 3 to smithi01785587-17 2024-09-21T09:41:49.057 INFO:tasks.rados.rados.0.smithi017.stdout:924: finishing write tid 4 to smithi01785587-17 2024-09-21T09:41:49.058 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 393 (ObjNum 377 snap 117 seq_num 377) dirty exists 2024-09-21T09:41:49.058 INFO:tasks.rados.rados.0.smithi017.stdout:924: left oid 17 (ObjNum 377 snap 117 seq_num 377) 2024-09-21T09:41:49.058 INFO:tasks.rados.rados.0.smithi017.stdout:925: expect (ObjNum 304 snap 91 seq_num 304) 2024-09-21T09:41:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[102913]: pgmap v15: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s rd, 808 KiB/s wr, 27 op/s 2024-09-21T09:41:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[102913]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:41:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[40737]: pgmap v15: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s rd, 808 KiB/s wr, 27 op/s 2024-09-21T09:41:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[40737]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:41:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:49 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:49.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:49 smithi134 ceph-mon[33541]: pgmap v15: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s rd, 808 KiB/s wr, 27 op/s 2024-09-21T09:41:49.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:49 smithi134 ceph-mon[33541]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:41:49.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:49 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:49 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:49 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:49 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:41:49.801 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:922: done (5 left) 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:923: done (4 left) 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:924: done (3 left) 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:925: done (2 left) 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:927: done (1 left) 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:928: done (0 left) 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:929: rollback oid 28 current snap is 118 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 28 to 115 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:930: read oid 29 snap -1 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:930: expect deleted 2024-09-21T09:41:50.058 INFO:tasks.rados.rados.0.smithi017.stdout:931: rollback oid 42 current snap is 118 2024-09-21T09:41:50.059 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 117 2024-09-21T09:41:50.059 INFO:tasks.rados.rados.0.smithi017.stdout:932: read oid 46 snap -1 2024-09-21T09:41:50.059 INFO:tasks.rados.rados.0.smithi017.stdout:932: expect (ObjNum 210 snap 60 seq_num 210) 2024-09-21T09:41:50.059 INFO:tasks.rados.rados.0.smithi017.stdout:933: write oid 1 current snap is 118 2024-09-21T09:41:50.059 INFO:tasks.rados.rados.0.smithi017.stdout:933: seq_num 378 ranges {753046=545419,1786664=431793} 2024-09-21T09:41:50.059 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:41:50.059 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:41:50.059 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='client.? 172.21.15.17:0/2614273695' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-21T09:41:50.065 INFO:tasks.rados.rados.0.smithi017.stdout:933: writing smithi01785587-1 from 753046 to 1298465 tid 1 2024-09-21T09:41:50.071 INFO:tasks.rados.rados.0.smithi017.stdout:933: writing smithi01785587-1 from 1786664 to 2218457 tid 2 2024-09-21T09:41:50.071 INFO:tasks.rados.rados.0.smithi017.stdout:934: read oid 30 snap -1 2024-09-21T09:41:50.071 INFO:tasks.rados.rados.0.smithi017.stdout:934: expect deleted 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:935: delete oid 48 current snap is 118 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:929: finishing rollback tid 0 to smithi01785587-28 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:929: finishing rollback tid 1 to smithi01785587-28 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:929: finishing rollback tid 2 to smithi01785587-28 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 484 (ObjNum 358 snap 108 seq_num 358) dirty exists 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:931: finishing rollback tid 0 to smithi01785587-42 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:931: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:931: finishing rollback tid 2 to smithi01785587-42 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 322 (ObjNum 321 snap 97 seq_num 321) dirty exists 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:933: finishing write tid 1 to smithi01785587-1 2024-09-21T09:41:50.072 INFO:tasks.rados.rados.0.smithi017.stdout:932: expect (ObjNum 210 snap 60 seq_num 210) 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:933: finishing write tid 2 to smithi01785587-1 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:933: finishing write tid 3 to smithi01785587-1 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:933: finishing write tid 4 to smithi01785587-1 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 261 (ObjNum 378 snap 118 seq_num 378) dirty exists 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:933: left oid 1 (ObjNum 378 snap 118 seq_num 378) 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:929: done (6 left) 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:930: done (5 left) 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:931: done (4 left) 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:932: done (3 left) 2024-09-21T09:41:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:933: done (2 left) 2024-09-21T09:41:50.192 INFO:tasks.rados.rados.0.smithi017.stdout:934: done (1 left) 2024-09-21T09:41:50.192 INFO:tasks.rados.rados.0.smithi017.stdout:935: done (0 left) 2024-09-21T09:41:50.192 INFO:tasks.rados.rados.0.smithi017.stdout:936: snap_remove snap 112 2024-09-21T09:41:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[40737]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2614273695' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-21T09:41:50.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:50.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:50.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:41:50.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-21T09:41:50.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:50 smithi017 ceph-mon[102913]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:41:50.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: Updating smithi017:/etc/ceph/ceph.conf 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: Updating smithi134:/etc/ceph/ceph.conf 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: Updating smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: Updating smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:50.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/2614273695' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-21T09:41:50.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:41:50.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:41:50.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:41:50.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-21T09:41:50.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:50 smithi134 ceph-mon[33541]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:41:51.063 INFO:tasks.rados.rados.0.smithi017.stdout:936: done (0 left) 2024-09-21T09:41:51.063 INFO:tasks.rados.rados.0.smithi017.stdout:937: read oid 5 snap -1 2024-09-21T09:41:51.063 INFO:tasks.rados.rados.0.smithi017.stdout:937: expect deleted 2024-09-21T09:41:51.063 INFO:tasks.rados.rados.0.smithi017.stdout:938: delete oid 37 current snap is 118 2024-09-21T09:41:51.070 INFO:tasks.rados.rados.0.smithi017.stdout:937: done (1 left) 2024-09-21T09:41:51.070 INFO:tasks.rados.rados.0.smithi017.stdout:938: done (0 left) 2024-09-21T09:41:51.070 INFO:tasks.rados.rados.0.smithi017.stdout:939: write oid 9 current snap is 118 2024-09-21T09:41:51.070 INFO:tasks.rados.rados.0.smithi017.stdout:939: seq_num 381 ranges {403467=487519,1562110=579120} 2024-09-21T09:41:51.075 INFO:tasks.rados.rados.0.smithi017.stdout:939: writing smithi01785587-9 from 403467 to 890986 tid 1 2024-09-21T09:41:51.082 INFO:tasks.rados.rados.0.smithi017.stdout:939: writing smithi01785587-9 from 1562110 to 2141230 tid 2 2024-09-21T09:41:51.082 INFO:tasks.rados.rados.0.smithi017.stdout:940: read oid 42 snap -1 2024-09-21T09:41:51.082 INFO:tasks.rados.rados.0.smithi017.stdout:940: expect (ObjNum 321 snap 97 seq_num 321) 2024-09-21T09:41:51.082 INFO:tasks.rados.rados.0.smithi017.stdout:941: snap_remove snap 87 2024-09-21T09:41:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[102913]: pgmap v18: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s rd, 808 KiB/s wr, 27 op/s 2024-09-21T09:41:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[102913]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[102913]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:41:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[102913]: from='client.25694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:41:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:41:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[40737]: pgmap v18: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s rd, 808 KiB/s wr, 27 op/s 2024-09-21T09:41:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[40737]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[40737]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:41:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[40737]: from='client.25694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:41:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:41:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:51 smithi017 ceph-mon[40737]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:51 smithi134 ceph-mon[33541]: pgmap v18: 89 pgs: 89 active+clean; 205 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s rd, 808 KiB/s wr, 27 op/s 2024-09-21T09:41:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:51 smithi134 ceph-mon[33541]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:51 smithi134 ceph-mon[33541]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:41:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:51 smithi134 ceph-mon[33541]: from='client.25694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:51 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:41:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:51 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:41:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:51 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:41:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:51 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:41:52.083 INFO:tasks.rados.rados.0.smithi017.stdout:941: done (2 left) 2024-09-21T09:41:52.083 INFO:tasks.rados.rados.0.smithi017.stdout:942: write oid 47 current snap is 118 2024-09-21T09:41:52.083 INFO:tasks.rados.rados.0.smithi017.stdout:939: finishing write tid 1 to smithi01785587-9 2024-09-21T09:41:52.083 INFO:tasks.rados.rados.0.smithi017.stdout:942: seq_num 382 ranges {420274=562768,1620592=439131} 2024-09-21T09:41:52.089 INFO:tasks.rados.rados.0.smithi017.stdout:942: writing smithi01785587-47 from 420274 to 983042 tid 1 2024-09-21T09:41:52.095 INFO:tasks.rados.rados.0.smithi017.stdout:942: writing smithi01785587-47 from 1620592 to 2059723 tid 2 2024-09-21T09:41:52.095 INFO:tasks.rados.rados.0.smithi017.stdout:943: write oid 39 current snap is 118 2024-09-21T09:41:52.095 INFO:tasks.rados.rados.0.smithi017.stdout:939: finishing write tid 2 to smithi01785587-9 2024-09-21T09:41:52.095 INFO:tasks.rados.rados.0.smithi017.stdout:939: finishing write tid 3 to smithi01785587-9 2024-09-21T09:41:52.096 INFO:tasks.rados.rados.0.smithi017.stdout:939: finishing write tid 4 to smithi01785587-9 2024-09-21T09:41:52.096 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 171 (ObjNum 381 snap 118 seq_num 381) dirty exists 2024-09-21T09:41:52.096 INFO:tasks.rados.rados.0.smithi017.stdout:939: left oid 9 (ObjNum 381 snap 118 seq_num 381) 2024-09-21T09:41:52.096 INFO:tasks.rados.rados.0.smithi017.stdout:940: expect (ObjNum 321 snap 97 seq_num 321) 2024-09-21T09:41:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:52 smithi017 ceph-mon[102913]: Upgrade: Updating mon.c 2024-09-21T09:41:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:52 smithi017 ceph-mon[102913]: Deploying daemon mon.c on smithi017 2024-09-21T09:41:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:41:52 smithi017 ceph-mon[102913]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:41:52.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:52 smithi017 ceph-mon[40737]: Upgrade: Updating mon.c 2024-09-21T09:41:52.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:52 smithi017 ceph-mon[40737]: Deploying daemon mon.c on smithi017 2024-09-21T09:41:52.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:52 smithi017 ceph-mon[40737]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:41:52.330 INFO:tasks.rados.rados.0.smithi017.stdout:942: finishing write tid 1 to smithi01785587-47 2024-09-21T09:41:52.331 INFO:tasks.rados.rados.0.smithi017.stdout:942: finishing write tid 2 to smithi01785587-47 2024-09-21T09:41:52.331 INFO:tasks.rados.rados.0.smithi017.stdout:942: finishing write tid 3 to smithi01785587-47 2024-09-21T09:41:52.331 INFO:tasks.rados.rados.0.smithi017.stdout:942: finishing write tid 4 to smithi01785587-47 2024-09-21T09:41:52.331 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 334 (ObjNum 382 snap 118 seq_num 382) dirty exists 2024-09-21T09:41:52.331 INFO:tasks.rados.rados.0.smithi017.stdout:942: left oid 47 (ObjNum 382 snap 118 seq_num 382) 2024-09-21T09:41:52.331 INFO:tasks.rados.rados.0.smithi017.stdout:943: seq_num 383 ranges {437081=638017,1679074=582789} 2024-09-21T09:41:52.336 INFO:tasks.rados.rados.0.smithi017.stdout:943: writing smithi01785587-39 from 437081 to 1075098 tid 1 2024-09-21T09:41:52.343 INFO:tasks.rados.rados.0.smithi017.stdout:943: writing smithi01785587-39 from 1679074 to 2261863 tid 2 2024-09-21T09:41:52.343 INFO:tasks.rados.rados.0.smithi017.stdout:939: done (3 left) 2024-09-21T09:41:52.343 INFO:tasks.rados.rados.0.smithi017.stdout:940: done (2 left) 2024-09-21T09:41:52.343 INFO:tasks.rados.rados.0.smithi017.stdout:942: done (1 left) 2024-09-21T09:41:52.343 INFO:tasks.rados.rados.0.smithi017.stdout:944: rollback oid 24 current snap is 118 2024-09-21T09:41:52.343 INFO:tasks.rados.rados.0.smithi017.stdout:943: finishing write tid 1 to smithi01785587-39 2024-09-21T09:41:52.343 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 24 to 115 2024-09-21T09:41:52.343 INFO:tasks.rados.rados.0.smithi017.stdout:945: write oid 6 current snap is 118 2024-09-21T09:41:52.344 INFO:tasks.rados.rados.0.smithi017.stdout:945: seq_num 384 ranges {453888=713266,1737556=442800} 2024-09-21T09:41:52.349 INFO:tasks.rados.rados.0.smithi017.stdout:945: writing smithi01785587-6 from 453888 to 1167154 tid 1 2024-09-21T09:41:52.355 INFO:tasks.rados.rados.0.smithi017.stdout:945: writing smithi01785587-6 from 1737556 to 2180356 tid 2 2024-09-21T09:41:52.355 INFO:tasks.rados.rados.0.smithi017.stdout:946: snap_remove snap 104 2024-09-21T09:41:52.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:52 smithi134 ceph-mon[33541]: Upgrade: Updating mon.c 2024-09-21T09:41:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:52 smithi134 ceph-mon[33541]: Deploying daemon mon.c on smithi017 2024-09-21T09:41:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:41:52 smithi134 ceph-mon[33541]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:41:52.810 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:41:52.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:52 smithi017 systemd[1]: Stopping Ceph mon.c for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:53.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:52 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c[40714]: 2024-09-21T09:41:52.970+0000 7fbff8772640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:41:53.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:52 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c[40714]: 2024-09-21T09:41:52.970+0000 7fbff8772640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:41:53.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:53 smithi017 podman[104942]: 2024-09-21 09:41:53.112936782 +0000 UTC m=+0.261776981 container died 316af690367dd0f1d316a8d69361cc25a7800a5c81dbbce1a7b0d7692e902795 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:41:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:53 smithi017 podman[104942]: 2024-09-21 09:41:53.518799518 +0000 UTC m=+0.667639707 container cleanup 316af690367dd0f1d316a8d69361cc25a7800a5c81dbbce1a7b0d7692e902795 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-21T09:41:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:53 smithi017 bash[104942]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c 2024-09-21T09:41:54.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:53 smithi017 podman[104982]: 2024-09-21 09:41:53.745282683 +0000 UTC m=+0.627615668 container remove 316af690367dd0f1d316a8d69361cc25a7800a5c81dbbce1a7b0d7692e902795 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T09:41:54.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:53 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.c.service: Deactivated successfully. 2024-09-21T09:41:54.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:53 smithi017 systemd[1]: Stopped Ceph mon.c for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:54.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:53 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.c.service: Consumed 9.063s CPU time. 2024-09-21T09:41:54.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 systemd[1]: Starting Ceph mon.c for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:54.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 podman[105113]: 2024-09-21 09:41:54.194564782 +0000 UTC m=+0.016501543 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:41:54.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 podman[105113]: 2024-09-21 09:41:54.31293767 +0000 UTC m=+0.134874420 container create 013cf9450c578dd09408b41d25b12e9b5b3de35d41bd8a0cd0adc72bc322c210 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:41:54.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:54 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:54.149Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:55.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:41:54] "GET /metrics HTTP/1.1" 200 33569 "" "Prometheus/2.51.0" 2024-09-21T09:41:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 podman[105113]: 2024-09-21 09:41:54.699030548 +0000 UTC m=+0.520967297 container init 013cf9450c578dd09408b41d25b12e9b5b3de35d41bd8a0cd0adc72bc322c210 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T09:41:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 podman[105113]: 2024-09-21 09:41:54.704675798 +0000 UTC m=+0.526612547 container start 013cf9450c578dd09408b41d25b12e9b5b3de35d41bd8a0cd0adc72bc322c210 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T09:41:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:41:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: pidfile_write: ignore empty --pid-file 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: load: jerasure load: lrc 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Git sha 0 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: DB SUMMARY 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: DB Session ID: SC8ME80RNZLCPHYCMPLQ 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1091 Bytes 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 10622172 ; 2024-09-21T09:41:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:41:55.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:41:55.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:41:55.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:41:55.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:41:55.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:41:55.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.env: 0x55a1aeb7f120 2024-09-21T09:41:55.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:41:55.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.info_log: 0x55a1b0797700 2024-09-21T09:41:55.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:41:55.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.statistics: (nil) 2024-09-21T09:41:55.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.use_fsync: 0 2024-09-21T09:41:55.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:41:55.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:41:55.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:41:55.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.db_log_dir: 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.wal_dir: 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:41:55.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.write_buffer_manager: 0x55a1b079a1e0 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:41:55.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.unordered_write: 0 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.row_cache: None 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.wal_filter: None 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.wal_compression: 0 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:41:55.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:41:55.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_open_files: -1 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:41:55.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Compression algorithms supported: 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: kZSTD supported: 0 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-21T09:41:55.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:41:55.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:41:55.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.merge_operator: 2024-09-21T09:41:55.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_filter: None 2024-09-21T09:41:55.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:41:55.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:41:55.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:41:55.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:41:55.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a1b0797320) 2024-09-21T09:41:55.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:41:55.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:41:55.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:41:55.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:41:55.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_type: 0 2024-09-21T09:41:55.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_index_type: 0 2024-09-21T09:41:55.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_shortening: 1 2024-09-21T09:41:55.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:41:55.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: checksum: 4 2024-09-21T09:41:55.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: no_block_cache: 0 2024-09-21T09:41:55.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache: 0x55a1b078b610 2024-09-21T09:41:55.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:41:55.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_options: 2024-09-21T09:41:55.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: capacity : 536870912 2024-09-21T09:41:55.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: num_shard_bits : 4 2024-09-21T09:41:55.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: strict_capacity_limit : 0 2024-09-21T09:41:55.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:41:55.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_compressed: (nil) 2024-09-21T09:41:55.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: persistent_cache: (nil) 2024-09-21T09:41:55.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size: 4096 2024-09-21T09:41:55.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size_deviation: 10 2024-09-21T09:41:55.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_restart_interval: 16 2024-09-21T09:41:55.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_block_restart_interval: 1 2024-09-21T09:41:55.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: metadata_block_size: 4096 2024-09-21T09:41:55.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: partition_filters: 0 2024-09-21T09:41:55.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: use_delta_encoding: 1 2024-09-21T09:41:55.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: filter_policy: bloomfilter 2024-09-21T09:41:55.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: whole_key_filtering: 1 2024-09-21T09:41:55.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: verify_compression: 0 2024-09-21T09:41:55.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:41:55.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: format_version: 5 2024-09-21T09:41:55.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_index_compression: 1 2024-09-21T09:41:55.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_align: 0 2024-09-21T09:41:55.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_auto_readahead_size: 262144 2024-09-21T09:41:55.219 INFO:journalctl@ceph.mon.c.smithi017.stdout: prepopulate_block_cache: 0 2024-09-21T09:41:55.219 INFO:journalctl@ceph.mon.c.smithi017.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:41:55.219 INFO:journalctl@ceph.mon.c.smithi017.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:41:55.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:41:55.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:41:55.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression: NoCompression 2024-09-21T09:41:55.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:41:55.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:41:55.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:41:55.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.num_levels: 7 2024-09-21T09:41:55.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:41:55.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:41:55.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:41:55.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:41:55.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:41:55.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:41:55.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:41:55.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:41:55.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:41:55.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:41:55.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:41:55.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:41:55.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:41:55.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:41:55.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:41:55.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:41:55.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:41:55.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:41:55.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:41:55.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:41:55.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:41:55.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:41:55.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:41:55.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:41:55.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:41:55.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:41:55.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:41:55.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:41:55.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:41:55.225 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:41:55.225 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:41:55.225 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:41:55.225 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:41:55.225 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:41:55.225 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:41:55.225 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:41:55.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:41:55.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:41:55.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:41:55.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:41:55.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:41:55.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:41:55.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:41:55.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:41:55.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:41:55.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:41:55.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:41:55.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:41:55.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:41:55.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:41:55.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:41:55.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:41:55.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:41:55.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:41:55.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:41:55.229 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:41:55.229 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:41:55.229 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:41:55.229 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:41:55.229 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:41:55.229 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:41:55.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:41:55.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:41:55.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:41:55.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:41:55.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.ttl: 2592000 2024-09-21T09:41:55.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:41:55.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:41:55.231 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:41:55.231 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:41:55.231 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:41:55.231 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:41:55.231 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:41:55.231 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:41:55.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:41:55.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:41:55.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:41:55.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:41:55.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:41:55.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-21T09:41:55.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:41:55.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13410, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:41:55.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-21T09:41:55.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7a97661d-3375-4595-82c7-c73b69f29e2c 2024-09-21T09:41:55.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911714733976, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-21T09:41:55.233 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-21T09:41:55.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911714758185, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 6549942, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13407, "largest_seqno": 16975, "table_properties": {"data_size": 6533900, "index_size": 11027, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3973, "raw_key_size": 35947, "raw_average_key_size": 22, "raw_value_size": 6500965, "raw_average_value_size": 4135, "num_data_blocks": 479, "num_entries": 1572, "num_filter_entries": 1572, "num_deletions": 304, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911714, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7a97661d-3375-4595-82c7-c73b69f29e2c", "db_session_id": "SC8ME80RNZLCPHYCMPLQ", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:41:55.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911714758316, "job": 1, "event": "recovery_finished"} 2024-09-21T09:41:55.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-21T09:41:55.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:41:55.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:41:55.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a1b07bae00 2024-09-21T09:41:55.235 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: DB pointer 0x55a1b08bc000 2024-09-21T09:41:55.235 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:41:55.235 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:41:55.235 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** DB Stats ** 2024-09-21T09:41:55.235 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:41:55.235 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:41:55.235 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:41:55.236 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:41:55.236 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:41:55.236 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:41:55.236 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:41:55.236 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:41:55.236 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:41:55.237 INFO:journalctl@ceph.mon.c.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:41:55.237 INFO:journalctl@ceph.mon.c.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:41:55.237 INFO:journalctl@ceph.mon.c.smithi017.stdout: L0 1/0 6.25 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 705.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:41:55.237 INFO:journalctl@ceph.mon.c.smithi017.stdout: L6 1/0 7.47 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:41:55.237 INFO:journalctl@ceph.mon.c.smithi017.stdout: Sum 2/0 13.71 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 705.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:41:55.237 INFO:journalctl@ceph.mon.c.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 705.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:41:55.238 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:41:55.238 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2024-09-21T09:41:55.238 INFO:journalctl@ceph.mon.c.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:41:55.238 INFO:journalctl@ceph.mon.c.smithi017.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:41:55.238 INFO:journalctl@ceph.mon.c.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 705.3 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:41:55.238 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:41:55.239 INFO:journalctl@ceph.mon.c.smithi017.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:41:55.239 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:41:55.239 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:41:55.239 INFO:journalctl@ceph.mon.c.smithi017.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-09-21T09:41:55.239 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:41:55.239 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:41:55.240 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:41:55.240 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:41:55.240 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative compaction: 0.01 GB write, 228.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:41:55.240 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval compaction: 0.01 GB write, 228.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:41:55.240 INFO:journalctl@ceph.mon.c.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:41:55.240 INFO:journalctl@ceph.mon.c.smithi017.stdout: Block cache BinnedLRUCache@0x55a1b078b610#7 capacity: 512.00 MB usage: 16.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T09:41:55.240 INFO:journalctl@ceph.mon.c.smithi017.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-09-21T09:41:55.241 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:41:55.241 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:41:55.241 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: starting mon.c rank 2 at public addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] at bind addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:41:55.241 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: mon.c@-1(???) e3 preinit fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:41:55.241 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: mon.c@-1(???).mds e8 new map 2024-09-21T09:41:55.241 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: mon.c@-1(???).mds e8 print_map 2024-09-21T09:41:55.242 INFO:journalctl@ceph.mon.c.smithi017.stdout: e8 2024-09-21T09:41:55.242 INFO:journalctl@ceph.mon.c.smithi017.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:41:55.242 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:41:55.242 INFO:journalctl@ceph.mon.c.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:41:55.242 INFO:journalctl@ceph.mon.c.smithi017.stdout: legacy client fscid: 1 2024-09-21T09:41:55.242 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:41:55.243 INFO:journalctl@ceph.mon.c.smithi017.stdout: Filesystem 'foo' (1) 2024-09-21T09:41:55.243 INFO:journalctl@ceph.mon.c.smithi017.stdout: fs_name foo 2024-09-21T09:41:55.243 INFO:journalctl@ceph.mon.c.smithi017.stdout: epoch 7 2024-09-21T09:41:55.243 INFO:journalctl@ceph.mon.c.smithi017.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:41:55.243 INFO:journalctl@ceph.mon.c.smithi017.stdout: created 2024-09-21T09:36:43.123303+0000 2024-09-21T09:41:55.243 INFO:journalctl@ceph.mon.c.smithi017.stdout: modified 2024-09-21T09:36:59.247371+0000 2024-09-21T09:41:55.243 INFO:journalctl@ceph.mon.c.smithi017.stdout: tableserver 0 2024-09-21T09:41:55.244 INFO:journalctl@ceph.mon.c.smithi017.stdout: root 0 2024-09-21T09:41:55.244 INFO:journalctl@ceph.mon.c.smithi017.stdout: session_timeout 60 2024-09-21T09:41:55.244 INFO:journalctl@ceph.mon.c.smithi017.stdout: session_autoclose 300 2024-09-21T09:41:55.244 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_file_size 1099511627776 2024-09-21T09:41:55.244 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_xattr_size 65536 2024-09-21T09:41:55.244 INFO:journalctl@ceph.mon.c.smithi017.stdout: required_client_features {} 2024-09-21T09:41:55.245 INFO:journalctl@ceph.mon.c.smithi017.stdout: last_failure 0 2024-09-21T09:41:55.245 INFO:journalctl@ceph.mon.c.smithi017.stdout: last_failure_osd_epoch 0 2024-09-21T09:41:55.245 INFO:journalctl@ceph.mon.c.smithi017.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:41:55.245 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_mds 1 2024-09-21T09:41:55.245 INFO:journalctl@ceph.mon.c.smithi017.stdout: in 0 2024-09-21T09:41:55.245 INFO:journalctl@ceph.mon.c.smithi017.stdout: up {0=14655} 2024-09-21T09:41:55.245 INFO:journalctl@ceph.mon.c.smithi017.stdout: failed 2024-09-21T09:41:55.246 INFO:journalctl@ceph.mon.c.smithi017.stdout: damaged 2024-09-21T09:41:55.246 INFO:journalctl@ceph.mon.c.smithi017.stdout: stopped 2024-09-21T09:41:55.246 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_pools [4] 2024-09-21T09:41:55.246 INFO:journalctl@ceph.mon.c.smithi017.stdout: metadata_pool 3 2024-09-21T09:41:55.246 INFO:journalctl@ceph.mon.c.smithi017.stdout: inline_data disabled 2024-09-21T09:41:55.246 INFO:journalctl@ceph.mon.c.smithi017.stdout: balancer 2024-09-21T09:41:55.247 INFO:journalctl@ceph.mon.c.smithi017.stdout: bal_rank_mask -1 2024-09-21T09:41:55.247 INFO:journalctl@ceph.mon.c.smithi017.stdout: standby_count_wanted 1 2024-09-21T09:41:55.247 INFO:journalctl@ceph.mon.c.smithi017.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:41:55.247 INFO:journalctl@ceph.mon.c.smithi017.stdout: [mds.foo.smithi017.coodep{0:14655} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:41:55.247 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:41:55.247 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:41:55.248 INFO:journalctl@ceph.mon.c.smithi017.stdout: Standby daemons: 2024-09-21T09:41:55.248 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2024-09-21T09:41:55.248 INFO:journalctl@ceph.mon.c.smithi017.stdout: [mds.foo.smithi134.bwovvb{-1:24493} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.134:6832/3208820776,v1:172.21.15.134:6833/3208820776] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:41:55.248 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: mon.c@-1(???).osd e299 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T09:41:55.248 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: mon.c@-1(???).osd e299 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:41:55.248 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: mon.c@-1(???).osd e299 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:41:55.249 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: mon.c@-1(???).osd e299 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:41:55.249 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 ceph-mon[105151]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-21T09:41:55.249 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 bash[105113]: 013cf9450c578dd09408b41d25b12e9b5b3de35d41bd8a0cd0adc72bc322c210 2024-09-21T09:41:55.249 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:41:54 smithi017 systemd[1]: Started Ceph mon.c for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:55.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:55.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:57.246 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:41:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:41:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:41:59.988 INFO:tasks.rados.rados.0.smithi017.stdout:946: done (3 left) 2024-09-21T09:41:59.988 INFO:tasks.rados.rados.0.smithi017.stdout:947: snap_create 2024-09-21T09:41:59.989 INFO:tasks.rados.rados.0.smithi017.stdout:944: finishing rollback tid 1 to smithi01785587-24 2024-09-21T09:41:59.989 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 247 (ObjNum 129 snap 0 seq_num 2399895984) dirty dne 2024-09-21T09:41:59.989 INFO:tasks.rados.rados.0.smithi017.stdout:943: finishing write tid 2 to smithi01785587-39 2024-09-21T09:41:59.989 INFO:tasks.rados.rados.0.smithi017.stdout:943: finishing write tid 3 to smithi01785587-39 2024-09-21T09:41:59.989 INFO:tasks.rados.rados.0.smithi017.stdout:943: finishing write tid 4 to smithi01785587-39 2024-09-21T09:41:59.989 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 481 (ObjNum 383 snap 118 seq_num 383) dirty exists 2024-09-21T09:41:59.989 INFO:tasks.rados.rados.0.smithi017.stdout:943: left oid 39 (ObjNum 383 snap 118 seq_num 383) 2024-09-21T09:41:59.989 INFO:tasks.rados.rados.0.smithi017.stdout:945: finishing write tid 1 to smithi01785587-6 2024-09-21T09:41:59.989 INFO:tasks.rados.rados.0.smithi017.stdout:945: finishing write tid 2 to smithi01785587-6 2024-09-21T09:41:59.990 INFO:tasks.rados.rados.0.smithi017.stdout:945: finishing write tid 3 to smithi01785587-6 2024-09-21T09:41:59.990 INFO:tasks.rados.rados.0.smithi017.stdout:945: finishing write tid 4 to smithi01785587-6 2024-09-21T09:41:59.990 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 271 (ObjNum 384 snap 118 seq_num 384) dirty exists 2024-09-21T09:41:59.990 INFO:tasks.rados.rados.0.smithi017.stdout:945: left oid 6 (ObjNum 384 snap 118 seq_num 384) 2024-09-21T09:42:00.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:01.003 INFO:tasks.rados.rados.0.smithi017.stdout:943: done (3 left) 2024-09-21T09:42:01.003 INFO:tasks.rados.rados.0.smithi017.stdout:944: done (2 left) 2024-09-21T09:42:01.003 INFO:tasks.rados.rados.0.smithi017.stdout:945: done (1 left) 2024-09-21T09:42:01.003 INFO:tasks.rados.rados.0.smithi017.stdout:947: done (0 left) 2024-09-21T09:42:01.004 INFO:tasks.rados.rados.0.smithi017.stdout:948: snap_remove snap 115 2024-09-21T09:42:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: mon.c calling monitor election 2024-09-21T09:42:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: mon.b calling monitor election 2024-09-21T09:42:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: mon.a calling monitor election 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: monmap epoch 3 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: last_changed 2024-09-21T09:31:25.701063+0000 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: created 2024-09-21T09:29:44.848123+0000 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: min_mon_release 17 (quincy) 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: election_strategy: 1 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: 1: [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon.b 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:42:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: mgrmap e39: y(active, since 21s), standbys: x 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[102913]: overall HEALTH_OK 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: mon.c calling monitor election 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: mon.b calling monitor election 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: mon.a calling monitor election 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: monmap epoch 3 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: last_changed 2024-09-21T09:31:25.701063+0000 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: created 2024-09-21T09:29:44.848123+0000 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: min_mon_release 17 (quincy) 2024-09-21T09:42:01.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: election_strategy: 1 2024-09-21T09:42:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-21T09:42:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: 1: [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon.b 2024-09-21T09:42:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-21T09:42:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:42:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:42:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: mgrmap e39: y(active, since 21s), standbys: x 2024-09-21T09:42:01.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:01 smithi017 ceph-mon[105151]: overall HEALTH_OK 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: mon.c calling monitor election 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: mon.b calling monitor election 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: mon.a calling monitor election 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: monmap epoch 3 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: last_changed 2024-09-21T09:31:25.701063+0000 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: created 2024-09-21T09:29:44.848123+0000 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: min_mon_release 17 (quincy) 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: election_strategy: 1 2024-09-21T09:42:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-21T09:42:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: 1: [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon.b 2024-09-21T09:42:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-21T09:42:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:42:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:42:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: mgrmap e39: y(active, since 21s), standbys: x 2024-09-21T09:42:01.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:01 smithi134 ceph-mon[33541]: overall HEALTH_OK 2024-09-21T09:42:01.510 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:42:02.013 INFO:tasks.rados.rados.0.smithi017.stdout:948: done (0 left) 2024-09-21T09:42:02.013 INFO:tasks.rados.rados.0.smithi017.stdout:949: delete oid 50 current snap is 119 2024-09-21T09:42:02.019 INFO:tasks.rados.rados.0.smithi017.stdout:949: done (0 left) 2024-09-21T09:42:02.019 INFO:tasks.rados.rados.0.smithi017.stdout:950: delete oid 37 current snap is 119 2024-09-21T09:42:02.021 INFO:tasks.rados.rados.0.smithi017.stdout:950: done (0 left) 2024-09-21T09:42:02.021 INFO:tasks.rados.rados.0.smithi017.stdout:951: write oid 39 current snap is 119 2024-09-21T09:42:02.022 INFO:tasks.rados.rados.0.smithi017.stdout:951: seq_num 387 ranges {504309=539013,1796649=590127} 2024-09-21T09:42:02.027 INFO:tasks.rados.rados.0.smithi017.stdout:951: writing smithi01785587-39 from 504309 to 1043322 tid 1 2024-09-21T09:42:02.035 INFO:tasks.rados.rados.0.smithi017.stdout:951: writing smithi01785587-39 from 1796649 to 2386776 tid 2 2024-09-21T09:42:02.035 INFO:tasks.rados.rados.0.smithi017.stdout:952: rollback oid 32 current snap is 119 2024-09-21T09:42:02.035 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 118 2024-09-21T09:42:02.035 INFO:tasks.rados.rados.0.smithi017.stdout:951: finishing write tid 1 to smithi01785587-39 2024-09-21T09:42:02.035 INFO:tasks.rados.rados.0.smithi017.stdout:953: snap_remove snap 116 2024-09-21T09:42:02.337 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:02 smithi017 ceph-mon[105151]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:42:02.337 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3256200052' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-21T09:42:02.337 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:02 smithi017 ceph-mon[102913]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:42:02.337 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3256200052' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-21T09:42:02.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch ps' 2024-09-21T09:42:02.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:02 smithi134 ceph-mon[33541]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:42:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:02 smithi134 ceph-mon[33541]: from='client.? 172.21.15.17:0/3256200052' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-21T09:42:03.020 INFO:tasks.rados.rados.0.smithi017.stdout:953: done (2 left) 2024-09-21T09:42:03.020 INFO:tasks.rados.rados.0.smithi017.stdout:954: write oid 19 current snap is 119 2024-09-21T09:42:03.020 INFO:tasks.rados.rados.0.smithi017.stdout:951: finishing write tid 2 to smithi01785587-39 2024-09-21T09:42:03.020 INFO:tasks.rados.rados.0.smithi017.stdout:952: finishing rollback tid 0 to smithi01785587-32 2024-09-21T09:42:03.020 INFO:tasks.rados.rados.0.smithi017.stdout:951: finishing write tid 3 to smithi01785587-39 2024-09-21T09:42:03.021 INFO:tasks.rados.rados.0.smithi017.stdout:952: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:42:03.021 INFO:tasks.rados.rados.0.smithi017.stdout:952: finishing rollback tid 2 to smithi01785587-32 2024-09-21T09:42:03.021 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 455 (ObjNum 305 snap 91 seq_num 305) dirty exists 2024-09-21T09:42:03.021 INFO:tasks.rados.rados.0.smithi017.stdout:951: finishing write tid 4 to smithi01785587-39 2024-09-21T09:42:03.021 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 500 (ObjNum 387 snap 119 seq_num 387) dirty exists 2024-09-21T09:42:03.021 INFO:tasks.rados.rados.0.smithi017.stdout:951: left oid 39 (ObjNum 387 snap 119 seq_num 387) 2024-09-21T09:42:03.021 INFO:tasks.rados.rados.0.smithi017.stdout:954: seq_num 388 ranges {521116=730615,1971484=549632} 2024-09-21T09:42:03.029 INFO:tasks.rados.rados.0.smithi017.stdout:954: writing smithi01785587-19 from 521116 to 1251731 tid 1 2024-09-21T09:42:03.033 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:42:03.037 INFO:tasks.rados.rados.0.smithi017.stdout:954: writing smithi01785587-19 from 1971484 to 2521116 tid 2 2024-09-21T09:42:03.037 INFO:tasks.rados.rados.0.smithi017.stdout:951: done (2 left) 2024-09-21T09:42:03.037 INFO:tasks.rados.rados.0.smithi017.stdout:952: done (1 left) 2024-09-21T09:42:03.037 INFO:tasks.rados.rados.0.smithi017.stdout:955: snap_remove snap 117 2024-09-21T09:42:03.272 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:03 smithi017 ceph-mon[102913]: pgmap v29: 89 pgs: 89 active+clean; 179 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 543 KiB/s wr, 3 op/s 2024-09-21T09:42:03.273 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:03 smithi017 ceph-mon[102913]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:42:03.273 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:03 smithi017 ceph-mon[105151]: pgmap v29: 89 pgs: 89 active+clean; 179 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 543 KiB/s wr, 3 op/s 2024-09-21T09:42:03.273 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:03 smithi017 ceph-mon[105151]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:42:03.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:03 smithi134 ceph-mon[33541]: pgmap v29: 89 pgs: 89 active+clean; 179 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 543 KiB/s wr, 3 op/s 2024-09-21T09:42:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:03 smithi134 ceph-mon[33541]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:42:04.024 INFO:tasks.rados.rados.0.smithi017.stdout:955: done (1 left) 2024-09-21T09:42:04.024 INFO:tasks.rados.rados.0.smithi017.stdout:956: rollback oid 36 current snap is 119 2024-09-21T09:42:04.024 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 36 to 98 2024-09-21T09:42:04.025 INFO:tasks.rados.rados.0.smithi017.stdout:957: write oid 17 current snap is 119 2024-09-21T09:42:04.025 INFO:tasks.rados.rados.0.smithi017.stdout:957: seq_num 389 ranges {537923=405864,1513613=593796} 2024-09-21T09:42:04.030 INFO:tasks.rados.rados.0.smithi017.stdout:957: writing smithi01785587-17 from 537923 to 943787 tid 1 2024-09-21T09:42:04.036 INFO:tasks.rados.rados.0.smithi017.stdout:957: writing smithi01785587-17 from 1513613 to 2107409 tid 2 2024-09-21T09:42:04.036 INFO:tasks.rados.rados.0.smithi017.stdout:958: delete oid 24 current snap is 119 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:954: finishing write tid 1 to smithi01785587-19 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:954: finishing write tid 2 to smithi01785587-19 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:954: finishing write tid 3 to smithi01785587-19 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:954: finishing write tid 4 to smithi01785587-19 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 506 (ObjNum 388 snap 119 seq_num 388) dirty exists 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:954: left oid 19 (ObjNum 388 snap 119 seq_num 388) 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:956: finishing rollback tid 1 to smithi01785587-36 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:956: finishing rollback tid 2 to smithi01785587-36 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 512 (ObjNum 313 snap 93 seq_num 313) dirty exists 2024-09-21T09:42:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:957: finishing write tid 1 to smithi01785587-17 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:957: finishing write tid 2 to smithi01785587-17 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:957: finishing write tid 3 to smithi01785587-17 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:957: finishing write tid 4 to smithi01785587-17 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 407 (ObjNum 389 snap 119 seq_num 389) dirty exists 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:957: left oid 17 (ObjNum 389 snap 119 seq_num 389) 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:954: done (3 left) 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:956: done (2 left) 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:957: done (1 left) 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:958: done (0 left) 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:959: write oid 43 current snap is 119 2024-09-21T09:42:04.041 INFO:tasks.rados.rados.0.smithi017.stdout:959: seq_num 391 ranges {571537=556362,1630577=597465} 2024-09-21T09:42:04.047 INFO:tasks.rados.rados.0.smithi017.stdout:959: writing smithi01785587-43 from 571537 to 1127899 tid 1 2024-09-21T09:42:04.053 INFO:tasks.rados.rados.0.smithi017.stdout:959: writing smithi01785587-43 from 1630577 to 2228042 tid 2 2024-09-21T09:42:04.053 INFO:tasks.rados.rados.0.smithi017.stdout:960: snap_remove snap 118 2024-09-21T09:42:04.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:04 smithi017 ceph-mon[102913]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:42:04.358 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:04 smithi017 ceph-mon[105151]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:42:04.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:04 smithi134 ceph-mon[33541]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:42:04.395 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:05.021 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:42:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:42:04] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.51.0" 2024-09-21T09:42:05.021 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:42:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:42:04.770+0000 7f3131349640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T09:42:05.028 INFO:tasks.rados.rados.0.smithi017.stdout:960: done (1 left) 2024-09-21T09:42:05.028 INFO:tasks.rados.rados.0.smithi017.stdout:961: snap_create 2024-09-21T09:42:05.028 INFO:tasks.rados.rados.0.smithi017.stdout:959: finishing write tid 1 to smithi01785587-43 2024-09-21T09:42:05.028 INFO:tasks.rados.rados.0.smithi017.stdout:959: finishing write tid 2 to smithi01785587-43 2024-09-21T09:42:05.028 INFO:tasks.rados.rados.0.smithi017.stdout:959: finishing write tid 3 to smithi01785587-43 2024-09-21T09:42:05.028 INFO:tasks.rados.rados.0.smithi017.stdout:959: finishing write tid 4 to smithi01785587-43 2024-09-21T09:42:05.028 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 288 (ObjNum 391 snap 119 seq_num 391) dirty exists 2024-09-21T09:42:05.028 INFO:tasks.rados.rados.0.smithi017.stdout:959: left oid 43 (ObjNum 391 snap 119 seq_num 391) 2024-09-21T09:42:05.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:05 smithi017 ceph-mon[102913]: pgmap v32: 89 pgs: 89 active+clean; 179 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:05.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:05 smithi017 ceph-mon[102913]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:42:05.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:05 smithi017 ceph-mon[105151]: pgmap v32: 89 pgs: 89 active+clean; 179 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:05.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:05 smithi017 ceph-mon[105151]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:42:05.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:05 smithi134 ceph-mon[33541]: pgmap v32: 89 pgs: 89 active+clean; 179 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:05 smithi134 ceph-mon[33541]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:42:05.763 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:42:05.763 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (2m) 21s ago 6m 19.9M - 0.27.0 11f11916f8cd 0a4d8256f8d7 2024-09-21T09:42:05.763 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi134 *:3000 running (117s) 20s ago 6m 80.2M - 9.4.7 954c08fa6188 8d2c807b0308 2024-09-21T09:42:05.763 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.coodep smithi017 running (5m) 21s ago 5m 16.7M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 81acf97499eb 2024-09-21T09:42:05.763 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi134.bwovvb smithi134 running (5m) 20s ago 5m 16.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be bf739a2a98de 2024-09-21T09:42:05.763 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi134 *:8443,9283,8765 running (47s) 20s ago 10m 451M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 4feb5ade269b 2024-09-21T09:42:05.763 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (75s) 21s ago 12m 488M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 14c61243c535 2024-09-21T09:42:05.763 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (33s) 21s ago 12m 34.1M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0590143b06cf 2024-09-21T09:42:05.763 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi134 running (10m) 20s ago 10m 56.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 7eb897902574 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 starting - - - 2048M 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (2m) 21s ago 5m 11.3M - 1.7.0 72c9c2088986 057b63034a9e 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi134 *:9100 running (110s) 20s ago 5m 10.0M - 1.7.0 72c9c2088986 c6b046d44374 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (10m) 21s ago 10m 85.0M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 958bb6e5f55d 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (9m) 21s ago 9m 66.4M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5e9c6f25d8a5 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (9m) 21s ago 9m 88.4M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b87d82e4dc74 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (8m) 21s ago 8m 64.3M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 44d3b78acaa4 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi134 running (8m) 20s ago 8m 85.6M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 7ece30084c41 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi134 running (8m) 20s ago 8m 74.2M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5dec2255fcd7 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi134 running (7m) 20s ago 7m 63.9M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 80d4436a0b8a 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi134 running (7m) 20s ago 7m 69.3M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed2e26ef5698 2024-09-21T09:42:05.764 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi134 *:9095 running (52s) 20s ago 6m 43.9M - 2.51.0 1d3b7f56885b 061a3294723d 2024-09-21T09:42:05.993 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:05 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:05.993 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:05 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:06.032 INFO:tasks.rados.rados.0.smithi017.stdout:959: done (1 left) 2024-09-21T09:42:06.032 INFO:tasks.rados.rados.0.smithi017.stdout:961: done (0 left) 2024-09-21T09:42:06.032 INFO:tasks.rados.rados.0.smithi017.stdout:962: snap_create 2024-09-21T09:42:06.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:06 smithi017 ceph-mon[102913]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:42:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:06 smithi017 ceph-mon[105151]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:42:06.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:06 smithi134 ceph-mon[33541]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:42:06.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade pause' 2024-09-21T09:42:07.034 INFO:tasks.rados.rados.0.smithi017.stdout:962: done (0 left) 2024-09-21T09:42:07.034 INFO:tasks.rados.rados.0.smithi017.stdout:963: write oid 31 current snap is 121 2024-09-21T09:42:07.034 INFO:tasks.rados.rados.0.smithi017.stdout:963: seq_num 392 ranges {588344=631611,1689059=741123} 2024-09-21T09:42:07.041 INFO:tasks.rados.rados.0.smithi017.stdout:963: writing smithi01785587-31 from 588344 to 1219955 tid 1 2024-09-21T09:42:07.049 INFO:tasks.rados.rados.0.smithi017.stdout:963: writing smithi01785587-31 from 1689059 to 2430182 tid 2 2024-09-21T09:42:07.049 INFO:tasks.rados.rados.0.smithi017.stdout:964: write oid 7 current snap is 121 2024-09-21T09:42:07.049 INFO:tasks.rados.rados.0.smithi017.stdout:963: finishing write tid 1 to smithi01785587-31 2024-09-21T09:42:07.049 INFO:tasks.rados.rados.0.smithi017.stdout:964: seq_num 393 ranges {605151=706860,2031188=573963} 2024-09-21T09:42:07.056 INFO:tasks.rados.rados.0.smithi017.stdout:964: writing smithi01785587-7 from 605151 to 1312011 tid 1 2024-09-21T09:42:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:964: writing smithi01785587-7 from 2031188 to 2605151 tid 2 2024-09-21T09:42:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:965: write oid 45 current snap is 121 2024-09-21T09:42:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:963: finishing write tid 2 to smithi01785587-31 2024-09-21T09:42:07.063 INFO:tasks.rados.rados.0.smithi017.stdout:963: finishing write tid 3 to smithi01785587-31 2024-09-21T09:42:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:965: seq_num 394 ranges {621958=782109,2089670=532288} 2024-09-21T09:42:07.071 INFO:tasks.rados.rados.0.smithi017.stdout:965: writing smithi01785587-45 from 621958 to 1404067 tid 1 2024-09-21T09:42:07.078 INFO:tasks.rados.rados.0.smithi017.stdout:965: writing smithi01785587-45 from 2089670 to 2621958 tid 2 2024-09-21T09:42:07.078 INFO:tasks.rados.rados.0.smithi017.stdout:966: read oid 9 snap -1 2024-09-21T09:42:07.079 INFO:tasks.rados.rados.0.smithi017.stdout:966: expect (ObjNum 381 snap 118 seq_num 381) 2024-09-21T09:42:07.079 INFO:tasks.rados.rados.0.smithi017.stdout:967: write oid 48 current snap is 121 2024-09-21T09:42:07.079 INFO:tasks.rados.rados.0.smithi017.stdout:967: seq_num 395 ranges {638765=457358,1748152=604803} 2024-09-21T09:42:07.084 INFO:tasks.rados.rados.0.smithi017.stdout:967: writing smithi01785587-48 from 638765 to 1096123 tid 1 2024-09-21T09:42:07.092 INFO:tasks.rados.rados.0.smithi017.stdout:967: writing smithi01785587-48 from 1748152 to 2352955 tid 2 2024-09-21T09:42:07.092 INFO:tasks.rados.rados.0.smithi017.stdout:968: read oid 41 snap -1 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:968: expect deleted 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:969: read oid 24 snap -1 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:969: expect deleted 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:970: rollback oid 33 current snap is 121 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 98 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:963: finishing write tid 4 to smithi01785587-31 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 481 (ObjNum 392 snap 121 seq_num 392) dirty exists 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:963: left oid 31 (ObjNum 392 snap 121 seq_num 392) 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:964: finishing write tid 1 to smithi01785587-7 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:964: finishing write tid 2 to smithi01785587-7 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:964: finishing write tid 3 to smithi01785587-7 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:964: finishing write tid 4 to smithi01785587-7 2024-09-21T09:42:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 508 (ObjNum 393 snap 121 seq_num 393) dirty exists 2024-09-21T09:42:07.094 INFO:tasks.rados.rados.0.smithi017.stdout:964: left oid 7 (ObjNum 393 snap 121 seq_num 393) 2024-09-21T09:42:07.094 INFO:tasks.rados.rados.0.smithi017.stdout:965: finishing write tid 1 to smithi01785587-45 2024-09-21T09:42:07.094 INFO:tasks.rados.rados.0.smithi017.stdout:965: finishing write tid 2 to smithi01785587-45 2024-09-21T09:42:07.094 INFO:tasks.rados.rados.0.smithi017.stdout:965: finishing write tid 3 to smithi01785587-45 2024-09-21T09:42:07.094 INFO:tasks.rados.rados.0.smithi017.stdout:965: finishing write tid 4 to smithi01785587-45 2024-09-21T09:42:07.094 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 413 (ObjNum 394 snap 121 seq_num 394) dirty exists 2024-09-21T09:42:07.094 INFO:tasks.rados.rados.0.smithi017.stdout:965: left oid 45 (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:42:07.094 INFO:tasks.rados.rados.0.smithi017.stdout:966: expect (ObjNum 381 snap 118 seq_num 381) 2024-09-21T09:42:07.178 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:967: finishing write tid 1 to smithi01785587-48 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:963: done (7 left) 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:964: done (6 left) 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:965: done (5 left) 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:966: done (4 left) 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:971: delete oid 4 current snap is 121 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:967: finishing write tid 2 to smithi01785587-48 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:970: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:967: finishing write tid 3 to smithi01785587-48 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:970: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:42:07.185 INFO:tasks.rados.rados.0.smithi017.stdout:967: finishing write tid 4 to smithi01785587-48 2024-09-21T09:42:07.186 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 515 (ObjNum 395 snap 121 seq_num 395) dirty exists 2024-09-21T09:42:07.186 INFO:tasks.rados.rados.0.smithi017.stdout:967: left oid 48 (ObjNum 395 snap 121 seq_num 395) 2024-09-21T09:42:07.186 INFO:tasks.rados.rados.0.smithi017.stdout:970: finishing rollback tid 2 to smithi01785587-33 2024-09-21T09:42:07.186 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 506 (ObjNum 310 snap 91 seq_num 310) dirty exists 2024-09-21T09:42:07.187 INFO:tasks.rados.rados.0.smithi017.stdout:967: done (4 left) 2024-09-21T09:42:07.187 INFO:tasks.rados.rados.0.smithi017.stdout:968: done (3 left) 2024-09-21T09:42:07.187 INFO:tasks.rados.rados.0.smithi017.stdout:969: done (2 left) 2024-09-21T09:42:07.187 INFO:tasks.rados.rados.0.smithi017.stdout:970: done (1 left) 2024-09-21T09:42:07.187 INFO:tasks.rados.rados.0.smithi017.stdout:971: done (0 left) 2024-09-21T09:42:07.187 INFO:tasks.rados.rados.0.smithi017.stdout:972: read oid 32 snap -1 2024-09-21T09:42:07.187 INFO:tasks.rados.rados.0.smithi017.stdout:972: expect (ObjNum 305 snap 91 seq_num 305) 2024-09-21T09:42:07.188 INFO:tasks.rados.rados.0.smithi017.stdout:973: rollback oid 16 current snap is 121 2024-09-21T09:42:07.188 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 16 to 98 2024-09-21T09:42:07.188 INFO:tasks.rados.rados.0.smithi017.stdout:974: snap_remove snap 111 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[102913]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[102913]: pgmap v35: 89 pgs: 89 active+clean; 179 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[102913]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[105151]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[105151]: pgmap v35: 89 pgs: 89 active+clean; 179 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[105151]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:42:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:06 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:07 smithi134 ceph-mon[33541]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:07 smithi134 ceph-mon[33541]: pgmap v35: 89 pgs: 89 active+clean; 179 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:07 smithi134 ceph-mon[33541]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:42:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:07 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:07 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:07 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:07.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:07 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:08.022 INFO:tasks.rados.rados.0.smithi017.stdout:974: done (2 left) 2024-09-21T09:42:08.022 INFO:tasks.rados.rados.0.smithi017.stdout:975: snap_remove snap 119 2024-09-21T09:42:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:08 smithi017 ceph-mon[102913]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:42:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:08 smithi017 ceph-mon[102913]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:42:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:08 smithi017 ceph-mon[105151]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:42:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:08 smithi017 ceph-mon[105151]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:42:08.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:08 smithi134 ceph-mon[33541]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:42:08.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:08 smithi134 ceph-mon[33541]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:42:09.023 INFO:tasks.rados.rados.0.smithi017.stdout:975: done (2 left) 2024-09-21T09:42:09.024 INFO:tasks.rados.rados.0.smithi017.stdout:976: write oid 46 current snap is 121 2024-09-21T09:42:09.024 INFO:tasks.rados.rados.0.smithi017.stdout:973: finishing rollback tid 0 to smithi01785587-16 2024-09-21T09:42:09.024 INFO:tasks.rados.rados.0.smithi017.stdout:973: finishing rollback tid 1 to smithi01785587-16 2024-09-21T09:42:09.024 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 511 (ObjNum 305 snap 0 seq_num 3571356120) dirty dne 2024-09-21T09:42:09.024 INFO:tasks.rados.rados.0.smithi017.stdout:972: expect (ObjNum 305 snap 91 seq_num 305) 2024-09-21T09:42:09.232 INFO:tasks.rados.rados.0.smithi017.stdout:976: seq_num 397 ranges {672379=724209,1865116=608472} 2024-09-21T09:42:09.239 INFO:tasks.rados.rados.0.smithi017.stdout:976: writing smithi01785587-46 from 672379 to 1396588 tid 1 2024-09-21T09:42:09.246 INFO:tasks.rados.rados.0.smithi017.stdout:976: writing smithi01785587-46 from 1865116 to 2473588 tid 2 2024-09-21T09:42:09.246 INFO:tasks.rados.rados.0.smithi017.stdout:972: done (2 left) 2024-09-21T09:42:09.246 INFO:tasks.rados.rados.0.smithi017.stdout:973: done (1 left) 2024-09-21T09:42:09.247 INFO:tasks.rados.rados.0.smithi017.stdout:977: snap_remove snap 120 2024-09-21T09:42:09.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:09 smithi017 ceph-mon[102913]: pgmap v38: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:42:09.310 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:09 smithi017 ceph-mon[102913]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:42:09.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:09 smithi017 ceph-mon[105151]: pgmap v38: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:42:09.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:09 smithi017 ceph-mon[105151]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:42:09.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:09 smithi134 ceph-mon[33541]: pgmap v38: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:42:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:09 smithi134 ceph-mon[33541]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:42:10.048 INFO:tasks.rados.rados.0.smithi017.stdout:977: done (1 left) 2024-09-21T09:42:10.048 INFO:tasks.rados.rados.0.smithi017.stdout:978: snap_remove snap 98 2024-09-21T09:42:10.276 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:42:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:11.055 INFO:tasks.rados.rados.0.smithi017.stdout:978: done (1 left) 2024-09-21T09:42:11.056 INFO:tasks.rados.rados.0.smithi017.stdout:979: delete oid 34 current snap is 121 2024-09-21T09:42:11.056 INFO:tasks.rados.rados.0.smithi017.stdout:976: finishing write tid 1 to smithi01785587-46 2024-09-21T09:42:11.056 INFO:tasks.rados.rados.0.smithi017.stdout:976: finishing write tid 2 to smithi01785587-46 2024-09-21T09:42:11.056 INFO:tasks.rados.rados.0.smithi017.stdout:976: finishing write tid 3 to smithi01785587-46 2024-09-21T09:42:11.056 INFO:tasks.rados.rados.0.smithi017.stdout:976: finishing write tid 4 to smithi01785587-46 2024-09-21T09:42:11.056 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 467 (ObjNum 397 snap 121 seq_num 397) dirty exists 2024-09-21T09:42:11.056 INFO:tasks.rados.rados.0.smithi017.stdout:976: left oid 46 (ObjNum 397 snap 121 seq_num 397) 2024-09-21T09:42:11.066 INFO:tasks.rados.rados.0.smithi017.stdout:976: done (1 left) 2024-09-21T09:42:11.066 INFO:tasks.rados.rados.0.smithi017.stdout:979: done (0 left) 2024-09-21T09:42:11.066 INFO:tasks.rados.rados.0.smithi017.stdout:980: snap_remove snap 105 2024-09-21T09:42:11.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'sleep 60' 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[102913]: pgmap v41: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[102913]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[102913]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[102913]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[105151]: pgmap v41: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[105151]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[105151]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:11.301 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[105151]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:42:11.302 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:11 smithi134 ceph-mon[33541]: pgmap v41: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:42:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:11 smithi134 ceph-mon[33541]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:42:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:11 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:11 smithi134 ceph-mon[33541]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:11 smithi134 ceph-mon[33541]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:42:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:11 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:12.063 INFO:tasks.rados.rados.0.smithi017.stdout:980: done (0 left) 2024-09-21T09:42:12.063 INFO:tasks.rados.rados.0.smithi017.stdout:981: snap_remove snap 103 2024-09-21T09:42:12.127 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:42:12.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:12 smithi017 ceph-mon[102913]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:42:12.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:12 smithi017 ceph-mon[105151]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:42:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:12 smithi134 ceph-mon[33541]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:42:13.022 INFO:tasks.rados.rados.0.smithi017.stdout:981: done (0 left) 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:982: rollback oid 30 current snap is 121 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:982: done (0 left) 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:983: read oid 17 snap -1 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:983: expect (ObjNum 389 snap 119 seq_num 389) 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:984: rollback oid 21 current snap is 121 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:984: done (1 left) 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:985: read oid 22 snap -1 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:985: expect deleted 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:986: read oid 47 snap -1 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:986: expect (ObjNum 382 snap 118 seq_num 382) 2024-09-21T09:42:13.023 INFO:tasks.rados.rados.0.smithi017.stdout:987: rollback oid 27 current snap is 121 2024-09-21T09:42:13.024 INFO:tasks.rados.rados.0.smithi017.stdout:987: done (3 left) 2024-09-21T09:42:13.024 INFO:tasks.rados.rados.0.smithi017.stdout:988: write oid 32 current snap is 121 2024-09-21T09:42:13.024 INFO:tasks.rados.rados.0.smithi017.stdout:988: seq_num 399 ranges {705993=474707,1582080=612141} 2024-09-21T09:42:13.029 INFO:tasks.rados.rados.0.smithi017.stdout:988: writing smithi01785587-32 from 705993 to 1180700 tid 1 2024-09-21T09:42:13.037 INFO:tasks.rados.rados.0.smithi017.stdout:988: writing smithi01785587-32 from 1582080 to 2194221 tid 2 2024-09-21T09:42:13.037 INFO:tasks.rados.rados.0.smithi017.stdout:989: snap_create 2024-09-21T09:42:13.037 INFO:tasks.rados.rados.0.smithi017.stdout:983: expect (ObjNum 389 snap 119 seq_num 389) 2024-09-21T09:42:13.159 INFO:tasks.rados.rados.0.smithi017.stdout:986: expect (ObjNum 382 snap 118 seq_num 382) 2024-09-21T09:42:13.293 INFO:tasks.rados.rados.0.smithi017.stdout:988: finishing write tid 1 to smithi01785587-32 2024-09-21T09:42:13.293 INFO:tasks.rados.rados.0.smithi017.stdout:988: finishing write tid 2 to smithi01785587-32 2024-09-21T09:42:13.293 INFO:tasks.rados.rados.0.smithi017.stdout:988: finishing write tid 3 to smithi01785587-32 2024-09-21T09:42:13.293 INFO:tasks.rados.rados.0.smithi017.stdout:988: finishing write tid 4 to smithi01785587-32 2024-09-21T09:42:13.293 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 492 (ObjNum 399 snap 121 seq_num 399) dirty exists 2024-09-21T09:42:13.293 INFO:tasks.rados.rados.0.smithi017.stdout:988: left oid 32 (ObjNum 399 snap 121 seq_num 399) 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: pgmap v44: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:13.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[105151]: osdmap e313: 8 total, 8 up, 8 in 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: pgmap v44: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.322 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:13.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:13.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:13.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:13 smithi017 ceph-mon[102913]: osdmap e313: 8 total, 8 up, 8 in 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: pgmap v44: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:42:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:13 smithi134 ceph-mon[33541]: osdmap e313: 8 total, 8 up, 8 in 2024-09-21T09:42:14.023 INFO:tasks.rados.rados.0.smithi017.stdout:983: done (4 left) 2024-09-21T09:42:14.023 INFO:tasks.rados.rados.0.smithi017.stdout:985: done (3 left) 2024-09-21T09:42:14.024 INFO:tasks.rados.rados.0.smithi017.stdout:986: done (2 left) 2024-09-21T09:42:14.024 INFO:tasks.rados.rados.0.smithi017.stdout:988: done (1 left) 2024-09-21T09:42:14.024 INFO:tasks.rados.rados.0.smithi017.stdout:989: done (0 left) 2024-09-21T09:42:14.024 INFO:tasks.rados.rados.0.smithi017.stdout:990: read oid 44 snap 121 2024-09-21T09:42:14.024 INFO:tasks.rados.rados.0.smithi017.stdout:990: expect (ObjNum 323 snap 98 seq_num 323) 2024-09-21T09:42:14.024 INFO:tasks.rados.rados.0.smithi017.stdout:991: rollback oid 49 current snap is 122 2024-09-21T09:42:14.024 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 121 2024-09-21T09:42:14.024 INFO:tasks.rados.rados.0.smithi017.stdout:992: write oid 5 current snap is 122 2024-09-21T09:42:14.024 INFO:tasks.rados.rados.0.smithi017.stdout:992: seq_num 400 ranges {722800=549956,2040562=682238} 2024-09-21T09:42:14.030 INFO:tasks.rados.rados.0.smithi017.stdout:992: writing smithi01785587-5 from 722800 to 1272756 tid 1 2024-09-21T09:42:14.039 INFO:tasks.rados.rados.0.smithi017.stdout:992: writing smithi01785587-5 from 2040562 to 2722800 tid 2 2024-09-21T09:42:14.040 INFO:tasks.rados.rados.0.smithi017.stdout:993: read oid 13 snap -1 2024-09-21T09:42:14.040 INFO:tasks.rados.rados.0.smithi017.stdout:993: expect deleted 2024-09-21T09:42:14.040 INFO:tasks.rados.rados.0.smithi017.stdout:994: write oid 34 current snap is 122 2024-09-21T09:42:14.040 INFO:tasks.rados.rados.0.smithi017.stdout:991: finishing rollback tid 0 to smithi01785587-49 2024-09-21T09:42:14.040 INFO:tasks.rados.rados.0.smithi017.stdout:994: seq_num 401 ranges {739607=625205,1982691=499457} 2024-09-21T09:42:14.047 INFO:tasks.rados.rados.0.smithi017.stdout:994: writing smithi01785587-34 from 739607 to 1364812 tid 1 2024-09-21T09:42:14.053 INFO:tasks.rados.rados.0.smithi017.stdout:994: writing smithi01785587-34 from 1982691 to 2482148 tid 2 2024-09-21T09:42:14.053 INFO:tasks.rados.rados.0.smithi017.stdout:991: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:42:14.054 INFO:tasks.rados.rados.0.smithi017.stdout:991: finishing rollback tid 2 to smithi01785587-49 2024-09-21T09:42:14.054 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 521 (ObjNum 341 snap 103 seq_num 341) dirty exists 2024-09-21T09:42:14.054 INFO:tasks.rados.rados.0.smithi017.stdout:995: read oid 45 snap 121 2024-09-21T09:42:14.054 INFO:tasks.rados.rados.0.smithi017.stdout:995: expect (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:42:14.054 INFO:tasks.rados.rados.0.smithi017.stdout:991: done (5 left) 2024-09-21T09:42:14.054 INFO:tasks.rados.rados.0.smithi017.stdout:996: snap_create 2024-09-21T09:42:14.054 INFO:tasks.rados.rados.0.smithi017.stdout:990: expect (ObjNum 323 snap 98 seq_num 323) 2024-09-21T09:42:14.209 INFO:tasks.rados.rados.0.smithi017.stdout:992: finishing write tid 1 to smithi01785587-5 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:992: finishing write tid 2 to smithi01785587-5 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:992: finishing write tid 3 to smithi01785587-5 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:992: finishing write tid 4 to smithi01785587-5 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 504 (ObjNum 400 snap 122 seq_num 400) dirty exists 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:992: left oid 5 (ObjNum 400 snap 122 seq_num 400) 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:994: finishing write tid 1 to smithi01785587-34 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:994: finishing write tid 2 to smithi01785587-34 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:994: finishing write tid 3 to smithi01785587-34 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:994: finishing write tid 4 to smithi01785587-34 2024-09-21T09:42:14.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 524 (ObjNum 401 snap 122 seq_num 401) dirty exists 2024-09-21T09:42:14.211 INFO:tasks.rados.rados.0.smithi017.stdout:994: left oid 34 (ObjNum 401 snap 122 seq_num 401) 2024-09-21T09:42:14.211 INFO:tasks.rados.rados.0.smithi017.stdout:995: expect (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:42:14.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:14.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:15.021 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:42:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:42:14] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.51.0" 2024-09-21T09:42:15.028 INFO:tasks.rados.rados.0.smithi017.stdout:990: done (5 left) 2024-09-21T09:42:15.028 INFO:tasks.rados.rados.0.smithi017.stdout:992: done (4 left) 2024-09-21T09:42:15.028 INFO:tasks.rados.rados.0.smithi017.stdout:993: done (3 left) 2024-09-21T09:42:15.028 INFO:tasks.rados.rados.0.smithi017.stdout:994: done (2 left) 2024-09-21T09:42:15.028 INFO:tasks.rados.rados.0.smithi017.stdout:995: done (1 left) 2024-09-21T09:42:15.028 INFO:tasks.rados.rados.0.smithi017.stdout:996: done (0 left) 2024-09-21T09:42:15.028 INFO:tasks.rados.rados.0.smithi017.stdout:997: snap_create 2024-09-21T09:42:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:15 smithi017 ceph-mon[102913]: pgmap v47: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-21T09:42:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:15 smithi017 ceph-mon[102913]: osdmap e314: 8 total, 8 up, 8 in 2024-09-21T09:42:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:15 smithi017 ceph-mon[105151]: pgmap v47: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-21T09:42:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:15 smithi017 ceph-mon[105151]: osdmap e314: 8 total, 8 up, 8 in 2024-09-21T09:42:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:15 smithi134 ceph-mon[33541]: pgmap v47: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-21T09:42:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:15 smithi134 ceph-mon[33541]: osdmap e314: 8 total, 8 up, 8 in 2024-09-21T09:42:15.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:15.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:16.034 INFO:tasks.rados.rados.0.smithi017.stdout:997: done (0 left) 2024-09-21T09:42:16.035 INFO:tasks.rados.rados.0.smithi017.stdout:998: rollback oid 43 current snap is 124 2024-09-21T09:42:16.035 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 122 2024-09-21T09:42:16.035 INFO:tasks.rados.rados.0.smithi017.stdout:999: rollback oid 11 current snap is 124 2024-09-21T09:42:16.035 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 123 2024-09-21T09:42:16.035 INFO:tasks.rados.rados.0.smithi017.stdout:1000: snap_create 2024-09-21T09:42:16.037 INFO:tasks.rados.rados.0.smithi017.stdout:999: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:42:16.037 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 0 (ObjNum 177 snap 0 seq_num 2399941504) dirty dne 2024-09-21T09:42:16.040 INFO:tasks.rados.rados.0.smithi017.stdout:998: finishing rollback tid 0 to smithi01785587-43 2024-09-21T09:42:16.040 INFO:tasks.rados.rados.0.smithi017.stdout:998: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:42:16.040 INFO:tasks.rados.rados.0.smithi017.stdout:998: finishing rollback tid 2 to smithi01785587-43 2024-09-21T09:42:16.040 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 308 (ObjNum 391 snap 119 seq_num 391) dirty exists 2024-09-21T09:42:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:16 smithi017 ceph-mon[102913]: osdmap e315: 8 total, 8 up, 8 in 2024-09-21T09:42:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:16 smithi017 ceph-mon[105151]: osdmap e315: 8 total, 8 up, 8 in 2024-09-21T09:42:16.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:16 smithi134 ceph-mon[33541]: osdmap e315: 8 total, 8 up, 8 in 2024-09-21T09:42:17.036 INFO:tasks.rados.rados.0.smithi017.stdout:998: done (2 left) 2024-09-21T09:42:17.036 INFO:tasks.rados.rados.0.smithi017.stdout:999: done (1 left) 2024-09-21T09:42:17.036 INFO:tasks.rados.rados.0.smithi017.stdout:1000: done (0 left) 2024-09-21T09:42:17.036 INFO:tasks.rados.rados.0.smithi017.stdout:1001: delete oid 21 current snap is 125 2024-09-21T09:42:17.043 INFO:tasks.rados.rados.0.smithi017.stdout:1001: done (0 left) 2024-09-21T09:42:17.043 INFO:tasks.rados.rados.0.smithi017.stdout:1002: snap_remove snap 122 2024-09-21T09:42:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:17 smithi134 ceph-mon[33541]: pgmap v50: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:17 smithi134 ceph-mon[33541]: osdmap e316: 8 total, 8 up, 8 in 2024-09-21T09:42:17.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:17 smithi017 ceph-mon[102913]: pgmap v50: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:17 smithi017 ceph-mon[102913]: osdmap e316: 8 total, 8 up, 8 in 2024-09-21T09:42:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:17 smithi017 ceph-mon[105151]: pgmap v50: 89 pgs: 89 active+clean; 180 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:17 smithi017 ceph-mon[105151]: osdmap e316: 8 total, 8 up, 8 in 2024-09-21T09:42:18.045 INFO:tasks.rados.rados.0.smithi017.stdout:1002: done (0 left) 2024-09-21T09:42:18.045 INFO:tasks.rados.rados.0.smithi017.stdout:1003: read oid 31 snap -1 2024-09-21T09:42:18.045 INFO:tasks.rados.rados.0.smithi017.stdout:1003: expect (ObjNum 392 snap 121 seq_num 392) 2024-09-21T09:42:18.045 INFO:tasks.rados.rados.0.smithi017.stdout:1004: snap_create 2024-09-21T09:42:18.053 INFO:tasks.rados.rados.0.smithi017.stdout:1003: expect (ObjNum 392 snap 121 seq_num 392) 2024-09-21T09:42:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:18 smithi017 ceph-mon[102913]: osdmap e317: 8 total, 8 up, 8 in 2024-09-21T09:42:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:18 smithi017 ceph-mon[105151]: osdmap e317: 8 total, 8 up, 8 in 2024-09-21T09:42:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:18 smithi134 ceph-mon[33541]: osdmap e317: 8 total, 8 up, 8 in 2024-09-21T09:42:19.048 INFO:tasks.rados.rados.0.smithi017.stdout:1003: done (1 left) 2024-09-21T09:42:19.048 INFO:tasks.rados.rados.0.smithi017.stdout:1004: done (0 left) 2024-09-21T09:42:19.048 INFO:tasks.rados.rados.0.smithi017.stdout:1005: read oid 29 snap 123 2024-09-21T09:42:19.048 INFO:tasks.rados.rados.0.smithi017.stdout:1005: expect deleted 2024-09-21T09:42:19.048 INFO:tasks.rados.rados.0.smithi017.stdout:1006: delete oid 9 current snap is 126 2024-09-21T09:42:19.055 INFO:tasks.rados.rados.0.smithi017.stdout:1005: done (1 left) 2024-09-21T09:42:19.055 INFO:tasks.rados.rados.0.smithi017.stdout:1006: done (0 left) 2024-09-21T09:42:19.055 INFO:tasks.rados.rados.0.smithi017.stdout:1007: read oid 39 snap 125 2024-09-21T09:42:19.055 INFO:tasks.rados.rados.0.smithi017.stdout:1007: expect (ObjNum 387 snap 119 seq_num 387) 2024-09-21T09:42:19.055 INFO:tasks.rados.rados.0.smithi017.stdout:1008: read oid 16 snap -1 2024-09-21T09:42:19.055 INFO:tasks.rados.rados.0.smithi017.stdout:1008: expect deleted 2024-09-21T09:42:19.055 INFO:tasks.rados.rados.0.smithi017.stdout:1009: rollback oid 14 current snap is 126 2024-09-21T09:42:19.055 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 124 2024-09-21T09:42:19.055 INFO:tasks.rados.rados.0.smithi017.stdout:1010: delete oid 48 current snap is 126 2024-09-21T09:42:19.059 INFO:tasks.rados.rados.0.smithi017.stdout:1008: done (3 left) 2024-09-21T09:42:19.059 INFO:tasks.rados.rados.0.smithi017.stdout:1010: done (2 left) 2024-09-21T09:42:19.059 INFO:tasks.rados.rados.0.smithi017.stdout:1011: read oid 46 snap -1 2024-09-21T09:42:19.059 INFO:tasks.rados.rados.0.smithi017.stdout:1011: expect (ObjNum 397 snap 121 seq_num 397) 2024-09-21T09:42:19.059 INFO:tasks.rados.rados.0.smithi017.stdout:1012: write oid 38 current snap is 126 2024-09-21T09:42:19.059 INFO:tasks.rados.rados.0.smithi017.stdout:1012: seq_num 405 ranges {406835=642554,1532972=506795,2772312=34523} 2024-09-21T09:42:19.065 INFO:tasks.rados.rados.0.smithi017.stdout:1012: writing smithi01785587-38 from 406835 to 1049389 tid 1 2024-09-21T09:42:19.070 INFO:tasks.rados.rados.0.smithi017.stdout:1012: writing smithi01785587-38 from 1532972 to 2039767 tid 2 2024-09-21T09:42:19.075 INFO:tasks.rados.rados.0.smithi017.stdout:1012: writing smithi01785587-38 from 2772312 to 2806835 tid 3 2024-09-21T09:42:19.075 INFO:tasks.rados.rados.0.smithi017.stdout:1013: rollback oid 29 current snap is 126 2024-09-21T09:42:19.075 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 29 to 124 2024-09-21T09:42:19.075 INFO:tasks.rados.rados.0.smithi017.stdout:1014: read oid 22 snap 124 2024-09-21T09:42:19.076 INFO:tasks.rados.rados.0.smithi017.stdout:1014: expect deleted 2024-09-21T09:42:19.076 INFO:tasks.rados.rados.0.smithi017.stdout:1015: delete oid 48 current snap is 126 2024-09-21T09:42:19.076 INFO:tasks.rados.rados.0.smithi017.stdout:1007: expect (ObjNum 387 snap 119 seq_num 387) 2024-09-21T09:42:19.257 INFO:tasks.rados.rados.0.smithi017.stdout:1009: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:42:19.257 INFO:tasks.rados.rados.0.smithi017.stdout:1009: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:42:19.258 INFO:tasks.rados.rados.0.smithi017.stdout:1009: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:42:19.258 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 508 (ObjNum 320 snap 95 seq_num 320) dirty exists 2024-09-21T09:42:19.258 INFO:tasks.rados.rados.0.smithi017.stdout:1011: expect (ObjNum 397 snap 121 seq_num 397) 2024-09-21T09:42:19.374 INFO:tasks.rados.rados.0.smithi017.stdout:1012: finishing write tid 1 to smithi01785587-38 2024-09-21T09:42:19.374 INFO:tasks.rados.rados.0.smithi017.stdout:1012: finishing write tid 2 to smithi01785587-38 2024-09-21T09:42:19.374 INFO:tasks.rados.rados.0.smithi017.stdout:1013: finishing rollback tid 1 to smithi01785587-29 2024-09-21T09:42:19.374 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:42:19.374 INFO:tasks.rados.rados.0.smithi017.stdout:1012: finishing write tid 3 to smithi01785587-38 2024-09-21T09:42:19.374 INFO:tasks.rados.rados.0.smithi017.stdout:1012: finishing write tid 4 to smithi01785587-38 2024-09-21T09:42:19.374 INFO:tasks.rados.rados.0.smithi017.stdout:1012: finishing write tid 5 to smithi01785587-38 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 315 (ObjNum 405 snap 126 seq_num 405) dirty exists 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1012: left oid 38 (ObjNum 405 snap 126 seq_num 405) 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1007: done (6 left) 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1009: done (5 left) 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1011: done (4 left) 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1012: done (3 left) 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1013: done (2 left) 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1014: done (1 left) 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1015: done (0 left) 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1016: rollback oid 22 current snap is 126 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 22 to 125 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1017: rollback oid 14 current snap is 126 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 121 2024-09-21T09:42:19.375 INFO:tasks.rados.rados.0.smithi017.stdout:1018: write oid 40 current snap is 126 2024-09-21T09:42:19.376 INFO:tasks.rados.rados.0.smithi017.stdout:1018: seq_num 407 ranges {440449=793052,1933583=510464} 2024-09-21T09:42:19.380 INFO:tasks.rados.rados.0.smithi017.stdout:1018: writing smithi01785587-40 from 440449 to 1233501 tid 1 2024-09-21T09:42:19.387 INFO:tasks.rados.rados.0.smithi017.stdout:1018: writing smithi01785587-40 from 1933583 to 2444047 tid 2 2024-09-21T09:42:19.387 INFO:tasks.rados.rados.0.smithi017.stdout:1019: read oid 49 snap -1 2024-09-21T09:42:19.387 INFO:tasks.rados.rados.0.smithi017.stdout:1019: expect (ObjNum 341 snap 103 seq_num 341) 2024-09-21T09:42:19.387 INFO:tasks.rados.rados.0.smithi017.stdout:1020: snap_remove snap 123 2024-09-21T09:42:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:19 smithi017 ceph-mon[102913]: pgmap v53: 89 pgs: 89 active+clean; 88 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 841 KiB/s wr, 13 op/s 2024-09-21T09:42:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:19 smithi017 ceph-mon[102913]: osdmap e318: 8 total, 8 up, 8 in 2024-09-21T09:42:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:19 smithi017 ceph-mon[105151]: pgmap v53: 89 pgs: 89 active+clean; 88 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 841 KiB/s wr, 13 op/s 2024-09-21T09:42:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:19 smithi017 ceph-mon[105151]: osdmap e318: 8 total, 8 up, 8 in 2024-09-21T09:42:19.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:19 smithi134 ceph-mon[33541]: pgmap v53: 89 pgs: 89 active+clean; 88 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 841 KiB/s wr, 13 op/s 2024-09-21T09:42:19.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:19 smithi134 ceph-mon[33541]: osdmap e318: 8 total, 8 up, 8 in 2024-09-21T09:42:20.058 INFO:tasks.rados.rados.0.smithi017.stdout:1020: done (4 left) 2024-09-21T09:42:20.058 INFO:tasks.rados.rados.0.smithi017.stdout:1021: write oid 12 current snap is 126 2024-09-21T09:42:20.058 INFO:tasks.rados.rados.0.smithi017.stdout:1016: finishing rollback tid 1 to smithi01785587-22 2024-09-21T09:42:20.059 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:42:20.059 INFO:tasks.rados.rados.0.smithi017.stdout:1021: seq_num 408 ranges {457256=468301,1592065=770475,2854581=2675} 2024-09-21T09:42:20.063 INFO:tasks.rados.rados.0.smithi017.stdout:1021: writing smithi01785587-12 from 457256 to 925557 tid 1 2024-09-21T09:42:20.072 INFO:tasks.rados.rados.0.smithi017.stdout:1021: writing smithi01785587-12 from 1592065 to 2362540 tid 2 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:1021: writing smithi01785587-12 from 2854581 to 2857256 tid 3 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:1016: done (4 left) 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:1022: read oid 1 snap -1 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:1022: expect (ObjNum 378 snap 118 seq_num 378) 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:1023: delete oid 24 current snap is 126 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:1017: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:1017: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:1017: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 511 (ObjNum 320 snap 95 seq_num 320) dirty exists 2024-09-21T09:42:20.076 INFO:tasks.rados.rados.0.smithi017.stdout:1018: finishing write tid 1 to smithi01785587-40 2024-09-21T09:42:20.077 INFO:tasks.rados.rados.0.smithi017.stdout:1018: finishing write tid 2 to smithi01785587-40 2024-09-21T09:42:20.077 INFO:tasks.rados.rados.0.smithi017.stdout:1018: finishing write tid 3 to smithi01785587-40 2024-09-21T09:42:20.077 INFO:tasks.rados.rados.0.smithi017.stdout:1018: finishing write tid 4 to smithi01785587-40 2024-09-21T09:42:20.077 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 546 (ObjNum 407 snap 126 seq_num 407) dirty exists 2024-09-21T09:42:20.077 INFO:tasks.rados.rados.0.smithi017.stdout:1018: left oid 40 (ObjNum 407 snap 126 seq_num 407) 2024-09-21T09:42:20.077 INFO:tasks.rados.rados.0.smithi017.stdout:1019: expect (ObjNum 341 snap 103 seq_num 341) 2024-09-21T09:42:20.308 INFO:tasks.rados.rados.0.smithi017.stdout:1021: finishing write tid 1 to smithi01785587-12 2024-09-21T09:42:20.308 INFO:tasks.rados.rados.0.smithi017.stdout:1021: finishing write tid 2 to smithi01785587-12 2024-09-21T09:42:20.308 INFO:tasks.rados.rados.0.smithi017.stdout:1021: finishing write tid 3 to smithi01785587-12 2024-09-21T09:42:20.308 INFO:tasks.rados.rados.0.smithi017.stdout:1021: finishing write tid 4 to smithi01785587-12 2024-09-21T09:42:20.308 INFO:tasks.rados.rados.0.smithi017.stdout:1021: finishing write tid 5 to smithi01785587-12 2024-09-21T09:42:20.308 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 373 (ObjNum 408 snap 126 seq_num 408) dirty exists 2024-09-21T09:42:20.309 INFO:tasks.rados.rados.0.smithi017.stdout:1021: left oid 12 (ObjNum 408 snap 126 seq_num 408) 2024-09-21T09:42:20.309 INFO:tasks.rados.rados.0.smithi017.stdout:1022: expect (ObjNum 378 snap 118 seq_num 378) 2024-09-21T09:42:20.388 INFO:tasks.rados.rados.0.smithi017.stdout:1017: done (5 left) 2024-09-21T09:42:20.388 INFO:tasks.rados.rados.0.smithi017.stdout:1018: done (4 left) 2024-09-21T09:42:20.389 INFO:tasks.rados.rados.0.smithi017.stdout:1019: done (3 left) 2024-09-21T09:42:20.389 INFO:tasks.rados.rados.0.smithi017.stdout:1021: done (2 left) 2024-09-21T09:42:20.389 INFO:tasks.rados.rados.0.smithi017.stdout:1022: done (1 left) 2024-09-21T09:42:20.389 INFO:tasks.rados.rados.0.smithi017.stdout:1023: done (0 left) 2024-09-21T09:42:20.389 INFO:tasks.rados.rados.0.smithi017.stdout:1024: read oid 14 snap 124 2024-09-21T09:42:20.389 INFO:tasks.rados.rados.0.smithi017.stdout:1024: expect (ObjNum 320 snap 95 seq_num 320) 2024-09-21T09:42:20.390 INFO:tasks.rados.rados.0.smithi017.stdout:1025: write oid 15 current snap is 126 2024-09-21T09:42:20.390 INFO:tasks.rados.rados.0.smithi017.stdout:1025: seq_num 410 ranges {490870=618799,1592676=774144,2793074=97796} 2024-09-21T09:42:20.394 INFO:tasks.rados.rados.0.smithi017.stdout:1025: writing smithi01785587-15 from 490870 to 1109669 tid 1 2024-09-21T09:42:20.401 INFO:tasks.rados.rados.0.smithi017.stdout:1025: writing smithi01785587-15 from 1592676 to 2366820 tid 2 2024-09-21T09:42:20.404 INFO:tasks.rados.rados.0.smithi017.stdout:1025: writing smithi01785587-15 from 2793074 to 2890870 tid 3 2024-09-21T09:42:20.404 INFO:tasks.rados.rados.0.smithi017.stdout:1026: read oid 9 snap -1 2024-09-21T09:42:20.404 INFO:tasks.rados.rados.0.smithi017.stdout:1026: expect deleted 2024-09-21T09:42:20.405 INFO:tasks.rados.rados.0.smithi017.stdout:1027: rollback oid 27 current snap is 126 2024-09-21T09:42:20.405 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 125 2024-09-21T09:42:20.405 INFO:tasks.rados.rados.0.smithi017.stdout:1024: expect (ObjNum 320 snap 95 seq_num 320) 2024-09-21T09:42:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:20 smithi017 ceph-mon[102913]: osdmap e319: 8 total, 8 up, 8 in 2024-09-21T09:42:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:20 smithi017 ceph-mon[105151]: osdmap e319: 8 total, 8 up, 8 in 2024-09-21T09:42:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:20 smithi134 ceph-mon[33541]: osdmap e319: 8 total, 8 up, 8 in 2024-09-21T09:42:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:20.515 INFO:tasks.rados.rados.0.smithi017.stdout:1025: finishing write tid 1 to smithi01785587-15 2024-09-21T09:42:20.515 INFO:tasks.rados.rados.0.smithi017.stdout:1027: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:42:20.515 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 0 (ObjNum 65 snap 0 seq_num 2399865792) dirty dne 2024-09-21T09:42:20.515 INFO:tasks.rados.rados.0.smithi017.stdout:1025: finishing write tid 2 to smithi01785587-15 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:1025: finishing write tid 3 to smithi01785587-15 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:1025: finishing write tid 4 to smithi01785587-15 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:1025: finishing write tid 5 to smithi01785587-15 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 202 (ObjNum 410 snap 126 seq_num 410) dirty exists 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:1025: left oid 15 (ObjNum 410 snap 126 seq_num 410) 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:1024: done (3 left) 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:1025: done (2 left) 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:1026: done (1 left) 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:1027: done (0 left) 2024-09-21T09:42:20.516 INFO:tasks.rados.rados.0.smithi017.stdout:1028: snap_create 2024-09-21T09:42:21.063 INFO:tasks.rados.rados.0.smithi017.stdout:1028: done (0 left) 2024-09-21T09:42:21.063 INFO:tasks.rados.rados.0.smithi017.stdout:1029: read oid 1 snap 125 2024-09-21T09:42:21.063 INFO:tasks.rados.rados.0.smithi017.stdout:1029: expect (ObjNum 378 snap 118 seq_num 378) 2024-09-21T09:42:21.064 INFO:tasks.rados.rados.0.smithi017.stdout:1030: write oid 28 current snap is 127 2024-09-21T09:42:21.064 INFO:tasks.rados.rados.0.smithi017.stdout:1030: seq_num 411 ranges {507677=410401,1367511=517802,2420497=487180} 2024-09-21T09:42:21.068 INFO:tasks.rados.rados.0.smithi017.stdout:1030: writing smithi01785587-28 from 507677 to 918078 tid 1 2024-09-21T09:42:21.074 INFO:tasks.rados.rados.0.smithi017.stdout:1030: writing smithi01785587-28 from 1367511 to 1885313 tid 2 2024-09-21T09:42:21.080 INFO:tasks.rados.rados.0.smithi017.stdout:1030: writing smithi01785587-28 from 2420497 to 2907677 tid 3 2024-09-21T09:42:21.080 INFO:tasks.rados.rados.0.smithi017.stdout:1031: write oid 26 current snap is 127 2024-09-21T09:42:21.080 INFO:tasks.rados.rados.0.smithi017.stdout:1031: seq_num 412 ranges {524484=485650,1425993=661460,2847920=76564} 2024-09-21T09:42:21.085 INFO:tasks.rados.rados.0.smithi017.stdout:1031: writing smithi01785587-26 from 524484 to 1010134 tid 1 2024-09-21T09:42:21.091 INFO:tasks.rados.rados.0.smithi017.stdout:1031: writing smithi01785587-26 from 1425993 to 2087453 tid 2 2024-09-21T09:42:21.096 INFO:tasks.rados.rados.0.smithi017.stdout:1031: writing smithi01785587-26 from 2847920 to 2924484 tid 3 2024-09-21T09:42:21.096 INFO:tasks.rados.rados.0.smithi017.stdout:1032: snap_create 2024-09-21T09:42:21.096 INFO:tasks.rados.rados.0.smithi017.stdout:1029: expect (ObjNum 378 snap 118 seq_num 378) 2024-09-21T09:42:21.176 INFO:tasks.rados.rados.0.smithi017.stdout:1030: finishing write tid 1 to smithi01785587-28 2024-09-21T09:42:21.176 INFO:tasks.rados.rados.0.smithi017.stdout:1030: finishing write tid 2 to smithi01785587-28 2024-09-21T09:42:21.176 INFO:tasks.rados.rados.0.smithi017.stdout:1030: finishing write tid 3 to smithi01785587-28 2024-09-21T09:42:21.176 INFO:tasks.rados.rados.0.smithi017.stdout:1030: finishing write tid 4 to smithi01785587-28 2024-09-21T09:42:21.176 INFO:tasks.rados.rados.0.smithi017.stdout:1030: finishing write tid 5 to smithi01785587-28 2024-09-21T09:42:21.177 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 558 (ObjNum 411 snap 127 seq_num 411) dirty exists 2024-09-21T09:42:21.177 INFO:tasks.rados.rados.0.smithi017.stdout:1030: left oid 28 (ObjNum 411 snap 127 seq_num 411) 2024-09-21T09:42:21.177 INFO:tasks.rados.rados.0.smithi017.stdout:1031: finishing write tid 1 to smithi01785587-26 2024-09-21T09:42:21.177 INFO:tasks.rados.rados.0.smithi017.stdout:1031: finishing write tid 2 to smithi01785587-26 2024-09-21T09:42:21.177 INFO:tasks.rados.rados.0.smithi017.stdout:1031: finishing write tid 3 to smithi01785587-26 2024-09-21T09:42:21.177 INFO:tasks.rados.rados.0.smithi017.stdout:1031: finishing write tid 4 to smithi01785587-26 2024-09-21T09:42:21.177 INFO:tasks.rados.rados.0.smithi017.stdout:1031: finishing write tid 5 to smithi01785587-26 2024-09-21T09:42:21.177 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 563 (ObjNum 412 snap 127 seq_num 412) dirty exists 2024-09-21T09:42:21.177 INFO:tasks.rados.rados.0.smithi017.stdout:1031: left oid 26 (ObjNum 412 snap 127 seq_num 412) 2024-09-21T09:42:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:21 smithi017 ceph-mon[102913]: pgmap v56: 89 pgs: 89 active+clean; 88 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 841 KiB/s wr, 13 op/s 2024-09-21T09:42:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:21 smithi017 ceph-mon[102913]: osdmap e320: 8 total, 8 up, 8 in 2024-09-21T09:42:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:21 smithi017 ceph-mon[105151]: pgmap v56: 89 pgs: 89 active+clean; 88 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 841 KiB/s wr, 13 op/s 2024-09-21T09:42:21.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:21 smithi017 ceph-mon[105151]: osdmap e320: 8 total, 8 up, 8 in 2024-09-21T09:42:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:21 smithi134 ceph-mon[33541]: pgmap v56: 89 pgs: 89 active+clean; 88 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 841 KiB/s wr, 13 op/s 2024-09-21T09:42:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:21 smithi134 ceph-mon[33541]: osdmap e320: 8 total, 8 up, 8 in 2024-09-21T09:42:22.079 INFO:tasks.rados.rados.0.smithi017.stdout:1029: done (3 left) 2024-09-21T09:42:22.079 INFO:tasks.rados.rados.0.smithi017.stdout:1030: done (2 left) 2024-09-21T09:42:22.079 INFO:tasks.rados.rados.0.smithi017.stdout:1031: done (1 left) 2024-09-21T09:42:22.080 INFO:tasks.rados.rados.0.smithi017.stdout:1032: done (0 left) 2024-09-21T09:42:22.080 INFO:tasks.rados.rados.0.smithi017.stdout:1033: delete oid 26 current snap is 128 2024-09-21T09:42:22.083 INFO:tasks.rados.rados.0.smithi017.stdout:1033: done (0 left) 2024-09-21T09:42:22.084 INFO:tasks.rados.rados.0.smithi017.stdout:1034: write oid 47 current snap is 128 2024-09-21T09:42:22.084 INFO:tasks.rados.rados.0.smithi017.stdout:1034: seq_num 414 ranges {558098=636148,1826604=665129} 2024-09-21T09:42:22.090 INFO:tasks.rados.rados.0.smithi017.stdout:1034: writing smithi01785587-47 from 558098 to 1194246 tid 1 2024-09-21T09:42:22.098 INFO:tasks.rados.rados.0.smithi017.stdout:1034: writing smithi01785587-47 from 1826604 to 2491733 tid 2 2024-09-21T09:42:22.098 INFO:tasks.rados.rados.0.smithi017.stdout:1035: write oid 20 current snap is 128 2024-09-21T09:42:22.098 INFO:tasks.rados.rados.0.smithi017.stdout:1034: finishing write tid 1 to smithi01785587-47 2024-09-21T09:42:22.098 INFO:tasks.rados.rados.0.smithi017.stdout:1035: seq_num 415 ranges {574905=711397,1885086=525140,2930189=44716} 2024-09-21T09:42:22.105 INFO:tasks.rados.rados.0.smithi017.stdout:1035: writing smithi01785587-20 from 574905 to 1286302 tid 1 2024-09-21T09:42:22.111 INFO:tasks.rados.rados.0.smithi017.stdout:1035: writing smithi01785587-20 from 1885086 to 2410226 tid 2 2024-09-21T09:42:22.114 INFO:tasks.rados.rados.0.smithi017.stdout:1035: writing smithi01785587-20 from 2930189 to 2974905 tid 3 2024-09-21T09:42:22.114 INFO:tasks.rados.rados.0.smithi017.stdout:1036: snap_create 2024-09-21T09:42:22.114 INFO:tasks.rados.rados.0.smithi017.stdout:1034: finishing write tid 2 to smithi01785587-47 2024-09-21T09:42:22.115 INFO:tasks.rados.rados.0.smithi017.stdout:1034: finishing write tid 3 to smithi01785587-47 2024-09-21T09:42:22.115 INFO:tasks.rados.rados.0.smithi017.stdout:1034: finishing write tid 4 to smithi01785587-47 2024-09-21T09:42:22.115 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 377 (ObjNum 414 snap 128 seq_num 414) dirty exists 2024-09-21T09:42:22.115 INFO:tasks.rados.rados.0.smithi017.stdout:1034: left oid 47 (ObjNum 414 snap 128 seq_num 414) 2024-09-21T09:42:22.115 INFO:tasks.rados.rados.0.smithi017.stdout:1035: finishing write tid 1 to smithi01785587-20 2024-09-21T09:42:22.115 INFO:tasks.rados.rados.0.smithi017.stdout:1035: finishing write tid 2 to smithi01785587-20 2024-09-21T09:42:22.116 INFO:tasks.rados.rados.0.smithi017.stdout:1035: finishing write tid 3 to smithi01785587-20 2024-09-21T09:42:22.116 INFO:tasks.rados.rados.0.smithi017.stdout:1035: finishing write tid 4 to smithi01785587-20 2024-09-21T09:42:22.116 INFO:tasks.rados.rados.0.smithi017.stdout:1035: finishing write tid 5 to smithi01785587-20 2024-09-21T09:42:22.116 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 223 (ObjNum 415 snap 128 seq_num 415) dirty exists 2024-09-21T09:42:22.116 INFO:tasks.rados.rados.0.smithi017.stdout:1035: left oid 20 (ObjNum 415 snap 128 seq_num 415) 2024-09-21T09:42:22.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:22 smithi017 ceph-mon[102913]: osdmap e321: 8 total, 8 up, 8 in 2024-09-21T09:42:22.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:22 smithi017 ceph-mon[105151]: osdmap e321: 8 total, 8 up, 8 in 2024-09-21T09:42:22.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:22 smithi134 ceph-mon[33541]: osdmap e321: 8 total, 8 up, 8 in 2024-09-21T09:42:23.024 INFO:tasks.rados.rados.0.smithi017.stdout:1034: done (2 left) 2024-09-21T09:42:23.025 INFO:tasks.rados.rados.0.smithi017.stdout:1035: done (1 left) 2024-09-21T09:42:23.025 INFO:tasks.rados.rados.0.smithi017.stdout:1036: done (0 left) 2024-09-21T09:42:23.025 INFO:tasks.rados.rados.0.smithi017.stdout:1037: read oid 36 snap 125 2024-09-21T09:42:23.025 INFO:tasks.rados.rados.0.smithi017.stdout:1037: expect (ObjNum 313 snap 93 seq_num 313) 2024-09-21T09:42:23.025 INFO:tasks.rados.rados.0.smithi017.stdout:1038: write oid 8 current snap is 129 2024-09-21T09:42:23.025 INFO:tasks.rados.rados.0.smithi017.stdout:1038: seq_num 416 ranges {591712=786646,1943568=668798} 2024-09-21T09:42:23.032 INFO:tasks.rados.rados.0.smithi017.stdout:1038: writing smithi01785587-8 from 591712 to 1378358 tid 1 2024-09-21T09:42:23.039 INFO:tasks.rados.rados.0.smithi017.stdout:1038: writing smithi01785587-8 from 1943568 to 2612366 tid 2 2024-09-21T09:42:23.040 INFO:tasks.rados.rados.0.smithi017.stdout:1039: write oid 16 current snap is 129 2024-09-21T09:42:23.040 INFO:tasks.rados.rados.0.smithi017.stdout:1039: seq_num 417 ranges {608519=461895,1602050=528809,2585035=423484} 2024-09-21T09:42:23.045 INFO:tasks.rados.rados.0.smithi017.stdout:1039: writing smithi01785587-16 from 608519 to 1070414 tid 1 2024-09-21T09:42:23.051 INFO:tasks.rados.rados.0.smithi017.stdout:1039: writing smithi01785587-16 from 1602050 to 2130859 tid 2 2024-09-21T09:42:23.056 INFO:tasks.rados.rados.0.smithi017.stdout:1039: writing smithi01785587-16 from 2585035 to 3008519 tid 3 2024-09-21T09:42:23.056 INFO:tasks.rados.rados.0.smithi017.stdout:1040: rollback oid 9 current snap is 129 2024-09-21T09:42:23.057 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 9 to 127 2024-09-21T09:42:23.057 INFO:tasks.rados.rados.0.smithi017.stdout:1037: expect (ObjNum 313 snap 93 seq_num 313) 2024-09-21T09:42:23.217 INFO:tasks.rados.rados.0.smithi017.stdout:1038: finishing write tid 1 to smithi01785587-8 2024-09-21T09:42:23.217 INFO:tasks.rados.rados.0.smithi017.stdout:1038: finishing write tid 2 to smithi01785587-8 2024-09-21T09:42:23.217 INFO:tasks.rados.rados.0.smithi017.stdout:1038: finishing write tid 3 to smithi01785587-8 2024-09-21T09:42:23.217 INFO:tasks.rados.rados.0.smithi017.stdout:1038: finishing write tid 4 to smithi01785587-8 2024-09-21T09:42:23.217 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 495 (ObjNum 416 snap 129 seq_num 416) dirty exists 2024-09-21T09:42:23.217 INFO:tasks.rados.rados.0.smithi017.stdout:1038: left oid 8 (ObjNum 416 snap 129 seq_num 416) 2024-09-21T09:42:23.217 INFO:tasks.rados.rados.0.smithi017.stdout:1039: finishing write tid 1 to smithi01785587-16 2024-09-21T09:42:23.217 INFO:tasks.rados.rados.0.smithi017.stdout:1039: finishing write tid 2 to smithi01785587-16 2024-09-21T09:42:23.217 INFO:tasks.rados.rados.0.smithi017.stdout:1037: done (3 left) 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1038: done (2 left) 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1041: read oid 21 snap -1 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1041: expect deleted 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1042: write oid 2 current snap is 129 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1039: finishing write tid 3 to smithi01785587-16 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1039: finishing write tid 4 to smithi01785587-16 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1039: finishing write tid 5 to smithi01785587-16 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 569 (ObjNum 417 snap 129 seq_num 417) dirty exists 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1039: left oid 16 (ObjNum 417 snap 129 seq_num 417) 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1040: finishing rollback tid 1 to smithi01785587-9 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 193 (ObjNum 33 snap 0 seq_num 2384147832) dirty dne 2024-09-21T09:42:23.218 INFO:tasks.rados.rados.0.smithi017.stdout:1042: seq_num 418 ranges {625326=537144,1944179=672467} 2024-09-21T09:42:23.223 INFO:tasks.rados.rados.0.smithi017.stdout:1042: writing smithi01785587-2 from 625326 to 1162470 tid 1 2024-09-21T09:42:23.231 INFO:tasks.rados.rados.0.smithi017.stdout:1042: writing smithi01785587-2 from 1944179 to 2616646 tid 2 2024-09-21T09:42:23.231 INFO:tasks.rados.rados.0.smithi017.stdout:1039: done (3 left) 2024-09-21T09:42:23.231 INFO:tasks.rados.rados.0.smithi017.stdout:1040: done (2 left) 2024-09-21T09:42:23.231 INFO:tasks.rados.rados.0.smithi017.stdout:1043: snap_create 2024-09-21T09:42:23.231 INFO:tasks.rados.rados.0.smithi017.stdout:1042: finishing write tid 1 to smithi01785587-2 2024-09-21T09:42:23.234 INFO:tasks.rados.rados.0.smithi017.stdout:1042: finishing write tid 2 to smithi01785587-2 2024-09-21T09:42:23.234 INFO:tasks.rados.rados.0.smithi017.stdout:1042: finishing write tid 3 to smithi01785587-2 2024-09-21T09:42:23.234 INFO:tasks.rados.rados.0.smithi017.stdout:1042: finishing write tid 4 to smithi01785587-2 2024-09-21T09:42:23.234 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 200 (ObjNum 418 snap 129 seq_num 418) dirty exists 2024-09-21T09:42:23.234 INFO:tasks.rados.rados.0.smithi017.stdout:1042: left oid 2 (ObjNum 418 snap 129 seq_num 418) 2024-09-21T09:42:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:23 smithi017 ceph-mon[105151]: pgmap v59: 89 pgs: 89 active+clean; 94 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 21 op/s 2024-09-21T09:42:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:23 smithi017 ceph-mon[105151]: osdmap e322: 8 total, 8 up, 8 in 2024-09-21T09:42:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:23 smithi017 ceph-mon[105151]: osdmap e323: 8 total, 8 up, 8 in 2024-09-21T09:42:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:23 smithi017 ceph-mon[102913]: pgmap v59: 89 pgs: 89 active+clean; 94 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 21 op/s 2024-09-21T09:42:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:23 smithi017 ceph-mon[102913]: osdmap e322: 8 total, 8 up, 8 in 2024-09-21T09:42:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:23 smithi017 ceph-mon[102913]: osdmap e323: 8 total, 8 up, 8 in 2024-09-21T09:42:23.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:23 smithi134 ceph-mon[33541]: pgmap v59: 89 pgs: 89 active+clean; 94 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 21 op/s 2024-09-21T09:42:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:23 smithi134 ceph-mon[33541]: osdmap e322: 8 total, 8 up, 8 in 2024-09-21T09:42:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:23 smithi134 ceph-mon[33541]: osdmap e323: 8 total, 8 up, 8 in 2024-09-21T09:42:24.078 INFO:tasks.rados.rados.0.smithi017.stdout:1041: done (2 left) 2024-09-21T09:42:24.079 INFO:tasks.rados.rados.0.smithi017.stdout:1042: done (1 left) 2024-09-21T09:42:24.079 INFO:tasks.rados.rados.0.smithi017.stdout:1043: done (0 left) 2024-09-21T09:42:24.079 INFO:tasks.rados.rados.0.smithi017.stdout:1044: rollback oid 27 current snap is 130 2024-09-21T09:42:24.079 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 129 2024-09-21T09:42:24.079 INFO:tasks.rados.rados.0.smithi017.stdout:1045: write oid 39 current snap is 130 2024-09-21T09:42:24.079 INFO:tasks.rados.rados.0.smithi017.stdout:1045: seq_num 419 ranges {642133=728746,2119014=532478} 2024-09-21T09:42:24.086 INFO:tasks.rados.rados.0.smithi017.stdout:1045: writing smithi01785587-39 from 642133 to 1370879 tid 1 2024-09-21T09:42:24.093 INFO:tasks.rados.rados.0.smithi017.stdout:1045: writing smithi01785587-39 from 2119014 to 2651492 tid 2 2024-09-21T09:42:24.093 INFO:tasks.rados.rados.0.smithi017.stdout:1046: write oid 38 current snap is 130 2024-09-21T09:42:24.093 INFO:tasks.rados.rados.0.smithi017.stdout:1044: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:42:24.093 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 0 (ObjNum -113271584 snap 32650 seq_num 304) dirty dne 2024-09-21T09:42:24.093 INFO:tasks.rados.rados.0.smithi017.stdout:1045: finishing write tid 1 to smithi01785587-39 2024-09-21T09:42:24.094 INFO:tasks.rados.rados.0.smithi017.stdout:1046: seq_num 420 ranges {658940=403995,1777496=676136,2950951=107989} 2024-09-21T09:42:24.099 INFO:tasks.rados.rados.0.smithi017.stdout:1046: writing smithi01785587-38 from 658940 to 1062935 tid 1 2024-09-21T09:42:24.106 INFO:tasks.rados.rados.0.smithi017.stdout:1046: writing smithi01785587-38 from 1777496 to 2453632 tid 2 2024-09-21T09:42:24.110 INFO:tasks.rados.rados.0.smithi017.stdout:1046: writing smithi01785587-38 from 2950951 to 3058940 tid 3 2024-09-21T09:42:24.110 INFO:tasks.rados.rados.0.smithi017.stdout:1044: done (2 left) 2024-09-21T09:42:24.110 INFO:tasks.rados.rados.0.smithi017.stdout:1047: write oid 32 current snap is 130 2024-09-21T09:42:24.111 INFO:tasks.rados.rados.0.smithi017.stdout:1045: finishing write tid 2 to smithi01785587-39 2024-09-21T09:42:24.111 INFO:tasks.rados.rados.0.smithi017.stdout:1045: finishing write tid 3 to smithi01785587-39 2024-09-21T09:42:24.111 INFO:tasks.rados.rados.0.smithi017.stdout:1045: finishing write tid 4 to smithi01785587-39 2024-09-21T09:42:24.111 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 552 (ObjNum 419 snap 130 seq_num 419) dirty exists 2024-09-21T09:42:24.111 INFO:tasks.rados.rados.0.smithi017.stdout:1045: left oid 39 (ObjNum 419 snap 130 seq_num 419) 2024-09-21T09:42:24.111 INFO:tasks.rados.rados.0.smithi017.stdout:1046: finishing write tid 1 to smithi01785587-38 2024-09-21T09:42:24.111 INFO:tasks.rados.rados.0.smithi017.stdout:1046: finishing write tid 2 to smithi01785587-38 2024-09-21T09:42:24.111 INFO:tasks.rados.rados.0.smithi017.stdout:1047: seq_num 421 ranges {675747=479244,1835978=536147} 2024-09-21T09:42:24.117 INFO:tasks.rados.rados.0.smithi017.stdout:1047: writing smithi01785587-32 from 675747 to 1154991 tid 1 2024-09-21T09:42:24.124 INFO:tasks.rados.rados.0.smithi017.stdout:1047: writing smithi01785587-32 from 1835978 to 2372125 tid 2 2024-09-21T09:42:24.124 INFO:tasks.rados.rados.0.smithi017.stdout:1045: done (2 left) 2024-09-21T09:42:24.124 INFO:tasks.rados.rados.0.smithi017.stdout:1048: read oid 11 snap 124 2024-09-21T09:42:24.124 INFO:tasks.rados.rados.0.smithi017.stdout:1048: expect deleted 2024-09-21T09:42:24.124 INFO:tasks.rados.rados.0.smithi017.stdout:1049: rollback oid 36 current snap is 130 2024-09-21T09:42:24.124 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 36 to 128 2024-09-21T09:42:24.124 INFO:tasks.rados.rados.0.smithi017.stdout:1046: finishing write tid 3 to smithi01785587-38 2024-09-21T09:42:24.125 INFO:tasks.rados.rados.0.smithi017.stdout:1046: finishing write tid 4 to smithi01785587-38 2024-09-21T09:42:24.125 INFO:tasks.rados.rados.0.smithi017.stdout:1046: finishing write tid 5 to smithi01785587-38 2024-09-21T09:42:24.125 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 323 (ObjNum 420 snap 130 seq_num 420) dirty exists 2024-09-21T09:42:24.125 INFO:tasks.rados.rados.0.smithi017.stdout:1046: left oid 38 (ObjNum 420 snap 130 seq_num 420) 2024-09-21T09:42:24.125 INFO:tasks.rados.rados.0.smithi017.stdout:1047: finishing write tid 1 to smithi01785587-32 2024-09-21T09:42:24.125 INFO:tasks.rados.rados.0.smithi017.stdout:1046: done (3 left) 2024-09-21T09:42:24.125 INFO:tasks.rados.rados.0.smithi017.stdout:1050: snap_create 2024-09-21T09:42:24.127 INFO:tasks.rados.rados.0.smithi017.stdout:1049: finishing rollback tid 0 to smithi01785587-36 2024-09-21T09:42:24.127 INFO:tasks.rados.rados.0.smithi017.stdout:1049: finishing rollback tid 1 to smithi01785587-36 2024-09-21T09:42:24.127 INFO:tasks.rados.rados.0.smithi017.stdout:1047: finishing write tid 2 to smithi01785587-32 2024-09-21T09:42:24.127 INFO:tasks.rados.rados.0.smithi017.stdout:1047: finishing write tid 3 to smithi01785587-32 2024-09-21T09:42:24.127 INFO:tasks.rados.rados.0.smithi017.stdout:1049: finishing rollback tid 2 to smithi01785587-36 2024-09-21T09:42:24.127 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 556 (ObjNum 313 snap 93 seq_num 313) dirty exists 2024-09-21T09:42:24.127 INFO:tasks.rados.rados.0.smithi017.stdout:1047: finishing write tid 4 to smithi01785587-32 2024-09-21T09:42:24.127 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 499 (ObjNum 421 snap 130 seq_num 421) dirty exists 2024-09-21T09:42:24.128 INFO:tasks.rados.rados.0.smithi017.stdout:1047: left oid 32 (ObjNum 421 snap 130 seq_num 421) 2024-09-21T09:42:24.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:25.076 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:42:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:42:24] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.51.0" 2024-09-21T09:42:25.085 INFO:tasks.rados.rados.0.smithi017.stdout:1047: done (3 left) 2024-09-21T09:42:25.085 INFO:tasks.rados.rados.0.smithi017.stdout:1048: done (2 left) 2024-09-21T09:42:25.085 INFO:tasks.rados.rados.0.smithi017.stdout:1049: done (1 left) 2024-09-21T09:42:25.085 INFO:tasks.rados.rados.0.smithi017.stdout:1050: done (0 left) 2024-09-21T09:42:25.085 INFO:tasks.rados.rados.0.smithi017.stdout:1051: snap_remove snap 124 2024-09-21T09:42:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:25 smithi017 ceph-mon[105151]: pgmap v62: 89 pgs: 89 active+clean; 94 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 21 op/s 2024-09-21T09:42:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:25 smithi017 ceph-mon[105151]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T09:42:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:25 smithi017 ceph-mon[102913]: pgmap v62: 89 pgs: 89 active+clean; 94 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 21 op/s 2024-09-21T09:42:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:25 smithi017 ceph-mon[102913]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T09:42:25.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:25 smithi134 ceph-mon[33541]: pgmap v62: 89 pgs: 89 active+clean; 94 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 21 op/s 2024-09-21T09:42:25.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:25 smithi134 ceph-mon[33541]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T09:42:25.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:25.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:26.094 INFO:tasks.rados.rados.0.smithi017.stdout:1051: done (0 left) 2024-09-21T09:42:26.094 INFO:tasks.rados.rados.0.smithi017.stdout:1052: snap_create 2024-09-21T09:42:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:26 smithi017 ceph-mon[102913]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:42:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:26 smithi017 ceph-mon[105151]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:42:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:26.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:26 smithi134 ceph-mon[33541]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:42:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:26 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:27.096 INFO:tasks.rados.rados.0.smithi017.stdout:1052: done (0 left) 2024-09-21T09:42:27.096 INFO:tasks.rados.rados.0.smithi017.stdout:1053: snap_remove snap 131 2024-09-21T09:42:27.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:27 smithi134 ceph-mon[33541]: pgmap v65: 89 pgs: 89 active+clean; 94 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:27 smithi134 ceph-mon[33541]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:42:27.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:27 smithi017 ceph-mon[102913]: pgmap v65: 89 pgs: 89 active+clean; 94 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:27 smithi017 ceph-mon[102913]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:42:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:27 smithi017 ceph-mon[105151]: pgmap v65: 89 pgs: 89 active+clean; 94 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:27 smithi017 ceph-mon[105151]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:42:28.102 INFO:tasks.rados.rados.0.smithi017.stdout:1053: done (0 left) 2024-09-21T09:42:28.102 INFO:tasks.rados.rados.0.smithi017.stdout:1054: delete oid 17 current snap is 132 2024-09-21T09:42:28.107 INFO:tasks.rados.rados.0.smithi017.stdout:1054: done (0 left) 2024-09-21T09:42:28.107 INFO:tasks.rados.rados.0.smithi017.stdout:1055: read oid 16 snap -1 2024-09-21T09:42:28.107 INFO:tasks.rados.rados.0.smithi017.stdout:1055: expect (ObjNum 417 snap 129 seq_num 417) 2024-09-21T09:42:28.107 INFO:tasks.rados.rados.0.smithi017.stdout:1056: read oid 42 snap -1 2024-09-21T09:42:28.107 INFO:tasks.rados.rados.0.smithi017.stdout:1056: expect (ObjNum 321 snap 97 seq_num 321) 2024-09-21T09:42:28.107 INFO:tasks.rados.rados.0.smithi017.stdout:1057: snap_remove snap 128 2024-09-21T09:42:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:28 smithi017 ceph-mon[102913]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:42:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:28 smithi017 ceph-mon[105151]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:42:28.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:28 smithi134 ceph-mon[33541]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:42:29.103 INFO:tasks.rados.rados.0.smithi017.stdout:1057: done (2 left) 2024-09-21T09:42:29.103 INFO:tasks.rados.rados.0.smithi017.stdout:1058: snap_create 2024-09-21T09:42:29.103 INFO:tasks.rados.rados.0.smithi017.stdout:1056: expect (ObjNum 321 snap 97 seq_num 321) 2024-09-21T09:42:29.337 INFO:tasks.rados.rados.0.smithi017.stdout:1055: expect (ObjNum 417 snap 129 seq_num 417) 2024-09-21T09:42:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:29 smithi017 ceph-mon[102913]: pgmap v68: 89 pgs: 89 active+clean; 112 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2024-09-21T09:42:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:29 smithi017 ceph-mon[102913]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:42:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:29 smithi017 ceph-mon[105151]: pgmap v68: 89 pgs: 89 active+clean; 112 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2024-09-21T09:42:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:29 smithi017 ceph-mon[105151]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:42:29.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:29 smithi134 ceph-mon[33541]: pgmap v68: 89 pgs: 89 active+clean; 112 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2024-09-21T09:42:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:29 smithi134 ceph-mon[33541]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:42:30.112 INFO:tasks.rados.rados.0.smithi017.stdout:1055: done (2 left) 2024-09-21T09:42:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1056: done (1 left) 2024-09-21T09:42:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1058: done (0 left) 2024-09-21T09:42:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1059: read oid 27 snap -1 2024-09-21T09:42:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1059: expect deleted 2024-09-21T09:42:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1060: read oid 41 snap -1 2024-09-21T09:42:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1060: expect deleted 2024-09-21T09:42:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1061: write oid 6 current snap is 133 2024-09-21T09:42:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1061: seq_num 423 ranges {709361=629742,1836589=539816,3033220=76141} 2024-09-21T09:42:30.119 INFO:tasks.rados.rados.0.smithi017.stdout:1061: writing smithi01785587-6 from 709361 to 1339103 tid 1 2024-09-21T09:42:30.125 INFO:tasks.rados.rados.0.smithi017.stdout:1061: writing smithi01785587-6 from 1836589 to 2376405 tid 2 2024-09-21T09:42:30.130 INFO:tasks.rados.rados.0.smithi017.stdout:1061: writing smithi01785587-6 from 3033220 to 3109361 tid 3 2024-09-21T09:42:30.130 INFO:tasks.rados.rados.0.smithi017.stdout:1062: delete oid 23 current snap is 133 2024-09-21T09:42:30.130 INFO:tasks.rados.rados.0.smithi017.stdout:1061: finishing write tid 1 to smithi01785587-6 2024-09-21T09:42:30.130 INFO:tasks.rados.rados.0.smithi017.stdout:1061: finishing write tid 2 to smithi01785587-6 2024-09-21T09:42:30.132 INFO:tasks.rados.rados.0.smithi017.stdout:1061: finishing write tid 3 to smithi01785587-6 2024-09-21T09:42:30.132 INFO:tasks.rados.rados.0.smithi017.stdout:1061: finishing write tid 4 to smithi01785587-6 2024-09-21T09:42:30.132 INFO:tasks.rados.rados.0.smithi017.stdout:1061: finishing write tid 5 to smithi01785587-6 2024-09-21T09:42:30.132 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 317 (ObjNum 423 snap 133 seq_num 423) dirty exists 2024-09-21T09:42:30.132 INFO:tasks.rados.rados.0.smithi017.stdout:1061: left oid 6 (ObjNum 423 snap 133 seq_num 423) 2024-09-21T09:42:30.136 INFO:tasks.rados.rados.0.smithi017.stdout:1059: done (3 left) 2024-09-21T09:42:30.136 INFO:tasks.rados.rados.0.smithi017.stdout:1060: done (2 left) 2024-09-21T09:42:30.136 INFO:tasks.rados.rados.0.smithi017.stdout:1061: done (1 left) 2024-09-21T09:42:30.136 INFO:tasks.rados.rados.0.smithi017.stdout:1062: done (0 left) 2024-09-21T09:42:30.136 INFO:tasks.rados.rados.0.smithi017.stdout:1063: read oid 37 snap 132 2024-09-21T09:42:30.137 INFO:tasks.rados.rados.0.smithi017.stdout:1063: expect deleted 2024-09-21T09:42:30.137 INFO:tasks.rados.rados.0.smithi017.stdout:1064: read oid 32 snap -1 2024-09-21T09:42:30.137 INFO:tasks.rados.rados.0.smithi017.stdout:1064: expect (ObjNum 421 snap 130 seq_num 421) 2024-09-21T09:42:30.137 INFO:tasks.rados.rados.0.smithi017.stdout:1065: read oid 45 snap -1 2024-09-21T09:42:30.137 INFO:tasks.rados.rados.0.smithi017.stdout:1065: expect (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:42:30.137 INFO:tasks.rados.rados.0.smithi017.stdout:1066: write oid 24 current snap is 133 2024-09-21T09:42:30.137 INFO:tasks.rados.rados.0.smithi017.stdout:1066: seq_num 425 ranges {742975=780240,1953553=427132,2971713=171262} 2024-09-21T09:42:30.144 INFO:tasks.rados.rados.0.smithi017.stdout:1066: writing smithi01785587-24 from 742975 to 1523215 tid 1 2024-09-21T09:42:30.149 INFO:tasks.rados.rados.0.smithi017.stdout:1066: writing smithi01785587-24 from 1953553 to 2380685 tid 2 2024-09-21T09:42:30.154 INFO:tasks.rados.rados.0.smithi017.stdout:1066: writing smithi01785587-24 from 2971713 to 3142975 tid 3 2024-09-21T09:42:30.154 INFO:tasks.rados.rados.0.smithi017.stdout:1067: read oid 48 snap -1 2024-09-21T09:42:30.154 INFO:tasks.rados.rados.0.smithi017.stdout:1067: expect deleted 2024-09-21T09:42:30.154 INFO:tasks.rados.rados.0.smithi017.stdout:1068: rollback oid 22 current snap is 133 2024-09-21T09:42:30.154 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 22 to 125 2024-09-21T09:42:30.154 INFO:tasks.rados.rados.0.smithi017.stdout:1065: expect (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:42:30.297 INFO:tasks.rados.rados.0.smithi017.stdout:1064: expect (ObjNum 421 snap 130 seq_num 421) 2024-09-21T09:42:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:30 smithi017 ceph-mon[102913]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:42:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:30 smithi017 ceph-mon[105151]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:42:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:30 smithi134 ceph-mon[33541]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:42:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:1066: finishing write tid 1 to smithi01785587-24 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:1066: finishing write tid 2 to smithi01785587-24 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:1068: finishing rollback tid 1 to smithi01785587-22 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:1063: done (5 left) 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:1064: done (4 left) 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:1065: done (3 left) 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:1067: done (2 left) 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:1068: done (1 left) 2024-09-21T09:42:30.589 INFO:tasks.rados.rados.0.smithi017.stdout:1069: read oid 37 snap -1 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:1069: expect deleted 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:1070: rollback oid 48 current snap is 133 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 129 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:1066: finishing write tid 3 to smithi01785587-24 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:1066: finishing write tid 4 to smithi01785587-24 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:1066: finishing write tid 5 to smithi01785587-24 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 331 (ObjNum 425 snap 133 seq_num 425) dirty exists 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:1066: left oid 24 (ObjNum 425 snap 133 seq_num 425) 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:1066: done (2 left) 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:1071: write oid 42 current snap is 133 2024-09-21T09:42:30.590 INFO:tasks.rados.rados.0.smithi017.stdout:1071: seq_num 426 ranges {759782=571842,2012035=687143} 2024-09-21T09:42:30.595 INFO:tasks.rados.rados.0.smithi017.stdout:1071: writing smithi01785587-42 from 759782 to 1331624 tid 1 2024-09-21T09:42:30.603 INFO:tasks.rados.rados.0.smithi017.stdout:1071: writing smithi01785587-42 from 2012035 to 2699178 tid 2 2024-09-21T09:42:30.603 INFO:tasks.rados.rados.0.smithi017.stdout:1072: snap_create 2024-09-21T09:42:30.603 INFO:tasks.rados.rados.0.smithi017.stdout:1070: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:42:30.603 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 541 (ObjNum 129 snap 0 seq_num 2399966944) dirty dne 2024-09-21T09:42:30.603 INFO:tasks.rados.rados.0.smithi017.stdout:1071: finishing write tid 1 to smithi01785587-42 2024-09-21T09:42:30.607 INFO:tasks.rados.rados.0.smithi017.stdout:1071: finishing write tid 2 to smithi01785587-42 2024-09-21T09:42:30.607 INFO:tasks.rados.rados.0.smithi017.stdout:1071: finishing write tid 3 to smithi01785587-42 2024-09-21T09:42:30.607 INFO:tasks.rados.rados.0.smithi017.stdout:1071: finishing write tid 4 to smithi01785587-42 2024-09-21T09:42:30.607 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 385 (ObjNum 426 snap 133 seq_num 426) dirty exists 2024-09-21T09:42:30.607 INFO:tasks.rados.rados.0.smithi017.stdout:1071: left oid 42 (ObjNum 426 snap 133 seq_num 426) 2024-09-21T09:42:31.118 INFO:tasks.rados.rados.0.smithi017.stdout:1069: done (3 left) 2024-09-21T09:42:31.118 INFO:tasks.rados.rados.0.smithi017.stdout:1070: done (2 left) 2024-09-21T09:42:31.118 INFO:tasks.rados.rados.0.smithi017.stdout:1071: done (1 left) 2024-09-21T09:42:31.118 INFO:tasks.rados.rados.0.smithi017.stdout:1072: done (0 left) 2024-09-21T09:42:31.118 INFO:tasks.rados.rados.0.smithi017.stdout:1073: snap_create 2024-09-21T09:42:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:31 smithi017 ceph-mon[102913]: pgmap v71: 89 pgs: 89 active+clean; 112 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2024-09-21T09:42:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:31 smithi017 ceph-mon[102913]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T09:42:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:31 smithi017 ceph-mon[105151]: pgmap v71: 89 pgs: 89 active+clean; 112 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2024-09-21T09:42:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:31 smithi017 ceph-mon[105151]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T09:42:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:31 smithi134 ceph-mon[33541]: pgmap v71: 89 pgs: 89 active+clean; 112 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2024-09-21T09:42:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:31 smithi134 ceph-mon[33541]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T09:42:32.131 INFO:tasks.rados.rados.0.smithi017.stdout:1073: done (0 left) 2024-09-21T09:42:32.132 INFO:tasks.rados.rados.0.smithi017.stdout:1074: read oid 47 snap -1 2024-09-21T09:42:32.132 INFO:tasks.rados.rados.0.smithi017.stdout:1074: expect (ObjNum 414 snap 128 seq_num 414) 2024-09-21T09:42:32.132 INFO:tasks.rados.rados.0.smithi017.stdout:1075: write oid 26 current snap is 135 2024-09-21T09:42:32.132 INFO:tasks.rados.rados.0.smithi017.stdout:1075: seq_num 427 ranges {776589=647091,2070517=430801,3026559=150030} 2024-09-21T09:42:32.139 INFO:tasks.rados.rados.0.smithi017.stdout:1075: writing smithi01785587-26 from 776589 to 1423680 tid 1 2024-09-21T09:42:32.145 INFO:tasks.rados.rados.0.smithi017.stdout:1075: writing smithi01785587-26 from 2070517 to 2501318 tid 2 2024-09-21T09:42:32.149 INFO:tasks.rados.rados.0.smithi017.stdout:1075: writing smithi01785587-26 from 3026559 to 3176589 tid 3 2024-09-21T09:42:32.149 INFO:tasks.rados.rados.0.smithi017.stdout:1076: write oid 49 current snap is 135 2024-09-21T09:42:32.149 INFO:tasks.rados.rados.0.smithi017.stdout:1076: seq_num 428 ranges {793396=722340,2128999=690812} 2024-09-21T09:42:32.157 INFO:tasks.rados.rados.0.smithi017.stdout:1076: writing smithi01785587-49 from 793396 to 1515736 tid 1 2024-09-21T09:42:32.164 INFO:tasks.rados.rados.0.smithi017.stdout:1076: writing smithi01785587-49 from 2128999 to 2819811 tid 2 2024-09-21T09:42:32.164 INFO:tasks.rados.rados.0.smithi017.stdout:1077: write oid 2 current snap is 135 2024-09-21T09:42:32.164 INFO:tasks.rados.rados.0.smithi017.stdout:1077: seq_num 429 ranges {410203=797589,1787481=434470,2681405=528798} 2024-09-21T09:42:32.171 INFO:tasks.rados.rados.0.smithi017.stdout:1077: writing smithi01785587-2 from 410203 to 1207792 tid 1 2024-09-21T09:42:32.176 INFO:tasks.rados.rados.0.smithi017.stdout:1077: writing smithi01785587-2 from 1787481 to 2221951 tid 2 2024-09-21T09:42:32.183 INFO:tasks.rados.rados.0.smithi017.stdout:1077: writing smithi01785587-2 from 2681405 to 3210203 tid 3 2024-09-21T09:42:32.183 INFO:tasks.rados.rados.0.smithi017.stdout:1078: read oid 36 snap -1 2024-09-21T09:42:32.183 INFO:tasks.rados.rados.0.smithi017.stdout:1078: expect (ObjNum 313 snap 93 seq_num 313) 2024-09-21T09:42:32.183 INFO:tasks.rados.rados.0.smithi017.stdout:1079: read oid 16 snap -1 2024-09-21T09:42:32.183 INFO:tasks.rados.rados.0.smithi017.stdout:1079: expect (ObjNum 417 snap 129 seq_num 417) 2024-09-21T09:42:32.183 INFO:tasks.rados.rados.0.smithi017.stdout:1080: rollback oid 4 current snap is 135 2024-09-21T09:42:32.183 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 4 to 121 2024-09-21T09:42:32.183 INFO:tasks.rados.rados.0.smithi017.stdout:1074: expect (ObjNum 414 snap 128 seq_num 414) 2024-09-21T09:42:32.362 INFO:tasks.rados.rados.0.smithi017.stdout:1075: finishing write tid 1 to smithi01785587-26 2024-09-21T09:42:32.362 INFO:tasks.rados.rados.0.smithi017.stdout:1075: finishing write tid 2 to smithi01785587-26 2024-09-21T09:42:32.362 INFO:tasks.rados.rados.0.smithi017.stdout:1075: finishing write tid 3 to smithi01785587-26 2024-09-21T09:42:32.362 INFO:tasks.rados.rados.0.smithi017.stdout:1075: finishing write tid 4 to smithi01785587-26 2024-09-21T09:42:32.362 INFO:tasks.rados.rados.0.smithi017.stdout:1075: finishing write tid 5 to smithi01785587-26 2024-09-21T09:42:32.362 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 578 (ObjNum 427 snap 135 seq_num 427) dirty exists 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1075: left oid 26 (ObjNum 427 snap 135 seq_num 427) 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1076: finishing write tid 1 to smithi01785587-49 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1076: finishing write tid 2 to smithi01785587-49 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1076: finishing write tid 3 to smithi01785587-49 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1074: done (6 left) 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1075: done (5 left) 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1081: write oid 21 current snap is 135 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1076: finishing write tid 4 to smithi01785587-49 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 536 (ObjNum 428 snap 135 seq_num 428) dirty exists 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1076: left oid 49 (ObjNum 428 snap 135 seq_num 428) 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1077: finishing write tid 1 to smithi01785587-2 2024-09-21T09:42:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:1081: seq_num 430 ranges {427010=472838,1329610=694481,2708828=518182} 2024-09-21T09:42:32.366 INFO:tasks.rados.rados.0.smithi017.stdout:1081: writing smithi01785587-21 from 427010 to 899848 tid 1 2024-09-21T09:42:32.372 INFO:tasks.rados.rados.0.smithi017.stdout:1081: writing smithi01785587-21 from 1329610 to 2024091 tid 2 2024-09-21T09:42:32.379 INFO:tasks.rados.rados.0.smithi017.stdout:1081: writing smithi01785587-21 from 2708828 to 3227010 tid 3 2024-09-21T09:42:32.379 INFO:tasks.rados.rados.0.smithi017.stdout:1076: done (5 left) 2024-09-21T09:42:32.379 INFO:tasks.rados.rados.0.smithi017.stdout:1082: snap_create 2024-09-21T09:42:32.379 INFO:tasks.rados.rados.0.smithi017.stdout:1077: finishing write tid 2 to smithi01785587-2 2024-09-21T09:42:32.379 INFO:tasks.rados.rados.0.smithi017.stdout:1077: finishing write tid 3 to smithi01785587-2 2024-09-21T09:42:32.380 INFO:tasks.rados.rados.0.smithi017.stdout:1077: finishing write tid 4 to smithi01785587-2 2024-09-21T09:42:32.380 INFO:tasks.rados.rados.0.smithi017.stdout:1077: finishing write tid 5 to smithi01785587-2 2024-09-21T09:42:32.380 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 211 (ObjNum 429 snap 135 seq_num 429) dirty exists 2024-09-21T09:42:32.380 INFO:tasks.rados.rados.0.smithi017.stdout:1077: left oid 2 (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:42:32.380 INFO:tasks.rados.rados.0.smithi017.stdout:1078: expect (ObjNum 313 snap 93 seq_num 313) 2024-09-21T09:42:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:32 smithi017 ceph-mon[102913]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T09:42:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:32 smithi017 ceph-mon[105151]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T09:42:32.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:32 smithi134 ceph-mon[33541]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T09:42:32.538 INFO:tasks.rados.rados.0.smithi017.stdout:1079: expect (ObjNum 417 snap 129 seq_num 417) 2024-09-21T09:42:32.637 INFO:tasks.rados.rados.0.smithi017.stdout:1080: finishing rollback tid 1 to smithi01785587-4 2024-09-21T09:42:32.638 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 0 (ObjNum 33 snap 0 seq_num 2384147832) dirty dne 2024-09-21T09:42:32.638 INFO:tasks.rados.rados.0.smithi017.stdout:1081: finishing write tid 1 to smithi01785587-21 2024-09-21T09:42:32.638 INFO:tasks.rados.rados.0.smithi017.stdout:1081: finishing write tid 2 to smithi01785587-21 2024-09-21T09:42:32.638 INFO:tasks.rados.rados.0.smithi017.stdout:1081: finishing write tid 3 to smithi01785587-21 2024-09-21T09:42:32.638 INFO:tasks.rados.rados.0.smithi017.stdout:1081: finishing write tid 4 to smithi01785587-21 2024-09-21T09:42:32.638 INFO:tasks.rados.rados.0.smithi017.stdout:1081: finishing write tid 5 to smithi01785587-21 2024-09-21T09:42:32.638 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 540 (ObjNum 430 snap 135 seq_num 430) dirty exists 2024-09-21T09:42:32.638 INFO:tasks.rados.rados.0.smithi017.stdout:1081: left oid 21 (ObjNum 430 snap 135 seq_num 430) 2024-09-21T09:42:33.028 INFO:tasks.rados.rados.0.smithi017.stdout:1077: done (5 left) 2024-09-21T09:42:33.028 INFO:tasks.rados.rados.0.smithi017.stdout:1078: done (4 left) 2024-09-21T09:42:33.028 INFO:tasks.rados.rados.0.smithi017.stdout:1079: done (3 left) 2024-09-21T09:42:33.028 INFO:tasks.rados.rados.0.smithi017.stdout:1080: done (2 left) 2024-09-21T09:42:33.028 INFO:tasks.rados.rados.0.smithi017.stdout:1081: done (1 left) 2024-09-21T09:42:33.028 INFO:tasks.rados.rados.0.smithi017.stdout:1082: done (0 left) 2024-09-21T09:42:33.028 INFO:tasks.rados.rados.0.smithi017.stdout:1083: rollback oid 17 current snap is 136 2024-09-21T09:42:33.028 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 135 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1084: rollback oid 23 current snap is 136 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 23 to 126 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1085: read oid 24 snap 129 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1085: expect deleted 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1086: read oid 20 snap -1 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1086: expect (ObjNum 415 snap 128 seq_num 415) 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1087: read oid 21 snap -1 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1087: expect (ObjNum 430 snap 135 seq_num 430) 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1088: read oid 34 snap -1 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1088: expect (ObjNum 401 snap 122 seq_num 401) 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1089: read oid 1 snap -1 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1089: expect (ObjNum 378 snap 118 seq_num 378) 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1090: write oid 48 current snap is 136 2024-09-21T09:42:33.029 INFO:tasks.rados.rados.0.smithi017.stdout:1090: seq_num 431 ranges {443817=548087,1788092=438139,3019898=223919} 2024-09-21T09:42:33.033 INFO:tasks.rados.rados.0.smithi017.stdout:1090: writing smithi01785587-48 from 443817 to 991904 tid 1 2024-09-21T09:42:33.041 INFO:tasks.rados.rados.0.smithi017.stdout:1090: writing smithi01785587-48 from 1788092 to 2226231 tid 2 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1090: writing smithi01785587-48 from 3019898 to 3243817 tid 3 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1091: rollback oid 2 current snap is 136 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 2 to 125 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1084: finishing rollback tid 1 to smithi01785587-23 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 0 (ObjNum -1895241504 snap 21949 seq_num 304) dirty dne 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1083: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 436 (ObjNum -1895282944 snap 21949 seq_num 304) dirty dne 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1083: done (8 left) 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1084: done (7 left) 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1085: done (6 left) 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1092: delete oid 35 current snap is 136 2024-09-21T09:42:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1087: expect (ObjNum 430 snap 135 seq_num 430) 2024-09-21T09:42:33.153 INFO:tasks.rados.rados.0.smithi017.stdout:1088: expect (ObjNum 401 snap 122 seq_num 401) 2024-09-21T09:42:33.246 INFO:tasks.rados.rados.0.smithi017.stdout:1089: expect (ObjNum 378 snap 118 seq_num 378) 2024-09-21T09:42:33.325 INFO:tasks.rados.rados.0.smithi017.stdout:1090: finishing write tid 1 to smithi01785587-48 2024-09-21T09:42:33.325 INFO:tasks.rados.rados.0.smithi017.stdout:1086: expect (ObjNum 415 snap 128 seq_num 415) 2024-09-21T09:42:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:33 smithi017 ceph-mon[102913]: pgmap v74: 89 pgs: 89 active+clean; 118 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 949 KiB/s wr, 12 op/s 2024-09-21T09:42:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:33 smithi017 ceph-mon[102913]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T09:42:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:33 smithi017 ceph-mon[102913]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T09:42:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:33 smithi017 ceph-mon[105151]: pgmap v74: 89 pgs: 89 active+clean; 118 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 949 KiB/s wr, 12 op/s 2024-09-21T09:42:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:33 smithi017 ceph-mon[105151]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T09:42:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:33 smithi017 ceph-mon[105151]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1090: finishing write tid 2 to smithi01785587-48 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1086: done (6 left) 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1087: done (5 left) 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1088: done (4 left) 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1089: done (3 left) 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1092: done (2 left) 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1093: rollback oid 49 current snap is 136 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 127 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1090: finishing write tid 3 to smithi01785587-48 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1090: finishing write tid 4 to smithi01785587-48 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:1090: finishing write tid 5 to smithi01785587-48 2024-09-21T09:42:33.452 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 551 (ObjNum 431 snap 136 seq_num 431) dirty exists 2024-09-21T09:42:33.453 INFO:tasks.rados.rados.0.smithi017.stdout:1090: left oid 48 (ObjNum 431 snap 136 seq_num 431) 2024-09-21T09:42:33.453 INFO:tasks.rados.rados.0.smithi017.stdout:1091: finishing rollback tid 0 to smithi01785587-2 2024-09-21T09:42:33.453 INFO:tasks.rados.rados.0.smithi017.stdout:1091: finishing rollback tid 1 to smithi01785587-2 2024-09-21T09:42:33.453 INFO:tasks.rados.rados.0.smithi017.stdout:1091: finishing rollback tid 2 to smithi01785587-2 2024-09-21T09:42:33.453 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 215 (ObjNum 374 snap 117 seq_num 374) dirty exists 2024-09-21T09:42:33.453 INFO:tasks.rados.rados.0.smithi017.stdout:1090: done (2 left) 2024-09-21T09:42:33.453 INFO:tasks.rados.rados.0.smithi017.stdout:1091: done (1 left) 2024-09-21T09:42:33.453 INFO:tasks.rados.rados.0.smithi017.stdout:1094: snap_create 2024-09-21T09:42:33.454 INFO:tasks.rados.rados.0.smithi017.stdout:1093: finishing rollback tid 0 to smithi01785587-49 2024-09-21T09:42:33.454 INFO:tasks.rados.rados.0.smithi017.stdout:1093: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:42:33.454 INFO:tasks.rados.rados.0.smithi017.stdout:1093: finishing rollback tid 2 to smithi01785587-49 2024-09-21T09:42:33.454 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 544 (ObjNum 341 snap 103 seq_num 341) dirty exists 2024-09-21T09:42:33.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:33 smithi134 ceph-mon[33541]: pgmap v74: 89 pgs: 89 active+clean; 118 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 949 KiB/s wr, 12 op/s 2024-09-21T09:42:33.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:33 smithi134 ceph-mon[33541]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T09:42:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:33 smithi134 ceph-mon[33541]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T09:42:34.140 INFO:tasks.rados.rados.0.smithi017.stdout:1093: done (1 left) 2024-09-21T09:42:34.141 INFO:tasks.rados.rados.0.smithi017.stdout:1094: done (0 left) 2024-09-21T09:42:34.141 INFO:tasks.rados.rados.0.smithi017.stdout:1095: rollback oid 11 current snap is 137 2024-09-21T09:42:34.141 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 126 2024-09-21T09:42:34.141 INFO:tasks.rados.rados.0.smithi017.stdout:1096: snap_remove snap 135 2024-09-21T09:42:34.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:35.143 INFO:tasks.rados.rados.0.smithi017.stdout:1096: done (1 left) 2024-09-21T09:42:35.143 INFO:tasks.rados.rados.0.smithi017.stdout:1097: write oid 26 current snap is 137 2024-09-21T09:42:35.143 INFO:tasks.rados.rados.0.smithi017.stdout:1095: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:42:35.143 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 0 (ObjNum -1895232576 snap 21949 seq_num 304) dirty dne 2024-09-21T09:42:35.143 INFO:tasks.rados.rados.0.smithi017.stdout:1097: seq_num 433 ranges {477431=698585,1905056=441808,3074744=202687} 2024-09-21T09:42:35.144 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:42:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:42:34] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-21T09:42:35.150 INFO:tasks.rados.rados.0.smithi017.stdout:1097: writing smithi01785587-26 from 477431 to 1176016 tid 1 2024-09-21T09:42:35.156 INFO:tasks.rados.rados.0.smithi017.stdout:1097: writing smithi01785587-26 from 1905056 to 2346864 tid 2 2024-09-21T09:42:35.162 INFO:tasks.rados.rados.0.smithi017.stdout:1097: writing smithi01785587-26 from 3074744 to 3277431 tid 3 2024-09-21T09:42:35.162 INFO:tasks.rados.rados.0.smithi017.stdout:1095: done (1 left) 2024-09-21T09:42:35.162 INFO:tasks.rados.rados.0.smithi017.stdout:1098: read oid 30 snap -1 2024-09-21T09:42:35.162 INFO:tasks.rados.rados.0.smithi017.stdout:1098: expect deleted 2024-09-21T09:42:35.162 INFO:tasks.rados.rados.0.smithi017.stdout:1099: write oid 10 current snap is 137 2024-09-21T09:42:35.163 INFO:tasks.rados.rados.0.smithi017.stdout:1097: finishing write tid 1 to smithi01785587-26 2024-09-21T09:42:35.163 INFO:tasks.rados.rados.0.smithi017.stdout:1097: finishing write tid 2 to smithi01785587-26 2024-09-21T09:42:35.163 INFO:tasks.rados.rados.0.smithi017.stdout:1099: seq_num 434 ranges {494238=490187,1563538=701819,2818520=475718} 2024-09-21T09:42:35.168 INFO:tasks.rados.rados.0.smithi017.stdout:1099: writing smithi01785587-10 from 494238 to 984425 tid 1 2024-09-21T09:42:35.175 INFO:tasks.rados.rados.0.smithi017.stdout:1099: writing smithi01785587-10 from 1563538 to 2265357 tid 2 2024-09-21T09:42:35.181 INFO:tasks.rados.rados.0.smithi017.stdout:1099: writing smithi01785587-10 from 2818520 to 3294238 tid 3 2024-09-21T09:42:35.181 INFO:tasks.rados.rados.0.smithi017.stdout:1100: write oid 5 current snap is 137 2024-09-21T09:42:35.181 INFO:tasks.rados.rados.0.smithi017.stdout:1097: finishing write tid 3 to smithi01785587-26 2024-09-21T09:42:35.181 INFO:tasks.rados.rados.0.smithi017.stdout:1100: seq_num 435 ranges {511045=565436,1622020=445477,2729590=556855} 2024-09-21T09:42:35.187 INFO:tasks.rados.rados.0.smithi017.stdout:1100: writing smithi01785587-5 from 511045 to 1076481 tid 1 2024-09-21T09:42:35.193 INFO:tasks.rados.rados.0.smithi017.stdout:1100: writing smithi01785587-5 from 1622020 to 2067497 tid 2 2024-09-21T09:42:35.200 INFO:tasks.rados.rados.0.smithi017.stdout:1100: writing smithi01785587-5 from 2729590 to 3286445 tid 3 2024-09-21T09:42:35.200 INFO:tasks.rados.rados.0.smithi017.stdout:1101: read oid 7 snap -1 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1101: expect (ObjNum 393 snap 121 seq_num 393) 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1102: rollback oid 24 current snap is 137 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 24 to 133 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1097: finishing write tid 4 to smithi01785587-26 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1097: finishing write tid 5 to smithi01785587-26 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 583 (ObjNum 433 snap 137 seq_num 433) dirty exists 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1097: left oid 26 (ObjNum 433 snap 137 seq_num 433) 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1099: finishing write tid 1 to smithi01785587-10 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1099: finishing write tid 2 to smithi01785587-10 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1097: done (5 left) 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1098: done (4 left) 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1103: delete oid 14 current snap is 137 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1099: finishing write tid 3 to smithi01785587-10 2024-09-21T09:42:35.201 INFO:tasks.rados.rados.0.smithi017.stdout:1099: finishing write tid 4 to smithi01785587-10 2024-09-21T09:42:35.202 INFO:tasks.rados.rados.0.smithi017.stdout:1099: finishing write tid 5 to smithi01785587-10 2024-09-21T09:42:35.202 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 44 (ObjNum 434 snap 137 seq_num 434) dirty exists 2024-09-21T09:42:35.202 INFO:tasks.rados.rados.0.smithi017.stdout:1099: left oid 10 (ObjNum 434 snap 137 seq_num 434) 2024-09-21T09:42:35.202 INFO:tasks.rados.rados.0.smithi017.stdout:1100: finishing write tid 1 to smithi01785587-5 2024-09-21T09:42:35.202 INFO:tasks.rados.rados.0.smithi017.stdout:1100: finishing write tid 2 to smithi01785587-5 2024-09-21T09:42:35.204 INFO:tasks.rados.rados.0.smithi017.stdout:1100: finishing write tid 3 to smithi01785587-5 2024-09-21T09:42:35.204 INFO:tasks.rados.rados.0.smithi017.stdout:1100: finishing write tid 4 to smithi01785587-5 2024-09-21T09:42:35.204 INFO:tasks.rados.rados.0.smithi017.stdout:1100: finishing write tid 5 to smithi01785587-5 2024-09-21T09:42:35.205 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 520 (ObjNum 435 snap 137 seq_num 435) dirty exists 2024-09-21T09:42:35.205 INFO:tasks.rados.rados.0.smithi017.stdout:1100: left oid 5 (ObjNum 435 snap 137 seq_num 435) 2024-09-21T09:42:35.205 INFO:tasks.rados.rados.0.smithi017.stdout:1101: expect (ObjNum 393 snap 121 seq_num 393) 2024-09-21T09:42:35.290 INFO:tasks.rados.rados.0.smithi017.stdout:1102: finishing rollback tid 0 to smithi01785587-24 2024-09-21T09:42:35.290 INFO:tasks.rados.rados.0.smithi017.stdout:1102: finishing rollback tid 1 to smithi01785587-24 2024-09-21T09:42:35.290 INFO:tasks.rados.rados.0.smithi017.stdout:1102: finishing rollback tid 2 to smithi01785587-24 2024-09-21T09:42:35.290 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 335 (ObjNum 425 snap 133 seq_num 425) dirty exists 2024-09-21T09:42:35.291 INFO:tasks.rados.rados.0.smithi017.stdout:1099: done (4 left) 2024-09-21T09:42:35.291 INFO:tasks.rados.rados.0.smithi017.stdout:1100: done (3 left) 2024-09-21T09:42:35.291 INFO:tasks.rados.rados.0.smithi017.stdout:1101: done (2 left) 2024-09-21T09:42:35.291 INFO:tasks.rados.rados.0.smithi017.stdout:1102: done (1 left) 2024-09-21T09:42:35.291 INFO:tasks.rados.rados.0.smithi017.stdout:1103: done (0 left) 2024-09-21T09:42:35.291 INFO:tasks.rados.rados.0.smithi017.stdout:1104: write oid 14 current snap is 137 2024-09-21T09:42:35.291 INFO:tasks.rados.rados.0.smithi017.stdout:1104: seq_num 437 ranges {544659=715934,1738984=449146,2784436=560223} 2024-09-21T09:42:35.297 INFO:tasks.rados.rados.0.smithi017.stdout:1104: writing smithi01785587-14 from 544659 to 1260593 tid 1 2024-09-21T09:42:35.302 INFO:tasks.rados.rados.0.smithi017.stdout:1104: writing smithi01785587-14 from 1738984 to 2188130 tid 2 2024-09-21T09:42:35.308 INFO:tasks.rados.rados.0.smithi017.stdout:1104: writing smithi01785587-14 from 2784436 to 3344659 tid 3 2024-09-21T09:42:35.308 INFO:tasks.rados.rados.0.smithi017.stdout:1105: write oid 31 current snap is 137 2024-09-21T09:42:35.308 INFO:tasks.rados.rados.0.smithi017.stdout:1105: seq_num 438 ranges {561466=791183,2081113=592804,3095506=265960} 2024-09-21T09:42:35.316 INFO:tasks.rados.rados.0.smithi017.stdout:1105: writing smithi01785587-31 from 561466 to 1352649 tid 1 2024-09-21T09:42:35.323 INFO:tasks.rados.rados.0.smithi017.stdout:1105: writing smithi01785587-31 from 2081113 to 2673917 tid 2 2024-09-21T09:42:35.327 INFO:tasks.rados.rados.0.smithi017.stdout:1105: writing smithi01785587-31 from 3095506 to 3361466 tid 3 2024-09-21T09:42:35.327 INFO:tasks.rados.rados.0.smithi017.stdout:1106: snap_create 2024-09-21T09:42:35.327 INFO:tasks.rados.rados.0.smithi017.stdout:1104: finishing write tid 1 to smithi01785587-14 2024-09-21T09:42:35.327 INFO:tasks.rados.rados.0.smithi017.stdout:1104: finishing write tid 2 to smithi01785587-14 2024-09-21T09:42:35.327 INFO:tasks.rados.rados.0.smithi017.stdout:1104: finishing write tid 3 to smithi01785587-14 2024-09-21T09:42:35.327 INFO:tasks.rados.rados.0.smithi017.stdout:1104: finishing write tid 4 to smithi01785587-14 2024-09-21T09:42:35.327 INFO:tasks.rados.rados.0.smithi017.stdout:1104: finishing write tid 5 to smithi01785587-14 2024-09-21T09:42:35.327 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 526 (ObjNum 437 snap 137 seq_num 437) dirty exists 2024-09-21T09:42:35.328 INFO:tasks.rados.rados.0.smithi017.stdout:1104: left oid 14 (ObjNum 437 snap 137 seq_num 437) 2024-09-21T09:42:35.328 INFO:tasks.rados.rados.0.smithi017.stdout:1105: finishing write tid 1 to smithi01785587-31 2024-09-21T09:42:35.328 INFO:tasks.rados.rados.0.smithi017.stdout:1105: finishing write tid 2 to smithi01785587-31 2024-09-21T09:42:35.329 INFO:tasks.rados.rados.0.smithi017.stdout:1105: finishing write tid 3 to smithi01785587-31 2024-09-21T09:42:35.329 INFO:tasks.rados.rados.0.smithi017.stdout:1105: finishing write tid 4 to smithi01785587-31 2024-09-21T09:42:35.329 INFO:tasks.rados.rados.0.smithi017.stdout:1105: finishing write tid 5 to smithi01785587-31 2024-09-21T09:42:35.329 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 531 (ObjNum 438 snap 137 seq_num 438) dirty exists 2024-09-21T09:42:35.329 INFO:tasks.rados.rados.0.smithi017.stdout:1105: left oid 31 (ObjNum 438 snap 137 seq_num 438) 2024-09-21T09:42:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:35 smithi017 ceph-mon[102913]: pgmap v77: 89 pgs: 89 active+clean; 118 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 949 KiB/s wr, 12 op/s 2024-09-21T09:42:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:35 smithi017 ceph-mon[102913]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T09:42:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:35 smithi017 ceph-mon[105151]: pgmap v77: 89 pgs: 89 active+clean; 118 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 949 KiB/s wr, 12 op/s 2024-09-21T09:42:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:35 smithi017 ceph-mon[105151]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T09:42:35.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:35 smithi134 ceph-mon[33541]: pgmap v77: 89 pgs: 89 active+clean; 118 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 949 KiB/s wr, 12 op/s 2024-09-21T09:42:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:35 smithi134 ceph-mon[33541]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T09:42:35.993 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:35.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:36.153 INFO:tasks.rados.rados.0.smithi017.stdout:1104: done (2 left) 2024-09-21T09:42:36.153 INFO:tasks.rados.rados.0.smithi017.stdout:1105: done (1 left) 2024-09-21T09:42:36.153 INFO:tasks.rados.rados.0.smithi017.stdout:1106: done (0 left) 2024-09-21T09:42:36.153 INFO:tasks.rados.rados.0.smithi017.stdout:1107: read oid 45 snap -1 2024-09-21T09:42:36.153 INFO:tasks.rados.rados.0.smithi017.stdout:1107: expect (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:42:36.153 INFO:tasks.rados.rados.0.smithi017.stdout:1108: snap_create 2024-09-21T09:42:36.158 INFO:tasks.rados.rados.0.smithi017.stdout:1107: expect (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:42:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:36 smithi017 ceph-mon[102913]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T09:42:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:36 smithi017 ceph-mon[105151]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T09:42:36.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:36 smithi134 ceph-mon[33541]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T09:42:37.155 INFO:tasks.rados.rados.0.smithi017.stdout:1107: done (1 left) 2024-09-21T09:42:37.155 INFO:tasks.rados.rados.0.smithi017.stdout:1108: done (0 left) 2024-09-21T09:42:37.155 INFO:tasks.rados.rados.0.smithi017.stdout:1109: read oid 37 snap -1 2024-09-21T09:42:37.155 INFO:tasks.rados.rados.0.smithi017.stdout:1109: expect deleted 2024-09-21T09:42:37.155 INFO:tasks.rados.rados.0.smithi017.stdout:1110: rollback oid 38 current snap is 139 2024-09-21T09:42:37.155 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 136 2024-09-21T09:42:37.155 INFO:tasks.rados.rados.0.smithi017.stdout:1111: write oid 1 current snap is 139 2024-09-21T09:42:37.155 INFO:tasks.rados.rados.0.smithi017.stdout:1111: seq_num 439 ranges {578273=466432,1739595=452815,2722929=655344} 2024-09-21T09:42:37.161 INFO:tasks.rados.rados.0.smithi017.stdout:1111: writing smithi01785587-1 from 578273 to 1044705 tid 1 2024-09-21T09:42:37.168 INFO:tasks.rados.rados.0.smithi017.stdout:1111: writing smithi01785587-1 from 1739595 to 2192410 tid 2 2024-09-21T09:42:37.174 INFO:tasks.rados.rados.0.smithi017.stdout:1111: writing smithi01785587-1 from 2722929 to 3378273 tid 3 2024-09-21T09:42:37.174 INFO:tasks.rados.rados.0.smithi017.stdout:1112: delete oid 7 current snap is 139 2024-09-21T09:42:37.175 INFO:tasks.rados.rados.0.smithi017.stdout:1110: finishing rollback tid 0 to smithi01785587-38 2024-09-21T09:42:37.175 INFO:tasks.rados.rados.0.smithi017.stdout:1110: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:42:37.175 INFO:tasks.rados.rados.0.smithi017.stdout:1110: finishing rollback tid 2 to smithi01785587-38 2024-09-21T09:42:37.175 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 339 (ObjNum 420 snap 130 seq_num 420) dirty exists 2024-09-21T09:42:37.175 INFO:tasks.rados.rados.0.smithi017.stdout:1111: finishing write tid 1 to smithi01785587-1 2024-09-21T09:42:37.175 INFO:tasks.rados.rados.0.smithi017.stdout:1111: finishing write tid 2 to smithi01785587-1 2024-09-21T09:42:37.177 INFO:tasks.rados.rados.0.smithi017.stdout:1109: done (3 left) 2024-09-21T09:42:37.177 INFO:tasks.rados.rados.0.smithi017.stdout:1110: done (2 left) 2024-09-21T09:42:37.177 INFO:tasks.rados.rados.0.smithi017.stdout:1112: done (1 left) 2024-09-21T09:42:37.177 INFO:tasks.rados.rados.0.smithi017.stdout:1113: rollback oid 48 current snap is 139 2024-09-21T09:42:37.177 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 125 2024-09-21T09:42:37.177 INFO:tasks.rados.rados.0.smithi017.stdout:1114: write oid 5 current snap is 139 2024-09-21T09:42:37.178 INFO:tasks.rados.rados.0.smithi017.stdout:1114: seq_num 441 ranges {611887=733283,1972912=456484,2894128=517759} 2024-09-21T09:42:37.184 INFO:tasks.rados.rados.0.smithi017.stdout:1114: writing smithi01785587-5 from 611887 to 1345170 tid 1 2024-09-21T09:42:37.190 INFO:tasks.rados.rados.0.smithi017.stdout:1114: writing smithi01785587-5 from 1972912 to 2429396 tid 2 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi017.stdout:1114: writing smithi01785587-5 from 2894128 to 3411887 tid 3 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi017.stdout:1115: write oid 29 current snap is 139 2024-09-21T09:42:37.196 INFO:tasks.rados.rados.0.smithi017.stdout:1115: seq_num 442 ranges {628694=408532,1515041=600142,2805198=552112} 2024-09-21T09:42:37.201 INFO:tasks.rados.rados.0.smithi017.stdout:1115: writing smithi01785587-29 from 628694 to 1037226 tid 1 2024-09-21T09:42:37.207 INFO:tasks.rados.rados.0.smithi017.stdout:1115: writing smithi01785587-29 from 1515041 to 2115183 tid 2 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1115: writing smithi01785587-29 from 2805198 to 3357310 tid 3 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1116: read oid 15 snap -1 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1116: expect (ObjNum 410 snap 126 seq_num 410) 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1117: read oid 16 snap -1 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1117: expect (ObjNum 417 snap 129 seq_num 417) 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1118: snap_create 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1113: finishing rollback tid 0 to smithi01785587-48 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1113: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1111: finishing write tid 3 to smithi01785587-1 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1111: finishing write tid 4 to smithi01785587-1 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:1113: finishing rollback tid 2 to smithi01785587-48 2024-09-21T09:42:37.215 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 555 (ObjNum 395 snap 121 seq_num 395) dirty exists 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1111: finishing write tid 5 to smithi01785587-1 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 322 (ObjNum 439 snap 139 seq_num 439) dirty exists 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1111: left oid 1 (ObjNum 439 snap 139 seq_num 439) 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1114: finishing write tid 1 to smithi01785587-5 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1114: finishing write tid 2 to smithi01785587-5 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1114: finishing write tid 3 to smithi01785587-5 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1114: finishing write tid 4 to smithi01785587-5 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1114: finishing write tid 5 to smithi01785587-5 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 536 (ObjNum 441 snap 139 seq_num 441) dirty exists 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1114: left oid 5 (ObjNum 441 snap 139 seq_num 441) 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1115: finishing write tid 1 to smithi01785587-29 2024-09-21T09:42:37.216 INFO:tasks.rados.rados.0.smithi017.stdout:1115: finishing write tid 2 to smithi01785587-29 2024-09-21T09:42:37.229 INFO:tasks.rados.rados.0.smithi017.stdout:1117: expect (ObjNum 417 snap 129 seq_num 417) 2024-09-21T09:42:37.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:37 smithi134 ceph-mon[33541]: pgmap v80: 89 pgs: 89 active+clean; 118 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:37.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:37 smithi134 ceph-mon[33541]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T09:42:37.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:37.329 INFO:tasks.rados.rados.0.smithi017.stdout:1115: finishing write tid 3 to smithi01785587-29 2024-09-21T09:42:37.329 INFO:tasks.rados.rados.0.smithi017.stdout:1115: finishing write tid 4 to smithi01785587-29 2024-09-21T09:42:37.329 INFO:tasks.rados.rados.0.smithi017.stdout:1115: finishing write tid 5 to smithi01785587-29 2024-09-21T09:42:37.329 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 229 (ObjNum 442 snap 139 seq_num 442) dirty exists 2024-09-21T09:42:37.329 INFO:tasks.rados.rados.0.smithi017.stdout:1115: left oid 29 (ObjNum 442 snap 139 seq_num 442) 2024-09-21T09:42:37.329 INFO:tasks.rados.rados.0.smithi017.stdout:1116: expect (ObjNum 410 snap 126 seq_num 410) 2024-09-21T09:42:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:37 smithi017 ceph-mon[102913]: pgmap v80: 89 pgs: 89 active+clean; 118 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:37 smithi017 ceph-mon[102913]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T09:42:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:37 smithi017 ceph-mon[105151]: pgmap v80: 89 pgs: 89 active+clean; 118 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:37 smithi017 ceph-mon[105151]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T09:42:38.167 INFO:tasks.rados.rados.0.smithi017.stdout:1111: done (6 left) 2024-09-21T09:42:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:1113: done (5 left) 2024-09-21T09:42:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:1114: done (4 left) 2024-09-21T09:42:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:1115: done (3 left) 2024-09-21T09:42:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:1116: done (2 left) 2024-09-21T09:42:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:1117: done (1 left) 2024-09-21T09:42:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:1118: done (0 left) 2024-09-21T09:42:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:1119: rollback oid 34 current snap is 140 2024-09-21T09:42:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 127 2024-09-21T09:42:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:1120: snap_create 2024-09-21T09:42:38.172 INFO:tasks.rados.rados.0.smithi017.stdout:1119: finishing rollback tid 0 to smithi01785587-34 2024-09-21T09:42:38.172 INFO:tasks.rados.rados.0.smithi017.stdout:1119: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:42:38.173 INFO:tasks.rados.rados.0.smithi017.stdout:1119: finishing rollback tid 2 to smithi01785587-34 2024-09-21T09:42:38.173 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 552 (ObjNum 401 snap 122 seq_num 401) dirty exists 2024-09-21T09:42:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:38 smithi017 ceph-mon[102913]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T09:42:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:38 smithi017 ceph-mon[105151]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T09:42:38.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:38 smithi134 ceph-mon[33541]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T09:42:39.167 INFO:tasks.rados.rados.0.smithi017.stdout:1119: done (1 left) 2024-09-21T09:42:39.167 INFO:tasks.rados.rados.0.smithi017.stdout:1120: done (0 left) 2024-09-21T09:42:39.167 INFO:tasks.rados.rados.0.smithi017.stdout:1121: snap_create 2024-09-21T09:42:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:39 smithi017 ceph-mon[102913]: pgmap v83: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.6 MiB/s wr, 34 op/s 2024-09-21T09:42:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:39 smithi017 ceph-mon[102913]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T09:42:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:39 smithi017 ceph-mon[105151]: pgmap v83: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.6 MiB/s wr, 34 op/s 2024-09-21T09:42:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:39 smithi017 ceph-mon[105151]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T09:42:39.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:39 smithi134 ceph-mon[33541]: pgmap v83: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.6 MiB/s wr, 34 op/s 2024-09-21T09:42:39.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:39 smithi134 ceph-mon[33541]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T09:42:40.177 INFO:tasks.rados.rados.0.smithi017.stdout:1121: done (0 left) 2024-09-21T09:42:40.177 INFO:tasks.rados.rados.0.smithi017.stdout:1122: write oid 16 current snap is 142 2024-09-21T09:42:40.177 INFO:tasks.rados.rados.0.smithi017.stdout:1122: seq_num 443 ranges {645501=483781,1573523=460153,2548974=763384} 2024-09-21T09:42:40.183 INFO:tasks.rados.rados.0.smithi017.stdout:1122: writing smithi01785587-16 from 645501 to 1129282 tid 1 2024-09-21T09:42:40.188 INFO:tasks.rados.rados.0.smithi017.stdout:1122: writing smithi01785587-16 from 1573523 to 2033676 tid 2 2024-09-21T09:42:40.196 INFO:tasks.rados.rados.0.smithi017.stdout:1122: writing smithi01785587-16 from 2548974 to 3312358 tid 3 2024-09-21T09:42:40.196 INFO:tasks.rados.rados.0.smithi017.stdout:1123: read oid 3 snap -1 2024-09-21T09:42:40.196 INFO:tasks.rados.rados.0.smithi017.stdout:1123: expect deleted 2024-09-21T09:42:40.197 INFO:tasks.rados.rados.0.smithi017.stdout:1124: snap_remove snap 129 2024-09-21T09:42:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:40 smithi017 ceph-mon[102913]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T09:42:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:40 smithi017 ceph-mon[105151]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T09:42:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:40 smithi134 ceph-mon[33541]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T09:42:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:41.190 INFO:tasks.rados.rados.0.smithi017.stdout:1124: done (2 left) 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1122: finishing write tid 1 to smithi01785587-16 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1122: finishing write tid 2 to smithi01785587-16 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1122: finishing write tid 3 to smithi01785587-16 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1122: finishing write tid 4 to smithi01785587-16 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1122: finishing write tid 5 to smithi01785587-16 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 588 (ObjNum 443 snap 142 seq_num 443) dirty exists 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1122: left oid 16 (ObjNum 443 snap 142 seq_num 443) 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1125: read oid 48 snap -1 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1125: expect (ObjNum 395 snap 121 seq_num 395) 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1122: done (2 left) 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1123: done (1 left) 2024-09-21T09:42:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1126: delete oid 12 current snap is 142 2024-09-21T09:42:41.199 INFO:tasks.rados.rados.0.smithi017.stdout:1126: done (1 left) 2024-09-21T09:42:41.199 INFO:tasks.rados.rados.0.smithi017.stdout:1127: snap_create 2024-09-21T09:42:41.200 INFO:tasks.rados.rados.0.smithi017.stdout:1125: expect (ObjNum 395 snap 121 seq_num 395) 2024-09-21T09:42:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:41 smithi017 ceph-mon[105151]: pgmap v86: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.6 MiB/s wr, 34 op/s 2024-09-21T09:42:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:41 smithi017 ceph-mon[105151]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T09:42:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:41 smithi017 ceph-mon[102913]: pgmap v86: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.6 MiB/s wr, 34 op/s 2024-09-21T09:42:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:41 smithi017 ceph-mon[102913]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T09:42:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:41.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:41 smithi134 ceph-mon[33541]: pgmap v86: 89 pgs: 89 active+clean; 140 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.6 MiB/s wr, 34 op/s 2024-09-21T09:42:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:41 smithi134 ceph-mon[33541]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T09:42:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:41 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:42.191 INFO:tasks.rados.rados.0.smithi017.stdout:1125: done (1 left) 2024-09-21T09:42:42.192 INFO:tasks.rados.rados.0.smithi017.stdout:1127: done (0 left) 2024-09-21T09:42:42.192 INFO:tasks.rados.rados.0.smithi017.stdout:1128: delete oid 24 current snap is 143 2024-09-21T09:42:42.196 INFO:tasks.rados.rados.0.smithi017.stdout:1128: done (0 left) 2024-09-21T09:42:42.196 INFO:tasks.rados.rados.0.smithi017.stdout:1129: read oid 43 snap 125 2024-09-21T09:42:42.196 INFO:tasks.rados.rados.0.smithi017.stdout:1129: expect (ObjNum 391 snap 119 seq_num 391) 2024-09-21T09:42:42.196 INFO:tasks.rados.rados.0.smithi017.stdout:1130: delete oid 39 current snap is 143 2024-09-21T09:42:42.198 INFO:tasks.rados.rados.0.smithi017.stdout:1130: done (1 left) 2024-09-21T09:42:42.198 INFO:tasks.rados.rados.0.smithi017.stdout:1131: snap_create 2024-09-21T09:42:42.205 INFO:tasks.rados.rados.0.smithi017.stdout:1129: expect (ObjNum 391 snap 119 seq_num 391) 2024-09-21T09:42:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:42 smithi017 ceph-mon[102913]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T09:42:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:42 smithi017 ceph-mon[105151]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T09:42:42.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:42 smithi134 ceph-mon[33541]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T09:42:43.030 INFO:tasks.rados.rados.0.smithi017.stdout:1129: done (1 left) 2024-09-21T09:42:43.030 INFO:tasks.rados.rados.0.smithi017.stdout:1131: done (0 left) 2024-09-21T09:42:43.030 INFO:tasks.rados.rados.0.smithi017.stdout:1132: read oid 6 snap -1 2024-09-21T09:42:43.030 INFO:tasks.rados.rados.0.smithi017.stdout:1132: expect (ObjNum 423 snap 133 seq_num 423) 2024-09-21T09:42:43.030 INFO:tasks.rados.rados.0.smithi017.stdout:1133: read oid 21 snap -1 2024-09-21T09:42:43.031 INFO:tasks.rados.rados.0.smithi017.stdout:1133: expect (ObjNum 430 snap 135 seq_num 430) 2024-09-21T09:42:43.031 INFO:tasks.rados.rados.0.smithi017.stdout:1134: snap_create 2024-09-21T09:42:43.037 INFO:tasks.rados.rados.0.smithi017.stdout:1133: expect (ObjNum 430 snap 135 seq_num 430) 2024-09-21T09:42:43.145 INFO:tasks.rados.rados.0.smithi017.stdout:1132: expect (ObjNum 423 snap 133 seq_num 423) 2024-09-21T09:42:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:43 smithi017 ceph-mon[105151]: pgmap v89: 89 pgs: 89 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:42:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:43 smithi017 ceph-mon[105151]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T09:42:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:43 smithi017 ceph-mon[105151]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:42:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:43 smithi017 ceph-mon[102913]: pgmap v89: 89 pgs: 89 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:42:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:43 smithi017 ceph-mon[102913]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T09:42:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:43 smithi017 ceph-mon[102913]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:42:43.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:43 smithi134 ceph-mon[33541]: pgmap v89: 89 pgs: 89 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:42:43.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:43 smithi134 ceph-mon[33541]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T09:42:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:43 smithi134 ceph-mon[33541]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:42:44.031 INFO:tasks.rados.rados.0.smithi017.stdout:1132: done (2 left) 2024-09-21T09:42:44.031 INFO:tasks.rados.rados.0.smithi017.stdout:1133: done (1 left) 2024-09-21T09:42:44.031 INFO:tasks.rados.rados.0.smithi017.stdout:1134: done (0 left) 2024-09-21T09:42:44.031 INFO:tasks.rados.rados.0.smithi017.stdout:1135: write oid 49 current snap is 145 2024-09-21T09:42:44.031 INFO:tasks.rados.rados.0.smithi017.stdout:1135: seq_num 447 ranges {712729=784777,2091098=467491,3342313=170416} 2024-09-21T09:42:44.039 INFO:tasks.rados.rados.0.smithi017.stdout:1135: writing smithi01785587-49 from 712729 to 1497506 tid 1 2024-09-21T09:42:44.045 INFO:tasks.rados.rados.0.smithi017.stdout:1135: writing smithi01785587-49 from 2091098 to 2558589 tid 2 2024-09-21T09:42:44.050 INFO:tasks.rados.rados.0.smithi017.stdout:1135: writing smithi01785587-49 from 3342313 to 3512729 tid 3 2024-09-21T09:42:44.051 INFO:tasks.rados.rados.0.smithi017.stdout:1136: snap_remove snap 142 2024-09-21T09:42:44.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:44.151Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:45.033 INFO:tasks.rados.rados.0.smithi017.stdout:1136: done (1 left) 2024-09-21T09:42:45.034 INFO:tasks.rados.rados.0.smithi017.stdout:1137: read oid 1 snap -1 2024-09-21T09:42:45.034 INFO:tasks.rados.rados.0.smithi017.stdout:1137: expect (ObjNum 439 snap 139 seq_num 439) 2024-09-21T09:42:45.034 INFO:tasks.rados.rados.0.smithi017.stdout:1138: read oid 40 snap 143 2024-09-21T09:42:45.034 INFO:tasks.rados.rados.0.smithi017.stdout:1138: expect (ObjNum 407 snap 126 seq_num 407) 2024-09-21T09:42:45.034 INFO:tasks.rados.rados.0.smithi017.stdout:1139: snap_remove snap 144 2024-09-21T09:42:45.035 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:42:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:42:44] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-21T09:42:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:45 smithi017 ceph-mon[102913]: pgmap v92: 89 pgs: 89 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:42:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:45 smithi017 ceph-mon[102913]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:42:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:45 smithi017 ceph-mon[105151]: pgmap v92: 89 pgs: 89 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:42:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:45 smithi017 ceph-mon[105151]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:42:45.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:45 smithi134 ceph-mon[33541]: pgmap v92: 89 pgs: 89 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:42:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:45 smithi134 ceph-mon[33541]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:42:45.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:45.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:46.044 INFO:tasks.rados.rados.0.smithi017.stdout:1139: done (3 left) 2024-09-21T09:42:46.044 INFO:tasks.rados.rados.0.smithi017.stdout:1140: read oid 37 snap 134 2024-09-21T09:42:46.044 INFO:tasks.rados.rados.0.smithi017.stdout:1140: expect deleted 2024-09-21T09:42:46.044 INFO:tasks.rados.rados.0.smithi017.stdout:1141: read oid 4 snap -1 2024-09-21T09:42:46.044 INFO:tasks.rados.rados.0.smithi017.stdout:1141: expect deleted 2024-09-21T09:42:46.044 INFO:tasks.rados.rados.0.smithi017.stdout:1135: finishing write tid 1 to smithi01785587-49 2024-09-21T09:42:46.045 INFO:tasks.rados.rados.0.smithi017.stdout:1135: finishing write tid 2 to smithi01785587-49 2024-09-21T09:42:46.045 INFO:tasks.rados.rados.0.smithi017.stdout:1135: finishing write tid 3 to smithi01785587-49 2024-09-21T09:42:46.045 INFO:tasks.rados.rados.0.smithi017.stdout:1135: finishing write tid 4 to smithi01785587-49 2024-09-21T09:42:46.045 INFO:tasks.rados.rados.0.smithi017.stdout:1135: finishing write tid 5 to smithi01785587-49 2024-09-21T09:42:46.045 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 563 (ObjNum 447 snap 145 seq_num 447) dirty exists 2024-09-21T09:42:46.045 INFO:tasks.rados.rados.0.smithi017.stdout:1135: left oid 49 (ObjNum 447 snap 145 seq_num 447) 2024-09-21T09:42:46.045 INFO:tasks.rados.rados.0.smithi017.stdout:1138: expect (ObjNum 407 snap 126 seq_num 407) 2024-09-21T09:42:46.333 INFO:tasks.rados.rados.0.smithi017.stdout:1137: expect (ObjNum 439 snap 139 seq_num 439) 2024-09-21T09:42:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:46 smithi017 ceph-mon[102913]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:42:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:46 smithi017 ceph-mon[105151]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:42:46.480 INFO:tasks.rados.rados.0.smithi017.stdout:1142: read oid 20 snap -1 2024-09-21T09:42:46.480 INFO:tasks.rados.rados.0.smithi017.stdout:1142: expect (ObjNum 415 snap 128 seq_num 415) 2024-09-21T09:42:46.480 INFO:tasks.rados.rados.0.smithi017.stdout:1135: done (5 left) 2024-09-21T09:42:46.480 INFO:tasks.rados.rados.0.smithi017.stdout:1137: done (4 left) 2024-09-21T09:42:46.480 INFO:tasks.rados.rados.0.smithi017.stdout:1138: done (3 left) 2024-09-21T09:42:46.480 INFO:tasks.rados.rados.0.smithi017.stdout:1140: done (2 left) 2024-09-21T09:42:46.480 INFO:tasks.rados.rados.0.smithi017.stdout:1141: done (1 left) 2024-09-21T09:42:46.480 INFO:tasks.rados.rados.0.smithi017.stdout:1143: write oid 47 current snap is 145 2024-09-21T09:42:46.480 INFO:tasks.rados.rados.0.smithi017.stdout:1143: seq_num 448 ranges {729536=460026,1749580=611149,2853383=452450} 2024-09-21T09:42:46.486 INFO:tasks.rados.rados.0.smithi017.stdout:1143: writing smithi01785587-47 from 729536 to 1189562 tid 1 2024-09-21T09:42:46.493 INFO:tasks.rados.rados.0.smithi017.stdout:1143: writing smithi01785587-47 from 1749580 to 2360729 tid 2 2024-09-21T09:42:46.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:46 smithi134 ceph-mon[33541]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:42:46.498 INFO:tasks.rados.rados.0.smithi017.stdout:1143: writing smithi01785587-47 from 2853383 to 3305833 tid 3 2024-09-21T09:42:46.498 INFO:tasks.rados.rados.0.smithi017.stdout:1144: read oid 17 snap -1 2024-09-21T09:42:46.498 INFO:tasks.rados.rados.0.smithi017.stdout:1144: expect deleted 2024-09-21T09:42:46.498 INFO:tasks.rados.rados.0.smithi017.stdout:1145: snap_remove snap 139 2024-09-21T09:42:47.049 INFO:tasks.rados.rados.0.smithi017.stdout:1145: done (3 left) 2024-09-21T09:42:47.049 INFO:tasks.rados.rados.0.smithi017.stdout:1146: read oid 36 snap -1 2024-09-21T09:42:47.049 INFO:tasks.rados.rados.0.smithi017.stdout:1146: expect (ObjNum 313 snap 93 seq_num 313) 2024-09-21T09:42:47.049 INFO:tasks.rados.rados.0.smithi017.stdout:1147: read oid 30 snap -1 2024-09-21T09:42:47.050 INFO:tasks.rados.rados.0.smithi017.stdout:1147: expect deleted 2024-09-21T09:42:47.050 INFO:tasks.rados.rados.0.smithi017.stdout:1148: delete oid 38 current snap is 145 2024-09-21T09:42:47.050 INFO:tasks.rados.rados.0.smithi017.stdout:1142: expect (ObjNum 415 snap 128 seq_num 415) 2024-09-21T09:42:47.176 INFO:tasks.rados.rados.0.smithi017.stdout:1143: finishing write tid 1 to smithi01785587-47 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1143: finishing write tid 2 to smithi01785587-47 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1143: finishing write tid 3 to smithi01785587-47 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1143: finishing write tid 4 to smithi01785587-47 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1143: finishing write tid 5 to smithi01785587-47 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 394 (ObjNum 448 snap 145 seq_num 448) dirty exists 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1143: left oid 47 (ObjNum 448 snap 145 seq_num 448) 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1142: done (5 left) 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1143: done (4 left) 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1144: done (3 left) 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1148: done (2 left) 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1149: snap_create 2024-09-21T09:42:47.177 INFO:tasks.rados.rados.0.smithi017.stdout:1146: expect (ObjNum 313 snap 93 seq_num 313) 2024-09-21T09:42:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:47 smithi134 ceph-mon[33541]: pgmap v95: 89 pgs: 89 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:47 smithi134 ceph-mon[33541]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:42:47.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:47 smithi017 ceph-mon[102913]: pgmap v95: 89 pgs: 89 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:47 smithi017 ceph-mon[102913]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:42:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:47 smithi017 ceph-mon[105151]: pgmap v95: 89 pgs: 89 active+clean; 151 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:42:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:47 smithi017 ceph-mon[105151]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:42:48.059 INFO:tasks.rados.rados.0.smithi017.stdout:1146: done (2 left) 2024-09-21T09:42:48.059 INFO:tasks.rados.rados.0.smithi017.stdout:1147: done (1 left) 2024-09-21T09:42:48.059 INFO:tasks.rados.rados.0.smithi017.stdout:1149: done (0 left) 2024-09-21T09:42:48.059 INFO:tasks.rados.rados.0.smithi017.stdout:1150: snap_remove snap 143 2024-09-21T09:42:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:48 smithi017 ceph-mon[102913]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T09:42:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:48 smithi017 ceph-mon[105151]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T09:42:48.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:48 smithi134 ceph-mon[33541]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T09:42:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1150: done (0 left) 2024-09-21T09:42:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1151: rollback oid 42 current snap is 146 2024-09-21T09:42:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 136 2024-09-21T09:42:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1152: read oid 44 snap -1 2024-09-21T09:42:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1152: expect (ObjNum 323 snap 98 seq_num 323) 2024-09-21T09:42:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1153: read oid 43 snap -1 2024-09-21T09:42:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1153: expect (ObjNum 391 snap 119 seq_num 391) 2024-09-21T09:42:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1154: write oid 22 current snap is 146 2024-09-21T09:42:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1154: seq_num 450 ranges {763150=726877,1982897=614818,3024582=474994} 2024-09-21T09:42:49.071 INFO:tasks.rados.rados.0.smithi017.stdout:1154: writing smithi01785587-22 from 763150 to 1490027 tid 1 2024-09-21T09:42:49.078 INFO:tasks.rados.rados.0.smithi017.stdout:1154: writing smithi01785587-22 from 1982897 to 2597715 tid 2 2024-09-21T09:42:49.083 INFO:tasks.rados.rados.0.smithi017.stdout:1154: writing smithi01785587-22 from 3024582 to 3499576 tid 3 2024-09-21T09:42:49.083 INFO:tasks.rados.rados.0.smithi017.stdout:1155: read oid 21 snap -1 2024-09-21T09:42:49.083 INFO:tasks.rados.rados.0.smithi017.stdout:1155: expect (ObjNum 430 snap 135 seq_num 430) 2024-09-21T09:42:49.083 INFO:tasks.rados.rados.0.smithi017.stdout:1156: write oid 48 current snap is 146 2024-09-21T09:42:49.083 INFO:tasks.rados.rados.0.smithi017.stdout:1156: seq_num 451 ranges {779957=402126,1925026=758476,3335652=244305} 2024-09-21T09:42:49.090 INFO:tasks.rados.rados.0.smithi017.stdout:1156: writing smithi01785587-48 from 779957 to 1182083 tid 1 2024-09-21T09:42:49.098 INFO:tasks.rados.rados.0.smithi017.stdout:1156: writing smithi01785587-48 from 1925026 to 2683502 tid 2 2024-09-21T09:42:49.103 INFO:tasks.rados.rados.0.smithi017.stdout:1156: writing smithi01785587-48 from 3335652 to 3579957 tid 3 2024-09-21T09:42:49.103 INFO:tasks.rados.rados.0.smithi017.stdout:1157: snap_create 2024-09-21T09:42:49.103 INFO:tasks.rados.rados.0.smithi017.stdout:1153: expect (ObjNum 391 snap 119 seq_num 391) 2024-09-21T09:42:49.189 INFO:tasks.rados.rados.0.smithi017.stdout:1151: finishing rollback tid 0 to smithi01785587-42 2024-09-21T09:42:49.189 INFO:tasks.rados.rados.0.smithi017.stdout:1151: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:42:49.189 INFO:tasks.rados.rados.0.smithi017.stdout:1152: expect (ObjNum 323 snap 98 seq_num 323) 2024-09-21T09:42:49.342 INFO:tasks.rados.rados.0.smithi017.stdout:1151: finishing rollback tid 2 to smithi01785587-42 2024-09-21T09:42:49.342 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 402 (ObjNum 426 snap 133 seq_num 426) dirty exists 2024-09-21T09:42:49.342 INFO:tasks.rados.rados.0.smithi017.stdout:1154: finishing write tid 1 to smithi01785587-22 2024-09-21T09:42:49.342 INFO:tasks.rados.rados.0.smithi017.stdout:1154: finishing write tid 2 to smithi01785587-22 2024-09-21T09:42:49.342 INFO:tasks.rados.rados.0.smithi017.stdout:1154: finishing write tid 3 to smithi01785587-22 2024-09-21T09:42:49.343 INFO:tasks.rados.rados.0.smithi017.stdout:1154: finishing write tid 4 to smithi01785587-22 2024-09-21T09:42:49.343 INFO:tasks.rados.rados.0.smithi017.stdout:1154: finishing write tid 5 to smithi01785587-22 2024-09-21T09:42:49.343 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 596 (ObjNum 450 snap 146 seq_num 450) dirty exists 2024-09-21T09:42:49.343 INFO:tasks.rados.rados.0.smithi017.stdout:1154: left oid 22 (ObjNum 450 snap 146 seq_num 450) 2024-09-21T09:42:49.343 INFO:tasks.rados.rados.0.smithi017.stdout:1155: expect (ObjNum 430 snap 135 seq_num 430) 2024-09-21T09:42:49.446 INFO:tasks.rados.rados.0.smithi017.stdout:1156: finishing write tid 1 to smithi01785587-48 2024-09-21T09:42:49.447 INFO:tasks.rados.rados.0.smithi017.stdout:1156: finishing write tid 2 to smithi01785587-48 2024-09-21T09:42:49.447 INFO:tasks.rados.rados.0.smithi017.stdout:1156: finishing write tid 3 to smithi01785587-48 2024-09-21T09:42:49.447 INFO:tasks.rados.rados.0.smithi017.stdout:1156: finishing write tid 4 to smithi01785587-48 2024-09-21T09:42:49.447 INFO:tasks.rados.rados.0.smithi017.stdout:1156: finishing write tid 5 to smithi01785587-48 2024-09-21T09:42:49.447 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 560 (ObjNum 451 snap 146 seq_num 451) dirty exists 2024-09-21T09:42:49.447 INFO:tasks.rados.rados.0.smithi017.stdout:1156: left oid 48 (ObjNum 451 snap 146 seq_num 451) 2024-09-21T09:42:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:49 smithi017 ceph-mon[102913]: pgmap v98: 89 pgs: 89 active+clean; 152 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 348 KiB/s wr, 15 op/s 2024-09-21T09:42:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:49 smithi017 ceph-mon[102913]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T09:42:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:49 smithi017 ceph-mon[105151]: pgmap v98: 89 pgs: 89 active+clean; 152 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 348 KiB/s wr, 15 op/s 2024-09-21T09:42:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:49 smithi017 ceph-mon[105151]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T09:42:49.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:49 smithi134 ceph-mon[33541]: pgmap v98: 89 pgs: 89 active+clean; 152 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 348 KiB/s wr, 15 op/s 2024-09-21T09:42:49.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:49 smithi134 ceph-mon[33541]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T09:42:50.070 INFO:tasks.rados.rados.0.smithi017.stdout:1151: done (6 left) 2024-09-21T09:42:50.070 INFO:tasks.rados.rados.0.smithi017.stdout:1152: done (5 left) 2024-09-21T09:42:50.070 INFO:tasks.rados.rados.0.smithi017.stdout:1153: done (4 left) 2024-09-21T09:42:50.070 INFO:tasks.rados.rados.0.smithi017.stdout:1154: done (3 left) 2024-09-21T09:42:50.070 INFO:tasks.rados.rados.0.smithi017.stdout:1155: done (2 left) 2024-09-21T09:42:50.070 INFO:tasks.rados.rados.0.smithi017.stdout:1156: done (1 left) 2024-09-21T09:42:50.070 INFO:tasks.rados.rados.0.smithi017.stdout:1157: done (0 left) 2024-09-21T09:42:50.070 INFO:tasks.rados.rados.0.smithi017.stdout:1158: write oid 7 current snap is 147 2024-09-21T09:42:50.070 INFO:tasks.rados.rados.0.smithi017.stdout:1158: seq_num 452 ranges {796764=477375,1983508=618487,3363075=233689} 2024-09-21T09:42:50.076 INFO:tasks.rados.rados.0.smithi017.stdout:1158: writing smithi01785587-7 from 796764 to 1274139 tid 1 2024-09-21T09:42:50.083 INFO:tasks.rados.rados.0.smithi017.stdout:1158: writing smithi01785587-7 from 1983508 to 2601995 tid 2 2024-09-21T09:42:50.089 INFO:tasks.rados.rados.0.smithi017.stdout:1158: writing smithi01785587-7 from 3363075 to 3596764 tid 3 2024-09-21T09:42:50.089 INFO:tasks.rados.rados.0.smithi017.stdout:1159: write oid 19 current snap is 147 2024-09-21T09:42:50.090 INFO:tasks.rados.rados.0.smithi017.stdout:1159: seq_num 453 ranges {413571=552624,1641990=762145,2990498=425163} 2024-09-21T09:42:50.095 INFO:tasks.rados.rados.0.smithi017.stdout:1159: writing smithi01785587-19 from 413571 to 966195 tid 1 2024-09-21T09:42:50.102 INFO:tasks.rados.rados.0.smithi017.stdout:1159: writing smithi01785587-19 from 1641990 to 2404135 tid 2 2024-09-21T09:42:50.108 INFO:tasks.rados.rados.0.smithi017.stdout:1159: writing smithi01785587-19 from 2990498 to 3415661 tid 3 2024-09-21T09:42:50.108 INFO:tasks.rados.rados.0.smithi017.stdout:1160: snap_create 2024-09-21T09:42:50.108 INFO:tasks.rados.rados.0.smithi017.stdout:1158: finishing write tid 1 to smithi01785587-7 2024-09-21T09:42:50.108 INFO:tasks.rados.rados.0.smithi017.stdout:1158: finishing write tid 2 to smithi01785587-7 2024-09-21T09:42:50.108 INFO:tasks.rados.rados.0.smithi017.stdout:1158: finishing write tid 3 to smithi01785587-7 2024-09-21T09:42:50.109 INFO:tasks.rados.rados.0.smithi017.stdout:1158: finishing write tid 4 to smithi01785587-7 2024-09-21T09:42:50.109 INFO:tasks.rados.rados.0.smithi017.stdout:1158: finishing write tid 5 to smithi01785587-7 2024-09-21T09:42:50.109 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 577 (ObjNum 452 snap 147 seq_num 452) dirty exists 2024-09-21T09:42:50.109 INFO:tasks.rados.rados.0.smithi017.stdout:1158: left oid 7 (ObjNum 452 snap 147 seq_num 452) 2024-09-21T09:42:50.109 INFO:tasks.rados.rados.0.smithi017.stdout:1159: finishing write tid 1 to smithi01785587-19 2024-09-21T09:42:50.109 INFO:tasks.rados.rados.0.smithi017.stdout:1159: finishing write tid 2 to smithi01785587-19 2024-09-21T09:42:50.111 INFO:tasks.rados.rados.0.smithi017.stdout:1159: finishing write tid 3 to smithi01785587-19 2024-09-21T09:42:50.111 INFO:tasks.rados.rados.0.smithi017.stdout:1159: finishing write tid 4 to smithi01785587-19 2024-09-21T09:42:50.111 INFO:tasks.rados.rados.0.smithi017.stdout:1159: finishing write tid 5 to smithi01785587-19 2024-09-21T09:42:50.111 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 565 (ObjNum 453 snap 147 seq_num 453) dirty exists 2024-09-21T09:42:50.111 INFO:tasks.rados.rados.0.smithi017.stdout:1159: left oid 19 (ObjNum 453 snap 147 seq_num 453) 2024-09-21T09:42:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:50 smithi017 ceph-mon[102913]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T09:42:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:50 smithi017 ceph-mon[105151]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T09:42:50.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:50 smithi134 ceph-mon[33541]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T09:42:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:51.076 INFO:tasks.rados.rados.0.smithi017.stdout:1158: done (2 left) 2024-09-21T09:42:51.076 INFO:tasks.rados.rados.0.smithi017.stdout:1159: done (1 left) 2024-09-21T09:42:51.076 INFO:tasks.rados.rados.0.smithi017.stdout:1160: done (0 left) 2024-09-21T09:42:51.076 INFO:tasks.rados.rados.0.smithi017.stdout:1161: read oid 47 snap 132 2024-09-21T09:42:51.076 INFO:tasks.rados.rados.0.smithi017.stdout:1161: expect (ObjNum 414 snap 128 seq_num 414) 2024-09-21T09:42:51.076 INFO:tasks.rados.rados.0.smithi017.stdout:1162: snap_remove snap 133 2024-09-21T09:42:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:51 smithi017 ceph-mon[102913]: pgmap v101: 89 pgs: 89 active+clean; 152 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 348 KiB/s wr, 15 op/s 2024-09-21T09:42:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:51 smithi017 ceph-mon[102913]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T09:42:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:51 smithi017 ceph-mon[105151]: pgmap v101: 89 pgs: 89 active+clean; 152 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 348 KiB/s wr, 15 op/s 2024-09-21T09:42:51.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:51 smithi017 ceph-mon[105151]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T09:42:51.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:51 smithi134 ceph-mon[33541]: pgmap v101: 89 pgs: 89 active+clean; 152 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 348 KiB/s wr, 15 op/s 2024-09-21T09:42:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:51 smithi134 ceph-mon[33541]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T09:42:52.088 INFO:tasks.rados.rados.0.smithi017.stdout:1161: expect (ObjNum 414 snap 128 seq_num 414) 2024-09-21T09:42:52.265 INFO:tasks.rados.rados.0.smithi017.stdout:1161: done (1 left) 2024-09-21T09:42:52.266 INFO:tasks.rados.rados.0.smithi017.stdout:1162: done (0 left) 2024-09-21T09:42:52.266 INFO:tasks.rados.rados.0.smithi017.stdout:1163: snap_remove snap 147 2024-09-21T09:42:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:52 smithi017 ceph-mon[102913]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T09:42:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:52 smithi017 ceph-mon[105151]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T09:42:52.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:52 smithi134 ceph-mon[33541]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T09:42:53.034 INFO:tasks.rados.rados.0.smithi017.stdout:1163: done (0 left) 2024-09-21T09:42:53.034 INFO:tasks.rados.rados.0.smithi017.stdout:1164: delete oid 27 current snap is 148 2024-09-21T09:42:53.038 INFO:tasks.rados.rados.0.smithi017.stdout:1164: done (0 left) 2024-09-21T09:42:53.038 INFO:tasks.rados.rados.0.smithi017.stdout:1165: rollback oid 19 current snap is 148 2024-09-21T09:42:53.039 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 19 to 141 2024-09-21T09:42:53.039 INFO:tasks.rados.rados.0.smithi017.stdout:1166: read oid 5 snap -1 2024-09-21T09:42:53.039 INFO:tasks.rados.rados.0.smithi017.stdout:1166: expect (ObjNum 441 snap 139 seq_num 441) 2024-09-21T09:42:53.039 INFO:tasks.rados.rados.0.smithi017.stdout:1167: write oid 1 current snap is 148 2024-09-21T09:42:53.039 INFO:tasks.rados.rados.0.smithi017.stdout:1167: seq_num 455 ranges {447185=703122,1642601=765814,2928991=447707} 2024-09-21T09:42:53.045 INFO:tasks.rados.rados.0.smithi017.stdout:1167: writing smithi01785587-1 from 447185 to 1150307 tid 1 2024-09-21T09:42:53.052 INFO:tasks.rados.rados.0.smithi017.stdout:1167: writing smithi01785587-1 from 1642601 to 2408415 tid 2 2024-09-21T09:42:53.058 INFO:tasks.rados.rados.0.smithi017.stdout:1167: writing smithi01785587-1 from 2928991 to 3376698 tid 3 2024-09-21T09:42:53.058 INFO:tasks.rados.rados.0.smithi017.stdout:1168: rollback oid 10 current snap is 148 2024-09-21T09:42:53.058 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 146 2024-09-21T09:42:53.058 INFO:tasks.rados.rados.0.smithi017.stdout:1165: finishing rollback tid 0 to smithi01785587-19 2024-09-21T09:42:53.058 INFO:tasks.rados.rados.0.smithi017.stdout:1165: finishing rollback tid 1 to smithi01785587-19 2024-09-21T09:42:53.058 INFO:tasks.rados.rados.0.smithi017.stdout:1165: finishing rollback tid 2 to smithi01785587-19 2024-09-21T09:42:53.058 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 570 (ObjNum 388 snap 119 seq_num 388) dirty exists 2024-09-21T09:42:53.058 INFO:tasks.rados.rados.0.smithi017.stdout:1166: expect (ObjNum 441 snap 139 seq_num 441) 2024-09-21T09:42:53.251 INFO:tasks.rados.rados.0.smithi017.stdout:1167: finishing write tid 1 to smithi01785587-1 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1167: finishing write tid 2 to smithi01785587-1 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1167: finishing write tid 3 to smithi01785587-1 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1167: finishing write tid 4 to smithi01785587-1 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1167: finishing write tid 5 to smithi01785587-1 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 333 (ObjNum 455 snap 148 seq_num 455) dirty exists 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1167: left oid 1 (ObjNum 455 snap 148 seq_num 455) 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1168: finishing rollback tid 0 to smithi01785587-10 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1168: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1165: done (3 left) 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1166: done (2 left) 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1167: done (1 left) 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1169: read oid 6 snap -1 2024-09-21T09:42:53.252 INFO:tasks.rados.rados.0.smithi017.stdout:1169: expect (ObjNum 423 snap 133 seq_num 423) 2024-09-21T09:42:53.253 INFO:tasks.rados.rados.0.smithi017.stdout:1170: write oid 14 current snap is 148 2024-09-21T09:42:53.253 INFO:tasks.rados.rados.0.smithi017.stdout:1168: finishing rollback tid 2 to smithi01785587-10 2024-09-21T09:42:53.253 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 52 (ObjNum 434 snap 137 seq_num 434) dirty exists 2024-09-21T09:42:53.253 INFO:tasks.rados.rados.0.smithi017.stdout:1170: seq_num 456 ranges {463992=778371,1701083=625825,2956414=658979} 2024-09-21T09:42:53.258 INFO:tasks.rados.rados.0.smithi017.stdout:1170: writing smithi01785587-14 from 463992 to 1242363 tid 1 2024-09-21T09:42:53.264 INFO:tasks.rados.rados.0.smithi017.stdout:1170: writing smithi01785587-14 from 1701083 to 2326908 tid 2 2024-09-21T09:42:53.272 INFO:tasks.rados.rados.0.smithi017.stdout:1170: writing smithi01785587-14 from 2956414 to 3615393 tid 3 2024-09-21T09:42:53.272 INFO:tasks.rados.rados.0.smithi017.stdout:1168: done (2 left) 2024-09-21T09:42:53.272 INFO:tasks.rados.rados.0.smithi017.stdout:1171: write oid 46 current snap is 148 2024-09-21T09:42:53.272 INFO:tasks.rados.rados.0.smithi017.stdout:1171: seq_num 457 ranges {480799=569973,1475918=769483,2700190=586604} 2024-09-21T09:42:53.277 INFO:tasks.rados.rados.0.smithi017.stdout:1171: writing smithi01785587-46 from 480799 to 1050772 tid 1 2024-09-21T09:42:53.284 INFO:tasks.rados.rados.0.smithi017.stdout:1171: writing smithi01785587-46 from 1475918 to 2245401 tid 2 2024-09-21T09:42:53.290 INFO:tasks.rados.rados.0.smithi017.stdout:1171: writing smithi01785587-46 from 2700190 to 3286794 tid 3 2024-09-21T09:42:53.290 INFO:tasks.rados.rados.0.smithi017.stdout:1172: write oid 1 current snap is 148 2024-09-21T09:42:53.290 INFO:tasks.rados.rados.0.smithi017.stdout:1170: finishing write tid 1 to smithi01785587-14 2024-09-21T09:42:53.291 INFO:tasks.rados.rados.0.smithi017.stdout:1169: expect (ObjNum 423 snap 133 seq_num 423) 2024-09-21T09:42:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:53 smithi017 ceph-mon[102913]: pgmap v104: 89 pgs: 89 active+clean; 165 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:42:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:53 smithi017 ceph-mon[102913]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T09:42:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:53 smithi017 ceph-mon[102913]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T09:42:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:53 smithi017 ceph-mon[105151]: pgmap v104: 89 pgs: 89 active+clean; 165 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:42:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:53 smithi017 ceph-mon[105151]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T09:42:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:53 smithi017 ceph-mon[105151]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T09:42:53.468 INFO:tasks.rados.rados.0.smithi017.stdout:1170: finishing write tid 2 to smithi01785587-14 2024-09-21T09:42:53.468 INFO:tasks.rados.rados.0.smithi017.stdout:1172: seq_num 458 ranges {497606=645222,1934400=629494,3243966=453640} 2024-09-21T09:42:53.474 INFO:tasks.rados.rados.0.smithi017.stdout:1172: writing smithi01785587-1 from 497606 to 1142828 tid 1 2024-09-21T09:42:53.482 INFO:tasks.rados.rados.0.smithi017.stdout:1172: writing smithi01785587-1 from 1934400 to 2563894 tid 2 2024-09-21T09:42:53.489 INFO:tasks.rados.rados.0.smithi017.stdout:1172: writing smithi01785587-1 from 3243966 to 3697606 tid 3 2024-09-21T09:42:53.489 INFO:tasks.rados.rados.0.smithi017.stdout:1169: done (3 left) 2024-09-21T09:42:53.489 INFO:tasks.rados.rados.0.smithi017.stdout:1173: rollback oid 6 current snap is 148 2024-09-21T09:42:53.489 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 145 2024-09-21T09:42:53.489 INFO:tasks.rados.rados.0.smithi017.stdout:1170: finishing write tid 3 to smithi01785587-14 2024-09-21T09:42:53.489 INFO:tasks.rados.rados.0.smithi017.stdout:1170: finishing write tid 4 to smithi01785587-14 2024-09-21T09:42:53.489 INFO:tasks.rados.rados.0.smithi017.stdout:1170: finishing write tid 5 to smithi01785587-14 2024-09-21T09:42:53.489 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 553 (ObjNum 456 snap 148 seq_num 456) dirty exists 2024-09-21T09:42:53.490 INFO:tasks.rados.rados.0.smithi017.stdout:1170: left oid 14 (ObjNum 456 snap 148 seq_num 456) 2024-09-21T09:42:53.490 INFO:tasks.rados.rados.0.smithi017.stdout:1171: finishing write tid 1 to smithi01785587-46 2024-09-21T09:42:53.490 INFO:tasks.rados.rados.0.smithi017.stdout:1171: finishing write tid 2 to smithi01785587-46 2024-09-21T09:42:53.490 INFO:tasks.rados.rados.0.smithi017.stdout:1171: finishing write tid 3 to smithi01785587-46 2024-09-21T09:42:53.490 INFO:tasks.rados.rados.0.smithi017.stdout:1170: done (3 left) 2024-09-21T09:42:53.490 INFO:tasks.rados.rados.0.smithi017.stdout:1174: write oid 39 current snap is 148 2024-09-21T09:42:53.490 INFO:tasks.rados.rados.0.smithi017.stdout:1171: finishing write tid 4 to smithi01785587-46 2024-09-21T09:42:53.490 INFO:tasks.rados.rados.0.smithi017.stdout:1174: seq_num 459 ranges {514413=720471,1876529=773152,3438683=275730} 2024-09-21T09:42:53.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:53 smithi134 ceph-mon[33541]: pgmap v104: 89 pgs: 89 active+clean; 165 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:42:53.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:53 smithi134 ceph-mon[33541]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T09:42:53.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:53 smithi134 ceph-mon[33541]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T09:42:53.495 INFO:tasks.rados.rados.0.smithi017.stdout:1174: writing smithi01785587-39 from 514413 to 1234884 tid 1 2024-09-21T09:42:53.503 INFO:tasks.rados.rados.0.smithi017.stdout:1174: writing smithi01785587-39 from 1876529 to 2649681 tid 2 2024-09-21T09:42:53.509 INFO:tasks.rados.rados.0.smithi017.stdout:1174: writing smithi01785587-39 from 3438683 to 3714413 tid 3 2024-09-21T09:42:53.509 INFO:tasks.rados.rados.0.smithi017.stdout:1175: snap_create 2024-09-21T09:42:53.509 INFO:tasks.rados.rados.0.smithi017.stdout:1171: finishing write tid 5 to smithi01785587-46 2024-09-21T09:42:53.509 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 512 (ObjNum 457 snap 148 seq_num 457) dirty exists 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1171: left oid 46 (ObjNum 457 snap 148 seq_num 457) 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1172: finishing write tid 1 to smithi01785587-1 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1172: finishing write tid 2 to smithi01785587-1 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1172: finishing write tid 3 to smithi01785587-1 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1172: finishing write tid 4 to smithi01785587-1 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1172: finishing write tid 5 to smithi01785587-1 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 337 (ObjNum 458 snap 148 seq_num 458) dirty exists 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1172: left oid 1 (ObjNum 458 snap 148 seq_num 458) 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1173: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1173: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:1173: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:42:53.510 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 341 (ObjNum 423 snap 133 seq_num 423) dirty exists 2024-09-21T09:42:53.511 INFO:tasks.rados.rados.0.smithi017.stdout:1174: finishing write tid 1 to smithi01785587-39 2024-09-21T09:42:53.511 INFO:tasks.rados.rados.0.smithi017.stdout:1174: finishing write tid 2 to smithi01785587-39 2024-09-21T09:42:53.511 INFO:tasks.rados.rados.0.smithi017.stdout:1174: finishing write tid 3 to smithi01785587-39 2024-09-21T09:42:53.511 INFO:tasks.rados.rados.0.smithi017.stdout:1174: finishing write tid 4 to smithi01785587-39 2024-09-21T09:42:53.511 INFO:tasks.rados.rados.0.smithi017.stdout:1174: finishing write tid 5 to smithi01785587-39 2024-09-21T09:42:53.511 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 582 (ObjNum 459 snap 148 seq_num 459) dirty exists 2024-09-21T09:42:53.511 INFO:tasks.rados.rados.0.smithi017.stdout:1174: left oid 39 (ObjNum 459 snap 148 seq_num 459) 2024-09-21T09:42:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:1171: done (4 left) 2024-09-21T09:42:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:1172: done (3 left) 2024-09-21T09:42:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:1173: done (2 left) 2024-09-21T09:42:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:1174: done (1 left) 2024-09-21T09:42:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:1175: done (0 left) 2024-09-21T09:42:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:1176: rollback oid 13 current snap is 149 2024-09-21T09:42:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 13 to 125 2024-09-21T09:42:54.087 INFO:tasks.rados.rados.0.smithi017.stdout:1177: read oid 18 snap 136 2024-09-21T09:42:54.088 INFO:tasks.rados.rados.0.smithi017.stdout:1177: expect (ObjNum 365 snap 114 seq_num 365) 2024-09-21T09:42:54.088 INFO:tasks.rados.rados.0.smithi017.stdout:1178: delete oid 24 current snap is 149 2024-09-21T09:42:54.089 INFO:tasks.rados.rados.0.smithi017.stdout:1176: finishing rollback tid 1 to smithi01785587-13 2024-09-21T09:42:54.089 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 0 (ObjNum -113271584 snap 32650 seq_num 304) dirty dne 2024-09-21T09:42:54.089 INFO:tasks.rados.rados.0.smithi017.stdout:1176: done (2 left) 2024-09-21T09:42:54.090 INFO:tasks.rados.rados.0.smithi017.stdout:1178: done (1 left) 2024-09-21T09:42:54.090 INFO:tasks.rados.rados.0.smithi017.stdout:1179: snap_create 2024-09-21T09:42:54.097 INFO:tasks.rados.rados.0.smithi017.stdout:1177: expect (ObjNum 365 snap 114 seq_num 365) 2024-09-21T09:42:54.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:54 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:55.085 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:42:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:42:54] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-21T09:42:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1177: done (1 left) 2024-09-21T09:42:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1179: done (0 left) 2024-09-21T09:42:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1180: snap_remove snap 126 2024-09-21T09:42:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:55 smithi017 ceph-mon[105151]: pgmap v107: 89 pgs: 89 active+clean; 165 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:42:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:55 smithi017 ceph-mon[105151]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T09:42:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:55 smithi017 ceph-mon[102913]: pgmap v107: 89 pgs: 89 active+clean; 165 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:42:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:55 smithi017 ceph-mon[102913]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T09:42:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:55 smithi134 ceph-mon[33541]: pgmap v107: 89 pgs: 89 active+clean; 165 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-09-21T09:42:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:55 smithi134 ceph-mon[33541]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T09:42:55.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:55.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:56.102 INFO:tasks.rados.rados.0.smithi017.stdout:1180: done (0 left) 2024-09-21T09:42:56.103 INFO:tasks.rados.rados.0.smithi017.stdout:1181: snap_remove snap 132 2024-09-21T09:42:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:56 smithi017 ceph-mon[102913]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T09:42:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:56 smithi017 ceph-mon[105151]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T09:42:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:56.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:56 smithi134 ceph-mon[33541]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T09:42:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:56 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:57.105 INFO:tasks.rados.rados.0.smithi017.stdout:1181: done (0 left) 2024-09-21T09:42:57.105 INFO:tasks.rados.rados.0.smithi017.stdout:1182: rollback oid 2 current snap is 150 2024-09-21T09:42:57.105 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 2 to 138 2024-09-21T09:42:57.105 INFO:tasks.rados.rados.0.smithi017.stdout:1183: write oid 26 current snap is 150 2024-09-21T09:42:57.105 INFO:tasks.rados.rados.0.smithi017.stdout:1183: seq_num 461 ranges {548027=470969,1593493=776821,3093529=631692} 2024-09-21T09:42:57.111 INFO:tasks.rados.rados.0.smithi017.stdout:1183: writing smithi01785587-26 from 548027 to 1018996 tid 1 2024-09-21T09:42:57.120 INFO:tasks.rados.rados.0.smithi017.stdout:1183: writing smithi01785587-26 from 1593493 to 2370314 tid 2 2024-09-21T09:42:57.128 INFO:tasks.rados.rados.0.smithi017.stdout:1183: writing smithi01785587-26 from 3093529 to 3725221 tid 3 2024-09-21T09:42:57.128 INFO:tasks.rados.rados.0.smithi017.stdout:1184: rollback oid 44 current snap is 150 2024-09-21T09:42:57.128 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 148 2024-09-21T09:42:57.128 INFO:tasks.rados.rados.0.smithi017.stdout:1182: finishing rollback tid 0 to smithi01785587-2 2024-09-21T09:42:57.128 INFO:tasks.rados.rados.0.smithi017.stdout:1182: finishing rollback tid 1 to smithi01785587-2 2024-09-21T09:42:57.128 INFO:tasks.rados.rados.0.smithi017.stdout:1182: finishing rollback tid 2 to smithi01785587-2 2024-09-21T09:42:57.128 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 227 (ObjNum 374 snap 117 seq_num 374) dirty exists 2024-09-21T09:42:57.128 INFO:tasks.rados.rados.0.smithi017.stdout:1183: finishing write tid 1 to smithi01785587-26 2024-09-21T09:42:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:1183: finishing write tid 2 to smithi01785587-26 2024-09-21T09:42:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:1182: done (2 left) 2024-09-21T09:42:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:1185: rollback oid 14 current snap is 150 2024-09-21T09:42:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 148 2024-09-21T09:42:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:1186: snap_remove snap 140 2024-09-21T09:42:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:57 smithi134 ceph-mon[33541]: pgmap v110: 89 pgs: 89 active+clean; 167 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 891 KiB/s wr, 5 op/s 2024-09-21T09:42:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:57 smithi134 ceph-mon[33541]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T09:42:57.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:42:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:42:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:42:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:57 smithi017 ceph-mon[105151]: pgmap v110: 89 pgs: 89 active+clean; 167 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 891 KiB/s wr, 5 op/s 2024-09-21T09:42:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:57 smithi017 ceph-mon[105151]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T09:42:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:57 smithi017 ceph-mon[102913]: pgmap v110: 89 pgs: 89 active+clean; 167 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 891 KiB/s wr, 5 op/s 2024-09-21T09:42:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:57 smithi017 ceph-mon[102913]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T09:42:58.120 INFO:tasks.rados.rados.0.smithi017.stdout:1183: finishing write tid 3 to smithi01785587-26 2024-09-21T09:42:58.120 INFO:tasks.rados.rados.0.smithi017.stdout:1183: finishing write tid 4 to smithi01785587-26 2024-09-21T09:42:58.120 INFO:tasks.rados.rados.0.smithi017.stdout:1184: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:42:58.120 INFO:tasks.rados.rados.0.smithi017.stdout:1183: finishing write tid 5 to smithi01785587-26 2024-09-21T09:42:58.120 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 609 (ObjNum 461 snap 150 seq_num 461) dirty exists 2024-09-21T09:42:58.120 INFO:tasks.rados.rados.0.smithi017.stdout:1183: left oid 26 (ObjNum 461 snap 150 seq_num 461) 2024-09-21T09:42:58.120 INFO:tasks.rados.rados.0.smithi017.stdout:1184: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1184: finishing rollback tid 2 to smithi01785587-44 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 569 (ObjNum 323 snap 98 seq_num 323) dirty exists 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1185: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1185: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1185: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 573 (ObjNum 456 snap 148 seq_num 456) dirty exists 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1183: done (3 left) 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1184: done (2 left) 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1185: done (1 left) 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1186: done (0 left) 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1187: write oid 46 current snap is 150 2024-09-21T09:42:58.121 INFO:tasks.rados.rados.0.smithi017.stdout:1187: seq_num 462 ranges {564834=546218,1651975=520479,2720952=559317} 2024-09-21T09:42:58.126 INFO:tasks.rados.rados.0.smithi017.stdout:1187: writing smithi01785587-46 from 564834 to 1111052 tid 1 2024-09-21T09:42:58.133 INFO:tasks.rados.rados.0.smithi017.stdout:1187: writing smithi01785587-46 from 1651975 to 2172454 tid 2 2024-09-21T09:42:58.139 INFO:tasks.rados.rados.0.smithi017.stdout:1187: writing smithi01785587-46 from 2720952 to 3280269 tid 3 2024-09-21T09:42:58.140 INFO:tasks.rados.rados.0.smithi017.stdout:1188: snap_remove snap 127 2024-09-21T09:42:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:58 smithi017 ceph-mon[102913]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T09:42:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:58 smithi017 ceph-mon[105151]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T09:42:58.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:58 smithi134 ceph-mon[33541]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1188: done (1 left) 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1189: rollback oid 49 current snap is 150 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 148 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1187: finishing write tid 1 to smithi01785587-46 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1187: finishing write tid 2 to smithi01785587-46 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1187: finishing write tid 3 to smithi01785587-46 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1187: finishing write tid 4 to smithi01785587-46 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1187: finishing write tid 5 to smithi01785587-46 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 525 (ObjNum 462 snap 150 seq_num 462) dirty exists 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1187: left oid 46 (ObjNum 462 snap 150 seq_num 462) 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1187: done (1 left) 2024-09-21T09:42:59.120 INFO:tasks.rados.rados.0.smithi017.stdout:1190: delete oid 2 current snap is 150 2024-09-21T09:42:59.125 INFO:tasks.rados.rados.0.smithi017.stdout:1190: done (1 left) 2024-09-21T09:42:59.125 INFO:tasks.rados.rados.0.smithi017.stdout:1191: rollback oid 22 current snap is 150 2024-09-21T09:42:59.126 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 22 to 138 2024-09-21T09:42:59.126 INFO:tasks.rados.rados.0.smithi017.stdout:1192: snap_create 2024-09-21T09:42:59.131 INFO:tasks.rados.rados.0.smithi017.stdout:1189: finishing rollback tid 0 to smithi01785587-49 2024-09-21T09:42:59.131 INFO:tasks.rados.rados.0.smithi017.stdout:1189: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:42:59.131 INFO:tasks.rados.rados.0.smithi017.stdout:1189: finishing rollback tid 2 to smithi01785587-49 2024-09-21T09:42:59.132 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 601 (ObjNum 447 snap 145 seq_num 447) dirty exists 2024-09-21T09:42:59.132 INFO:tasks.rados.rados.0.smithi017.stdout:1191: finishing rollback tid 0 to smithi01785587-22 2024-09-21T09:42:59.133 INFO:tasks.rados.rados.0.smithi017.stdout:1191: finishing rollback tid 1 to smithi01785587-22 2024-09-21T09:42:59.134 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 618 (ObjNum 305 snap 0 seq_num 1540944057) dirty dne 2024-09-21T09:42:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:59 smithi017 ceph-mon[102913]: pgmap v113: 89 pgs: 89 active+clean; 174 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:42:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:42:59 smithi017 ceph-mon[102913]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T09:42:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:59 smithi017 ceph-mon[105151]: pgmap v113: 89 pgs: 89 active+clean; 174 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:42:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:42:59 smithi017 ceph-mon[105151]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T09:42:59.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:59 smithi134 ceph-mon[33541]: pgmap v113: 89 pgs: 89 active+clean; 174 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:42:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:42:59 smithi134 ceph-mon[33541]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T09:43:00.127 INFO:tasks.rados.rados.0.smithi017.stdout:1189: done (2 left) 2024-09-21T09:43:00.127 INFO:tasks.rados.rados.0.smithi017.stdout:1191: done (1 left) 2024-09-21T09:43:00.127 INFO:tasks.rados.rados.0.smithi017.stdout:1192: done (0 left) 2024-09-21T09:43:00.127 INFO:tasks.rados.rados.0.smithi017.stdout:1193: delete oid 11 current snap is 151 2024-09-21T09:43:00.131 INFO:tasks.rados.rados.0.smithi017.stdout:1193: done (0 left) 2024-09-21T09:43:00.131 INFO:tasks.rados.rados.0.smithi017.stdout:1194: delete oid 37 current snap is 151 2024-09-21T09:43:00.136 INFO:tasks.rados.rados.0.smithi017.stdout:1194: done (0 left) 2024-09-21T09:43:00.136 INFO:tasks.rados.rados.0.smithi017.stdout:1195: read oid 46 snap -1 2024-09-21T09:43:00.136 INFO:tasks.rados.rados.0.smithi017.stdout:1195: expect (ObjNum 462 snap 150 seq_num 462) 2024-09-21T09:43:00.136 INFO:tasks.rados.rados.0.smithi017.stdout:1196: rollback oid 6 current snap is 151 2024-09-21T09:43:00.136 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 145 2024-09-21T09:43:00.136 INFO:tasks.rados.rados.0.smithi017.stdout:1197: snap_create 2024-09-21T09:43:00.140 INFO:tasks.rados.rados.0.smithi017.stdout:1195: expect (ObjNum 462 snap 150 seq_num 462) 2024-09-21T09:43:00.369 INFO:tasks.rados.rados.0.smithi017.stdout:1196: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:43:00.369 INFO:tasks.rados.rados.0.smithi017.stdout:1196: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:43:00.369 INFO:tasks.rados.rados.0.smithi017.stdout:1196: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:43:00.369 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 361 (ObjNum 423 snap 133 seq_num 423) dirty exists 2024-09-21T09:43:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:00 smithi017 ceph-mon[102913]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:43:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:00 smithi017 ceph-mon[105151]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:43:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:00 smithi134 ceph-mon[33541]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:43:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:01.131 INFO:tasks.rados.rados.0.smithi017.stdout:1195: done (2 left) 2024-09-21T09:43:01.132 INFO:tasks.rados.rados.0.smithi017.stdout:1196: done (1 left) 2024-09-21T09:43:01.132 INFO:tasks.rados.rados.0.smithi017.stdout:1197: done (0 left) 2024-09-21T09:43:01.132 INFO:tasks.rados.rados.0.smithi017.stdout:1198: write oid 29 current snap is 152 2024-09-21T09:43:01.132 INFO:tasks.rados.rados.0.smithi017.stdout:1198: seq_num 466 ranges {632062=563567,1885903=527817,2830644=720758} 2024-09-21T09:43:01.138 INFO:tasks.rados.rados.0.smithi017.stdout:1198: writing smithi01785587-29 from 632062 to 1195629 tid 1 2024-09-21T09:43:01.144 INFO:tasks.rados.rados.0.smithi017.stdout:1198: writing smithi01785587-29 from 1885903 to 2413720 tid 2 2024-09-21T09:43:01.151 INFO:tasks.rados.rados.0.smithi017.stdout:1198: writing smithi01785587-29 from 2830644 to 3551402 tid 3 2024-09-21T09:43:01.151 INFO:tasks.rados.rados.0.smithi017.stdout:1199: write oid 41 current snap is 152 2024-09-21T09:43:01.151 INFO:tasks.rados.rados.0.smithi017.stdout:1199: seq_num 467 ranges {648869=638816,1828032=787828,3141714=532030} 2024-09-21T09:43:01.158 INFO:tasks.rados.rados.0.smithi017.stdout:1199: writing smithi01785587-41 from 648869 to 1287685 tid 1 2024-09-21T09:43:01.166 INFO:tasks.rados.rados.0.smithi017.stdout:1199: writing smithi01785587-41 from 1828032 to 2615860 tid 2 2024-09-21T09:43:01.172 INFO:tasks.rados.rados.0.smithi017.stdout:1199: writing smithi01785587-41 from 3141714 to 3673744 tid 3 2024-09-21T09:43:01.172 INFO:tasks.rados.rados.0.smithi017.stdout:1200: read oid 24 snap -1 2024-09-21T09:43:01.172 INFO:tasks.rados.rados.0.smithi017.stdout:1200: expect deleted 2024-09-21T09:43:01.172 INFO:tasks.rados.rados.0.smithi017.stdout:1201: snap_remove snap 137 2024-09-21T09:43:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:01 smithi017 ceph-mon[105151]: pgmap v116: 89 pgs: 89 active+clean; 174 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:43:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:01 smithi017 ceph-mon[105151]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:43:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:01 smithi017 ceph-mon[102913]: pgmap v116: 89 pgs: 89 active+clean; 174 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:43:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:01 smithi017 ceph-mon[102913]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:43:01.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:01 smithi134 ceph-mon[33541]: pgmap v116: 89 pgs: 89 active+clean; 174 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:43:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:01 smithi134 ceph-mon[33541]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1201: done (3 left) 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1202: snap_create 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1198: finishing write tid 1 to smithi01785587-29 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1198: finishing write tid 2 to smithi01785587-29 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1198: finishing write tid 3 to smithi01785587-29 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1198: finishing write tid 4 to smithi01785587-29 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1198: finishing write tid 5 to smithi01785587-29 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 238 (ObjNum 466 snap 152 seq_num 466) dirty exists 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1198: left oid 29 (ObjNum 466 snap 152 seq_num 466) 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1199: finishing write tid 1 to smithi01785587-41 2024-09-21T09:43:02.138 INFO:tasks.rados.rados.0.smithi017.stdout:1199: finishing write tid 2 to smithi01785587-41 2024-09-21T09:43:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:1199: finishing write tid 3 to smithi01785587-41 2024-09-21T09:43:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:1199: finishing write tid 4 to smithi01785587-41 2024-09-21T09:43:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:1199: finishing write tid 5 to smithi01785587-41 2024-09-21T09:43:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 206 (ObjNum 467 snap 152 seq_num 467) dirty exists 2024-09-21T09:43:02.139 INFO:tasks.rados.rados.0.smithi017.stdout:1199: left oid 41 (ObjNum 467 snap 152 seq_num 467) 2024-09-21T09:43:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:02 smithi017 ceph-mon[102913]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:43:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:02 smithi017 ceph-mon[105151]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:43:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:02 smithi134 ceph-mon[33541]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:43:03.035 INFO:tasks.rados.rados.0.smithi017.stdout:1198: done (3 left) 2024-09-21T09:43:03.035 INFO:tasks.rados.rados.0.smithi017.stdout:1199: done (2 left) 2024-09-21T09:43:03.035 INFO:tasks.rados.rados.0.smithi017.stdout:1200: done (1 left) 2024-09-21T09:43:03.035 INFO:tasks.rados.rados.0.smithi017.stdout:1202: done (0 left) 2024-09-21T09:43:03.036 INFO:tasks.rados.rados.0.smithi017.stdout:1203: read oid 37 snap -1 2024-09-21T09:43:03.036 INFO:tasks.rados.rados.0.smithi017.stdout:1203: expect deleted 2024-09-21T09:43:03.036 INFO:tasks.rados.rados.0.smithi017.stdout:1204: snap_remove snap 136 2024-09-21T09:43:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:03 smithi017 ceph-mon[102913]: pgmap v119: 89 pgs: 89 active+clean; 175 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 856 KiB/s wr, 7 op/s 2024-09-21T09:43:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:03 smithi017 ceph-mon[102913]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:43:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:03 smithi017 ceph-mon[102913]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:43:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:03 smithi017 ceph-mon[105151]: pgmap v119: 89 pgs: 89 active+clean; 175 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 856 KiB/s wr, 7 op/s 2024-09-21T09:43:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:03 smithi017 ceph-mon[105151]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:43:03.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:03 smithi017 ceph-mon[105151]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:43:03.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:03 smithi134 ceph-mon[33541]: pgmap v119: 89 pgs: 89 active+clean; 175 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 856 KiB/s wr, 7 op/s 2024-09-21T09:43:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:03 smithi134 ceph-mon[33541]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:43:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:03 smithi134 ceph-mon[33541]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:43:04.037 INFO:tasks.rados.rados.0.smithi017.stdout:1204: done (1 left) 2024-09-21T09:43:04.038 INFO:tasks.rados.rados.0.smithi017.stdout:1205: rollback oid 41 current snap is 153 2024-09-21T09:43:04.038 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 145 2024-09-21T09:43:04.038 INFO:tasks.rados.rados.0.smithi017.stdout:1203: done (1 left) 2024-09-21T09:43:04.038 INFO:tasks.rados.rados.0.smithi017.stdout:1206: rollback oid 10 current snap is 153 2024-09-21T09:43:04.038 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 146 2024-09-21T09:43:04.038 INFO:tasks.rados.rados.0.smithi017.stdout:1207: snap_remove snap 138 2024-09-21T09:43:04.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:05.034 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:43:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:43:04] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-21T09:43:05.042 INFO:tasks.rados.rados.0.smithi017.stdout:1207: done (2 left) 2024-09-21T09:43:05.043 INFO:tasks.rados.rados.0.smithi017.stdout:1208: write oid 20 current snap is 153 2024-09-21T09:43:05.043 INFO:tasks.rados.rados.0.smithi017.stdout:1205: finishing rollback tid 0 to smithi01785587-41 2024-09-21T09:43:05.043 INFO:tasks.rados.rados.0.smithi017.stdout:1208: seq_num 468 ranges {665676=714065,1886514=531486,3169137=696539} 2024-09-21T09:43:05.050 INFO:tasks.rados.rados.0.smithi017.stdout:1208: writing smithi01785587-20 from 665676 to 1379741 tid 1 2024-09-21T09:43:05.055 INFO:tasks.rados.rados.0.smithi017.stdout:1208: writing smithi01785587-20 from 1886514 to 2418000 tid 2 2024-09-21T09:43:05.065 INFO:tasks.rados.rados.0.smithi017.stdout:1208: writing smithi01785587-20 from 3169137 to 3865676 tid 3 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:1209: rollback oid 44 current snap is 153 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 152 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:1206: finishing rollback tid 0 to smithi01785587-10 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:1205: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 209 (ObjNum 257 snap 0 seq_num 4181695952) dirty dne 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:1206: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:1206: finishing rollback tid 2 to smithi01785587-10 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 66 (ObjNum 434 snap 137 seq_num 434) dirty exists 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:1208: finishing write tid 1 to smithi01785587-20 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:1208: finishing write tid 2 to smithi01785587-20 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:1205: done (3 left) 2024-09-21T09:43:05.066 INFO:tasks.rados.rados.0.smithi017.stdout:1206: done (2 left) 2024-09-21T09:43:05.067 INFO:tasks.rados.rados.0.smithi017.stdout:1210: rollback oid 28 current snap is 153 2024-09-21T09:43:05.067 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 28 to 125 2024-09-21T09:43:05.067 INFO:tasks.rados.rados.0.smithi017.stdout:1211: delete oid 2 current snap is 153 2024-09-21T09:43:05.067 INFO:tasks.rados.rados.0.smithi017.stdout:1211: done (3 left) 2024-09-21T09:43:05.067 INFO:tasks.rados.rados.0.smithi017.stdout:1212: write oid 8 current snap is 153 2024-09-21T09:43:05.067 INFO:tasks.rados.rados.0.smithi017.stdout:1212: seq_num 470 ranges {699290=464563,1603478=535155,2823983=765846} 2024-09-21T09:43:05.073 INFO:tasks.rados.rados.0.smithi017.stdout:1212: writing smithi01785587-8 from 699290 to 1163853 tid 1 2024-09-21T09:43:05.080 INFO:tasks.rados.rados.0.smithi017.stdout:1212: writing smithi01785587-8 from 1603478 to 2138633 tid 2 2024-09-21T09:43:05.089 INFO:tasks.rados.rados.0.smithi017.stdout:1212: writing smithi01785587-8 from 2823983 to 3589829 tid 3 2024-09-21T09:43:05.089 INFO:tasks.rados.rados.0.smithi017.stdout:1213: snap_remove snap 121 2024-09-21T09:43:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:05 smithi017 ceph-mon[102913]: pgmap v122: 89 pgs: 89 active+clean; 175 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 856 KiB/s wr, 7 op/s 2024-09-21T09:43:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:05 smithi017 ceph-mon[102913]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T09:43:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:05 smithi017 ceph-mon[105151]: pgmap v122: 89 pgs: 89 active+clean; 175 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 856 KiB/s wr, 7 op/s 2024-09-21T09:43:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:05 smithi017 ceph-mon[105151]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T09:43:05.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:05 smithi134 ceph-mon[33541]: pgmap v122: 89 pgs: 89 active+clean; 175 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 856 KiB/s wr, 7 op/s 2024-09-21T09:43:05.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:05 smithi134 ceph-mon[33541]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T09:43:05.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:05 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:05.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:05 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:06.049 INFO:tasks.rados.rados.0.smithi017.stdout:1213: done (4 left) 2024-09-21T09:43:06.049 INFO:tasks.rados.rados.0.smithi017.stdout:1214: write oid 31 current snap is 153 2024-09-21T09:43:06.049 INFO:tasks.rados.rados.0.smithi017.stdout:1214: seq_num 471 ranges {716097=539812,1945607=795166,3535053=381044} 2024-09-21T09:43:06.057 INFO:tasks.rados.rados.0.smithi017.stdout:1214: writing smithi01785587-31 from 716097 to 1255909 tid 1 2024-09-21T09:43:06.066 INFO:tasks.rados.rados.0.smithi017.stdout:1214: writing smithi01785587-31 from 1945607 to 2740773 tid 2 2024-09-21T09:43:06.074 INFO:tasks.rados.rados.0.smithi017.stdout:1214: writing smithi01785587-31 from 3535053 to 3916097 tid 3 2024-09-21T09:43:06.074 INFO:tasks.rados.rados.0.smithi017.stdout:1215: rollback oid 25 current snap is 153 2024-09-21T09:43:06.074 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 146 2024-09-21T09:43:06.074 INFO:tasks.rados.rados.0.smithi017.stdout:1209: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:43:06.074 INFO:tasks.rados.rados.0.smithi017.stdout:1209: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:43:06.074 INFO:tasks.rados.rados.0.smithi017.stdout:1208: finishing write tid 3 to smithi01785587-20 2024-09-21T09:43:06.074 INFO:tasks.rados.rados.0.smithi017.stdout:1210: finishing rollback tid 0 to smithi01785587-28 2024-09-21T09:43:06.074 INFO:tasks.rados.rados.0.smithi017.stdout:1209: finishing rollback tid 2 to smithi01785587-44 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 609 (ObjNum 323 snap 98 seq_num 323) dirty exists 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1208: finishing write tid 4 to smithi01785587-20 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1210: finishing rollback tid 1 to smithi01785587-28 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1208: finishing write tid 5 to smithi01785587-20 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 243 (ObjNum 468 snap 153 seq_num 468) dirty exists 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1208: left oid 20 (ObjNum 468 snap 153 seq_num 468) 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1210: finishing rollback tid 2 to smithi01785587-28 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 634 (ObjNum 358 snap 108 seq_num 358) dirty exists 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1212: finishing write tid 1 to smithi01785587-8 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1212: finishing write tid 2 to smithi01785587-8 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1212: finishing write tid 3 to smithi01785587-8 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1212: finishing write tid 4 to smithi01785587-8 2024-09-21T09:43:06.075 INFO:tasks.rados.rados.0.smithi017.stdout:1212: finishing write tid 5 to smithi01785587-8 2024-09-21T09:43:06.076 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 541 (ObjNum 470 snap 153 seq_num 470) dirty exists 2024-09-21T09:43:06.076 INFO:tasks.rados.rados.0.smithi017.stdout:1212: left oid 8 (ObjNum 470 snap 153 seq_num 470) 2024-09-21T09:43:06.076 INFO:tasks.rados.rados.0.smithi017.stdout:1214: finishing write tid 1 to smithi01785587-31 2024-09-21T09:43:06.076 INFO:tasks.rados.rados.0.smithi017.stdout:1214: finishing write tid 2 to smithi01785587-31 2024-09-21T09:43:06.076 INFO:tasks.rados.rados.0.smithi017.stdout:1208: done (5 left) 2024-09-21T09:43:06.076 INFO:tasks.rados.rados.0.smithi017.stdout:1209: done (4 left) 2024-09-21T09:43:06.076 INFO:tasks.rados.rados.0.smithi017.stdout:1210: done (3 left) 2024-09-21T09:43:06.076 INFO:tasks.rados.rados.0.smithi017.stdout:1212: done (2 left) 2024-09-21T09:43:06.076 INFO:tasks.rados.rados.0.smithi017.stdout:1216: snap_remove snap 151 2024-09-21T09:43:06.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:06 smithi017 ceph-mon[102913]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T09:43:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:06 smithi017 ceph-mon[105151]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T09:43:06.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:06 smithi134 ceph-mon[33541]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T09:43:07.057 INFO:tasks.rados.rados.0.smithi017.stdout:1216: done (2 left) 2024-09-21T09:43:07.058 INFO:tasks.rados.rados.0.smithi017.stdout:1217: delete oid 45 current snap is 153 2024-09-21T09:43:07.058 INFO:tasks.rados.rados.0.smithi017.stdout:1215: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:43:07.058 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:43:07.058 INFO:tasks.rados.rados.0.smithi017.stdout:1214: finishing write tid 3 to smithi01785587-31 2024-09-21T09:43:07.058 INFO:tasks.rados.rados.0.smithi017.stdout:1214: finishing write tid 4 to smithi01785587-31 2024-09-21T09:43:07.058 INFO:tasks.rados.rados.0.smithi017.stdout:1214: finishing write tid 5 to smithi01785587-31 2024-09-21T09:43:07.058 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 620 (ObjNum 471 snap 153 seq_num 471) dirty exists 2024-09-21T09:43:07.058 INFO:tasks.rados.rados.0.smithi017.stdout:1214: left oid 31 (ObjNum 471 snap 153 seq_num 471) 2024-09-21T09:43:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1214: done (2 left) 2024-09-21T09:43:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1215: done (1 left) 2024-09-21T09:43:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1217: done (0 left) 2024-09-21T09:43:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1218: write oid 50 current snap is 153 2024-09-21T09:43:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1218: seq_num 473 ranges {749711=406663,1778924=682482,3189899=716015} 2024-09-21T09:43:07.070 INFO:tasks.rados.rados.0.smithi017.stdout:1218: writing smithi01785587-50 from 749711 to 1156374 tid 1 2024-09-21T09:43:07.077 INFO:tasks.rados.rados.0.smithi017.stdout:1218: writing smithi01785587-50 from 1778924 to 2461406 tid 2 2024-09-21T09:43:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1218: writing smithi01785587-50 from 3189899 to 3905914 tid 3 2024-09-21T09:43:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1219: read oid 31 snap 134 2024-09-21T09:43:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1219: expect (ObjNum 392 snap 121 seq_num 392) 2024-09-21T09:43:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1220: write oid 34 current snap is 153 2024-09-21T09:43:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1218: finishing write tid 1 to smithi01785587-50 2024-09-21T09:43:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1218: finishing write tid 2 to smithi01785587-50 2024-09-21T09:43:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1220: seq_num 474 ranges {766518=481912,1837406=542493,2933675=527287,3876584=89934} 2024-09-21T09:43:07.093 INFO:tasks.rados.rados.0.smithi017.stdout:1220: writing smithi01785587-34 from 766518 to 1248430 tid 1 2024-09-21T09:43:07.099 INFO:tasks.rados.rados.0.smithi017.stdout:1220: writing smithi01785587-34 from 1837406 to 2379899 tid 2 2024-09-21T09:43:07.105 INFO:tasks.rados.rados.0.smithi017.stdout:1220: writing smithi01785587-34 from 2933675 to 3460962 tid 3 2024-09-21T09:43:07.108 INFO:tasks.rados.rados.0.smithi017.stdout:1220: writing smithi01785587-34 from 3876584 to 3966518 tid 4 2024-09-21T09:43:07.109 INFO:tasks.rados.rados.0.smithi017.stdout:1221: write oid 10 current snap is 153 2024-09-21T09:43:07.109 INFO:tasks.rados.rados.0.smithi017.stdout:1221: seq_num 475 ranges {783325=557161,1779535=686151,3244745=454912} 2024-09-21T09:43:07.115 INFO:tasks.rados.rados.0.smithi017.stdout:1221: writing smithi01785587-10 from 783325 to 1340486 tid 1 2024-09-21T09:43:07.122 INFO:tasks.rados.rados.0.smithi017.stdout:1221: writing smithi01785587-10 from 1779535 to 2465686 tid 2 2024-09-21T09:43:07.129 INFO:tasks.rados.rados.0.smithi017.stdout:1221: writing smithi01785587-10 from 3244745 to 3699657 tid 3 2024-09-21T09:43:07.129 INFO:tasks.rados.rados.0.smithi017.stdout:1222: write oid 42 current snap is 153 2024-09-21T09:43:07.129 INFO:tasks.rados.rados.0.smithi017.stdout:1222: seq_num 476 ranges {400132=632410,1438017=546162} 2024-09-21T09:43:07.135 INFO:tasks.rados.rados.0.smithi017.stdout:1222: writing smithi01785587-42 from 400132 to 1032542 tid 1 2024-09-21T09:43:07.140 INFO:tasks.rados.rados.0.smithi017.stdout:1222: writing smithi01785587-42 from 1438017 to 1984179 tid 2 2024-09-21T09:43:07.140 INFO:tasks.rados.rados.0.smithi017.stdout:1223: snap_remove snap 150 2024-09-21T09:43:07.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:07 smithi134 ceph-mon[33541]: pgmap v125: 89 pgs: 89 active+clean; 175 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:43:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:07 smithi134 ceph-mon[33541]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T09:43:07.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:06 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:07 smithi017 ceph-mon[102913]: pgmap v125: 89 pgs: 89 active+clean; 175 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:43:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:07 smithi017 ceph-mon[102913]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T09:43:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:07 smithi017 ceph-mon[105151]: pgmap v125: 89 pgs: 89 active+clean; 175 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:43:07.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:07 smithi017 ceph-mon[105151]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T09:43:08.067 INFO:tasks.rados.rados.0.smithi017.stdout:1223: done (5 left) 2024-09-21T09:43:08.067 INFO:tasks.rados.rados.0.smithi017.stdout:1224: snap_remove snap 148 2024-09-21T09:43:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:08 smithi017 ceph-mon[102913]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T09:43:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:08 smithi017 ceph-mon[105151]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T09:43:08.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:08 smithi134 ceph-mon[33541]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T09:43:09.061 INFO:tasks.rados.rados.0.smithi017.stdout:1224: done (5 left) 2024-09-21T09:43:09.061 INFO:tasks.rados.rados.0.smithi017.stdout:1225: delete oid 18 current snap is 153 2024-09-21T09:43:09.062 INFO:tasks.rados.rados.0.smithi017.stdout:1218: finishing write tid 3 to smithi01785587-50 2024-09-21T09:43:09.062 INFO:tasks.rados.rados.0.smithi017.stdout:1218: finishing write tid 4 to smithi01785587-50 2024-09-21T09:43:09.062 INFO:tasks.rados.rados.0.smithi017.stdout:1218: finishing write tid 5 to smithi01785587-50 2024-09-21T09:43:09.062 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 384 (ObjNum 473 snap 153 seq_num 473) dirty exists 2024-09-21T09:43:09.062 INFO:tasks.rados.rados.0.smithi017.stdout:1218: left oid 50 (ObjNum 473 snap 153 seq_num 473) 2024-09-21T09:43:09.062 INFO:tasks.rados.rados.0.smithi017.stdout:1219: expect (ObjNum 392 snap 121 seq_num 392) 2024-09-21T09:43:09.211 INFO:tasks.rados.rados.0.smithi017.stdout:1220: finishing write tid 1 to smithi01785587-34 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1220: finishing write tid 2 to smithi01785587-34 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1220: finishing write tid 3 to smithi01785587-34 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1220: finishing write tid 4 to smithi01785587-34 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1220: finishing write tid 5 to smithi01785587-34 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1220: finishing write tid 6 to smithi01785587-34 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 637 (ObjNum 474 snap 153 seq_num 474) dirty exists 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1220: left oid 34 (ObjNum 474 snap 153 seq_num 474) 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1221: finishing write tid 1 to smithi01785587-10 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1221: finishing write tid 2 to smithi01785587-10 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1221: finishing write tid 3 to smithi01785587-10 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1221: finishing write tid 4 to smithi01785587-10 2024-09-21T09:43:09.212 INFO:tasks.rados.rados.0.smithi017.stdout:1221: finishing write tid 5 to smithi01785587-10 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 78 (ObjNum 475 snap 153 seq_num 475) dirty exists 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1221: left oid 10 (ObjNum 475 snap 153 seq_num 475) 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1222: finishing write tid 1 to smithi01785587-42 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1222: finishing write tid 2 to smithi01785587-42 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1222: finishing write tid 3 to smithi01785587-42 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1218: done (5 left) 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1219: done (4 left) 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1220: done (3 left) 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1221: done (2 left) 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1225: done (1 left) 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1226: delete oid 34 current snap is 153 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1222: finishing write tid 4 to smithi01785587-42 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 462 (ObjNum 476 snap 153 seq_num 476) dirty exists 2024-09-21T09:43:09.213 INFO:tasks.rados.rados.0.smithi017.stdout:1222: left oid 42 (ObjNum 476 snap 153 seq_num 476) 2024-09-21T09:43:09.215 INFO:tasks.rados.rados.0.smithi017.stdout:1222: done (1 left) 2024-09-21T09:43:09.215 INFO:tasks.rados.rados.0.smithi017.stdout:1226: done (0 left) 2024-09-21T09:43:09.215 INFO:tasks.rados.rados.0.smithi017.stdout:1227: write oid 2 current snap is 153 2024-09-21T09:43:09.215 INFO:tasks.rados.rados.0.smithi017.stdout:1227: seq_num 479 ranges {450553=574510,1613463=437090} 2024-09-21T09:43:09.221 INFO:tasks.rados.rados.0.smithi017.stdout:1227: writing smithi01785587-2 from 450553 to 1025063 tid 1 2024-09-21T09:43:09.227 INFO:tasks.rados.rados.0.smithi017.stdout:1227: writing smithi01785587-2 from 1613463 to 2050553 tid 2 2024-09-21T09:43:09.227 INFO:tasks.rados.rados.0.smithi017.stdout:1228: snap_remove snap 146 2024-09-21T09:43:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:09 smithi017 ceph-mon[102913]: pgmap v128: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-21T09:43:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:09 smithi017 ceph-mon[102913]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T09:43:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:09 smithi017 ceph-mon[105151]: pgmap v128: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-21T09:43:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:09 smithi017 ceph-mon[105151]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T09:43:09.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:09 smithi134 ceph-mon[33541]: pgmap v128: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-21T09:43:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:09 smithi134 ceph-mon[33541]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T09:43:10.072 INFO:tasks.rados.rados.0.smithi017.stdout:1228: done (1 left) 2024-09-21T09:43:10.072 INFO:tasks.rados.rados.0.smithi017.stdout:1229: rollback oid 34 current snap is 153 2024-09-21T09:43:10.072 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 130 2024-09-21T09:43:10.072 INFO:tasks.rados.rados.0.smithi017.stdout:1227: finishing write tid 1 to smithi01785587-2 2024-09-21T09:43:10.072 INFO:tasks.rados.rados.0.smithi017.stdout:1227: finishing write tid 2 to smithi01785587-2 2024-09-21T09:43:10.073 INFO:tasks.rados.rados.0.smithi017.stdout:1227: finishing write tid 3 to smithi01785587-2 2024-09-21T09:43:10.073 INFO:tasks.rados.rados.0.smithi017.stdout:1227: finishing write tid 4 to smithi01785587-2 2024-09-21T09:43:10.073 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 250 (ObjNum 479 snap 153 seq_num 479) dirty exists 2024-09-21T09:43:10.073 INFO:tasks.rados.rados.0.smithi017.stdout:1227: left oid 2 (ObjNum 479 snap 153 seq_num 479) 2024-09-21T09:43:10.073 INFO:tasks.rados.rados.0.smithi017.stdout:1227: done (1 left) 2024-09-21T09:43:10.073 INFO:tasks.rados.rados.0.smithi017.stdout:1230: read oid 15 snap -1 2024-09-21T09:43:10.073 INFO:tasks.rados.rados.0.smithi017.stdout:1230: expect (ObjNum 410 snap 126 seq_num 410) 2024-09-21T09:43:10.073 INFO:tasks.rados.rados.0.smithi017.stdout:1231: rollback oid 31 current snap is 153 2024-09-21T09:43:10.074 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 31 to 152 2024-09-21T09:43:10.074 INFO:tasks.rados.rados.0.smithi017.stdout:1232: read oid 25 snap -1 2024-09-21T09:43:10.074 INFO:tasks.rados.rados.0.smithi017.stdout:1232: expect deleted 2024-09-21T09:43:10.074 INFO:tasks.rados.rados.0.smithi017.stdout:1233: rollback oid 17 current snap is 153 2024-09-21T09:43:10.074 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 152 2024-09-21T09:43:10.074 INFO:tasks.rados.rados.0.smithi017.stdout:1234: write oid 46 current snap is 153 2024-09-21T09:43:10.074 INFO:tasks.rados.rados.0.smithi017.stdout:1234: seq_num 480 ranges {467360=649759,1671945=395415} 2024-09-21T09:43:10.081 INFO:tasks.rados.rados.0.smithi017.stdout:1234: writing smithi01785587-46 from 467360 to 1117119 tid 1 2024-09-21T09:43:10.086 INFO:tasks.rados.rados.0.smithi017.stdout:1234: writing smithi01785587-46 from 1671945 to 2067360 tid 2 2024-09-21T09:43:10.086 INFO:tasks.rados.rados.0.smithi017.stdout:1235: read oid 27 snap -1 2024-09-21T09:43:10.086 INFO:tasks.rados.rados.0.smithi017.stdout:1235: expect deleted 2024-09-21T09:43:10.087 INFO:tasks.rados.rados.0.smithi017.stdout:1236: write oid 10 current snap is 153 2024-09-21T09:43:10.087 INFO:tasks.rados.rados.0.smithi017.stdout:1236: seq_num 481 ranges {484167=725008,1730427=353740} 2024-09-21T09:43:10.093 INFO:tasks.rados.rados.0.smithi017.stdout:1236: writing smithi01785587-10 from 484167 to 1209175 tid 1 2024-09-21T09:43:10.098 INFO:tasks.rados.rados.0.smithi017.stdout:1236: writing smithi01785587-10 from 1730427 to 2084167 tid 2 2024-09-21T09:43:10.098 INFO:tasks.rados.rados.0.smithi017.stdout:1237: write oid 29 current snap is 153 2024-09-21T09:43:10.098 INFO:tasks.rados.rados.0.smithi017.stdout:1237: seq_num 482 ranges {500974=400257,1388909=557169} 2024-09-21T09:43:10.103 INFO:tasks.rados.rados.0.smithi017.stdout:1237: writing smithi01785587-29 from 500974 to 901231 tid 1 2024-09-21T09:43:10.110 INFO:tasks.rados.rados.0.smithi017.stdout:1237: writing smithi01785587-29 from 1388909 to 1946078 tid 2 2024-09-21T09:43:10.110 INFO:tasks.rados.rados.0.smithi017.stdout:1238: write oid 36 current snap is 153 2024-09-21T09:43:10.110 INFO:tasks.rados.rados.0.smithi017.stdout:1238: seq_num 483 ranges {517781=475506,1731038=386743} 2024-09-21T09:43:10.114 INFO:tasks.rados.rados.0.smithi017.stdout:1238: writing smithi01785587-36 from 517781 to 993287 tid 1 2024-09-21T09:43:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1238: writing smithi01785587-36 from 1731038 to 2117781 tid 2 2024-09-21T09:43:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1239: snap_create 2024-09-21T09:43:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1233: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:43:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 436 (ObjNum 49 snap 0 seq_num 2384147568) dirty dne 2024-09-21T09:43:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1229: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:43:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1230: expect (ObjNum 410 snap 126 seq_num 410) 2024-09-21T09:43:10.216 INFO:tasks.rados.rados.0.smithi017.stdout:1229: finishing rollback tid 2 to smithi01785587-34 2024-09-21T09:43:10.216 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 646 (ObjNum 401 snap 122 seq_num 401) dirty exists 2024-09-21T09:43:10.216 INFO:tasks.rados.rados.0.smithi017.stdout:1231: finishing rollback tid 0 to smithi01785587-31 2024-09-21T09:43:10.216 INFO:tasks.rados.rados.0.smithi017.stdout:1231: finishing rollback tid 1 to smithi01785587-31 2024-09-21T09:43:10.216 INFO:tasks.rados.rados.0.smithi017.stdout:1231: finishing rollback tid 2 to smithi01785587-31 2024-09-21T09:43:10.216 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 637 (ObjNum 438 snap 137 seq_num 438) dirty exists 2024-09-21T09:43:10.216 INFO:tasks.rados.rados.0.smithi017.stdout:1234: finishing write tid 1 to smithi01785587-46 2024-09-21T09:43:10.217 INFO:tasks.rados.rados.0.smithi017.stdout:1234: finishing write tid 2 to smithi01785587-46 2024-09-21T09:43:10.217 INFO:tasks.rados.rados.0.smithi017.stdout:1234: finishing write tid 3 to smithi01785587-46 2024-09-21T09:43:10.217 INFO:tasks.rados.rados.0.smithi017.stdout:1234: finishing write tid 4 to smithi01785587-46 2024-09-21T09:43:10.217 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 561 (ObjNum 480 snap 153 seq_num 480) dirty exists 2024-09-21T09:43:10.217 INFO:tasks.rados.rados.0.smithi017.stdout:1234: left oid 46 (ObjNum 480 snap 153 seq_num 480) 2024-09-21T09:43:10.217 INFO:tasks.rados.rados.0.smithi017.stdout:1236: finishing write tid 1 to smithi01785587-10 2024-09-21T09:43:10.217 INFO:tasks.rados.rados.0.smithi017.stdout:1236: finishing write tid 2 to smithi01785587-10 2024-09-21T09:43:10.217 INFO:tasks.rados.rados.0.smithi017.stdout:1236: finishing write tid 3 to smithi01785587-10 2024-09-21T09:43:10.218 INFO:tasks.rados.rados.0.smithi017.stdout:1236: finishing write tid 4 to smithi01785587-10 2024-09-21T09:43:10.218 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 87 (ObjNum 481 snap 153 seq_num 481) dirty exists 2024-09-21T09:43:10.218 INFO:tasks.rados.rados.0.smithi017.stdout:1236: left oid 10 (ObjNum 481 snap 153 seq_num 481) 2024-09-21T09:43:10.218 INFO:tasks.rados.rados.0.smithi017.stdout:1237: finishing write tid 1 to smithi01785587-29 2024-09-21T09:43:10.218 INFO:tasks.rados.rados.0.smithi017.stdout:1237: finishing write tid 2 to smithi01785587-29 2024-09-21T09:43:10.218 INFO:tasks.rados.rados.0.smithi017.stdout:1237: finishing write tid 3 to smithi01785587-29 2024-09-21T09:43:10.218 INFO:tasks.rados.rados.0.smithi017.stdout:1237: finishing write tid 4 to smithi01785587-29 2024-09-21T09:43:10.218 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 265 (ObjNum 482 snap 153 seq_num 482) dirty exists 2024-09-21T09:43:10.218 INFO:tasks.rados.rados.0.smithi017.stdout:1237: left oid 29 (ObjNum 482 snap 153 seq_num 482) 2024-09-21T09:43:10.219 INFO:tasks.rados.rados.0.smithi017.stdout:1238: finishing write tid 1 to smithi01785587-36 2024-09-21T09:43:10.219 INFO:tasks.rados.rados.0.smithi017.stdout:1238: finishing write tid 2 to smithi01785587-36 2024-09-21T09:43:10.219 INFO:tasks.rados.rados.0.smithi017.stdout:1238: finishing write tid 3 to smithi01785587-36 2024-09-21T09:43:10.219 INFO:tasks.rados.rados.0.smithi017.stdout:1238: finishing write tid 4 to smithi01785587-36 2024-09-21T09:43:10.219 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 611 (ObjNum 483 snap 153 seq_num 483) dirty exists 2024-09-21T09:43:10.219 INFO:tasks.rados.rados.0.smithi017.stdout:1238: left oid 36 (ObjNum 483 snap 153 seq_num 483) 2024-09-21T09:43:10.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:10 smithi017 ceph-mon[102913]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T09:43:10.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:10 smithi017 ceph-mon[105151]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T09:43:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:10 smithi134 ceph-mon[33541]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T09:43:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:11.077 INFO:tasks.rados.rados.0.smithi017.stdout:1229: done (10 left) 2024-09-21T09:43:11.077 INFO:tasks.rados.rados.0.smithi017.stdout:1230: done (9 left) 2024-09-21T09:43:11.077 INFO:tasks.rados.rados.0.smithi017.stdout:1231: done (8 left) 2024-09-21T09:43:11.077 INFO:tasks.rados.rados.0.smithi017.stdout:1232: done (7 left) 2024-09-21T09:43:11.078 INFO:tasks.rados.rados.0.smithi017.stdout:1233: done (6 left) 2024-09-21T09:43:11.078 INFO:tasks.rados.rados.0.smithi017.stdout:1234: done (5 left) 2024-09-21T09:43:11.078 INFO:tasks.rados.rados.0.smithi017.stdout:1235: done (4 left) 2024-09-21T09:43:11.078 INFO:tasks.rados.rados.0.smithi017.stdout:1236: done (3 left) 2024-09-21T09:43:11.078 INFO:tasks.rados.rados.0.smithi017.stdout:1237: done (2 left) 2024-09-21T09:43:11.078 INFO:tasks.rados.rados.0.smithi017.stdout:1238: done (1 left) 2024-09-21T09:43:11.078 INFO:tasks.rados.rados.0.smithi017.stdout:1239: done (0 left) 2024-09-21T09:43:11.078 INFO:tasks.rados.rados.0.smithi017.stdout:1240: delete oid 32 current snap is 154 2024-09-21T09:43:11.081 INFO:tasks.rados.rados.0.smithi017.stdout:1240: done (0 left) 2024-09-21T09:43:11.081 INFO:tasks.rados.rados.0.smithi017.stdout:1241: delete oid 41 current snap is 154 2024-09-21T09:43:11.083 INFO:tasks.rados.rados.0.smithi017.stdout:1241: done (0 left) 2024-09-21T09:43:11.083 INFO:tasks.rados.rados.0.smithi017.stdout:1242: snap_create 2024-09-21T09:43:11.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:11 smithi017 ceph-mon[102913]: pgmap v131: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-21T09:43:11.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:11 smithi017 ceph-mon[102913]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T09:43:11.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:11.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:11 smithi017 ceph-mon[105151]: pgmap v131: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-21T09:43:11.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:11 smithi017 ceph-mon[105151]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T09:43:11.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:11 smithi134 ceph-mon[33541]: pgmap v131: 89 pgs: 89 active+clean; 181 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 7 op/s 2024-09-21T09:43:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:11 smithi134 ceph-mon[33541]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T09:43:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:11 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:12.090 INFO:tasks.rados.rados.0.smithi017.stdout:1242: done (0 left) 2024-09-21T09:43:12.090 INFO:tasks.rados.rados.0.smithi017.stdout:1243: delete oid 34 current snap is 155 2024-09-21T09:43:12.093 INFO:tasks.rados.rados.0.smithi017.stdout:1243: done (0 left) 2024-09-21T09:43:12.093 INFO:tasks.rados.rados.0.smithi017.stdout:1244: write oid 46 current snap is 155 2024-09-21T09:43:12.093 INFO:tasks.rados.rados.0.smithi017.stdout:1244: seq_num 487 ranges {585009=492855,1564966=620043} 2024-09-21T09:43:12.099 INFO:tasks.rados.rados.0.smithi017.stdout:1244: writing smithi01785587-46 from 585009 to 1077864 tid 1 2024-09-21T09:43:12.105 INFO:tasks.rados.rados.0.smithi017.stdout:1244: writing smithi01785587-46 from 1564966 to 2185009 tid 2 2024-09-21T09:43:12.105 INFO:tasks.rados.rados.0.smithi017.stdout:1245: write oid 48 current snap is 155 2024-09-21T09:43:12.105 INFO:tasks.rados.rados.0.smithi017.stdout:1245: seq_num 488 ranges {601816=568104,1623448=451823} 2024-09-21T09:43:12.111 INFO:tasks.rados.rados.0.smithi017.stdout:1245: writing smithi01785587-48 from 601816 to 1169920 tid 1 2024-09-21T09:43:12.116 INFO:tasks.rados.rados.0.smithi017.stdout:1245: writing smithi01785587-48 from 1623448 to 2075271 tid 2 2024-09-21T09:43:12.116 INFO:tasks.rados.rados.0.smithi017.stdout:1246: write oid 18 current snap is 155 2024-09-21T09:43:12.117 INFO:tasks.rados.rados.0.smithi017.stdout:1246: seq_num 489 ranges {618623=643353,1681930=536693} 2024-09-21T09:43:12.123 INFO:tasks.rados.rados.0.smithi017.stdout:1246: writing smithi01785587-18 from 618623 to 1261976 tid 1 2024-09-21T09:43:12.129 INFO:tasks.rados.rados.0.smithi017.stdout:1246: writing smithi01785587-18 from 1681930 to 2218623 tid 2 2024-09-21T09:43:12.129 INFO:tasks.rados.rados.0.smithi017.stdout:1247: snap_remove snap 130 2024-09-21T09:43:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:12 smithi017 ceph-mon[102913]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T09:43:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:12 smithi017 ceph-mon[105151]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T09:43:12.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:12 smithi134 ceph-mon[33541]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T09:43:13.036 INFO:tasks.rados.rados.0.smithi017.stdout:1247: done (3 left) 2024-09-21T09:43:13.036 INFO:tasks.rados.rados.0.smithi017.stdout:1248: write oid 6 current snap is 155 2024-09-21T09:43:13.036 INFO:tasks.rados.rados.0.smithi017.stdout:1244: finishing write tid 1 to smithi01785587-46 2024-09-21T09:43:13.037 INFO:tasks.rados.rados.0.smithi017.stdout:1248: seq_num 490 ranges {635430=718602,2140412=95018} 2024-09-21T09:43:13.045 INFO:tasks.rados.rados.0.smithi017.stdout:1248: writing smithi01785587-6 from 635430 to 1354032 tid 1 2024-09-21T09:43:13.051 INFO:tasks.rados.rados.0.smithi017.stdout:1248: writing smithi01785587-6 from 2140412 to 2235430 tid 2 2024-09-21T09:43:13.051 INFO:tasks.rados.rados.0.smithi017.stdout:1249: snap_create 2024-09-21T09:43:13.051 INFO:tasks.rados.rados.0.smithi017.stdout:1244: finishing write tid 2 to smithi01785587-46 2024-09-21T09:43:13.051 INFO:tasks.rados.rados.0.smithi017.stdout:1244: finishing write tid 3 to smithi01785587-46 2024-09-21T09:43:13.051 INFO:tasks.rados.rados.0.smithi017.stdout:1244: finishing write tid 4 to smithi01785587-46 2024-09-21T09:43:13.051 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 567 (ObjNum 487 snap 155 seq_num 487) dirty exists 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1244: left oid 46 (ObjNum 487 snap 155 seq_num 487) 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1245: finishing write tid 1 to smithi01785587-48 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1245: finishing write tid 2 to smithi01785587-48 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1245: finishing write tid 3 to smithi01785587-48 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1245: finishing write tid 4 to smithi01785587-48 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 602 (ObjNum 488 snap 155 seq_num 488) dirty exists 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1245: left oid 48 (ObjNum 488 snap 155 seq_num 488) 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1246: finishing write tid 1 to smithi01785587-18 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1246: finishing write tid 2 to smithi01785587-18 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1246: finishing write tid 3 to smithi01785587-18 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1246: finishing write tid 4 to smithi01785587-18 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 105 (ObjNum 489 snap 155 seq_num 489) dirty exists 2024-09-21T09:43:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1246: left oid 18 (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:43:13.053 INFO:tasks.rados.rados.0.smithi017.stdout:1248: finishing write tid 1 to smithi01785587-6 2024-09-21T09:43:13.053 INFO:tasks.rados.rados.0.smithi017.stdout:1248: finishing write tid 2 to smithi01785587-6 2024-09-21T09:43:13.053 INFO:tasks.rados.rados.0.smithi017.stdout:1248: finishing write tid 3 to smithi01785587-6 2024-09-21T09:43:13.053 INFO:tasks.rados.rados.0.smithi017.stdout:1248: finishing write tid 4 to smithi01785587-6 2024-09-21T09:43:13.053 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 395 (ObjNum 490 snap 155 seq_num 490) dirty exists 2024-09-21T09:43:13.053 INFO:tasks.rados.rados.0.smithi017.stdout:1248: left oid 6 (ObjNum 490 snap 155 seq_num 490) 2024-09-21T09:43:13.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:13 smithi134 ceph-mon[33541]: pgmap v134: 89 pgs: 89 active+clean; 192 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 3.1 MiB/s wr, 20 op/s 2024-09-21T09:43:13.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:13 smithi134 ceph-mon[33541]: osdmap e372: 8 total, 8 up, 8 in 2024-09-21T09:43:13.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:13 smithi134 ceph-mon[33541]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:43:13.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:13 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:13.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:13 smithi134 ceph-mon[33541]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T09:43:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[105151]: pgmap v134: 89 pgs: 89 active+clean; 192 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 3.1 MiB/s wr, 20 op/s 2024-09-21T09:43:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[105151]: osdmap e372: 8 total, 8 up, 8 in 2024-09-21T09:43:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[105151]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:43:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[105151]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T09:43:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[102913]: pgmap v134: 89 pgs: 89 active+clean; 192 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 3.1 MiB/s wr, 20 op/s 2024-09-21T09:43:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[102913]: osdmap e372: 8 total, 8 up, 8 in 2024-09-21T09:43:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[102913]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:43:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:13 smithi017 ceph-mon[102913]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T09:43:14.037 INFO:tasks.rados.rados.0.smithi017.stdout:1244: done (4 left) 2024-09-21T09:43:14.038 INFO:tasks.rados.rados.0.smithi017.stdout:1245: done (3 left) 2024-09-21T09:43:14.038 INFO:tasks.rados.rados.0.smithi017.stdout:1246: done (2 left) 2024-09-21T09:43:14.038 INFO:tasks.rados.rados.0.smithi017.stdout:1248: done (1 left) 2024-09-21T09:43:14.038 INFO:tasks.rados.rados.0.smithi017.stdout:1249: done (0 left) 2024-09-21T09:43:14.038 INFO:tasks.rados.rados.0.smithi017.stdout:1250: snap_remove snap 152 2024-09-21T09:43:14.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:14.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade resume' 2024-09-21T09:43:15.048 INFO:tasks.rados.rados.0.smithi017.stdout:1250: done (0 left) 2024-09-21T09:43:15.049 INFO:tasks.rados.rados.0.smithi017.stdout:1251: read oid 18 snap -1 2024-09-21T09:43:15.049 INFO:tasks.rados.rados.0.smithi017.stdout:1251: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:43:15.049 INFO:tasks.rados.rados.0.smithi017.stdout:1252: snap_remove snap 155 2024-09-21T09:43:15.049 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:15 smithi017 ceph-mon[105151]: pgmap v137: 89 pgs: 89 active+clean; 192 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 3.1 MiB/s wr, 20 op/s 2024-09-21T09:43:15.049 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:15 smithi017 ceph-mon[105151]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T09:43:15.049 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:43:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:43:14] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-21T09:43:15.313 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:15 smithi017 ceph-mon[102913]: pgmap v137: 89 pgs: 89 active+clean; 192 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 3.1 MiB/s wr, 20 op/s 2024-09-21T09:43:15.313 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:15 smithi017 ceph-mon[102913]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T09:43:15.340 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:15 smithi134 ceph-mon[33541]: pgmap v137: 89 pgs: 89 active+clean; 192 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 3.1 MiB/s wr, 20 op/s 2024-09-21T09:43:15.340 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:15 smithi134 ceph-mon[33541]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T09:43:15.760 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:15.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:15.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:16.058 INFO:tasks.rados.rados.0.smithi017.stdout:1252: done (1 left) 2024-09-21T09:43:16.059 INFO:tasks.rados.rados.0.smithi017.stdout:1253: rollback oid 32 current snap is 156 2024-09-21T09:43:16.059 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 145 2024-09-21T09:43:16.059 INFO:tasks.rados.rados.0.smithi017.stdout:1251: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:43:16.132 INFO:tasks.rados.rados.0.smithi017.stdout:1253: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:43:16.132 INFO:tasks.rados.rados.0.smithi017.stdout:1253: finishing rollback tid 2 to smithi01785587-32 2024-09-21T09:43:16.132 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 581 (ObjNum 421 snap 130 seq_num 421) dirty exists 2024-09-21T09:43:16.132 INFO:tasks.rados.rados.0.smithi017.stdout:1251: done (1 left) 2024-09-21T09:43:16.133 INFO:tasks.rados.rados.0.smithi017.stdout:1253: done (0 left) 2024-09-21T09:43:16.133 INFO:tasks.rados.rados.0.smithi017.stdout:1254: write oid 37 current snap is 156 2024-09-21T09:43:16.133 INFO:tasks.rados.rados.0.smithi017.stdout:1254: seq_num 491 ranges {652237=793851,2198894=53343} 2024-09-21T09:43:16.140 INFO:tasks.rados.rados.0.smithi017.stdout:1254: writing smithi01785587-37 from 652237 to 1446088 tid 1 2024-09-21T09:43:16.145 INFO:tasks.rados.rados.0.smithi017.stdout:1254: writing smithi01785587-37 from 2198894 to 2252237 tid 2 2024-09-21T09:43:16.145 INFO:tasks.rados.rados.0.smithi017.stdout:1255: write oid 39 current snap is 156 2024-09-21T09:43:16.145 INFO:tasks.rados.rados.0.smithi017.stdout:1255: seq_num 492 ranges {669044=469100,1741023=459161} 2024-09-21T09:43:16.151 INFO:tasks.rados.rados.0.smithi017.stdout:1255: writing smithi01785587-39 from 669044 to 1138144 tid 1 2024-09-21T09:43:16.157 INFO:tasks.rados.rados.0.smithi017.stdout:1255: writing smithi01785587-39 from 1741023 to 2200184 tid 2 2024-09-21T09:43:16.158 INFO:tasks.rados.rados.0.smithi017.stdout:1256: delete oid 38 current snap is 156 2024-09-21T09:43:16.158 INFO:tasks.rados.rados.0.smithi017.stdout:1254: finishing write tid 1 to smithi01785587-37 2024-09-21T09:43:16.158 INFO:tasks.rados.rados.0.smithi017.stdout:1254: finishing write tid 2 to smithi01785587-37 2024-09-21T09:43:16.158 INFO:tasks.rados.rados.0.smithi017.stdout:1254: finishing write tid 3 to smithi01785587-37 2024-09-21T09:43:16.158 INFO:tasks.rados.rados.0.smithi017.stdout:1254: finishing write tid 4 to smithi01785587-37 2024-09-21T09:43:16.158 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 620 (ObjNum 491 snap 156 seq_num 491) dirty exists 2024-09-21T09:43:16.158 INFO:tasks.rados.rados.0.smithi017.stdout:1254: left oid 37 (ObjNum 491 snap 156 seq_num 491) 2024-09-21T09:43:16.158 INFO:tasks.rados.rados.0.smithi017.stdout:1255: finishing write tid 1 to smithi01785587-39 2024-09-21T09:43:16.160 INFO:tasks.rados.rados.0.smithi017.stdout:1254: done (2 left) 2024-09-21T09:43:16.160 INFO:tasks.rados.rados.0.smithi017.stdout:1256: done (1 left) 2024-09-21T09:43:16.160 INFO:tasks.rados.rados.0.smithi017.stdout:1257: rollback oid 25 current snap is 156 2024-09-21T09:43:16.160 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 149 2024-09-21T09:43:16.160 INFO:tasks.rados.rados.0.smithi017.stdout:1258: read oid 7 snap -1 2024-09-21T09:43:16.160 INFO:tasks.rados.rados.0.smithi017.stdout:1258: expect (ObjNum 452 snap 147 seq_num 452) 2024-09-21T09:43:16.161 INFO:tasks.rados.rados.0.smithi017.stdout:1259: delete oid 33 current snap is 156 2024-09-21T09:43:16.161 INFO:tasks.rados.rados.0.smithi017.stdout:1255: finishing write tid 2 to smithi01785587-39 2024-09-21T09:43:16.161 INFO:tasks.rados.rados.0.smithi017.stdout:1255: finishing write tid 3 to smithi01785587-39 2024-09-21T09:43:16.161 INFO:tasks.rados.rados.0.smithi017.stdout:1255: finishing write tid 4 to smithi01785587-39 2024-09-21T09:43:16.161 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 624 (ObjNum 492 snap 156 seq_num 492) dirty exists 2024-09-21T09:43:16.161 INFO:tasks.rados.rados.0.smithi017.stdout:1255: left oid 39 (ObjNum 492 snap 156 seq_num 492) 2024-09-21T09:43:16.161 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:16 smithi017 ceph-mon[102913]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T09:43:16.161 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:16 smithi017 ceph-mon[105151]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T09:43:16.162 INFO:tasks.rados.rados.0.smithi017.stdout:1255: done (3 left) 2024-09-21T09:43:16.162 INFO:tasks.rados.rados.0.smithi017.stdout:1259: done (2 left) 2024-09-21T09:43:16.162 INFO:tasks.rados.rados.0.smithi017.stdout:1260: write oid 21 current snap is 156 2024-09-21T09:43:16.162 INFO:tasks.rados.rados.0.smithi017.stdout:1260: seq_num 495 ranges {719465=411200,1632822=686643} 2024-09-21T09:43:16.169 INFO:tasks.rados.rados.0.smithi017.stdout:1260: writing smithi01785587-21 from 719465 to 1130665 tid 1 2024-09-21T09:43:16.175 INFO:tasks.rados.rados.0.smithi017.stdout:1260: writing smithi01785587-21 from 1632822 to 2319465 tid 2 2024-09-21T09:43:16.176 INFO:tasks.rados.rados.0.smithi017.stdout:1261: snap_create 2024-09-21T09:43:16.176 INFO:tasks.rados.rados.0.smithi017.stdout:1257: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:43:16.176 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 0 (ObjNum 129 snap 0 seq_num 2399724616) dirty dne 2024-09-21T09:43:16.176 INFO:tasks.rados.rados.0.smithi017.stdout:1258: expect (ObjNum 452 snap 147 seq_num 452) 2024-09-21T09:43:16.300 INFO:tasks.rados.rados.0.smithi017.stdout:1260: finishing write tid 1 to smithi01785587-21 2024-09-21T09:43:16.301 INFO:tasks.rados.rados.0.smithi017.stdout:1260: finishing write tid 2 to smithi01785587-21 2024-09-21T09:43:16.301 INFO:tasks.rados.rados.0.smithi017.stdout:1260: finishing write tid 3 to smithi01785587-21 2024-09-21T09:43:16.301 INFO:tasks.rados.rados.0.smithi017.stdout:1260: finishing write tid 4 to smithi01785587-21 2024-09-21T09:43:16.301 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 664 (ObjNum 495 snap 156 seq_num 495) dirty exists 2024-09-21T09:43:16.301 INFO:tasks.rados.rados.0.smithi017.stdout:1260: left oid 21 (ObjNum 495 snap 156 seq_num 495) 2024-09-21T09:43:16.319 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:16 smithi134 ceph-mon[33541]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T09:43:17.057 INFO:tasks.rados.rados.0.smithi017.stdout:1257: done (3 left) 2024-09-21T09:43:17.057 INFO:tasks.rados.rados.0.smithi017.stdout:1258: done (2 left) 2024-09-21T09:43:17.057 INFO:tasks.rados.rados.0.smithi017.stdout:1260: done (1 left) 2024-09-21T09:43:17.057 INFO:tasks.rados.rados.0.smithi017.stdout:1261: done (0 left) 2024-09-21T09:43:17.057 INFO:tasks.rados.rados.0.smithi017.stdout:1262: write oid 44 current snap is 157 2024-09-21T09:43:17.058 INFO:tasks.rados.rados.0.smithi017.stdout:1262: seq_num 496 ranges {736272=486449,1974951=361321} 2024-09-21T09:43:17.063 INFO:tasks.rados.rados.0.smithi017.stdout:1262: writing smithi01785587-44 from 736272 to 1222721 tid 1 2024-09-21T09:43:17.070 INFO:tasks.rados.rados.0.smithi017.stdout:1262: writing smithi01785587-44 from 1974951 to 2336272 tid 2 2024-09-21T09:43:17.071 INFO:tasks.rados.rados.0.smithi017.stdout:1263: read oid 49 snap 141 2024-09-21T09:43:17.071 INFO:tasks.rados.rados.0.smithi017.stdout:1263: expect (ObjNum 341 snap 103 seq_num 341) 2024-09-21T09:43:17.071 INFO:tasks.rados.rados.0.smithi017.stdout:1264: rollback oid 32 current snap is 157 2024-09-21T09:43:17.071 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 145 2024-09-21T09:43:17.071 INFO:tasks.rados.rados.0.smithi017.stdout:1262: finishing write tid 1 to smithi01785587-44 2024-09-21T09:43:17.071 INFO:tasks.rados.rados.0.smithi017.stdout:1265: read oid 5 snap -1 2024-09-21T09:43:17.071 INFO:tasks.rados.rados.0.smithi017.stdout:1265: expect (ObjNum 441 snap 139 seq_num 441) 2024-09-21T09:43:17.071 INFO:tasks.rados.rados.0.smithi017.stdout:1266: snap_remove snap 154 2024-09-21T09:43:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:17 smithi134 ceph-mon[33541]: pgmap v140: 89 pgs: 89 active+clean; 193 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 199 KiB/s wr, 1 op/s 2024-09-21T09:43:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:17 smithi134 ceph-mon[33541]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:43:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:17 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:17 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:17.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:17.326 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:17 smithi017 ceph-mon[102913]: pgmap v140: 89 pgs: 89 active+clean; 193 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 199 KiB/s wr, 1 op/s 2024-09-21T09:43:17.326 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:17 smithi017 ceph-mon[102913]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:43:17.326 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:17 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:17.326 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:17 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:17.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:17 smithi017 ceph-mon[105151]: pgmap v140: 89 pgs: 89 active+clean; 193 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 199 KiB/s wr, 1 op/s 2024-09-21T09:43:17.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:17 smithi017 ceph-mon[105151]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:43:17.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:17 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:17.327 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:17 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:18.067 INFO:tasks.rados.rados.0.smithi017.stdout:1266: done (4 left) 2024-09-21T09:43:18.067 INFO:tasks.rados.rados.0.smithi017.stdout:1267: write oid 21 current snap is 157 2024-09-21T09:43:18.067 INFO:tasks.rados.rados.0.smithi017.stdout:1267: seq_num 497 ranges {753079=561698,2033433=319646} 2024-09-21T09:43:18.076 INFO:tasks.rados.rados.0.smithi017.stdout:1267: writing smithi01785587-21 from 753079 to 1314777 tid 1 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:1267: writing smithi01785587-21 from 2033433 to 2353079 tid 2 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:1268: rollback oid 31 current snap is 157 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 31 to 153 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:1262: finishing write tid 2 to smithi01785587-44 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:1262: finishing write tid 3 to smithi01785587-44 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:1262: finishing write tid 4 to smithi01785587-44 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 659 (ObjNum 496 snap 157 seq_num 496) dirty exists 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:1262: left oid 44 (ObjNum 496 snap 157 seq_num 496) 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:1264: finishing rollback tid 0 to smithi01785587-32 2024-09-21T09:43:18.083 INFO:tasks.rados.rados.0.smithi017.stdout:1264: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:43:18.084 INFO:tasks.rados.rados.0.smithi017.stdout:1265: expect (ObjNum 441 snap 139 seq_num 441) 2024-09-21T09:43:18.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:18 smithi017 ceph-mon[102913]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:43:18.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:18 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:18.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:18 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:18.284 INFO:teuthology.orchestra.run.smithi017.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:43:18.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:18 smithi017 ceph-mon[105151]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:43:18.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:18 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:18.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:18 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:18.287 INFO:tasks.rados.rados.0.smithi017.stdout:1264: finishing rollback tid 2 to smithi01785587-32 2024-09-21T09:43:18.287 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 585 (ObjNum 421 snap 130 seq_num 421) dirty exists 2024-09-21T09:43:18.288 INFO:tasks.rados.rados.0.smithi017.stdout:1263: expect (ObjNum 341 snap 103 seq_num 341) 2024-09-21T09:43:18.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:18 smithi134 ceph-mon[33541]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:43:18.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:18 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:18 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:18.522 INFO:tasks.rados.rados.0.smithi017.stdout:1267: finishing write tid 1 to smithi01785587-21 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1262: done (5 left) 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1263: done (4 left) 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1264: done (3 left) 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1265: done (2 left) 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1269: write oid 45 current snap is 157 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1267: finishing write tid 2 to smithi01785587-21 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1267: finishing write tid 3 to smithi01785587-21 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1267: finishing write tid 4 to smithi01785587-21 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 672 (ObjNum 497 snap 157 seq_num 497) dirty exists 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1267: left oid 21 (ObjNum 497 snap 157 seq_num 497) 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1268: finishing rollback tid 0 to smithi01785587-31 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1268: finishing rollback tid 1 to smithi01785587-31 2024-09-21T09:43:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:1268: finishing rollback tid 2 to smithi01785587-31 2024-09-21T09:43:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 665 (ObjNum 438 snap 137 seq_num 438) dirty exists 2024-09-21T09:43:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:1269: seq_num 498 ranges {769886=636947,2091915=277971} 2024-09-21T09:43:18.530 INFO:tasks.rados.rados.0.smithi017.stdout:1269: writing smithi01785587-45 from 769886 to 1406833 tid 1 2024-09-21T09:43:18.535 INFO:tasks.rados.rados.0.smithi017.stdout:1269: writing smithi01785587-45 from 2091915 to 2369886 tid 2 2024-09-21T09:43:18.535 INFO:tasks.rados.rados.0.smithi017.stdout:1267: done (2 left) 2024-09-21T09:43:18.535 INFO:tasks.rados.rados.0.smithi017.stdout:1268: done (1 left) 2024-09-21T09:43:18.536 INFO:tasks.rados.rados.0.smithi017.stdout:1270: delete oid 30 current snap is 157 2024-09-21T09:43:18.536 INFO:tasks.rados.rados.0.smithi017.stdout:1269: finishing write tid 1 to smithi01785587-45 2024-09-21T09:43:18.537 INFO:tasks.rados.rados.0.smithi017.stdout:1270: done (1 left) 2024-09-21T09:43:18.537 INFO:tasks.rados.rados.0.smithi017.stdout:1271: read oid 22 snap 153 2024-09-21T09:43:18.537 INFO:tasks.rados.rados.0.smithi017.stdout:1271: expect deleted 2024-09-21T09:43:18.537 INFO:tasks.rados.rados.0.smithi017.stdout:1272: write oid 39 current snap is 157 2024-09-21T09:43:18.537 INFO:tasks.rados.rados.0.smithi017.stdout:1272: seq_num 500 ranges {403500=787445,1692526=473837} 2024-09-21T09:43:18.543 INFO:tasks.rados.rados.0.smithi017.stdout:1272: writing smithi01785587-39 from 403500 to 1190945 tid 1 2024-09-21T09:43:18.549 INFO:tasks.rados.rados.0.smithi017.stdout:1272: writing smithi01785587-39 from 1692526 to 2166363 tid 2 2024-09-21T09:43:18.549 INFO:tasks.rados.rados.0.smithi017.stdout:1273: write oid 36 current snap is 157 2024-09-21T09:43:18.549 INFO:tasks.rados.rados.0.smithi017.stdout:1269: finishing write tid 2 to smithi01785587-45 2024-09-21T09:43:18.549 INFO:tasks.rados.rados.0.smithi017.stdout:1273: seq_num 501 ranges {420307=462694,1351008=617495} 2024-09-21T09:43:18.554 INFO:tasks.rados.rados.0.smithi017.stdout:1273: writing smithi01785587-36 from 420307 to 883001 tid 1 2024-09-21T09:43:18.560 INFO:tasks.rados.rados.0.smithi017.stdout:1273: writing smithi01785587-36 from 1351008 to 1968503 tid 2 2024-09-21T09:43:18.560 INFO:tasks.rados.rados.0.smithi017.stdout:1274: snap_create 2024-09-21T09:43:18.560 INFO:tasks.rados.rados.0.smithi017.stdout:1269: finishing write tid 3 to smithi01785587-45 2024-09-21T09:43:18.560 INFO:tasks.rados.rados.0.smithi017.stdout:1269: finishing write tid 4 to smithi01785587-45 2024-09-21T09:43:18.560 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 466 (ObjNum 498 snap 157 seq_num 498) dirty exists 2024-09-21T09:43:18.560 INFO:tasks.rados.rados.0.smithi017.stdout:1269: left oid 45 (ObjNum 498 snap 157 seq_num 498) 2024-09-21T09:43:18.561 INFO:tasks.rados.rados.0.smithi017.stdout:1272: finishing write tid 1 to smithi01785587-39 2024-09-21T09:43:18.561 INFO:tasks.rados.rados.0.smithi017.stdout:1272: finishing write tid 2 to smithi01785587-39 2024-09-21T09:43:18.561 INFO:tasks.rados.rados.0.smithi017.stdout:1272: finishing write tid 3 to smithi01785587-39 2024-09-21T09:43:18.561 INFO:tasks.rados.rados.0.smithi017.stdout:1272: finishing write tid 4 to smithi01785587-39 2024-09-21T09:43:18.561 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 630 (ObjNum 500 snap 157 seq_num 500) dirty exists 2024-09-21T09:43:18.561 INFO:tasks.rados.rados.0.smithi017.stdout:1272: left oid 39 (ObjNum 500 snap 157 seq_num 500) 2024-09-21T09:43:18.561 INFO:tasks.rados.rados.0.smithi017.stdout:1273: finishing write tid 1 to smithi01785587-36 2024-09-21T09:43:18.563 INFO:tasks.rados.rados.0.smithi017.stdout:1273: finishing write tid 2 to smithi01785587-36 2024-09-21T09:43:18.563 INFO:tasks.rados.rados.0.smithi017.stdout:1273: finishing write tid 3 to smithi01785587-36 2024-09-21T09:43:18.563 INFO:tasks.rados.rados.0.smithi017.stdout:1273: finishing write tid 4 to smithi01785587-36 2024-09-21T09:43:18.563 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 634 (ObjNum 501 snap 157 seq_num 501) dirty exists 2024-09-21T09:43:18.563 INFO:tasks.rados.rados.0.smithi017.stdout:1273: left oid 36 (ObjNum 501 snap 157 seq_num 501) 2024-09-21T09:43:18.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for all mons to upgrade' 2024-09-21T09:43:19.133 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: pgmap v143: 89 pgs: 89 active+clean; 190 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s rd, 1008 KiB/s wr, 8 op/s 2024-09-21T09:43:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:43:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:43:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[105151]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: pgmap v143: 89 pgs: 89 active+clean; 190 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s rd, 1008 KiB/s wr, 8 op/s 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:19.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:43:19.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:19.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:19.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:19.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:43:19.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:43:19.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:19 smithi017 ceph-mon[102913]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: pgmap v143: 89 pgs: 89 active+clean; 190 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s rd, 1008 KiB/s wr, 8 op/s 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:19.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:19.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:43:19.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:43:19.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:19 smithi134 ceph-mon[33541]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:1269: done (4 left) 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:1271: done (3 left) 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:1272: done (2 left) 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:1273: done (1 left) 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:1274: done (0 left) 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:1275: rollback oid 42 current snap is 158 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 157 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:1276: rollback oid 39 current snap is 158 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 134 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:1277: write oid 21 current snap is 158 2024-09-21T09:43:19.288 INFO:tasks.rados.rados.0.smithi017.stdout:1277: seq_num 502 ranges {437114=654296,1525843=477506} 2024-09-21T09:43:19.294 INFO:tasks.rados.rados.0.smithi017.stdout:1277: writing smithi01785587-21 from 437114 to 1091410 tid 1 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:1277: writing smithi01785587-21 from 1525843 to 2003349 tid 2 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:1278: read oid 50 snap -1 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:1278: expect (ObjNum 473 snap 153 seq_num 473) 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:1279: write oid 38 current snap is 158 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:1275: finishing rollback tid 0 to smithi01785587-42 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:1275: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:1275: finishing rollback tid 2 to smithi01785587-42 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 480 (ObjNum 476 snap 153 seq_num 476) dirty exists 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:1276: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:43:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:1279: seq_num 503 ranges {453921=729545,1584325=621164} 2024-09-21T09:43:19.306 INFO:tasks.rados.rados.0.smithi017.stdout:1279: writing smithi01785587-38 from 453921 to 1183466 tid 1 2024-09-21T09:43:19.312 INFO:tasks.rados.rados.0.smithi017.stdout:1279: writing smithi01785587-38 from 1584325 to 2205489 tid 2 2024-09-21T09:43:19.312 INFO:tasks.rados.rados.0.smithi017.stdout:1275: done (4 left) 2024-09-21T09:43:19.312 INFO:tasks.rados.rados.0.smithi017.stdout:1280: read oid 30 snap -1 2024-09-21T09:43:19.313 INFO:tasks.rados.rados.0.smithi017.stdout:1280: expect deleted 2024-09-21T09:43:19.313 INFO:tasks.rados.rados.0.smithi017.stdout:1281: write oid 36 current snap is 158 2024-09-21T09:43:19.313 INFO:tasks.rados.rados.0.smithi017.stdout:1276: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:43:19.313 INFO:tasks.rados.rados.0.smithi017.stdout:1276: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:43:19.313 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 638 (ObjNum 419 snap 130 seq_num 419) dirty exists 2024-09-21T09:43:19.313 INFO:tasks.rados.rados.0.smithi017.stdout:1277: finishing write tid 1 to smithi01785587-21 2024-09-21T09:43:19.313 INFO:tasks.rados.rados.0.smithi017.stdout:1281: seq_num 504 ranges {470728=404794,1526454=481175} 2024-09-21T09:43:19.317 INFO:tasks.rados.rados.0.smithi017.stdout:1281: writing smithi01785587-36 from 470728 to 875522 tid 1 2024-09-21T09:43:19.323 INFO:tasks.rados.rados.0.smithi017.stdout:1281: writing smithi01785587-36 from 1526454 to 2007629 tid 2 2024-09-21T09:43:19.323 INFO:tasks.rados.rados.0.smithi017.stdout:1276: done (5 left) 2024-09-21T09:43:19.323 INFO:tasks.rados.rados.0.smithi017.stdout:1282: write oid 22 current snap is 158 2024-09-21T09:43:19.324 INFO:tasks.rados.rados.0.smithi017.stdout:1282: seq_num 505 ranges {487535=480043,1584936=624833} 2024-09-21T09:43:19.329 INFO:tasks.rados.rados.0.smithi017.stdout:1282: writing smithi01785587-22 from 487535 to 967578 tid 1 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:1282: writing smithi01785587-22 from 1584936 to 2209769 tid 2 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:1283: read oid 23 snap 125 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:1283: expect deleted 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:1284: snap_create 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:1277: finishing write tid 2 to smithi01785587-21 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:1277: finishing write tid 3 to smithi01785587-21 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:1277: finishing write tid 4 to smithi01785587-21 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 676 (ObjNum 502 snap 158 seq_num 502) dirty exists 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:1277: left oid 21 (ObjNum 502 snap 158 seq_num 502) 2024-09-21T09:43:19.336 INFO:tasks.rados.rados.0.smithi017.stdout:1278: expect (ObjNum 473 snap 153 seq_num 473) 2024-09-21T09:43:19.469 INFO:tasks.rados.rados.0.smithi017.stdout:1279: finishing write tid 1 to smithi01785587-38 2024-09-21T09:43:19.469 INFO:tasks.rados.rados.0.smithi017.stdout:1279: finishing write tid 2 to smithi01785587-38 2024-09-21T09:43:19.469 INFO:tasks.rados.rados.0.smithi017.stdout:1279: finishing write tid 3 to smithi01785587-38 2024-09-21T09:43:19.469 INFO:tasks.rados.rados.0.smithi017.stdout:1279: finishing write tid 4 to smithi01785587-38 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 390 (ObjNum 503 snap 158 seq_num 503) dirty exists 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1279: left oid 38 (ObjNum 503 snap 158 seq_num 503) 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1281: finishing write tid 1 to smithi01785587-36 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1281: finishing write tid 2 to smithi01785587-36 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1281: finishing write tid 3 to smithi01785587-36 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1281: finishing write tid 4 to smithi01785587-36 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 642 (ObjNum 504 snap 158 seq_num 504) dirty exists 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1281: left oid 36 (ObjNum 504 snap 158 seq_num 504) 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1282: finishing write tid 1 to smithi01785587-22 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1282: finishing write tid 2 to smithi01785587-22 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1282: finishing write tid 3 to smithi01785587-22 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1282: finishing write tid 4 to smithi01785587-22 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 667 (ObjNum 505 snap 158 seq_num 505) dirty exists 2024-09-21T09:43:19.470 INFO:tasks.rados.rados.0.smithi017.stdout:1282: left oid 22 (ObjNum 505 snap 158 seq_num 505) 2024-09-21T09:43:20.292 INFO:tasks.rados.rados.0.smithi017.stdout:1277: done (7 left) 2024-09-21T09:43:20.292 INFO:tasks.rados.rados.0.smithi017.stdout:1278: done (6 left) 2024-09-21T09:43:20.292 INFO:tasks.rados.rados.0.smithi017.stdout:1279: done (5 left) 2024-09-21T09:43:20.292 INFO:tasks.rados.rados.0.smithi017.stdout:1280: done (4 left) 2024-09-21T09:43:20.292 INFO:tasks.rados.rados.0.smithi017.stdout:1281: done (3 left) 2024-09-21T09:43:20.293 INFO:tasks.rados.rados.0.smithi017.stdout:1282: done (2 left) 2024-09-21T09:43:20.293 INFO:tasks.rados.rados.0.smithi017.stdout:1283: done (1 left) 2024-09-21T09:43:20.293 INFO:tasks.rados.rados.0.smithi017.stdout:1284: done (0 left) 2024-09-21T09:43:20.293 INFO:tasks.rados.rados.0.smithi017.stdout:1285: snap_create 2024-09-21T09:43:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[102913]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:43:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[105151]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:43:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:20.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:20.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:20 smithi134 ceph-mon[33541]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:43:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:20 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:20 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:20 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:20 smithi134 ceph-mon[33541]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:20.879 INFO:teuthology.orchestra.run.smithi017.stdout:wait for all mons to upgrade 2024-09-21T09:43:21.293 INFO:tasks.rados.rados.0.smithi017.stdout:1285: done (0 left) 2024-09-21T09:43:21.294 INFO:tasks.rados.rados.0.smithi017.stdout:1286: read oid 13 snap -1 2024-09-21T09:43:21.294 INFO:tasks.rados.rados.0.smithi017.stdout:1286: expect deleted 2024-09-21T09:43:21.294 INFO:tasks.rados.rados.0.smithi017.stdout:1287: read oid 44 snap -1 2024-09-21T09:43:21.294 INFO:tasks.rados.rados.0.smithi017.stdout:1287: expect (ObjNum 496 snap 157 seq_num 496) 2024-09-21T09:43:21.294 INFO:tasks.rados.rados.0.smithi017.stdout:1288: snap_remove snap 153 2024-09-21T09:43:21.410 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:21 smithi134 ceph-mon[33541]: pgmap v146: 89 pgs: 89 active+clean; 190 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s rd, 810 KiB/s wr, 6 op/s 2024-09-21T09:43:21.410 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:21 smithi134 ceph-mon[33541]: Upgrade: Updating mon.b 2024-09-21T09:43:21.410 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:21 smithi134 ceph-mon[33541]: Deploying daemon mon.b on smithi134 2024-09-21T09:43:21.410 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:21 smithi134 ceph-mon[33541]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:43:21.500 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:21 smithi017 ceph-mon[102913]: pgmap v146: 89 pgs: 89 active+clean; 190 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s rd, 810 KiB/s wr, 6 op/s 2024-09-21T09:43:21.501 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:21 smithi017 ceph-mon[102913]: Upgrade: Updating mon.b 2024-09-21T09:43:21.501 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:21 smithi017 ceph-mon[102913]: Deploying daemon mon.b on smithi134 2024-09-21T09:43:21.501 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:21 smithi017 ceph-mon[102913]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:43:21.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:21 smithi017 ceph-mon[105151]: pgmap v146: 89 pgs: 89 active+clean; 190 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s rd, 810 KiB/s wr, 6 op/s 2024-09-21T09:43:21.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:21 smithi017 ceph-mon[105151]: Upgrade: Updating mon.b 2024-09-21T09:43:21.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:21 smithi017 ceph-mon[105151]: Deploying daemon mon.b on smithi134 2024-09-21T09:43:21.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:21 smithi017 ceph-mon[105151]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:43:21.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-21T09:43:21.788 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:21.950 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:21 smithi134 systemd[1]: Stopping Ceph mon.b for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:43:22.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b[33518]: 2024-09-21T09:43:21.947+0000 7fead7b71640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:43:22.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b[33518]: 2024-09-21T09:43:21.947+0000 7fead7b71640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:43:22.474 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:22 smithi134 podman[89891]: 2024-09-21 09:43:22.21330539 +0000 UTC m=+0.382152191 container died 7eb897902574747f6cc24082034ad673109b48354c205cbe8fbd65103e838f45 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T09:43:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:22 smithi134 podman[89891]: 2024-09-21 09:43:22.570745537 +0000 UTC m=+0.739592379 container cleanup 7eb897902574747f6cc24082034ad673109b48354c205cbe8fbd65103e838f45 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux ) 2024-09-21T09:43:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:22 smithi134 bash[89891]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b 2024-09-21T09:43:23.208 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:22 smithi134 podman[89908]: 2024-09-21 09:43:22.872114987 +0000 UTC m=+0.653500826 container remove 7eb897902574747f6cc24082034ad673109b48354c205cbe8fbd65103e838f45 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-21T09:43:23.208 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.b.service: Deactivated successfully. 2024-09-21T09:43:23.208 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 systemd[1]: Stopped Ceph mon.b for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:43:23.208 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.b.service: Consumed 10.223s CPU time. 2024-09-21T09:43:23.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 systemd[1]: Starting Ceph mon.b for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:43:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 podman[90047]: 2024-09-21 09:43:23.317371624 +0000 UTC m=+0.015649869 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:43:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 podman[90047]: 2024-09-21 09:43:23.448294705 +0000 UTC m=+0.146572999 container create 10638b01de94ace450167417aa0e40829016d1580c224b86af361cb1241dcb26 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:43:23.927 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:43:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 podman[90047]: 2024-09-21 09:43:23.684000547 +0000 UTC m=+0.382278802 container init 10638b01de94ace450167417aa0e40829016d1580c224b86af361cb1241dcb26 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T09:43:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 podman[90047]: 2024-09-21 09:43:23.68918366 +0000 UTC m=+0.387461905 container start 10638b01de94ace450167417aa0e40829016d1580c224b86af361cb1241dcb26 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T09:43:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:43:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: pidfile_write: ignore empty --pid-file 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: load: jerasure load: lrc 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Git sha 0 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: DB SUMMARY 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: DB Session ID: 0V0VXQDYV38YE4TH7HGK 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1091 Bytes 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-21T09:43:23.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 14759776 ; 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.env: 0x5638f12d9120 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.info_log: 0x5638f3485700 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:43:23.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.statistics: (nil) 2024-09-21T09:43:23.998 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.use_fsync: 0 2024-09-21T09:43:23.998 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:43:23.998 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:43:23.998 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:43:23.998 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:43:23.998 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:43:23.998 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:43:23.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:43:23.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:43:23.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:43:23.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:43:23.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:43:23.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.db_log_dir: 2024-09-21T09:43:23.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.wal_dir: 2024-09-21T09:43:23.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:43:23.999 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:43:24.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:43:24.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:43:24.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:43:24.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:43:24.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:43:24.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:43:24.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.write_buffer_manager: 0x5638f34881e0 2024-09-21T09:43:24.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:43:24.000 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.unordered_write: 0 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:43:24.001 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.row_cache: None 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.wal_filter: None 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.wal_compression: 0 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:43:24.002 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:43:24.003 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_open_files: -1 2024-09-21T09:43:24.004 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Compression algorithms supported: 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: kZSTD supported: 0 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:43:24.005 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.merge_operator: 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_filter: None 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:43:24.006 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5638f3485320) 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout: index_type: 0 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout: data_block_index_type: 0 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout: index_shortening: 1 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:43:24.007 INFO:journalctl@ceph.mon.b.smithi134.stdout: checksum: 4 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: no_block_cache: 0 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: block_cache: 0x5638f3479610 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: block_cache_options: 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: capacity : 536870912 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: num_shard_bits : 4 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: strict_capacity_limit : 0 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: block_cache_compressed: (nil) 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: persistent_cache: (nil) 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: block_size: 4096 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: block_size_deviation: 10 2024-09-21T09:43:24.008 INFO:journalctl@ceph.mon.b.smithi134.stdout: block_restart_interval: 16 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: index_block_restart_interval: 1 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: metadata_block_size: 4096 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: partition_filters: 0 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: use_delta_encoding: 1 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: filter_policy: bloomfilter 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: whole_key_filtering: 1 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: verify_compression: 0 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: format_version: 5 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: enable_index_compression: 1 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: block_align: 0 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: max_auto_readahead_size: 262144 2024-09-21T09:43:24.009 INFO:journalctl@ceph.mon.b.smithi134.stdout: prepopulate_block_cache: 0 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression: NoCompression 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.num_levels: 7 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:43:24.010 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:43:24.011 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:43:24.012 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:43:24.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:43:24.014 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.ttl: 2592000 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:43:24.015 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:43:24.017 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13482, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d52dda91-6e63-459a-ba24-d4095ebfc7d4 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911803715453, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-21T09:43:24.018 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911803749791, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 9177884, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13479, "largest_seqno": 19079, "table_properties": {"data_size": 9153043, "index_size": 18097, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 5701, "raw_key_size": 51536, "raw_average_key_size": 22, "raw_value_size": 9103595, "raw_average_value_size": 4010, "num_data_blocks": 778, "num_entries": 2270, "num_filter_entries": 2270, "num_deletions": 322, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911803, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d52dda91-6e63-459a-ba24-d4095ebfc7d4", "db_session_id": "0V0VXQDYV38YE4TH7HGK", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911803749931, "job": 1, "event": "recovery_finished"} 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5638f34a8e00 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: DB pointer 0x5638f35aa000 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout: ** DB Stats ** 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:43:24.019 INFO:journalctl@ceph.mon.b.smithi134.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: ** Compaction Stats [default] ** 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: L0 1/0 8.75 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 573.6 0.02 0.00 1 0.015 0 0 0.0 0.0 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: L6 1/0 7.47 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: Sum 2/0 16.22 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 573.6 0.02 0.00 1 0.015 0 0 0.0 0.0 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 573.6 0.02 0.00 1 0.015 0 0 0.0 0.0 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: ** Compaction Stats [default] ** 2024-09-21T09:43:24.020 INFO:journalctl@ceph.mon.b.smithi134.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 573.6 0.02 0.00 1 0.015 0 0 0.0 0.0 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: Flush(GB): cumulative 0.009, interval 0.009 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: Cumulative compaction: 0.01 GB write, 228.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: Interval compaction: 0.01 GB write, 228.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: Block cache BinnedLRUCache@0x5638f3479610#7 capacity: 512.00 MB usage: 26.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-21T09:43:24.021 INFO:journalctl@ceph.mon.b.smithi134.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,6.03 KB,0.00115037%) IndexBlock(1,20.08 KB,0.0038296%) Misc(1,0.00 KB,0%) 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout: 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: starting mon.b rank 1 at public addrs [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] at bind addrs [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: mon.b@-1(???) e3 preinit fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: mon.b@-1(???).mds e8 new map 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: mon.b@-1(???).mds e8 print_map 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout: e8 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout: legacy client fscid: 1 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout: 2024-09-21T09:43:24.022 INFO:journalctl@ceph.mon.b.smithi134.stdout: Filesystem 'foo' (1) 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: fs_name foo 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: epoch 7 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: created 2024-09-21T09:36:43.123303+0000 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: modified 2024-09-21T09:36:59.247371+0000 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: tableserver 0 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: root 0 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: session_timeout 60 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: session_autoclose 300 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: max_file_size 1099511627776 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: max_xattr_size 65536 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: required_client_features {} 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: last_failure 0 2024-09-21T09:43:24.023 INFO:journalctl@ceph.mon.b.smithi134.stdout: last_failure_osd_epoch 0 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: max_mds 1 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: in 0 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: up {0=14655} 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: failed 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: damaged 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: stopped 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: data_pools [4] 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: metadata_pool 3 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: inline_data disabled 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: balancer 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: bal_rank_mask -1 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: standby_count_wanted 1 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: [mds.foo.smithi017.coodep{0:14655} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.17:6834/684133422,v1:172.21.15.17:6835/684133422] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:43:24.024 INFO:journalctl@ceph.mon.b.smithi134.stdout: 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout: 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout: Standby daemons: 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout: 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout: [mds.foo.smithi134.bwovvb{-1:24493} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.134:6832/3208820776,v1:172.21.15.134:6833/3208820776] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: mon.b@-1(???).osd e381 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: mon.b@-1(???).osd e381 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: mon.b@-1(???).osd e381 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: mon.b@-1(???).osd e381 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 ceph-mon[90085]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 bash[90047]: 10638b01de94ace450167417aa0e40829016d1580c224b86af361cb1241dcb26 2024-09-21T09:43:24.025 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:23 smithi134 systemd[1]: Started Ceph mon.b for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:43:24.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:25.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:43:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:43:24] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.51.0" 2024-09-21T09:43:25.993 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:25.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:26.317 INFO:tasks.rados.rados.0.smithi017.stdout:1288: done (2 left) 2024-09-21T09:43:26.318 INFO:tasks.rados.rados.0.smithi017.stdout:1289: read oid 35 snap -1 2024-09-21T09:43:26.318 INFO:tasks.rados.rados.0.smithi017.stdout:1289: expect deleted 2024-09-21T09:43:26.318 INFO:tasks.rados.rados.0.smithi017.stdout:1290: rollback oid 29 current snap is 160 2024-09-21T09:43:26.318 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 29 to 134 2024-09-21T09:43:26.318 INFO:tasks.rados.rados.0.smithi017.stdout:1291: write oid 32 current snap is 160 2024-09-21T09:43:26.318 INFO:tasks.rados.rados.0.smithi017.stdout:1291: seq_num 506 ranges {504342=555292,1643418=484844} 2024-09-21T09:43:26.324 INFO:tasks.rados.rados.0.smithi017.stdout:1291: writing smithi01785587-32 from 504342 to 1059634 tid 1 2024-09-21T09:43:26.330 INFO:tasks.rados.rados.0.smithi017.stdout:1291: writing smithi01785587-32 from 1643418 to 2128262 tid 2 2024-09-21T09:43:26.331 INFO:tasks.rados.rados.0.smithi017.stdout:1292: write oid 39 current snap is 160 2024-09-21T09:43:26.331 INFO:tasks.rados.rados.0.smithi017.stdout:1292: seq_num 507 ranges {521149=630541,1701900=628502} 2024-09-21T09:43:26.338 INFO:tasks.rados.rados.0.smithi017.stdout:1292: writing smithi01785587-39 from 521149 to 1151690 tid 1 2024-09-21T09:43:26.344 INFO:tasks.rados.rados.0.smithi017.stdout:1292: writing smithi01785587-39 from 1701900 to 2330402 tid 2 2024-09-21T09:43:26.344 INFO:tasks.rados.rados.0.smithi017.stdout:1293: write oid 40 current snap is 160 2024-09-21T09:43:26.345 INFO:tasks.rados.rados.0.smithi017.stdout:1287: expect (ObjNum 496 snap 157 seq_num 496) 2024-09-21T09:43:26.473 INFO:tasks.rados.rados.0.smithi017.stdout:1290: finishing rollback tid 0 to smithi01785587-29 2024-09-21T09:43:26.473 INFO:tasks.rados.rados.0.smithi017.stdout:1290: finishing rollback tid 1 to smithi01785587-29 2024-09-21T09:43:26.473 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 275 (ObjNum 129 snap 0 seq_num 2399789672) dirty dne 2024-09-21T09:43:26.473 INFO:tasks.rados.rados.0.smithi017.stdout:1293: seq_num 508 ranges {537956=705790,1644029=772160} 2024-09-21T09:43:26.480 INFO:tasks.rados.rados.0.smithi017.stdout:1293: writing smithi01785587-40 from 537956 to 1243746 tid 1 2024-09-21T09:43:26.487 INFO:tasks.rados.rados.0.smithi017.stdout:1293: writing smithi01785587-40 from 1644029 to 2416189 tid 2 2024-09-21T09:43:26.487 INFO:tasks.rados.rados.0.smithi017.stdout:1286: done (6 left) 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1287: done (5 left) 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1289: done (4 left) 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1290: done (3 left) 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1294: read oid 17 snap -1 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1294: expect deleted 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1295: rollback oid 48 current snap is 160 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 134 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1291: finishing write tid 1 to smithi01785587-32 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1291: finishing write tid 2 to smithi01785587-32 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1291: finishing write tid 3 to smithi01785587-32 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1291: finishing write tid 4 to smithi01785587-32 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 595 (ObjNum 506 snap 160 seq_num 506) dirty exists 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1291: left oid 32 (ObjNum 506 snap 160 seq_num 506) 2024-09-21T09:43:26.488 INFO:tasks.rados.rados.0.smithi017.stdout:1291: done (4 left) 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1296: rollback oid 46 current snap is 160 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 156 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1292: finishing write tid 1 to smithi01785587-39 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1292: finishing write tid 2 to smithi01785587-39 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1292: finishing write tid 3 to smithi01785587-39 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1292: finishing write tid 4 to smithi01785587-39 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 650 (ObjNum 507 snap 160 seq_num 507) dirty exists 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1292: left oid 39 (ObjNum 507 snap 160 seq_num 507) 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1293: finishing write tid 1 to smithi01785587-40 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1292: done (4 left) 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1297: read oid 35 snap 156 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1297: expect deleted 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1298: read oid 13 snap -1 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1298: expect deleted 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1299: write oid 45 current snap is 160 2024-09-21T09:43:26.489 INFO:tasks.rados.rados.0.smithi017.stdout:1299: seq_num 509 ranges {554763=781039,2102511=452252} 2024-09-21T09:43:26.494 INFO:tasks.rados.rados.0.smithi017.stdout:1299: writing smithi01785587-45 from 554763 to 1335802 tid 1 2024-09-21T09:43:26.501 INFO:tasks.rados.rados.0.smithi017.stdout:1299: writing smithi01785587-45 from 2102511 to 2554763 tid 2 2024-09-21T09:43:26.501 INFO:tasks.rados.rados.0.smithi017.stdout:1300: read oid 2 snap -1 2024-09-21T09:43:26.501 INFO:tasks.rados.rados.0.smithi017.stdout:1300: expect (ObjNum 479 snap 153 seq_num 479) 2024-09-21T09:43:26.501 INFO:tasks.rados.rados.0.smithi017.stdout:1301: read oid 11 snap 157 2024-09-21T09:43:26.501 INFO:tasks.rados.rados.0.smithi017.stdout:1301: expect deleted 2024-09-21T09:43:26.502 INFO:tasks.rados.rados.0.smithi017.stdout:1302: snap_remove snap 125 2024-09-21T09:43:26.614 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:43:26.614 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:43:26.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: mgrmap e40: y(active, since 105s), standbys: x 2024-09-21T09:43:26.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: overall HEALTH_OK 2024-09-21T09:43:26.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: mon.b calling monitor election 2024-09-21T09:43:26.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: mon.c calling monitor election 2024-09-21T09:43:26.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:43:26.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:43:26.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: mon.a calling monitor election 2024-09-21T09:43:26.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: monmap epoch 4 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: last_changed 2024-09-21T09:43:25.307391+0000 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: created 2024-09-21T09:29:44.848123+0000 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: min_mon_release 19 (squid) 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: election_strategy: 1 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: 1: [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon.b 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: mgrmap e40: y(active, since 105s), standbys: x 2024-09-21T09:43:26.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: overall HEALTH_OK 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: mgrmap e40: y(active, since 105s), standbys: x 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: overall HEALTH_OK 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: mon.b calling monitor election 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: mon.c calling monitor election 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: mon.a calling monitor election 2024-09-21T09:43:26.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: monmap epoch 4 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: last_changed 2024-09-21T09:43:25.307391+0000 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: created 2024-09-21T09:29:44.848123+0000 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: min_mon_release 19 (squid) 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: election_strategy: 1 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: 1: [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon.b 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: mgrmap e40: y(active, since 105s), standbys: x 2024-09-21T09:43:26.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: overall HEALTH_OK 2024-09-21T09:43:26.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:26.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:26.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:26.708 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:43:26.708 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: mgrmap e40: y(active, since 105s), standbys: x 2024-09-21T09:43:26.708 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: overall HEALTH_OK 2024-09-21T09:43:26.708 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: mon.b calling monitor election 2024-09-21T09:43:26.708 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: mon.c calling monitor election 2024-09-21T09:43:26.708 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:43:26.708 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:43:26.708 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: mon.a calling monitor election 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: monmap epoch 4 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: last_changed 2024-09-21T09:43:25.307391+0000 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: created 2024-09-21T09:29:44.848123+0000 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: min_mon_release 19 (squid) 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: election_strategy: 1 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: 0: [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon.a 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: 1: [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon.b 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: 2: [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon.c 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: fsmap foo:1 {0=foo.smithi017.coodep=up:active} 1 up:standby 2024-09-21T09:43:26.709 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:43:26.710 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: mgrmap e40: y(active, since 105s), standbys: x 2024-09-21T09:43:26.710 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: overall HEALTH_OK 2024-09-21T09:43:26.710 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:26.710 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:26.710 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:26.983 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:27.337 INFO:tasks.rados.rados.0.smithi017.stdout:1302: done (9 left) 2024-09-21T09:43:27.337 INFO:tasks.rados.rados.0.smithi017.stdout:1303: snap_remove snap 159 2024-09-21T09:43:27.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:27 smithi017 ceph-mon[102913]: pgmap v151: 89 pgs: 89 active+clean; 216 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-09-21T09:43:27.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:27 smithi017 ceph-mon[102913]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T09:43:27.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:27 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3979028144' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-21T09:43:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:27 smithi017 ceph-mon[105151]: pgmap v151: 89 pgs: 89 active+clean; 216 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-09-21T09:43:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:27 smithi017 ceph-mon[105151]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T09:43:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:27 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3979028144' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-21T09:43:27.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch ps' 2024-09-21T09:43:27.635 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:27.636 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:27 smithi134 ceph-mon[90085]: pgmap v151: 89 pgs: 89 active+clean; 216 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-09-21T09:43:27.636 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:27 smithi134 ceph-mon[90085]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T09:43:27.636 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:27 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3979028144' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-09-21T09:43:28.039 INFO:tasks.rados.rados.0.smithi017.stdout:1303: done (9 left) 2024-09-21T09:43:28.039 INFO:tasks.rados.rados.0.smithi017.stdout:1304: write oid 34 current snap is 160 2024-09-21T09:43:28.039 INFO:tasks.rados.rados.0.smithi017.stdout:1304: seq_num 510 ranges {571570=572641,1877346=694224} 2024-09-21T09:43:28.045 INFO:tasks.rados.rados.0.smithi017.stdout:1304: writing smithi01785587-34 from 571570 to 1144211 tid 1 2024-09-21T09:43:28.054 INFO:tasks.rados.rados.0.smithi017.stdout:1304: writing smithi01785587-34 from 1877346 to 2571570 tid 2 2024-09-21T09:43:28.054 INFO:tasks.rados.rados.0.smithi017.stdout:1294: done (9 left) 2024-09-21T09:43:28.054 INFO:tasks.rados.rados.0.smithi017.stdout:1297: done (8 left) 2024-09-21T09:43:28.054 INFO:tasks.rados.rados.0.smithi017.stdout:1305: read oid 38 snap -1 2024-09-21T09:43:28.054 INFO:tasks.rados.rados.0.smithi017.stdout:1305: expect (ObjNum 503 snap 158 seq_num 503) 2024-09-21T09:43:28.054 INFO:tasks.rados.rados.0.smithi017.stdout:1306: rollback oid 19 current snap is 160 2024-09-21T09:43:28.054 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 19 to 145 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1295: finishing rollback tid 0 to smithi01785587-48 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1295: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 614 (ObjNum 129 snap 0 seq_num 2399779840) dirty dne 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1296: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1293: finishing write tid 2 to smithi01785587-40 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1293: finishing write tid 3 to smithi01785587-40 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1296: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1295: done (9 left) 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1298: done (8 left) 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1307: write oid 17 current snap is 160 2024-09-21T09:43:28.055 INFO:tasks.rados.rados.0.smithi017.stdout:1307: seq_num 511 ranges {588377=647890,1935828=635840} 2024-09-21T09:43:28.060 INFO:tasks.rados.rados.0.smithi017.stdout:1307: writing smithi01785587-17 from 588377 to 1236267 tid 1 2024-09-21T09:43:28.069 INFO:tasks.rados.rados.0.smithi017.stdout:1307: writing smithi01785587-17 from 1935828 to 2571668 tid 2 2024-09-21T09:43:28.069 INFO:tasks.rados.rados.0.smithi017.stdout:1308: write oid 43 current snap is 160 2024-09-21T09:43:28.069 INFO:tasks.rados.rados.0.smithi017.stdout:1293: finishing write tid 4 to smithi01785587-40 2024-09-21T09:43:28.069 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 654 (ObjNum 508 snap 160 seq_num 508) dirty exists 2024-09-21T09:43:28.069 INFO:tasks.rados.rados.0.smithi017.stdout:1293: left oid 40 (ObjNum 508 snap 160 seq_num 508) 2024-09-21T09:43:28.069 INFO:tasks.rados.rados.0.smithi017.stdout:1296: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:43:28.069 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 599 (ObjNum 487 snap 155 seq_num 487) dirty exists 2024-09-21T09:43:28.069 INFO:tasks.rados.rados.0.smithi017.stdout:1299: finishing write tid 1 to smithi01785587-45 2024-09-21T09:43:28.069 INFO:tasks.rados.rados.0.smithi017.stdout:1308: seq_num 512 ranges {605184=723139,1877957=727227} 2024-09-21T09:43:28.076 INFO:tasks.rados.rados.0.smithi017.stdout:1308: writing smithi01785587-43 from 605184 to 1328323 tid 1 2024-09-21T09:43:28.083 INFO:tasks.rados.rados.0.smithi017.stdout:1308: writing smithi01785587-43 from 1877957 to 2605184 tid 2 2024-09-21T09:43:28.083 INFO:tasks.rados.rados.0.smithi017.stdout:1293: done (9 left) 2024-09-21T09:43:28.083 INFO:tasks.rados.rados.0.smithi017.stdout:1296: done (8 left) 2024-09-21T09:43:28.083 INFO:tasks.rados.rados.0.smithi017.stdout:1309: snap_create 2024-09-21T09:43:28.083 INFO:tasks.rados.rados.0.smithi017.stdout:1299: finishing write tid 2 to smithi01785587-45 2024-09-21T09:43:28.083 INFO:tasks.rados.rados.0.smithi017.stdout:1299: finishing write tid 3 to smithi01785587-45 2024-09-21T09:43:28.083 INFO:tasks.rados.rados.0.smithi017.stdout:1299: finishing write tid 4 to smithi01785587-45 2024-09-21T09:43:28.084 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 470 (ObjNum 509 snap 160 seq_num 509) dirty exists 2024-09-21T09:43:28.084 INFO:tasks.rados.rados.0.smithi017.stdout:1299: left oid 45 (ObjNum 509 snap 160 seq_num 509) 2024-09-21T09:43:28.084 INFO:tasks.rados.rados.0.smithi017.stdout:1300: expect (ObjNum 479 snap 153 seq_num 479) 2024-09-21T09:43:28.151 INFO:tasks.rados.rados.0.smithi017.stdout:1304: finishing write tid 1 to smithi01785587-34 2024-09-21T09:43:28.151 INFO:tasks.rados.rados.0.smithi017.stdout:1305: expect (ObjNum 503 snap 158 seq_num 503) 2024-09-21T09:43:28.229 INFO:tasks.rados.rados.0.smithi017.stdout:1304: finishing write tid 2 to smithi01785587-34 2024-09-21T09:43:28.229 INFO:tasks.rados.rados.0.smithi017.stdout:1306: finishing rollback tid 0 to smithi01785587-19 2024-09-21T09:43:28.229 INFO:tasks.rados.rados.0.smithi017.stdout:1304: finishing write tid 3 to smithi01785587-34 2024-09-21T09:43:28.229 INFO:tasks.rados.rados.0.smithi017.stdout:1306: finishing rollback tid 1 to smithi01785587-19 2024-09-21T09:43:28.229 INFO:tasks.rados.rados.0.smithi017.stdout:1304: finishing write tid 4 to smithi01785587-34 2024-09-21T09:43:28.229 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 689 (ObjNum 510 snap 160 seq_num 510) dirty exists 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:1304: left oid 34 (ObjNum 510 snap 160 seq_num 510) 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:1306: finishing rollback tid 2 to smithi01785587-19 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 624 (ObjNum 388 snap 119 seq_num 388) dirty exists 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:1307: finishing write tid 1 to smithi01785587-17 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:1307: finishing write tid 2 to smithi01785587-17 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:1307: finishing write tid 3 to smithi01785587-17 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:1307: finishing write tid 4 to smithi01785587-17 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 479 (ObjNum 511 snap 160 seq_num 511) dirty exists 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:1307: left oid 17 (ObjNum 511 snap 160 seq_num 511) 2024-09-21T09:43:28.230 INFO:tasks.rados.rados.0.smithi017.stdout:1308: finishing write tid 1 to smithi01785587-43 2024-09-21T09:43:28.231 INFO:tasks.rados.rados.0.smithi017.stdout:1308: finishing write tid 2 to smithi01785587-43 2024-09-21T09:43:28.231 INFO:tasks.rados.rados.0.smithi017.stdout:1308: finishing write tid 3 to smithi01785587-43 2024-09-21T09:43:28.231 INFO:tasks.rados.rados.0.smithi017.stdout:1308: finishing write tid 4 to smithi01785587-43 2024-09-21T09:43:28.231 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 409 (ObjNum 512 snap 160 seq_num 512) dirty exists 2024-09-21T09:43:28.231 INFO:tasks.rados.rados.0.smithi017.stdout:1308: left oid 43 (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:43:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:28 smithi017 ceph-mon[102913]: osdmap e383: 8 total, 8 up, 8 in 2024-09-21T09:43:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:28 smithi017 ceph-mon[102913]: osdmap e384: 8 total, 8 up, 8 in 2024-09-21T09:43:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:28 smithi017 ceph-mon[105151]: osdmap e383: 8 total, 8 up, 8 in 2024-09-21T09:43:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:28 smithi017 ceph-mon[105151]: osdmap e384: 8 total, 8 up, 8 in 2024-09-21T09:43:28.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:28 smithi134 ceph-mon[90085]: osdmap e383: 8 total, 8 up, 8 in 2024-09-21T09:43:28.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:28 smithi134 ceph-mon[90085]: osdmap e384: 8 total, 8 up, 8 in 2024-09-21T09:43:29.040 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:43:28 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:43:28.761+0000 7f3131349640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:43:29.040 INFO:tasks.rados.rados.0.smithi017.stdout:1299: done (8 left) 2024-09-21T09:43:29.040 INFO:tasks.rados.rados.0.smithi017.stdout:1300: done (7 left) 2024-09-21T09:43:29.040 INFO:tasks.rados.rados.0.smithi017.stdout:1301: done (6 left) 2024-09-21T09:43:29.041 INFO:tasks.rados.rados.0.smithi017.stdout:1304: done (5 left) 2024-09-21T09:43:29.041 INFO:tasks.rados.rados.0.smithi017.stdout:1305: done (4 left) 2024-09-21T09:43:29.041 INFO:tasks.rados.rados.0.smithi017.stdout:1306: done (3 left) 2024-09-21T09:43:29.041 INFO:tasks.rados.rados.0.smithi017.stdout:1307: done (2 left) 2024-09-21T09:43:29.041 INFO:tasks.rados.rados.0.smithi017.stdout:1308: done (1 left) 2024-09-21T09:43:29.041 INFO:tasks.rados.rados.0.smithi017.stdout:1309: done (0 left) 2024-09-21T09:43:29.041 INFO:tasks.rados.rados.0.smithi017.stdout:1310: read oid 24 snap -1 2024-09-21T09:43:29.041 INFO:tasks.rados.rados.0.smithi017.stdout:1310: expect deleted 2024-09-21T09:43:29.041 INFO:tasks.rados.rados.0.smithi017.stdout:1311: delete oid 31 current snap is 161 2024-09-21T09:43:29.043 INFO:tasks.rados.rados.0.smithi017.stdout:1310: done (1 left) 2024-09-21T09:43:29.043 INFO:tasks.rados.rados.0.smithi017.stdout:1311: done (0 left) 2024-09-21T09:43:29.043 INFO:tasks.rados.rados.0.smithi017.stdout:1312: delete oid 41 current snap is 161 2024-09-21T09:43:29.046 INFO:tasks.rados.rados.0.smithi017.stdout:1312: done (0 left) 2024-09-21T09:43:29.046 INFO:tasks.rados.rados.0.smithi017.stdout:1313: snap_remove snap 134 2024-09-21T09:43:29.579 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (3m) 11s ago 7m 19.9M - 0.27.0 11f11916f8cd 0a4d8256f8d7 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi134 *:3000 running (3m) 12s ago 7m 85.1M - 9.4.7 954c08fa6188 8d2c807b0308 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.coodep smithi017 running (6m) 11s ago 6m 17.8M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 81acf97499eb 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi134.bwovvb smithi134 running (6m) 12s ago 6m 16.4M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be bf739a2a98de 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi134 *:8443,9283,8765 running (2m) 12s ago 11m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 4feb5ade269b 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (2m) 11s ago 13m 508M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 14c61243c535 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (116s) 11s ago 13m 46.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0590143b06cf 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi134 starting - - - 2048M 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (95s) 11s ago 12m 37.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 013cf9450c57 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (3m) 11s ago 7m 10.8M - 1.7.0 72c9c2088986 057b63034a9e 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi134 *:9100 running (3m) 12s ago 7m 11.3M - 1.7.0 72c9c2088986 c6b046d44374 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (11m) 11s ago 11m 94.4M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 958bb6e5f55d 2024-09-21T09:43:29.580 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (11m) 11s ago 11m 70.8M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5e9c6f25d8a5 2024-09-21T09:43:29.581 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (10m) 11s ago 10m 98.9M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b87d82e4dc74 2024-09-21T09:43:29.581 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (10m) 11s ago 10m 69.6M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 44d3b78acaa4 2024-09-21T09:43:29.581 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi134 running (9m) 12s ago 9m 96.0M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 7ece30084c41 2024-09-21T09:43:29.581 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi134 running (9m) 12s ago 9m 81.9M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5dec2255fcd7 2024-09-21T09:43:29.581 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi134 running (9m) 12s ago 9m 69.1M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 80d4436a0b8a 2024-09-21T09:43:29.581 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi134 running (8m) 12s ago 8m 73.4M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed2e26ef5698 2024-09-21T09:43:29.581 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi134 *:9095 running (2m) 12s ago 8m 48.1M - 2.51.0 1d3b7f56885b 061a3294723d 2024-09-21T09:43:29.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:29 smithi017 ceph-mon[102913]: pgmap v154: 89 pgs: 89 active+clean; 216 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 1 op/s 2024-09-21T09:43:29.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:29 smithi017 ceph-mon[102913]: osdmap e385: 8 total, 8 up, 8 in 2024-09-21T09:43:29.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:29 smithi017 ceph-mon[105151]: pgmap v154: 89 pgs: 89 active+clean; 216 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 1 op/s 2024-09-21T09:43:29.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:29 smithi017 ceph-mon[105151]: osdmap e385: 8 total, 8 up, 8 in 2024-09-21T09:43:29.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:29 smithi134 ceph-mon[90085]: pgmap v154: 89 pgs: 89 active+clean; 216 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s rd, 1 op/s 2024-09-21T09:43:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:29 smithi134 ceph-mon[90085]: osdmap e385: 8 total, 8 up, 8 in 2024-09-21T09:43:30.042 INFO:tasks.rados.rados.0.smithi017.stdout:1313: done (0 left) 2024-09-21T09:43:30.042 INFO:tasks.rados.rados.0.smithi017.stdout:1314: snap_create 2024-09-21T09:43:30.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade pause' 2024-09-21T09:43:30.460 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:30.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:30.832 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:30 smithi134 ceph-mon[90085]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:30.832 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.832 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.832 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:30 smithi134 ceph-mon[90085]: pgmap v157: 89 pgs: 89 active+clean; 216 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 2 op/s 2024-09-21T09:43:30.832 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:30 smithi134 ceph-mon[90085]: osdmap e386: 8 total, 8 up, 8 in 2024-09-21T09:43:30.833 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.833 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[102913]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[102913]: pgmap v157: 89 pgs: 89 active+clean; 216 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 2 op/s 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[102913]: osdmap e386: 8 total, 8 up, 8 in 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[105151]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[105151]: pgmap v157: 89 pgs: 89 active+clean; 216 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s rd, 2 op/s 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[105151]: osdmap e386: 8 total, 8 up, 8 in 2024-09-21T09:43:30.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:30.927 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:31.043 INFO:tasks.rados.rados.0.smithi017.stdout:1314: done (0 left) 2024-09-21T09:43:31.044 INFO:tasks.rados.rados.0.smithi017.stdout:1315: read oid 32 snap -1 2024-09-21T09:43:31.044 INFO:tasks.rados.rados.0.smithi017.stdout:1315: expect (ObjNum 506 snap 160 seq_num 506) 2024-09-21T09:43:31.044 INFO:tasks.rados.rados.0.smithi017.stdout:1316: rollback oid 49 current snap is 162 2024-09-21T09:43:31.044 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 145 2024-09-21T09:43:31.044 INFO:tasks.rados.rados.0.smithi017.stdout:1317: delete oid 50 current snap is 162 2024-09-21T09:43:31.049 INFO:tasks.rados.rados.0.smithi017.stdout:1317: done (2 left) 2024-09-21T09:43:31.049 INFO:tasks.rados.rados.0.smithi017.stdout:1318: read oid 37 snap 158 2024-09-21T09:43:31.049 INFO:tasks.rados.rados.0.smithi017.stdout:1318: expect (ObjNum 491 snap 156 seq_num 491) 2024-09-21T09:43:31.049 INFO:tasks.rados.rados.0.smithi017.stdout:1319: delete oid 28 current snap is 162 2024-09-21T09:43:31.049 INFO:tasks.rados.rados.0.smithi017.stdout:1316: finishing rollback tid 0 to smithi01785587-49 2024-09-21T09:43:31.050 INFO:tasks.rados.rados.0.smithi017.stdout:1315: expect (ObjNum 506 snap 160 seq_num 506) 2024-09-21T09:43:31.309 INFO:tasks.rados.rados.0.smithi017.stdout:1316: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:43:31.310 INFO:tasks.rados.rados.0.smithi017.stdout:1316: finishing rollback tid 2 to smithi01785587-49 2024-09-21T09:43:31.310 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 699 (ObjNum 447 snap 145 seq_num 447) dirty exists 2024-09-21T09:43:31.310 INFO:tasks.rados.rados.0.smithi017.stdout:1318: expect (ObjNum 491 snap 156 seq_num 491) 2024-09-21T09:43:31.387 INFO:tasks.rados.rados.0.smithi017.stdout:1315: done (3 left) 2024-09-21T09:43:31.387 INFO:tasks.rados.rados.0.smithi017.stdout:1316: done (2 left) 2024-09-21T09:43:31.387 INFO:tasks.rados.rados.0.smithi017.stdout:1318: done (1 left) 2024-09-21T09:43:31.387 INFO:tasks.rados.rados.0.smithi017.stdout:1319: done (0 left) 2024-09-21T09:43:31.388 INFO:tasks.rados.rados.0.smithi017.stdout:1320: read oid 3 snap 160 2024-09-21T09:43:31.388 INFO:tasks.rados.rados.0.smithi017.stdout:1320: expect deleted 2024-09-21T09:43:31.388 INFO:tasks.rados.rados.0.smithi017.stdout:1321: rollback oid 19 current snap is 162 2024-09-21T09:43:31.388 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 19 to 141 2024-09-21T09:43:31.388 INFO:tasks.rados.rados.0.smithi017.stdout:1322: write oid 23 current snap is 162 2024-09-21T09:43:31.388 INFO:tasks.rados.rados.0.smithi017.stdout:1322: seq_num 517 ranges {689219=415737,1770367=646847} 2024-09-21T09:43:31.393 INFO:tasks.rados.rados.0.smithi017.stdout:1322: writing smithi01785587-23 from 689219 to 1104956 tid 1 2024-09-21T09:43:31.400 INFO:tasks.rados.rados.0.smithi017.stdout:1322: writing smithi01785587-23 from 1770367 to 2417214 tid 2 2024-09-21T09:43:31.400 INFO:tasks.rados.rados.0.smithi017.stdout:1323: snap_create 2024-09-21T09:43:31.400 INFO:tasks.rados.rados.0.smithi017.stdout:1321: finishing rollback tid 0 to smithi01785587-19 2024-09-21T09:43:31.400 INFO:tasks.rados.rados.0.smithi017.stdout:1321: finishing rollback tid 1 to smithi01785587-19 2024-09-21T09:43:31.401 INFO:tasks.rados.rados.0.smithi017.stdout:1321: finishing rollback tid 2 to smithi01785587-19 2024-09-21T09:43:31.401 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 630 (ObjNum 388 snap 119 seq_num 388) dirty exists 2024-09-21T09:43:31.401 INFO:tasks.rados.rados.0.smithi017.stdout:1322: finishing write tid 1 to smithi01785587-23 2024-09-21T09:43:31.403 INFO:tasks.rados.rados.0.smithi017.stdout:1322: finishing write tid 2 to smithi01785587-23 2024-09-21T09:43:31.403 INFO:tasks.rados.rados.0.smithi017.stdout:1322: finishing write tid 3 to smithi01785587-23 2024-09-21T09:43:31.403 INFO:tasks.rados.rados.0.smithi017.stdout:1322: finishing write tid 4 to smithi01785587-23 2024-09-21T09:43:31.403 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 616 (ObjNum 517 snap 162 seq_num 517) dirty exists 2024-09-21T09:43:31.403 INFO:tasks.rados.rados.0.smithi017.stdout:1322: left oid 23 (ObjNum 517 snap 162 seq_num 517) 2024-09-21T09:43:32.048 INFO:tasks.rados.rados.0.smithi017.stdout:1320: done (3 left) 2024-09-21T09:43:32.049 INFO:tasks.rados.rados.0.smithi017.stdout:1321: done (2 left) 2024-09-21T09:43:32.049 INFO:tasks.rados.rados.0.smithi017.stdout:1322: done (1 left) 2024-09-21T09:43:32.049 INFO:tasks.rados.rados.0.smithi017.stdout:1323: done (0 left) 2024-09-21T09:43:32.049 INFO:tasks.rados.rados.0.smithi017.stdout:1324: delete oid 41 current snap is 163 2024-09-21T09:43:32.049 INFO:tasks.rados.rados.0.smithi017.stdout:1324: done (0 left) 2024-09-21T09:43:32.050 INFO:tasks.rados.rados.0.smithi017.stdout:1325: delete oid 45 current snap is 163 2024-09-21T09:43:32.053 INFO:tasks.rados.rados.0.smithi017.stdout:1325: done (0 left) 2024-09-21T09:43:32.053 INFO:tasks.rados.rados.0.smithi017.stdout:1326: snap_create 2024-09-21T09:43:32.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:32 smithi017 ceph-mon[105151]: osdmap e387: 8 total, 8 up, 8 in 2024-09-21T09:43:32.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:32 smithi017 ceph-mon[102913]: osdmap e387: 8 total, 8 up, 8 in 2024-09-21T09:43:32.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:32 smithi134 ceph-mon[90085]: osdmap e387: 8 total, 8 up, 8 in 2024-09-21T09:43:32.319 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:43:32.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'sleep 60' 2024-09-21T09:43:33.041 INFO:tasks.rados.rados.0.smithi017.stdout:1326: done (0 left) 2024-09-21T09:43:33.041 INFO:tasks.rados.rados.0.smithi017.stdout:1327: write oid 34 current snap is 164 2024-09-21T09:43:33.041 INFO:tasks.rados.rados.0.smithi017.stdout:1327: seq_num 520 ranges {739640=641484,1829460=794174} 2024-09-21T09:43:33.048 INFO:tasks.rados.rados.0.smithi017.stdout:1327: writing smithi01785587-34 from 739640 to 1381124 tid 1 2024-09-21T09:43:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1327: writing smithi01785587-34 from 1829460 to 2623634 tid 2 2024-09-21T09:43:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1328: read oid 50 snap -1 2024-09-21T09:43:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1328: expect deleted 2024-09-21T09:43:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1329: read oid 19 snap -1 2024-09-21T09:43:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1329: expect (ObjNum 388 snap 119 seq_num 388) 2024-09-21T09:43:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1330: snap_remove snap 162 2024-09-21T09:43:33.151 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:33 smithi134 ceph-mon[90085]: pgmap v160: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:43:33.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:33 smithi134 ceph-mon[90085]: osdmap e388: 8 total, 8 up, 8 in 2024-09-21T09:43:33.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:33 smithi134 ceph-mon[90085]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:33.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:33 smithi134 ceph-mon[90085]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:43:33.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:33 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:33.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:33 smithi134 ceph-mon[90085]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T09:43:33.165 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[105151]: pgmap v160: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:43:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[105151]: osdmap e388: 8 total, 8 up, 8 in 2024-09-21T09:43:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[105151]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[105151]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:43:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[105151]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T09:43:33.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[102913]: pgmap v160: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:43:33.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[102913]: osdmap e388: 8 total, 8 up, 8 in 2024-09-21T09:43:33.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[102913]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:33.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[102913]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:43:33.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:33.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:33 smithi017 ceph-mon[102913]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T09:43:34.051 INFO:tasks.rados.rados.0.smithi017.stdout:1330: done (3 left) 2024-09-21T09:43:34.052 INFO:tasks.rados.rados.0.smithi017.stdout:1331: write oid 8 current snap is 164 2024-09-21T09:43:34.052 INFO:tasks.rados.rados.0.smithi017.stdout:1327: finishing write tid 1 to smithi01785587-34 2024-09-21T09:43:34.052 INFO:tasks.rados.rados.0.smithi017.stdout:1331: seq_num 521 ranges {756447=716733,1887942=537832} 2024-09-21T09:43:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1331: writing smithi01785587-8 from 756447 to 1473180 tid 1 2024-09-21T09:43:34.068 INFO:tasks.rados.rados.0.smithi017.stdout:1331: writing smithi01785587-8 from 1887942 to 2425774 tid 2 2024-09-21T09:43:34.068 INFO:tasks.rados.rados.0.smithi017.stdout:1332: read oid 1 snap -1 2024-09-21T09:43:34.068 INFO:tasks.rados.rados.0.smithi017.stdout:1332: expect (ObjNum 458 snap 148 seq_num 458) 2024-09-21T09:43:34.068 INFO:tasks.rados.rados.0.smithi017.stdout:1333: read oid 43 snap -1 2024-09-21T09:43:34.069 INFO:tasks.rados.rados.0.smithi017.stdout:1333: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:43:34.069 INFO:tasks.rados.rados.0.smithi017.stdout:1334: read oid 21 snap -1 2024-09-21T09:43:34.069 INFO:tasks.rados.rados.0.smithi017.stdout:1334: expect (ObjNum 502 snap 158 seq_num 502) 2024-09-21T09:43:34.069 INFO:tasks.rados.rados.0.smithi017.stdout:1335: snap_create 2024-09-21T09:43:34.069 INFO:tasks.rados.rados.0.smithi017.stdout:1327: finishing write tid 2 to smithi01785587-34 2024-09-21T09:43:34.070 INFO:tasks.rados.rados.0.smithi017.stdout:1327: finishing write tid 3 to smithi01785587-34 2024-09-21T09:43:34.070 INFO:tasks.rados.rados.0.smithi017.stdout:1327: finishing write tid 4 to smithi01785587-34 2024-09-21T09:43:34.070 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 703 (ObjNum 520 snap 164 seq_num 520) dirty exists 2024-09-21T09:43:34.070 INFO:tasks.rados.rados.0.smithi017.stdout:1327: left oid 34 (ObjNum 520 snap 164 seq_num 520) 2024-09-21T09:43:34.070 INFO:tasks.rados.rados.0.smithi017.stdout:1329: expect (ObjNum 388 snap 119 seq_num 388) 2024-09-21T09:43:34.174 INFO:tasks.rados.rados.0.smithi017.stdout:1334: expect (ObjNum 502 snap 158 seq_num 502) 2024-09-21T09:43:34.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:34.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:34.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:34.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:34.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: Reconfiguring daemon mon.a on smithi017 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[102913]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:34.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:43:34.322 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:34.322 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:34.322 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:34.322 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: Reconfiguring daemon mon.a on smithi017 2024-09-21T09:43:34.322 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-mon[105151]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T09:43:34.351 INFO:tasks.rados.rados.0.smithi017.stdout:1331: finishing write tid 1 to smithi01785587-8 2024-09-21T09:43:34.351 INFO:tasks.rados.rados.0.smithi017.stdout:1331: finishing write tid 2 to smithi01785587-8 2024-09-21T09:43:34.351 INFO:tasks.rados.rados.0.smithi017.stdout:1331: finishing write tid 3 to smithi01785587-8 2024-09-21T09:43:34.351 INFO:tasks.rados.rados.0.smithi017.stdout:1331: finishing write tid 4 to smithi01785587-8 2024-09-21T09:43:34.351 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 620 (ObjNum 521 snap 164 seq_num 521) dirty exists 2024-09-21T09:43:34.351 INFO:tasks.rados.rados.0.smithi017.stdout:1331: left oid 8 (ObjNum 521 snap 164 seq_num 521) 2024-09-21T09:43:34.351 INFO:tasks.rados.rados.0.smithi017.stdout:1332: expect (ObjNum 458 snap 148 seq_num 458) 2024-09-21T09:43:34.407 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:34.407 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:34.407 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:34.407 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:34.408 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:34.408 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:34.408 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:43:34.408 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:34.408 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:34.408 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:34.408 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: Reconfiguring daemon mon.a on smithi017 2024-09-21T09:43:34.408 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:34 smithi134 ceph-mon[90085]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T09:43:34.602 INFO:tasks.rados.rados.0.smithi017.stdout:1333: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:43:35.052 INFO:tasks.rados.rados.0.smithi017.stdout:1327: done (7 left) 2024-09-21T09:43:35.052 INFO:tasks.rados.rados.0.smithi017.stdout:1328: done (6 left) 2024-09-21T09:43:35.052 INFO:tasks.rados.rados.0.smithi017.stdout:1329: done (5 left) 2024-09-21T09:43:35.052 INFO:tasks.rados.rados.0.smithi017.stdout:1331: done (4 left) 2024-09-21T09:43:35.053 INFO:tasks.rados.rados.0.smithi017.stdout:1332: done (3 left) 2024-09-21T09:43:35.053 INFO:tasks.rados.rados.0.smithi017.stdout:1333: done (2 left) 2024-09-21T09:43:35.053 INFO:tasks.rados.rados.0.smithi017.stdout:1334: done (1 left) 2024-09-21T09:43:35.053 INFO:tasks.rados.rados.0.smithi017.stdout:1335: done (0 left) 2024-09-21T09:43:35.053 INFO:tasks.rados.rados.0.smithi017.stdout:1336: write oid 19 current snap is 165 2024-09-21T09:43:35.053 INFO:tasks.rados.rados.0.smithi017.stdout:1336: seq_num 522 ranges {773254=791982,2346424=426830} 2024-09-21T09:43:35.060 INFO:tasks.rados.rados.0.smithi017.stdout:1336: writing smithi01785587-19 from 773254 to 1565236 tid 1 2024-09-21T09:43:35.068 INFO:tasks.rados.rados.0.smithi017.stdout:1336: writing smithi01785587-19 from 2346424 to 2773254 tid 2 2024-09-21T09:43:35.068 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:43:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:43:34] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.51.0" 2024-09-21T09:43:35.069 INFO:tasks.rados.rados.0.smithi017.stdout:1337: rollback oid 15 current snap is 165 2024-09-21T09:43:35.069 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 149 2024-09-21T09:43:35.069 INFO:tasks.rados.rados.0.smithi017.stdout:1338: read oid 38 snap -1 2024-09-21T09:43:35.069 INFO:tasks.rados.rados.0.smithi017.stdout:1338: expect (ObjNum 503 snap 158 seq_num 503) 2024-09-21T09:43:35.069 INFO:tasks.rados.rados.0.smithi017.stdout:1339: read oid 2 snap -1 2024-09-21T09:43:35.069 INFO:tasks.rados.rados.0.smithi017.stdout:1339: expect (ObjNum 479 snap 153 seq_num 479) 2024-09-21T09:43:35.069 INFO:tasks.rados.rados.0.smithi017.stdout:1340: delete oid 5 current snap is 165 2024-09-21T09:43:35.071 INFO:tasks.rados.rados.0.smithi017.stdout:1340: done (4 left) 2024-09-21T09:43:35.071 INFO:tasks.rados.rados.0.smithi017.stdout:1341: read oid 45 snap 158 2024-09-21T09:43:35.072 INFO:tasks.rados.rados.0.smithi017.stdout:1341: expect (ObjNum 498 snap 157 seq_num 498) 2024-09-21T09:43:35.072 INFO:tasks.rados.rados.0.smithi017.stdout:1342: read oid 7 snap -1 2024-09-21T09:43:35.072 INFO:tasks.rados.rados.0.smithi017.stdout:1342: expect (ObjNum 452 snap 147 seq_num 452) 2024-09-21T09:43:35.072 INFO:tasks.rados.rados.0.smithi017.stdout:1343: rollback oid 20 current snap is 165 2024-09-21T09:43:35.072 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 145 2024-09-21T09:43:35.072 INFO:tasks.rados.rados.0.smithi017.stdout:1344: rollback oid 16 current snap is 165 2024-09-21T09:43:35.072 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 16 to 161 2024-09-21T09:43:35.072 INFO:tasks.rados.rados.0.smithi017.stdout:1345: snap_remove snap 163 2024-09-21T09:43:35.377 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:35 smithi017 ceph-mon[102913]: pgmap v163: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:43:35.377 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:35 smithi017 ceph-mon[102913]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T09:43:35.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:35 smithi017 ceph-mon[105151]: pgmap v163: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:43:35.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:35 smithi017 ceph-mon[105151]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T09:43:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:35 smithi134 ceph-mon[90085]: pgmap v163: 89 pgs: 89 active+clean; 185 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:43:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:35 smithi134 ceph-mon[90085]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T09:43:35.993 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:35.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:36.054 INFO:tasks.rados.rados.0.smithi017.stdout:1345: done (8 left) 2024-09-21T09:43:36.054 INFO:tasks.rados.rados.0.smithi017.stdout:1346: snap_create 2024-09-21T09:43:36.054 INFO:tasks.rados.rados.0.smithi017.stdout:1339: expect (ObjNum 479 snap 153 seq_num 479) 2024-09-21T09:43:36.122 INFO:tasks.rados.rados.0.smithi017.stdout:1336: finishing write tid 1 to smithi01785587-19 2024-09-21T09:43:36.122 INFO:tasks.rados.rados.0.smithi017.stdout:1336: finishing write tid 2 to smithi01785587-19 2024-09-21T09:43:36.122 INFO:tasks.rados.rados.0.smithi017.stdout:1336: finishing write tid 3 to smithi01785587-19 2024-09-21T09:43:36.122 INFO:tasks.rados.rados.0.smithi017.stdout:1336: finishing write tid 4 to smithi01785587-19 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 634 (ObjNum 522 snap 165 seq_num 522) dirty exists 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:1336: left oid 19 (ObjNum 522 snap 165 seq_num 522) 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:1337: finishing rollback tid 0 to smithi01785587-15 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:1343: finishing rollback tid 0 to smithi01785587-20 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:1343: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:1337: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:1337: finishing rollback tid 2 to smithi01785587-15 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 218 (ObjNum 410 snap 126 seq_num 410) dirty exists 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:1343: finishing rollback tid 2 to smithi01785587-20 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 285 (ObjNum 415 snap 128 seq_num 415) dirty exists 2024-09-21T09:43:36.123 INFO:tasks.rados.rados.0.smithi017.stdout:1338: expect (ObjNum 503 snap 158 seq_num 503) 2024-09-21T09:43:36.204 INFO:tasks.rados.rados.0.smithi017.stdout:1341: expect (ObjNum 498 snap 157 seq_num 498) 2024-09-21T09:43:36.284 INFO:tasks.rados.rados.0.smithi017.stdout:1344: finishing rollback tid 0 to smithi01785587-16 2024-09-21T09:43:36.284 INFO:tasks.rados.rados.0.smithi017.stdout:1344: finishing rollback tid 1 to smithi01785587-16 2024-09-21T09:43:36.285 INFO:tasks.rados.rados.0.smithi017.stdout:1344: finishing rollback tid 2 to smithi01785587-16 2024-09-21T09:43:36.285 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 687 (ObjNum 443 snap 142 seq_num 443) dirty exists 2024-09-21T09:43:36.285 INFO:tasks.rados.rados.0.smithi017.stdout:1342: expect (ObjNum 452 snap 147 seq_num 452) 2024-09-21T09:43:36.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:36.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:36.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[105151]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:43:36.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:43:36.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:43:36.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:36.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[105151]: Reconfiguring daemon mgr.y on smithi017 2024-09-21T09:43:36.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[105151]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T09:43:36.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:36.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:36.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[102913]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:43:36.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:43:36.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:43:36.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:36.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[102913]: Reconfiguring daemon mgr.y on smithi017 2024-09-21T09:43:36.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:36 smithi017 ceph-mon[102913]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T09:43:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:36 smithi134 ceph-mon[90085]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:43:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:43:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:43:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:36 smithi134 ceph-mon[90085]: Reconfiguring daemon mgr.y on smithi017 2024-09-21T09:43:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:36 smithi134 ceph-mon[90085]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T09:43:37.054 INFO:tasks.rados.rados.0.smithi017.stdout:1336: done (8 left) 2024-09-21T09:43:37.054 INFO:tasks.rados.rados.0.smithi017.stdout:1337: done (7 left) 2024-09-21T09:43:37.054 INFO:tasks.rados.rados.0.smithi017.stdout:1338: done (6 left) 2024-09-21T09:43:37.054 INFO:tasks.rados.rados.0.smithi017.stdout:1339: done (5 left) 2024-09-21T09:43:37.055 INFO:tasks.rados.rados.0.smithi017.stdout:1341: done (4 left) 2024-09-21T09:43:37.055 INFO:tasks.rados.rados.0.smithi017.stdout:1342: done (3 left) 2024-09-21T09:43:37.055 INFO:tasks.rados.rados.0.smithi017.stdout:1343: done (2 left) 2024-09-21T09:43:37.055 INFO:tasks.rados.rados.0.smithi017.stdout:1344: done (1 left) 2024-09-21T09:43:37.055 INFO:tasks.rados.rados.0.smithi017.stdout:1346: done (0 left) 2024-09-21T09:43:37.055 INFO:tasks.rados.rados.0.smithi017.stdout:1347: delete oid 37 current snap is 166 2024-09-21T09:43:37.058 INFO:tasks.rados.rados.0.smithi017.stdout:1347: done (0 left) 2024-09-21T09:43:37.058 INFO:tasks.rados.rados.0.smithi017.stdout:1348: read oid 19 snap 149 2024-09-21T09:43:37.058 INFO:tasks.rados.rados.0.smithi017.stdout:1348: expect (ObjNum 388 snap 119 seq_num 388) 2024-09-21T09:43:37.058 INFO:tasks.rados.rados.0.smithi017.stdout:1349: write oid 21 current snap is 166 2024-09-21T09:43:37.058 INFO:tasks.rados.rados.0.smithi017.stdout:1349: seq_num 525 ranges {423675=734082,1721870=545170,2717777=105898} 2024-09-21T09:43:37.064 INFO:tasks.rados.rados.0.smithi017.stdout:1349: writing smithi01785587-21 from 423675 to 1157757 tid 1 2024-09-21T09:43:37.071 INFO:tasks.rados.rados.0.smithi017.stdout:1349: writing smithi01785587-21 from 1721870 to 2267040 tid 2 2024-09-21T09:43:37.074 INFO:tasks.rados.rados.0.smithi017.stdout:1349: writing smithi01785587-21 from 2717777 to 2823675 tid 3 2024-09-21T09:43:37.074 INFO:tasks.rados.rados.0.smithi017.stdout:1350: write oid 44 current snap is 166 2024-09-21T09:43:37.074 INFO:tasks.rados.rados.0.smithi017.stdout:1350: seq_num 526 ranges {440482=409331,1380352=405181,2461553=378929} 2024-09-21T09:43:37.079 INFO:tasks.rados.rados.0.smithi017.stdout:1350: writing smithi01785587-44 from 440482 to 849813 tid 1 2024-09-21T09:43:37.084 INFO:tasks.rados.rados.0.smithi017.stdout:1350: writing smithi01785587-44 from 1380352 to 1785533 tid 2 2024-09-21T09:43:37.090 INFO:tasks.rados.rados.0.smithi017.stdout:1350: writing smithi01785587-44 from 2461553 to 2840482 tid 3 2024-09-21T09:43:37.090 INFO:tasks.rados.rados.0.smithi017.stdout:1351: read oid 35 snap -1 2024-09-21T09:43:37.090 INFO:tasks.rados.rados.0.smithi017.stdout:1351: expect deleted 2024-09-21T09:43:37.090 INFO:tasks.rados.rados.0.smithi017.stdout:1352: read oid 50 snap -1 2024-09-21T09:43:37.090 INFO:tasks.rados.rados.0.smithi017.stdout:1352: expect deleted 2024-09-21T09:43:37.090 INFO:tasks.rados.rados.0.smithi017.stdout:1353: write oid 26 current snap is 166 2024-09-21T09:43:37.090 INFO:tasks.rados.rados.0.smithi017.stdout:1348: expect (ObjNum 388 snap 119 seq_num 388) 2024-09-21T09:43:37.194 INFO:tasks.rados.rados.0.smithi017.stdout:1349: finishing write tid 1 to smithi01785587-21 2024-09-21T09:43:37.194 INFO:tasks.rados.rados.0.smithi017.stdout:1349: finishing write tid 2 to smithi01785587-21 2024-09-21T09:43:37.194 INFO:tasks.rados.rados.0.smithi017.stdout:1349: finishing write tid 3 to smithi01785587-21 2024-09-21T09:43:37.194 INFO:tasks.rados.rados.0.smithi017.stdout:1349: finishing write tid 4 to smithi01785587-21 2024-09-21T09:43:37.194 INFO:tasks.rados.rados.0.smithi017.stdout:1349: finishing write tid 5 to smithi01785587-21 2024-09-21T09:43:37.194 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 712 (ObjNum 525 snap 166 seq_num 525) dirty exists 2024-09-21T09:43:37.194 INFO:tasks.rados.rados.0.smithi017.stdout:1349: left oid 21 (ObjNum 525 snap 166 seq_num 525) 2024-09-21T09:43:37.195 INFO:tasks.rados.rados.0.smithi017.stdout:1350: finishing write tid 1 to smithi01785587-44 2024-09-21T09:43:37.195 INFO:tasks.rados.rados.0.smithi017.stdout:1353: seq_num 527 ranges {457289=484580,1438834=548839,2772623=84666} 2024-09-21T09:43:37.199 INFO:tasks.rados.rados.0.smithi017.stdout:1353: writing smithi01785587-26 from 457289 to 941869 tid 1 2024-09-21T09:43:37.204 INFO:tasks.rados.rados.0.smithi017.stdout:1353: writing smithi01785587-26 from 1438834 to 1987673 tid 2 2024-09-21T09:43:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:1353: writing smithi01785587-26 from 2772623 to 2857289 tid 3 2024-09-21T09:43:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:1348: done (5 left) 2024-09-21T09:43:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:1349: done (4 left) 2024-09-21T09:43:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:1354: snap_remove snap 165 2024-09-21T09:43:37.243 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:37.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[102913]: pgmap v166: 89 pgs: 89 active+clean; 190 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 917 KiB/s wr, 21 op/s 2024-09-21T09:43:37.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:37.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:37.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:37.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:37.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:37.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[102913]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:43:37.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[105151]: pgmap v166: 89 pgs: 89 active+clean; 190 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 917 KiB/s wr, 21 op/s 2024-09-21T09:43:37.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:37.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:37.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:37.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:37.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:37.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:37 smithi017 ceph-mon[105151]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:43:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:37 smithi134 ceph-mon[90085]: pgmap v166: 89 pgs: 89 active+clean; 190 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 917 KiB/s wr, 21 op/s 2024-09-21T09:43:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:37 smithi134 ceph-mon[90085]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:1354: done (4 left) 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:1355: rollback oid 17 current snap is 166 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 164 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:1350: finishing write tid 2 to smithi01785587-44 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:1350: finishing write tid 3 to smithi01785587-44 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:1350: finishing write tid 4 to smithi01785587-44 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:1350: finishing write tid 5 to smithi01785587-44 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 696 (ObjNum 526 snap 166 seq_num 526) dirty exists 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:1350: left oid 44 (ObjNum 526 snap 166 seq_num 526) 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:1353: finishing write tid 1 to smithi01785587-26 2024-09-21T09:43:38.057 INFO:tasks.rados.rados.0.smithi017.stdout:1353: finishing write tid 2 to smithi01785587-26 2024-09-21T09:43:38.058 INFO:tasks.rados.rados.0.smithi017.stdout:1353: finishing write tid 3 to smithi01785587-26 2024-09-21T09:43:38.058 INFO:tasks.rados.rados.0.smithi017.stdout:1350: done (4 left) 2024-09-21T09:43:38.058 INFO:tasks.rados.rados.0.smithi017.stdout:1351: done (3 left) 2024-09-21T09:43:38.058 INFO:tasks.rados.rados.0.smithi017.stdout:1352: done (2 left) 2024-09-21T09:43:38.058 INFO:tasks.rados.rados.0.smithi017.stdout:1356: read oid 42 snap -1 2024-09-21T09:43:38.058 INFO:tasks.rados.rados.0.smithi017.stdout:1356: expect (ObjNum 476 snap 153 seq_num 476) 2024-09-21T09:43:38.058 INFO:tasks.rados.rados.0.smithi017.stdout:1357: write oid 24 current snap is 166 2024-09-21T09:43:38.058 INFO:tasks.rados.rados.0.smithi017.stdout:1357: seq_num 528 ranges {474096=559829,1780963=408850,2800046=74050} 2024-09-21T09:43:38.063 INFO:tasks.rados.rados.0.smithi017.stdout:1357: writing smithi01785587-24 from 474096 to 1033925 tid 1 2024-09-21T09:43:38.069 INFO:tasks.rados.rados.0.smithi017.stdout:1357: writing smithi01785587-24 from 1780963 to 2189813 tid 2 2024-09-21T09:43:38.074 INFO:tasks.rados.rados.0.smithi017.stdout:1357: writing smithi01785587-24 from 2800046 to 2874096 tid 3 2024-09-21T09:43:38.074 INFO:tasks.rados.rados.0.smithi017.stdout:1358: write oid 7 current snap is 166 2024-09-21T09:43:38.074 INFO:tasks.rados.rados.0.smithi017.stdout:1353: finishing write tid 4 to smithi01785587-26 2024-09-21T09:43:38.074 INFO:tasks.rados.rados.0.smithi017.stdout:1353: finishing write tid 5 to smithi01785587-26 2024-09-21T09:43:38.074 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 694 (ObjNum 527 snap 166 seq_num 527) dirty exists 2024-09-21T09:43:38.074 INFO:tasks.rados.rados.0.smithi017.stdout:1353: left oid 26 (ObjNum 527 snap 166 seq_num 527) 2024-09-21T09:43:38.074 INFO:tasks.rados.rados.0.smithi017.stdout:1358: seq_num 529 ranges {490903=635078,1839445=552508} 2024-09-21T09:43:38.079 INFO:tasks.rados.rados.0.smithi017.stdout:1358: writing smithi01785587-7 from 490903 to 1125981 tid 1 2024-09-21T09:43:38.087 INFO:tasks.rados.rados.0.smithi017.stdout:1358: writing smithi01785587-7 from 1839445 to 2391953 tid 2 2024-09-21T09:43:38.087 INFO:tasks.rados.rados.0.smithi017.stdout:1353: done (4 left) 2024-09-21T09:43:38.087 INFO:tasks.rados.rados.0.smithi017.stdout:1359: delete oid 16 current snap is 166 2024-09-21T09:43:38.087 INFO:tasks.rados.rados.0.smithi017.stdout:1356: expect (ObjNum 476 snap 153 seq_num 476) 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1355: finishing rollback tid 0 to smithi01785587-17 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1355: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1355: finishing rollback tid 2 to smithi01785587-17 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 489 (ObjNum 511 snap 160 seq_num 511) dirty exists 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1357: finishing write tid 1 to smithi01785587-24 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1355: done (4 left) 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1356: done (3 left) 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1359: done (2 left) 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1360: read oid 48 snap -1 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1360: expect deleted 2024-09-21T09:43:38.225 INFO:tasks.rados.rados.0.smithi017.stdout:1361: read oid 16 snap 157 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1361: expect (ObjNum 443 snap 142 seq_num 443) 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1362: snap_create 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1357: finishing write tid 2 to smithi01785587-24 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1357: finishing write tid 3 to smithi01785587-24 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1357: finishing write tid 4 to smithi01785587-24 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1357: finishing write tid 5 to smithi01785587-24 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 402 (ObjNum 528 snap 166 seq_num 528) dirty exists 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1357: left oid 24 (ObjNum 528 snap 166 seq_num 528) 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1358: finishing write tid 1 to smithi01785587-7 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1358: finishing write tid 2 to smithi01785587-7 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1358: finishing write tid 3 to smithi01785587-7 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1358: finishing write tid 4 to smithi01785587-7 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 718 (ObjNum 529 snap 166 seq_num 529) dirty exists 2024-09-21T09:43:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:1358: left oid 7 (ObjNum 529 snap 166 seq_num 529) 2024-09-21T09:43:38.235 INFO:tasks.rados.rados.0.smithi017.stdout:1361: expect (ObjNum 443 snap 142 seq_num 443) 2024-09-21T09:43:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:38 smithi017 ceph-mon[105151]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:43:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:38 smithi017 ceph-mon[105151]: Reconfiguring daemon mon.c on smithi017 2024-09-21T09:43:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:38 smithi017 ceph-mon[105151]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:43:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:38 smithi017 ceph-mon[102913]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:43:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:38 smithi017 ceph-mon[102913]: Reconfiguring daemon mon.c on smithi017 2024-09-21T09:43:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:38 smithi017 ceph-mon[102913]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:43:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:38 smithi134 ceph-mon[90085]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:43:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:38 smithi134 ceph-mon[90085]: Reconfiguring daemon mon.c on smithi017 2024-09-21T09:43:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:38 smithi134 ceph-mon[90085]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:43:39.057 INFO:tasks.rados.rados.0.smithi017.stdout:1357: done (4 left) 2024-09-21T09:43:39.057 INFO:tasks.rados.rados.0.smithi017.stdout:1358: done (3 left) 2024-09-21T09:43:39.057 INFO:tasks.rados.rados.0.smithi017.stdout:1360: done (2 left) 2024-09-21T09:43:39.057 INFO:tasks.rados.rados.0.smithi017.stdout:1361: done (1 left) 2024-09-21T09:43:39.057 INFO:tasks.rados.rados.0.smithi017.stdout:1362: done (0 left) 2024-09-21T09:43:39.057 INFO:tasks.rados.rados.0.smithi017.stdout:1363: read oid 17 snap -1 2024-09-21T09:43:39.057 INFO:tasks.rados.rados.0.smithi017.stdout:1363: expect (ObjNum 511 snap 160 seq_num 511) 2024-09-21T09:43:39.057 INFO:tasks.rados.rados.0.smithi017.stdout:1364: write oid 41 current snap is 167 2024-09-21T09:43:39.057 INFO:tasks.rados.rados.0.smithi017.stdout:1364: seq_num 531 ranges {524517=785576,1956409=556177} 2024-09-21T09:43:39.063 INFO:tasks.rados.rados.0.smithi017.stdout:1364: writing smithi01785587-41 from 524517 to 1310093 tid 1 2024-09-21T09:43:39.071 INFO:tasks.rados.rados.0.smithi017.stdout:1364: writing smithi01785587-41 from 1956409 to 2512586 tid 2 2024-09-21T09:43:39.071 INFO:tasks.rados.rados.0.smithi017.stdout:1365: snap_create 2024-09-21T09:43:39.071 INFO:tasks.rados.rados.0.smithi017.stdout:1363: expect (ObjNum 511 snap 160 seq_num 511) 2024-09-21T09:43:39.158 INFO:tasks.rados.rados.0.smithi017.stdout:1364: finishing write tid 1 to smithi01785587-41 2024-09-21T09:43:39.158 INFO:tasks.rados.rados.0.smithi017.stdout:1364: finishing write tid 2 to smithi01785587-41 2024-09-21T09:43:39.159 INFO:tasks.rados.rados.0.smithi017.stdout:1364: finishing write tid 3 to smithi01785587-41 2024-09-21T09:43:39.159 INFO:tasks.rados.rados.0.smithi017.stdout:1364: finishing write tid 4 to smithi01785587-41 2024-09-21T09:43:39.159 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 223 (ObjNum 531 snap 167 seq_num 531) dirty exists 2024-09-21T09:43:39.159 INFO:tasks.rados.rados.0.smithi017.stdout:1364: left oid 41 (ObjNum 531 snap 167 seq_num 531) 2024-09-21T09:43:39.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[102913]: pgmap v169: 89 pgs: 89 active+clean; 194 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.2 MiB/s wr, 30 op/s 2024-09-21T09:43:39.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[102913]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[102913]: Reconfiguring daemon osd.0 on smithi017 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[102913]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[105151]: pgmap v169: 89 pgs: 89 active+clean; 194 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.2 MiB/s wr, 30 op/s 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:39.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[105151]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:43:39.780 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:43:39.780 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:39.780 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[105151]: Reconfiguring daemon osd.0 on smithi017 2024-09-21T09:43:39.780 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:39 smithi017 ceph-mon[105151]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:43:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:39 smithi134 ceph-mon[90085]: pgmap v169: 89 pgs: 89 active+clean; 194 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.2 MiB/s wr, 30 op/s 2024-09-21T09:43:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:39 smithi134 ceph-mon[90085]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:43:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:43:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:39 smithi134 ceph-mon[90085]: Reconfiguring daemon osd.0 on smithi017 2024-09-21T09:43:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:39 smithi134 ceph-mon[90085]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:43:40.057 INFO:tasks.rados.rados.0.smithi017.stdout:1363: done (2 left) 2024-09-21T09:43:40.058 INFO:tasks.rados.rados.0.smithi017.stdout:1364: done (1 left) 2024-09-21T09:43:40.058 INFO:tasks.rados.rados.0.smithi017.stdout:1365: done (0 left) 2024-09-21T09:43:40.058 INFO:tasks.rados.rados.0.smithi017.stdout:1366: write oid 10 current snap is 168 2024-09-21T09:43:40.058 INFO:tasks.rados.rados.0.smithi017.stdout:1366: seq_num 532 ranges {541324=460825,1498538=416188,2393385=547939} 2024-09-21T09:43:40.063 INFO:tasks.rados.rados.0.smithi017.stdout:1366: writing smithi01785587-10 from 541324 to 1002149 tid 1 2024-09-21T09:43:40.068 INFO:tasks.rados.rados.0.smithi017.stdout:1366: writing smithi01785587-10 from 1498538 to 1914726 tid 2 2024-09-21T09:43:40.074 INFO:tasks.rados.rados.0.smithi017.stdout:1366: writing smithi01785587-10 from 2393385 to 2941324 tid 3 2024-09-21T09:43:40.074 INFO:tasks.rados.rados.0.smithi017.stdout:1367: delete oid 29 current snap is 168 2024-09-21T09:43:40.074 INFO:tasks.rados.rados.0.smithi017.stdout:1366: finishing write tid 1 to smithi01785587-10 2024-09-21T09:43:40.074 INFO:tasks.rados.rados.0.smithi017.stdout:1366: finishing write tid 2 to smithi01785587-10 2024-09-21T09:43:40.076 INFO:tasks.rados.rados.0.smithi017.stdout:1367: done (1 left) 2024-09-21T09:43:40.076 INFO:tasks.rados.rados.0.smithi017.stdout:1368: read oid 32 snap -1 2024-09-21T09:43:40.076 INFO:tasks.rados.rados.0.smithi017.stdout:1368: expect (ObjNum 506 snap 160 seq_num 506) 2024-09-21T09:43:40.076 INFO:tasks.rados.rados.0.smithi017.stdout:1369: snap_create 2024-09-21T09:43:40.078 INFO:tasks.rados.rados.0.smithi017.stdout:1368: expect (ObjNum 506 snap 160 seq_num 506) 2024-09-21T09:43:40.346 INFO:tasks.rados.rados.0.smithi017.stdout:1366: finishing write tid 3 to smithi01785587-10 2024-09-21T09:43:40.347 INFO:tasks.rados.rados.0.smithi017.stdout:1366: finishing write tid 4 to smithi01785587-10 2024-09-21T09:43:40.347 INFO:tasks.rados.rados.0.smithi017.stdout:1366: finishing write tid 5 to smithi01785587-10 2024-09-21T09:43:40.347 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 100 (ObjNum 532 snap 168 seq_num 532) dirty exists 2024-09-21T09:43:40.347 INFO:tasks.rados.rados.0.smithi017.stdout:1366: left oid 10 (ObjNum 532 snap 168 seq_num 532) 2024-09-21T09:43:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:41.059 INFO:tasks.rados.rados.0.smithi017.stdout:1366: done (2 left) 2024-09-21T09:43:41.059 INFO:tasks.rados.rados.0.smithi017.stdout:1368: done (1 left) 2024-09-21T09:43:41.060 INFO:tasks.rados.rados.0.smithi017.stdout:1369: done (0 left) 2024-09-21T09:43:41.060 INFO:tasks.rados.rados.0.smithi017.stdout:1370: read oid 41 snap -1 2024-09-21T09:43:41.060 INFO:tasks.rados.rados.0.smithi017.stdout:1370: expect (ObjNum 531 snap 167 seq_num 531) 2024-09-21T09:43:41.060 INFO:tasks.rados.rados.0.smithi017.stdout:1371: delete oid 20 current snap is 169 2024-09-21T09:43:41.065 INFO:tasks.rados.rados.0.smithi017.stdout:1370: expect (ObjNum 531 snap 167 seq_num 531) 2024-09-21T09:43:41.162 INFO:tasks.rados.rados.0.smithi017.stdout:1370: done (1 left) 2024-09-21T09:43:41.162 INFO:tasks.rados.rados.0.smithi017.stdout:1371: done (0 left) 2024-09-21T09:43:41.162 INFO:tasks.rados.rados.0.smithi017.stdout:1372: write oid 24 current snap is 169 2024-09-21T09:43:41.163 INFO:tasks.rados.rados.0.smithi017.stdout:1372: seq_num 535 ranges {591745=402925,1790337=563515} 2024-09-21T09:43:41.167 INFO:tasks.rados.rados.0.smithi017.stdout:1372: writing smithi01785587-24 from 591745 to 994670 tid 1 2024-09-21T09:43:41.175 INFO:tasks.rados.rados.0.smithi017.stdout:1372: writing smithi01785587-24 from 1790337 to 2353852 tid 2 2024-09-21T09:43:41.175 INFO:tasks.rados.rados.0.smithi017.stdout:1373: write oid 40 current snap is 169 2024-09-21T09:43:41.175 INFO:tasks.rados.rados.0.smithi017.stdout:1372: finishing write tid 1 to smithi01785587-24 2024-09-21T09:43:41.176 INFO:tasks.rados.rados.0.smithi017.stdout:1373: seq_num 536 ranges {608552=478174,1848819=707173} 2024-09-21T09:43:41.181 INFO:tasks.rados.rados.0.smithi017.stdout:1373: writing smithi01785587-40 from 608552 to 1086726 tid 1 2024-09-21T09:43:41.190 INFO:tasks.rados.rados.0.smithi017.stdout:1373: writing smithi01785587-40 from 1848819 to 2555992 tid 2 2024-09-21T09:43:41.190 INFO:tasks.rados.rados.0.smithi017.stdout:1374: rollback oid 11 current snap is 169 2024-09-21T09:43:41.190 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 149 2024-09-21T09:43:41.190 INFO:tasks.rados.rados.0.smithi017.stdout:1372: finishing write tid 2 to smithi01785587-24 2024-09-21T09:43:41.190 INFO:tasks.rados.rados.0.smithi017.stdout:1375: write oid 46 current snap is 169 2024-09-21T09:43:41.190 INFO:tasks.rados.rados.0.smithi017.stdout:1372: finishing write tid 3 to smithi01785587-24 2024-09-21T09:43:41.190 INFO:tasks.rados.rados.0.smithi017.stdout:1372: finishing write tid 4 to smithi01785587-24 2024-09-21T09:43:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 406 (ObjNum 535 snap 169 seq_num 535) dirty exists 2024-09-21T09:43:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1372: left oid 24 (ObjNum 535 snap 169 seq_num 535) 2024-09-21T09:43:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1373: finishing write tid 1 to smithi01785587-40 2024-09-21T09:43:41.191 INFO:tasks.rados.rados.0.smithi017.stdout:1375: seq_num 537 ranges {625359=553423,1790948=567184,2930500=94859} 2024-09-21T09:43:41.196 INFO:tasks.rados.rados.0.smithi017.stdout:1375: writing smithi01785587-46 from 625359 to 1178782 tid 1 2024-09-21T09:43:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[102913]: pgmap v172: 89 pgs: 89 active+clean; 194 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 298 KiB/s wr, 9 op/s 2024-09-21T09:43:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[102913]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:43:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:43:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[102913]: Reconfiguring daemon osd.1 on smithi017 2024-09-21T09:43:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[102913]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:43:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[105151]: pgmap v172: 89 pgs: 89 active+clean; 194 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 298 KiB/s wr, 9 op/s 2024-09-21T09:43:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[105151]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:43:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:43:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[105151]: Reconfiguring daemon osd.1 on smithi017 2024-09-21T09:43:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[105151]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:43:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:41.203 INFO:tasks.rados.rados.0.smithi017.stdout:1375: writing smithi01785587-46 from 1790948 to 2358132 tid 2 2024-09-21T09:43:41.207 INFO:tasks.rados.rados.0.smithi017.stdout:1375: writing smithi01785587-46 from 2930500 to 3025359 tid 3 2024-09-21T09:43:41.207 INFO:tasks.rados.rados.0.smithi017.stdout:1372: done (3 left) 2024-09-21T09:43:41.207 INFO:tasks.rados.rados.0.smithi017.stdout:1376: snap_create 2024-09-21T09:43:41.207 INFO:tasks.rados.rados.0.smithi017.stdout:1373: finishing write tid 2 to smithi01785587-40 2024-09-21T09:43:41.207 INFO:tasks.rados.rados.0.smithi017.stdout:1374: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:43:41.207 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 0 (ObjNum 33 snap 0 seq_num 2384147832) dirty dne 2024-09-21T09:43:41.207 INFO:tasks.rados.rados.0.smithi017.stdout:1373: finishing write tid 3 to smithi01785587-40 2024-09-21T09:43:41.207 INFO:tasks.rados.rados.0.smithi017.stdout:1373: finishing write tid 4 to smithi01785587-40 2024-09-21T09:43:41.208 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 678 (ObjNum 536 snap 169 seq_num 536) dirty exists 2024-09-21T09:43:41.208 INFO:tasks.rados.rados.0.smithi017.stdout:1373: left oid 40 (ObjNum 536 snap 169 seq_num 536) 2024-09-21T09:43:41.208 INFO:tasks.rados.rados.0.smithi017.stdout:1375: finishing write tid 1 to smithi01785587-46 2024-09-21T09:43:41.208 INFO:tasks.rados.rados.0.smithi017.stdout:1375: finishing write tid 2 to smithi01785587-46 2024-09-21T09:43:41.208 INFO:tasks.rados.rados.0.smithi017.stdout:1375: finishing write tid 3 to smithi01785587-46 2024-09-21T09:43:41.209 INFO:tasks.rados.rados.0.smithi017.stdout:1375: finishing write tid 4 to smithi01785587-46 2024-09-21T09:43:41.209 INFO:tasks.rados.rados.0.smithi017.stdout:1375: finishing write tid 5 to smithi01785587-46 2024-09-21T09:43:41.209 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 628 (ObjNum 537 snap 169 seq_num 537) dirty exists 2024-09-21T09:43:41.209 INFO:tasks.rados.rados.0.smithi017.stdout:1375: left oid 46 (ObjNum 537 snap 169 seq_num 537) 2024-09-21T09:43:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-mon[90085]: pgmap v172: 89 pgs: 89 active+clean; 194 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 298 KiB/s wr, 9 op/s 2024-09-21T09:43:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-mon[90085]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:43:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:43:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-mon[90085]: Reconfiguring daemon osd.1 on smithi017 2024-09-21T09:43:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-mon[90085]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:43:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:42.065 INFO:tasks.rados.rados.0.smithi017.stdout:1373: done (3 left) 2024-09-21T09:43:42.065 INFO:tasks.rados.rados.0.smithi017.stdout:1374: done (2 left) 2024-09-21T09:43:42.065 INFO:tasks.rados.rados.0.smithi017.stdout:1375: done (1 left) 2024-09-21T09:43:42.065 INFO:tasks.rados.rados.0.smithi017.stdout:1376: done (0 left) 2024-09-21T09:43:42.066 INFO:tasks.rados.rados.0.smithi017.stdout:1377: delete oid 36 current snap is 170 2024-09-21T09:43:42.067 INFO:tasks.rados.rados.0.smithi017.stdout:1377: done (0 left) 2024-09-21T09:43:42.067 INFO:tasks.rados.rados.0.smithi017.stdout:1378: read oid 1 snap 156 2024-09-21T09:43:42.067 INFO:tasks.rados.rados.0.smithi017.stdout:1378: expect (ObjNum 458 snap 148 seq_num 458) 2024-09-21T09:43:42.067 INFO:tasks.rados.rados.0.smithi017.stdout:1379: write oid 35 current snap is 170 2024-09-21T09:43:42.067 INFO:tasks.rados.rados.0.smithi017.stdout:1379: seq_num 539 ranges {658973=703921,1907912=570853,2985346=73627} 2024-09-21T09:43:42.075 INFO:tasks.rados.rados.0.smithi017.stdout:1379: writing smithi01785587-35 from 658973 to 1362894 tid 1 2024-09-21T09:43:42.082 INFO:tasks.rados.rados.0.smithi017.stdout:1379: writing smithi01785587-35 from 1907912 to 2478765 tid 2 2024-09-21T09:43:42.086 INFO:tasks.rados.rados.0.smithi017.stdout:1379: writing smithi01785587-35 from 2985346 to 3058973 tid 3 2024-09-21T09:43:42.086 INFO:tasks.rados.rados.0.smithi017.stdout:1380: read oid 45 snap -1 2024-09-21T09:43:42.086 INFO:tasks.rados.rados.0.smithi017.stdout:1380: expect deleted 2024-09-21T09:43:42.086 INFO:tasks.rados.rados.0.smithi017.stdout:1381: write oid 41 current snap is 170 2024-09-21T09:43:42.086 INFO:tasks.rados.rados.0.smithi017.stdout:1381: seq_num 540 ranges {675780=495523,1682747=714511,3012769=63011} 2024-09-21T09:43:42.092 INFO:tasks.rados.rados.0.smithi017.stdout:1381: writing smithi01785587-41 from 675780 to 1171303 tid 1 2024-09-21T09:43:42.099 INFO:tasks.rados.rados.0.smithi017.stdout:1381: writing smithi01785587-41 from 1682747 to 2397258 tid 2 2024-09-21T09:43:42.103 INFO:tasks.rados.rados.0.smithi017.stdout:1381: writing smithi01785587-41 from 3012769 to 3075780 tid 3 2024-09-21T09:43:42.103 INFO:tasks.rados.rados.0.smithi017.stdout:1382: delete oid 24 current snap is 170 2024-09-21T09:43:42.103 INFO:tasks.rados.rados.0.smithi017.stdout:1378: expect (ObjNum 458 snap 148 seq_num 458) 2024-09-21T09:43:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[105151]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:43:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[105151]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:43:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:43:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:42.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[105151]: Reconfiguring daemon osd.2 on smithi017 2024-09-21T09:43:42.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[102913]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:43:42.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:42.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:42.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[102913]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:43:42.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:43:42.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:42.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:42 smithi017 ceph-mon[102913]: Reconfiguring daemon osd.2 on smithi017 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:1379: finishing write tid 1 to smithi01785587-35 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:1379: finishing write tid 2 to smithi01785587-35 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:1379: finishing write tid 3 to smithi01785587-35 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:1379: finishing write tid 4 to smithi01785587-35 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:1379: finishing write tid 5 to smithi01785587-35 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 187 (ObjNum 539 snap 170 seq_num 539) dirty exists 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:1379: left oid 35 (ObjNum 539 snap 170 seq_num 539) 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:1381: finishing write tid 1 to smithi01785587-41 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:1378: done (4 left) 2024-09-21T09:43:42.365 INFO:tasks.rados.rados.0.smithi017.stdout:1379: done (3 left) 2024-09-21T09:43:42.366 INFO:tasks.rados.rados.0.smithi017.stdout:1380: done (2 left) 2024-09-21T09:43:42.366 INFO:tasks.rados.rados.0.smithi017.stdout:1382: done (1 left) 2024-09-21T09:43:42.366 INFO:tasks.rados.rados.0.smithi017.stdout:1383: snap_remove snap 161 2024-09-21T09:43:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:42 smithi134 ceph-mon[90085]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:43:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:42 smithi134 ceph-mon[90085]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:43:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:43:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:42 smithi134 ceph-mon[90085]: Reconfiguring daemon osd.2 on smithi017 2024-09-21T09:43:43.047 INFO:tasks.rados.rados.0.smithi017.stdout:1383: done (1 left) 2024-09-21T09:43:43.047 INFO:tasks.rados.rados.0.smithi017.stdout:1384: snap_create 2024-09-21T09:43:43.047 INFO:tasks.rados.rados.0.smithi017.stdout:1381: finishing write tid 2 to smithi01785587-41 2024-09-21T09:43:43.047 INFO:tasks.rados.rados.0.smithi017.stdout:1381: finishing write tid 3 to smithi01785587-41 2024-09-21T09:43:43.048 INFO:tasks.rados.rados.0.smithi017.stdout:1381: finishing write tid 4 to smithi01785587-41 2024-09-21T09:43:43.048 INFO:tasks.rados.rados.0.smithi017.stdout:1381: finishing write tid 5 to smithi01785587-41 2024-09-21T09:43:43.048 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 228 (ObjNum 540 snap 170 seq_num 540) dirty exists 2024-09-21T09:43:43.048 INFO:tasks.rados.rados.0.smithi017.stdout:1381: left oid 41 (ObjNum 540 snap 170 seq_num 540) 2024-09-21T09:43:43.063 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[105151]: pgmap v175: 89 pgs: 89 active+clean; 206 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-09-21T09:43:43.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[102913]: pgmap v175: 89 pgs: 89 active+clean; 206 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-09-21T09:43:43.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[102913]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T09:43:43.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:43.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:43.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:43:43.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:43.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[102913]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T09:43:43.320 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[105151]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T09:43:43.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:43.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:43.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:43:43.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:43.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:43 smithi017 ceph-mon[105151]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T09:43:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:43 smithi134 ceph-mon[90085]: pgmap v175: 89 pgs: 89 active+clean; 206 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-09-21T09:43:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:43 smithi134 ceph-mon[90085]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T09:43:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:43:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:43 smithi134 ceph-mon[90085]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T09:43:44.045 INFO:tasks.rados.rados.0.smithi017.stdout:1381: done (1 left) 2024-09-21T09:43:44.045 INFO:tasks.rados.rados.0.smithi017.stdout:1384: done (0 left) 2024-09-21T09:43:44.045 INFO:tasks.rados.rados.0.smithi017.stdout:1385: delete oid 44 current snap is 171 2024-09-21T09:43:44.049 INFO:tasks.rados.rados.0.smithi017.stdout:1385: done (0 left) 2024-09-21T09:43:44.049 INFO:tasks.rados.rados.0.smithi017.stdout:1386: write oid 12 current snap is 171 2024-09-21T09:43:44.049 INFO:tasks.rados.rados.0.smithi017.stdout:1386: seq_num 543 ranges {726201=721270,2141840=578191} 2024-09-21T09:43:44.057 INFO:tasks.rados.rados.0.smithi017.stdout:1386: writing smithi01785587-12 from 726201 to 1447471 tid 1 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:1386: writing smithi01785587-12 from 2141840 to 2720031 tid 2 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:1387: read oid 40 snap -1 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:1387: expect (ObjNum 536 snap 169 seq_num 536) 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:1388: rollback oid 2 current snap is 171 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 2 to 170 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:1386: finishing write tid 1 to smithi01785587-12 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:1389: rollback oid 42 current snap is 171 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 145 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:1390: read oid 45 snap -1 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:1390: expect deleted 2024-09-21T09:43:44.065 INFO:tasks.rados.rados.0.smithi017.stdout:1391: write oid 23 current snap is 171 2024-09-21T09:43:44.066 INFO:tasks.rados.rados.0.smithi017.stdout:1391: seq_num 544 ranges {743008=796519,2200322=721849} 2024-09-21T09:43:44.074 INFO:tasks.rados.rados.0.smithi017.stdout:1391: writing smithi01785587-23 from 743008 to 1539527 tid 1 2024-09-21T09:43:44.082 INFO:tasks.rados.rados.0.smithi017.stdout:1391: writing smithi01785587-23 from 2200322 to 2922171 tid 2 2024-09-21T09:43:44.082 INFO:tasks.rados.rados.0.smithi017.stdout:1392: snap_remove snap 169 2024-09-21T09:43:44.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:44 smithi017 ceph-mon[102913]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:43:44.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:44 smithi017 ceph-mon[102913]: Reconfiguring daemon osd.3 on smithi017 2024-09-21T09:43:44.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:44 smithi017 ceph-mon[102913]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T09:43:44.167 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:44 smithi017 ceph-mon[105151]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:43:44.167 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:44 smithi017 ceph-mon[105151]: Reconfiguring daemon osd.3 on smithi017 2024-09-21T09:43:44.167 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:44 smithi017 ceph-mon[105151]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T09:43:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:44 smithi134 ceph-mon[90085]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:43:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:44 smithi134 ceph-mon[90085]: Reconfiguring daemon osd.3 on smithi017 2024-09-21T09:43:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:44 smithi134 ceph-mon[90085]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T09:43:44.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:44.886 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:43:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:43:44] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.51.0" 2024-09-21T09:43:45.074 INFO:tasks.rados.rados.0.smithi017.stdout:1392: done (6 left) 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1393: delete oid 19 current snap is 171 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1386: finishing write tid 2 to smithi01785587-12 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1386: finishing write tid 3 to smithi01785587-12 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1386: finishing write tid 4 to smithi01785587-12 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 497 (ObjNum 543 snap 171 seq_num 543) dirty exists 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1386: left oid 12 (ObjNum 543 snap 171 seq_num 543) 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1389: finishing rollback tid 0 to smithi01785587-42 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1389: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1388: finishing rollback tid 0 to smithi01785587-2 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1388: finishing rollback tid 1 to smithi01785587-2 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1389: finishing rollback tid 2 to smithi01785587-42 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 501 (ObjNum 426 snap 133 seq_num 426) dirty exists 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:1388: finishing rollback tid 2 to smithi01785587-2 2024-09-21T09:43:45.075 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 264 (ObjNum 479 snap 153 seq_num 479) dirty exists 2024-09-21T09:43:45.076 INFO:tasks.rados.rados.0.smithi017.stdout:1387: expect (ObjNum 536 snap 169 seq_num 536) 2024-09-21T09:43:45.266 INFO:tasks.rados.rados.0.smithi017.stdout:1391: finishing write tid 1 to smithi01785587-23 2024-09-21T09:43:45.266 INFO:tasks.rados.rados.0.smithi017.stdout:1391: finishing write tid 2 to smithi01785587-23 2024-09-21T09:43:45.266 INFO:tasks.rados.rados.0.smithi017.stdout:1391: finishing write tid 3 to smithi01785587-23 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1391: finishing write tid 4 to smithi01785587-23 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 636 (ObjNum 544 snap 171 seq_num 544) dirty exists 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1391: left oid 23 (ObjNum 544 snap 171 seq_num 544) 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1386: done (6 left) 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1387: done (5 left) 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1388: done (4 left) 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1389: done (3 left) 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1390: done (2 left) 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1391: done (1 left) 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1393: done (0 left) 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1394: read oid 9 snap -1 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1394: expect deleted 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1395: read oid 16 snap -1 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1395: expect deleted 2024-09-21T09:43:45.267 INFO:tasks.rados.rados.0.smithi017.stdout:1396: write oid 24 current snap is 171 2024-09-21T09:43:45.268 INFO:tasks.rados.rados.0.smithi017.stdout:1396: seq_num 546 ranges {776622=547017,1800933=725518,2944601=232021} 2024-09-21T09:43:45.273 INFO:tasks.rados.rados.0.smithi017.stdout:1396: writing smithi01785587-24 from 776622 to 1323639 tid 1 2024-09-21T09:43:45.280 INFO:tasks.rados.rados.0.smithi017.stdout:1396: writing smithi01785587-24 from 1800933 to 2526451 tid 2 2024-09-21T09:43:45.283 INFO:tasks.rados.rados.0.smithi017.stdout:1396: writing smithi01785587-24 from 2944601 to 3176622 tid 3 2024-09-21T09:43:45.283 INFO:tasks.rados.rados.0.smithi017.stdout:1397: snap_remove snap 156 2024-09-21T09:43:45.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-mon[90085]: pgmap v178: 89 pgs: 89 active+clean; 206 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-09-21T09:43:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-mon[90085]: Reconfiguring mds.foo.smithi017.coodep (monmap changed)... 2024-09-21T09:43:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:43:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-mon[90085]: Reconfiguring daemon mds.foo.smithi017.coodep on smithi017 2024-09-21T09:43:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-mon[90085]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T09:43:45.556 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[102913]: pgmap v178: 89 pgs: 89 active+clean; 206 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-09-21T09:43:45.556 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:45.556 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[102913]: Reconfiguring mds.foo.smithi017.coodep (monmap changed)... 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[102913]: Reconfiguring daemon mds.foo.smithi017.coodep on smithi017 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[102913]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[105151]: pgmap v178: 89 pgs: 89 active+clean; 206 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[105151]: Reconfiguring mds.foo.smithi017.coodep (monmap changed)... 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.coodep", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:45.557 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[105151]: Reconfiguring daemon mds.foo.smithi017.coodep on smithi017 2024-09-21T09:43:45.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:45 smithi017 ceph-mon[105151]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T09:43:45.969 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:45.969 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:46.263 INFO:tasks.rados.rados.0.smithi017.stdout:1396: finishing write tid 1 to smithi01785587-24 2024-09-21T09:43:46.263 INFO:tasks.rados.rados.0.smithi017.stdout:1396: finishing write tid 2 to smithi01785587-24 2024-09-21T09:43:46.263 INFO:tasks.rados.rados.0.smithi017.stdout:1396: finishing write tid 3 to smithi01785587-24 2024-09-21T09:43:46.264 INFO:tasks.rados.rados.0.smithi017.stdout:1396: finishing write tid 4 to smithi01785587-24 2024-09-21T09:43:46.264 INFO:tasks.rados.rados.0.smithi017.stdout:1396: finishing write tid 5 to smithi01785587-24 2024-09-21T09:43:46.264 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 416 (ObjNum 546 snap 171 seq_num 546) dirty exists 2024-09-21T09:43:46.264 INFO:tasks.rados.rados.0.smithi017.stdout:1396: left oid 24 (ObjNum 546 snap 171 seq_num 546) 2024-09-21T09:43:46.264 INFO:tasks.rados.rados.0.smithi017.stdout:1394: done (3 left) 2024-09-21T09:43:46.264 INFO:tasks.rados.rados.0.smithi017.stdout:1395: done (2 left) 2024-09-21T09:43:46.264 INFO:tasks.rados.rados.0.smithi017.stdout:1396: done (1 left) 2024-09-21T09:43:46.264 INFO:tasks.rados.rados.0.smithi017.stdout:1397: done (0 left) 2024-09-21T09:43:46.264 INFO:tasks.rados.rados.0.smithi017.stdout:1398: snap_remove snap 168 2024-09-21T09:43:46.941 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:46.941 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:46.941 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-mon[90085]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:43:46.941 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:46.941 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:46.941 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:46.942 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-mon[90085]: Reconfiguring daemon mon.b on smithi134 2024-09-21T09:43:46.942 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-mon[90085]: pgmap v181: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:43:46.942 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-mon[90085]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T09:43:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[105151]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:43:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[105151]: Reconfiguring daemon mon.b on smithi134 2024-09-21T09:43:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[105151]: pgmap v181: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:43:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[105151]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T09:43:46.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:46.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:46.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[102913]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:43:46.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:43:46.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:43:46.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:46.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[102913]: Reconfiguring daemon mon.b on smithi134 2024-09-21T09:43:46.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[102913]: pgmap v181: 89 pgs: 89 active+clean; 218 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:43:46.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:46 smithi017 ceph-mon[102913]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T09:43:47.243 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:47.266 INFO:tasks.rados.rados.0.smithi017.stdout:1398: done (0 left) 2024-09-21T09:43:47.266 INFO:tasks.rados.rados.0.smithi017.stdout:1399: write oid 36 current snap is 171 2024-09-21T09:43:47.266 INFO:tasks.rados.rados.0.smithi017.stdout:1399: seq_num 547 ranges {793429=622266,1859415=469176,2972024=221405} 2024-09-21T09:43:47.274 INFO:tasks.rados.rados.0.smithi017.stdout:1399: writing smithi01785587-36 from 793429 to 1415695 tid 1 2024-09-21T09:43:47.279 INFO:tasks.rados.rados.0.smithi017.stdout:1399: writing smithi01785587-36 from 1859415 to 2328591 tid 2 2024-09-21T09:43:47.284 INFO:tasks.rados.rados.0.smithi017.stdout:1399: writing smithi01785587-36 from 2972024 to 3193429 tid 3 2024-09-21T09:43:47.284 INFO:tasks.rados.rados.0.smithi017.stdout:1400: snap_create 2024-09-21T09:43:47.284 INFO:tasks.rados.rados.0.smithi017.stdout:1399: finishing write tid 1 to smithi01785587-36 2024-09-21T09:43:47.284 INFO:tasks.rados.rados.0.smithi017.stdout:1399: finishing write tid 2 to smithi01785587-36 2024-09-21T09:43:47.286 INFO:tasks.rados.rados.0.smithi017.stdout:1399: finishing write tid 3 to smithi01785587-36 2024-09-21T09:43:47.286 INFO:tasks.rados.rados.0.smithi017.stdout:1399: finishing write tid 4 to smithi01785587-36 2024-09-21T09:43:47.287 INFO:tasks.rados.rados.0.smithi017.stdout:1399: finishing write tid 5 to smithi01785587-36 2024-09-21T09:43:47.287 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 704 (ObjNum 547 snap 171 seq_num 547) dirty exists 2024-09-21T09:43:47.287 INFO:tasks.rados.rados.0.smithi017.stdout:1399: left oid 36 (ObjNum 547 snap 171 seq_num 547) 2024-09-21T09:43:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:47 smithi134 ceph-mon[90085]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:43:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:43:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:43:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:47 smithi134 ceph-mon[90085]: Reconfiguring daemon mgr.x on smithi134 2024-09-21T09:43:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:47 smithi134 ceph-mon[90085]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T09:43:48.269 INFO:tasks.rados.rados.0.smithi017.stdout:1399: done (1 left) 2024-09-21T09:43:48.269 INFO:tasks.rados.rados.0.smithi017.stdout:1400: done (0 left) 2024-09-21T09:43:48.269 INFO:tasks.rados.rados.0.smithi017.stdout:1401: write oid 24 current snap is 172 2024-09-21T09:43:48.270 INFO:tasks.rados.rados.0.smithi017.stdout:1401: seq_num 548 ranges {410236=413868,1234250=729187,2715800=423063} 2024-09-21T09:43:48.274 INFO:tasks.rados.rados.0.smithi017.stdout:1401: writing smithi01785587-24 from 410236 to 824104 tid 1 2024-09-21T09:43:48.281 INFO:tasks.rados.rados.0.smithi017.stdout:1401: writing smithi01785587-24 from 1234250 to 1963437 tid 2 2024-09-21T09:43:48.288 INFO:tasks.rados.rados.0.smithi017.stdout:1401: writing smithi01785587-24 from 2715800 to 3138863 tid 3 2024-09-21T09:43:48.288 INFO:tasks.rados.rados.0.smithi017.stdout:1402: read oid 49 snap -1 2024-09-21T09:43:48.288 INFO:tasks.rados.rados.0.smithi017.stdout:1402: expect (ObjNum 447 snap 145 seq_num 447) 2024-09-21T09:43:48.288 INFO:tasks.rados.rados.0.smithi017.stdout:1403: write oid 9 current snap is 172 2024-09-21T09:43:48.288 INFO:tasks.rados.rados.0.smithi017.stdout:1401: finishing write tid 1 to smithi01785587-24 2024-09-21T09:43:48.288 INFO:tasks.rados.rados.0.smithi017.stdout:1401: finishing write tid 2 to smithi01785587-24 2024-09-21T09:43:48.288 INFO:tasks.rados.rados.0.smithi017.stdout:1403: seq_num 549 ranges {427043=489117,1576379=472845,2626870=600173} 2024-09-21T09:43:48.293 INFO:tasks.rados.rados.0.smithi017.stdout:1403: writing smithi01785587-9 from 427043 to 916160 tid 1 2024-09-21T09:43:48.299 INFO:tasks.rados.rados.0.smithi017.stdout:1403: writing smithi01785587-9 from 1576379 to 2049224 tid 2 2024-09-21T09:43:48.306 INFO:tasks.rados.rados.0.smithi017.stdout:1403: writing smithi01785587-9 from 2626870 to 3227043 tid 3 2024-09-21T09:43:48.306 INFO:tasks.rados.rados.0.smithi017.stdout:1404: read oid 12 snap 157 2024-09-21T09:43:48.306 INFO:tasks.rados.rados.0.smithi017.stdout:1404: expect deleted 2024-09-21T09:43:48.306 INFO:tasks.rados.rados.0.smithi017.stdout:1405: delete oid 10 current snap is 172 2024-09-21T09:43:48.306 INFO:tasks.rados.rados.0.smithi017.stdout:1401: finishing write tid 3 to smithi01785587-24 2024-09-21T09:43:48.306 INFO:tasks.rados.rados.0.smithi017.stdout:1401: finishing write tid 4 to smithi01785587-24 2024-09-21T09:43:48.307 INFO:tasks.rados.rados.0.smithi017.stdout:1401: finishing write tid 5 to smithi01785587-24 2024-09-21T09:43:48.307 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 425 (ObjNum 548 snap 172 seq_num 548) dirty exists 2024-09-21T09:43:48.307 INFO:tasks.rados.rados.0.smithi017.stdout:1401: left oid 24 (ObjNum 548 snap 172 seq_num 548) 2024-09-21T09:43:48.307 INFO:tasks.rados.rados.0.smithi017.stdout:1402: expect (ObjNum 447 snap 145 seq_num 447) 2024-09-21T09:43:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[105151]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:43:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:43:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:43:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[105151]: Reconfiguring daemon mgr.x on smithi134 2024-09-21T09:43:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[105151]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T09:43:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[102913]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:43:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:43:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:43:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[102913]: Reconfiguring daemon mgr.x on smithi134 2024-09-21T09:43:48.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:47 smithi017 ceph-mon[102913]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T09:43:48.554 INFO:tasks.rados.rados.0.smithi017.stdout:1403: finishing write tid 1 to smithi01785587-9 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1403: finishing write tid 2 to smithi01785587-9 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1403: finishing write tid 3 to smithi01785587-9 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1403: finishing write tid 4 to smithi01785587-9 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1403: finishing write tid 5 to smithi01785587-9 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 274 (ObjNum 549 snap 172 seq_num 549) dirty exists 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1403: left oid 9 (ObjNum 549 snap 172 seq_num 549) 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1401: done (4 left) 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1402: done (3 left) 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1403: done (2 left) 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1404: done (1 left) 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1405: done (0 left) 2024-09-21T09:43:48.555 INFO:tasks.rados.rados.0.smithi017.stdout:1406: snap_remove snap 164 2024-09-21T09:43:49.326 INFO:tasks.rados.rados.0.smithi017.stdout:1406: done (0 left) 2024-09-21T09:43:49.326 INFO:tasks.rados.rados.0.smithi017.stdout:1407: write oid 21 current snap is 172 2024-09-21T09:43:49.326 INFO:tasks.rados.rados.0.smithi017.stdout:1407: seq_num 551 ranges {460657=639615,1693343=476514,2681716=578941} 2024-09-21T09:43:49.333 INFO:tasks.rados.rados.0.smithi017.stdout:1407: writing smithi01785587-21 from 460657 to 1100272 tid 1 2024-09-21T09:43:49.339 INFO:tasks.rados.rados.0.smithi017.stdout:1407: writing smithi01785587-21 from 1693343 to 2169857 tid 2 2024-09-21T09:43:49.346 INFO:tasks.rados.rados.0.smithi017.stdout:1407: writing smithi01785587-21 from 2681716 to 3260657 tid 3 2024-09-21T09:43:49.346 INFO:tasks.rados.rados.0.smithi017.stdout:1408: read oid 25 snap -1 2024-09-21T09:43:49.346 INFO:tasks.rados.rados.0.smithi017.stdout:1408: expect deleted 2024-09-21T09:43:49.346 INFO:tasks.rados.rados.0.smithi017.stdout:1409: delete oid 19 current snap is 172 2024-09-21T09:43:49.346 INFO:tasks.rados.rados.0.smithi017.stdout:1407: finishing write tid 1 to smithi01785587-21 2024-09-21T09:43:49.346 INFO:tasks.rados.rados.0.smithi017.stdout:1407: finishing write tid 2 to smithi01785587-21 2024-09-21T09:43:49.347 INFO:tasks.rados.rados.0.smithi017.stdout:1408: done (2 left) 2024-09-21T09:43:49.347 INFO:tasks.rados.rados.0.smithi017.stdout:1409: done (1 left) 2024-09-21T09:43:49.347 INFO:tasks.rados.rados.0.smithi017.stdout:1410: delete oid 23 current snap is 172 2024-09-21T09:43:49.349 INFO:tasks.rados.rados.0.smithi017.stdout:1407: finishing write tid 3 to smithi01785587-21 2024-09-21T09:43:49.349 INFO:tasks.rados.rados.0.smithi017.stdout:1407: finishing write tid 4 to smithi01785587-21 2024-09-21T09:43:49.349 INFO:tasks.rados.rados.0.smithi017.stdout:1407: finishing write tid 5 to smithi01785587-21 2024-09-21T09:43:49.350 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 741 (ObjNum 551 snap 172 seq_num 551) dirty exists 2024-09-21T09:43:49.350 INFO:tasks.rados.rados.0.smithi017.stdout:1407: left oid 21 (ObjNum 551 snap 172 seq_num 551) 2024-09-21T09:43:49.350 INFO:tasks.rados.rados.0.smithi017.stdout:1407: done (1 left) 2024-09-21T09:43:49.350 INFO:tasks.rados.rados.0.smithi017.stdout:1410: done (0 left) 2024-09-21T09:43:49.350 INFO:tasks.rados.rados.0.smithi017.stdout:1411: write oid 34 current snap is 172 2024-09-21T09:43:49.350 INFO:tasks.rados.rados.0.smithi017.stdout:1411: seq_num 554 ranges {511078=465362,1752436=740194,3163985=147093} 2024-09-21T09:43:49.354 INFO:tasks.rados.rados.0.smithi017.stdout:1411: writing smithi01785587-34 from 511078 to 976440 tid 1 2024-09-21T09:43:49.363 INFO:tasks.rados.rados.0.smithi017.stdout:1411: writing smithi01785587-34 from 1752436 to 2492630 tid 2 2024-09-21T09:43:49.368 INFO:tasks.rados.rados.0.smithi017.stdout:1411: writing smithi01785587-34 from 3163985 to 3311078 tid 3 2024-09-21T09:43:49.368 INFO:tasks.rados.rados.0.smithi017.stdout:1412: write oid 8 current snap is 172 2024-09-21T09:43:49.368 INFO:tasks.rados.rados.0.smithi017.stdout:1411: finishing write tid 1 to smithi01785587-34 2024-09-21T09:43:49.368 INFO:tasks.rados.rados.0.smithi017.stdout:1411: finishing write tid 2 to smithi01785587-34 2024-09-21T09:43:49.368 INFO:tasks.rados.rados.0.smithi017.stdout:1412: seq_num 555 ranges {527885=656964,1927271=483852,3191408=136477} 2024-09-21T09:43:49.374 INFO:tasks.rados.rados.0.smithi017.stdout:1412: writing smithi01785587-8 from 527885 to 1184849 tid 1 2024-09-21T09:43:49.381 INFO:tasks.rados.rados.0.smithi017.stdout:1412: writing smithi01785587-8 from 1927271 to 2411123 tid 2 2024-09-21T09:43:49.386 INFO:tasks.rados.rados.0.smithi017.stdout:1412: writing smithi01785587-8 from 3191408 to 3327885 tid 3 2024-09-21T09:43:49.386 INFO:tasks.rados.rados.0.smithi017.stdout:1413: read oid 28 snap -1 2024-09-21T09:43:49.386 INFO:tasks.rados.rados.0.smithi017.stdout:1413: expect deleted 2024-09-21T09:43:49.386 INFO:tasks.rados.rados.0.smithi017.stdout:1414: snap_remove snap 160 2024-09-21T09:43:49.671 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:49 smithi134 ceph-mon[90085]: pgmap v184: 89 pgs: 89 active+clean; 223 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-09-21T09:43:49.671 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:49 smithi134 ceph-mon[90085]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T09:43:49.672 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:49.672 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:49.672 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:49 smithi134 ceph-mon[90085]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:43:49.672 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:43:49.672 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:49.672 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:49 smithi134 ceph-mon[90085]: Reconfiguring daemon osd.4 on smithi134 2024-09-21T09:43:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[105151]: pgmap v184: 89 pgs: 89 active+clean; 223 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-09-21T09:43:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[105151]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T09:43:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[105151]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:43:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:43:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[105151]: Reconfiguring daemon osd.4 on smithi134 2024-09-21T09:43:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[102913]: pgmap v184: 89 pgs: 89 active+clean; 223 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-09-21T09:43:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[102913]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T09:43:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[102913]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:43:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:43:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:49.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:49 smithi017 ceph-mon[102913]: Reconfiguring daemon osd.4 on smithi134 2024-09-21T09:43:50.332 INFO:tasks.rados.rados.0.smithi017.stdout:1414: done (3 left) 2024-09-21T09:43:50.332 INFO:tasks.rados.rados.0.smithi017.stdout:1415: read oid 17 snap -1 2024-09-21T09:43:50.332 INFO:tasks.rados.rados.0.smithi017.stdout:1415: expect (ObjNum 511 snap 160 seq_num 511) 2024-09-21T09:43:50.332 INFO:tasks.rados.rados.0.smithi017.stdout:1416: snap_remove snap 170 2024-09-21T09:43:50.458 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:50 smithi134 ceph-mon[90085]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T09:43:50.458 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:50 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:50.458 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:50 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:50.458 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:50 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:43:50.458 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:50 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:50.458 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[105151]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T09:43:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:43:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[102913]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T09:43:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:43:50.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:51.334 INFO:tasks.rados.rados.0.smithi017.stdout:1416: done (4 left) 2024-09-21T09:43:51.334 INFO:tasks.rados.rados.0.smithi017.stdout:1417: snap_create 2024-09-21T09:43:51.336 INFO:tasks.rados.rados.0.smithi017.stdout:1411: finishing write tid 3 to smithi01785587-34 2024-09-21T09:43:51.336 INFO:tasks.rados.rados.0.smithi017.stdout:1411: finishing write tid 4 to smithi01785587-34 2024-09-21T09:43:51.336 INFO:tasks.rados.rados.0.smithi017.stdout:1411: finishing write tid 5 to smithi01785587-34 2024-09-21T09:43:51.336 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 746 (ObjNum 554 snap 172 seq_num 554) dirty exists 2024-09-21T09:43:51.336 INFO:tasks.rados.rados.0.smithi017.stdout:1411: left oid 34 (ObjNum 554 snap 172 seq_num 554) 2024-09-21T09:43:51.336 INFO:tasks.rados.rados.0.smithi017.stdout:1412: finishing write tid 1 to smithi01785587-8 2024-09-21T09:43:51.337 INFO:tasks.rados.rados.0.smithi017.stdout:1412: finishing write tid 2 to smithi01785587-8 2024-09-21T09:43:51.337 INFO:tasks.rados.rados.0.smithi017.stdout:1412: finishing write tid 3 to smithi01785587-8 2024-09-21T09:43:51.337 INFO:tasks.rados.rados.0.smithi017.stdout:1412: finishing write tid 4 to smithi01785587-8 2024-09-21T09:43:51.337 INFO:tasks.rados.rados.0.smithi017.stdout:1412: finishing write tid 5 to smithi01785587-8 2024-09-21T09:43:51.337 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 659 (ObjNum 555 snap 172 seq_num 555) dirty exists 2024-09-21T09:43:51.337 INFO:tasks.rados.rados.0.smithi017.stdout:1412: left oid 8 (ObjNum 555 snap 172 seq_num 555) 2024-09-21T09:43:51.337 INFO:tasks.rados.rados.0.smithi017.stdout:1415: expect (ObjNum 511 snap 160 seq_num 511) 2024-09-21T09:43:51.598 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:51 smithi134 ceph-mon[90085]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:43:51.598 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:51 smithi134 ceph-mon[90085]: Reconfiguring daemon osd.5 on smithi134 2024-09-21T09:43:51.598 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:51 smithi134 ceph-mon[90085]: pgmap v187: 89 pgs: 89 active+clean; 223 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 930 KiB/s wr, 9 op/s 2024-09-21T09:43:51.598 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:51 smithi134 ceph-mon[90085]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T09:43:51.598 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:51.598 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:51.598 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:43:51.598 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[102913]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:43:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[102913]: Reconfiguring daemon osd.5 on smithi134 2024-09-21T09:43:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[102913]: pgmap v187: 89 pgs: 89 active+clean; 223 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 930 KiB/s wr, 9 op/s 2024-09-21T09:43:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[102913]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T09:43:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:43:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[105151]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:43:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[105151]: Reconfiguring daemon osd.5 on smithi134 2024-09-21T09:43:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[105151]: pgmap v187: 89 pgs: 89 active+clean; 223 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 930 KiB/s wr, 9 op/s 2024-09-21T09:43:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[105151]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T09:43:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:43:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:52.344 INFO:tasks.rados.rados.0.smithi017.stdout:1411: done (4 left) 2024-09-21T09:43:52.344 INFO:tasks.rados.rados.0.smithi017.stdout:1412: done (3 left) 2024-09-21T09:43:52.344 INFO:tasks.rados.rados.0.smithi017.stdout:1413: done (2 left) 2024-09-21T09:43:52.344 INFO:tasks.rados.rados.0.smithi017.stdout:1415: done (1 left) 2024-09-21T09:43:52.344 INFO:tasks.rados.rados.0.smithi017.stdout:1417: done (0 left) 2024-09-21T09:43:52.344 INFO:tasks.rados.rados.0.smithi017.stdout:1418: delete oid 36 current snap is 173 2024-09-21T09:43:52.348 INFO:tasks.rados.rados.0.smithi017.stdout:1418: done (0 left) 2024-09-21T09:43:52.348 INFO:tasks.rados.rados.0.smithi017.stdout:1419: read oid 29 snap 166 2024-09-21T09:43:52.348 INFO:tasks.rados.rados.0.smithi017.stdout:1419: expect deleted 2024-09-21T09:43:52.348 INFO:tasks.rados.rados.0.smithi017.stdout:1420: write oid 27 current snap is 173 2024-09-21T09:43:52.348 INFO:tasks.rados.rados.0.smithi017.stdout:1420: seq_num 557 ranges {561499=407462,1527882=487521,2729901=557217} 2024-09-21T09:43:52.353 INFO:tasks.rados.rados.0.smithi017.stdout:1420: writing smithi01785587-27 from 561499 to 968961 tid 1 2024-09-21T09:43:52.358 INFO:tasks.rados.rados.0.smithi017.stdout:1420: writing smithi01785587-27 from 1527882 to 2015403 tid 2 2024-09-21T09:43:52.366 INFO:tasks.rados.rados.0.smithi017.stdout:1420: writing smithi01785587-27 from 2729901 to 3287118 tid 3 2024-09-21T09:43:52.366 INFO:tasks.rados.rados.0.smithi017.stdout:1421: read oid 43 snap -1 2024-09-21T09:43:52.366 INFO:tasks.rados.rados.0.smithi017.stdout:1421: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:43:52.366 INFO:tasks.rados.rados.0.smithi017.stdout:1422: delete oid 44 current snap is 173 2024-09-21T09:43:52.366 INFO:tasks.rados.rados.0.smithi017.stdout:1420: finishing write tid 1 to smithi01785587-27 2024-09-21T09:43:52.366 INFO:tasks.rados.rados.0.smithi017.stdout:1420: finishing write tid 2 to smithi01785587-27 2024-09-21T09:43:52.368 INFO:tasks.rados.rados.0.smithi017.stdout:1419: done (3 left) 2024-09-21T09:43:52.368 INFO:tasks.rados.rados.0.smithi017.stdout:1422: done (2 left) 2024-09-21T09:43:52.368 INFO:tasks.rados.rados.0.smithi017.stdout:1423: write oid 14 current snap is 173 2024-09-21T09:43:52.368 INFO:tasks.rados.rados.0.smithi017.stdout:1423: seq_num 559 ranges {595113=557960,1644846=491190,2784747=610366} 2024-09-21T09:43:52.374 INFO:tasks.rados.rados.0.smithi017.stdout:1423: writing smithi01785587-14 from 595113 to 1153073 tid 1 2024-09-21T09:43:52.380 INFO:tasks.rados.rados.0.smithi017.stdout:1423: writing smithi01785587-14 from 1644846 to 2136036 tid 2 2024-09-21T09:43:52.387 INFO:tasks.rados.rados.0.smithi017.stdout:1423: writing smithi01785587-14 from 2784747 to 3395113 tid 3 2024-09-21T09:43:52.387 INFO:tasks.rados.rados.0.smithi017.stdout:1424: snap_remove snap 167 2024-09-21T09:43:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:52 smithi017 ceph-mon[102913]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:43:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:52 smithi017 ceph-mon[102913]: Reconfiguring daemon osd.6 on smithi134 2024-09-21T09:43:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:52 smithi017 ceph-mon[102913]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T09:43:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:52 smithi017 ceph-mon[105151]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:43:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:52 smithi017 ceph-mon[105151]: Reconfiguring daemon osd.6 on smithi134 2024-09-21T09:43:52.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:52 smithi017 ceph-mon[105151]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T09:43:52.593 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:52 smithi134 ceph-mon[90085]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:43:52.593 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:52 smithi134 ceph-mon[90085]: Reconfiguring daemon osd.6 on smithi134 2024-09-21T09:43:52.593 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:52 smithi134 ceph-mon[90085]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T09:43:53.047 INFO:tasks.rados.rados.0.smithi017.stdout:1424: done (3 left) 2024-09-21T09:43:53.047 INFO:tasks.rados.rados.0.smithi017.stdout:1425: read oid 18 snap -1 2024-09-21T09:43:53.047 INFO:tasks.rados.rados.0.smithi017.stdout:1425: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:43:53.047 INFO:tasks.rados.rados.0.smithi017.stdout:1426: write oid 12 current snap is 173 2024-09-21T09:43:53.047 INFO:tasks.rados.rados.0.smithi017.stdout:1426: seq_num 560 ranges {611920=633209,1703328=634848,2812170=507386} 2024-09-21T09:43:53.055 INFO:tasks.rados.rados.0.smithi017.stdout:1426: writing smithi01785587-12 from 611920 to 1245129 tid 1 2024-09-21T09:43:53.062 INFO:tasks.rados.rados.0.smithi017.stdout:1426: writing smithi01785587-12 from 1703328 to 2338176 tid 2 2024-09-21T09:43:53.068 INFO:tasks.rados.rados.0.smithi017.stdout:1426: writing smithi01785587-12 from 2812170 to 3319556 tid 3 2024-09-21T09:43:53.068 INFO:tasks.rados.rados.0.smithi017.stdout:1427: write oid 34 current snap is 173 2024-09-21T09:43:53.068 INFO:tasks.rados.rados.0.smithi017.stdout:1420: finishing write tid 3 to smithi01785587-27 2024-09-21T09:43:53.068 INFO:tasks.rados.rados.0.smithi017.stdout:1427: seq_num 561 ranges {628727=708458,2045457=494859,3123240=305487} 2024-09-21T09:43:53.075 INFO:tasks.rados.rados.0.smithi017.stdout:1427: writing smithi01785587-34 from 628727 to 1337185 tid 1 2024-09-21T09:43:53.082 INFO:tasks.rados.rados.0.smithi017.stdout:1427: writing smithi01785587-34 from 2045457 to 2540316 tid 2 2024-09-21T09:43:53.087 INFO:tasks.rados.rados.0.smithi017.stdout:1427: writing smithi01785587-34 from 3123240 to 3428727 tid 3 2024-09-21T09:43:53.087 INFO:tasks.rados.rados.0.smithi017.stdout:1428: read oid 35 snap -1 2024-09-21T09:43:53.087 INFO:tasks.rados.rados.0.smithi017.stdout:1428: expect (ObjNum 539 snap 170 seq_num 539) 2024-09-21T09:43:53.087 INFO:tasks.rados.rados.0.smithi017.stdout:1429: write oid 47 current snap is 173 2024-09-21T09:43:53.088 INFO:tasks.rados.rados.0.smithi017.stdout:1420: finishing write tid 4 to smithi01785587-27 2024-09-21T09:43:53.088 INFO:tasks.rados.rados.0.smithi017.stdout:1420: finishing write tid 5 to smithi01785587-27 2024-09-21T09:43:53.088 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 503 (ObjNum 557 snap 173 seq_num 557) dirty exists 2024-09-21T09:43:53.088 INFO:tasks.rados.rados.0.smithi017.stdout:1420: left oid 27 (ObjNum 557 snap 173 seq_num 557) 2024-09-21T09:43:53.088 INFO:tasks.rados.rados.0.smithi017.stdout:1421: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:43:53.198 INFO:tasks.rados.rados.0.smithi017.stdout:1423: finishing write tid 1 to smithi01785587-14 2024-09-21T09:43:53.198 INFO:tasks.rados.rados.0.smithi017.stdout:1423: finishing write tid 2 to smithi01785587-14 2024-09-21T09:43:53.198 INFO:tasks.rados.rados.0.smithi017.stdout:1423: finishing write tid 3 to smithi01785587-14 2024-09-21T09:43:53.198 INFO:tasks.rados.rados.0.smithi017.stdout:1423: finishing write tid 4 to smithi01785587-14 2024-09-21T09:43:53.198 INFO:tasks.rados.rados.0.smithi017.stdout:1423: finishing write tid 5 to smithi01785587-14 2024-09-21T09:43:53.198 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 732 (ObjNum 559 snap 173 seq_num 559) dirty exists 2024-09-21T09:43:53.198 INFO:tasks.rados.rados.0.smithi017.stdout:1423: left oid 14 (ObjNum 559 snap 173 seq_num 559) 2024-09-21T09:43:53.198 INFO:tasks.rados.rados.0.smithi017.stdout:1425: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:43:53.269 INFO:tasks.rados.rados.0.smithi017.stdout:1426: finishing write tid 1 to smithi01785587-12 2024-09-21T09:43:53.269 INFO:tasks.rados.rados.0.smithi017.stdout:1426: finishing write tid 2 to smithi01785587-12 2024-09-21T09:43:53.269 INFO:tasks.rados.rados.0.smithi017.stdout:1426: finishing write tid 3 to smithi01785587-12 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1426: finishing write tid 4 to smithi01785587-12 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1426: finishing write tid 5 to smithi01785587-12 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 520 (ObjNum 560 snap 173 seq_num 560) dirty exists 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1426: left oid 12 (ObjNum 560 snap 173 seq_num 560) 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1427: finishing write tid 1 to smithi01785587-34 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1427: finishing write tid 2 to smithi01785587-34 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1427: finishing write tid 3 to smithi01785587-34 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1427: finishing write tid 4 to smithi01785587-34 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1427: finishing write tid 5 to smithi01785587-34 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 767 (ObjNum 561 snap 173 seq_num 561) dirty exists 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1427: left oid 34 (ObjNum 561 snap 173 seq_num 561) 2024-09-21T09:43:53.270 INFO:tasks.rados.rados.0.smithi017.stdout:1428: expect (ObjNum 539 snap 170 seq_num 539) 2024-09-21T09:43:53.372 INFO:tasks.rados.rados.0.smithi017.stdout:1429: seq_num 562 ranges {645534=783707,2103939=638517,3150663=294871} 2024-09-21T09:43:53.380 INFO:tasks.rados.rados.0.smithi017.stdout:1429: writing smithi01785587-47 from 645534 to 1429241 tid 1 2024-09-21T09:43:53.387 INFO:tasks.rados.rados.0.smithi017.stdout:1429: writing smithi01785587-47 from 2103939 to 2742456 tid 2 2024-09-21T09:43:53.392 INFO:tasks.rados.rados.0.smithi017.stdout:1429: writing smithi01785587-47 from 3150663 to 3445534 tid 3 2024-09-21T09:43:53.392 INFO:tasks.rados.rados.0.smithi017.stdout:1420: done (7 left) 2024-09-21T09:43:53.392 INFO:tasks.rados.rados.0.smithi017.stdout:1421: done (6 left) 2024-09-21T09:43:53.392 INFO:tasks.rados.rados.0.smithi017.stdout:1423: done (5 left) 2024-09-21T09:43:53.392 INFO:tasks.rados.rados.0.smithi017.stdout:1425: done (4 left) 2024-09-21T09:43:53.392 INFO:tasks.rados.rados.0.smithi017.stdout:1426: done (3 left) 2024-09-21T09:43:53.393 INFO:tasks.rados.rados.0.smithi017.stdout:1427: done (2 left) 2024-09-21T09:43:53.393 INFO:tasks.rados.rados.0.smithi017.stdout:1428: done (1 left) 2024-09-21T09:43:53.393 INFO:tasks.rados.rados.0.smithi017.stdout:1430: rollback oid 1 current snap is 173 2024-09-21T09:43:53.393 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 1 to 166 2024-09-21T09:43:53.393 INFO:tasks.rados.rados.0.smithi017.stdout:1429: finishing write tid 1 to smithi01785587-47 2024-09-21T09:43:53.393 INFO:tasks.rados.rados.0.smithi017.stdout:1429: finishing write tid 2 to smithi01785587-47 2024-09-21T09:43:53.393 INFO:tasks.rados.rados.0.smithi017.stdout:1431: write oid 30 current snap is 173 2024-09-21T09:43:53.393 INFO:tasks.rados.rados.0.smithi017.stdout:1431: seq_num 563 ranges {662341=575309,1878774=498528,2894439=457555} 2024-09-21T09:43:53.398 INFO:tasks.rados.rados.0.smithi017.stdout:1431: writing smithi01785587-30 from 662341 to 1237650 tid 1 2024-09-21T09:43:53.405 INFO:tasks.rados.rados.0.smithi017.stdout:1431: writing smithi01785587-30 from 1878774 to 2377302 tid 2 2024-09-21T09:43:53.410 INFO:tasks.rados.rados.0.smithi017.stdout:1431: writing smithi01785587-30 from 2894439 to 3351994 tid 3 2024-09-21T09:43:53.410 INFO:tasks.rados.rados.0.smithi017.stdout:1432: write oid 45 current snap is 173 2024-09-21T09:43:53.410 INFO:tasks.rados.rados.0.smithi017.stdout:1432: seq_num 564 ranges {679148=650558,1937256=642186,3321862=157286} 2024-09-21T09:43:53.417 INFO:tasks.rados.rados.0.smithi017.stdout:1432: writing smithi01785587-45 from 679148 to 1329706 tid 1 2024-09-21T09:43:53.424 INFO:tasks.rados.rados.0.smithi017.stdout:1432: writing smithi01785587-45 from 1937256 to 2579442 tid 2 2024-09-21T09:43:53.429 INFO:tasks.rados.rados.0.smithi017.stdout:1432: writing smithi01785587-45 from 3321862 to 3479148 tid 3 2024-09-21T09:43:53.429 INFO:tasks.rados.rados.0.smithi017.stdout:1433: snap_remove snap 141 2024-09-21T09:43:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[105151]: pgmap v190: 89 pgs: 89 active+clean; 227 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:43:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[105151]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T09:43:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[105151]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[102913]: pgmap v190: 89 pgs: 89 active+clean; 227 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[102913]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:53 smithi017 ceph-mon[102913]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T09:43:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:53 smithi134 ceph-mon[90085]: pgmap v190: 89 pgs: 89 active+clean; 227 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:43:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:53 smithi134 ceph-mon[90085]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T09:43:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:53 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:53 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:53 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:43:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:53 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:53 smithi134 ceph-mon[90085]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T09:43:54.350 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:54 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:54.352 INFO:tasks.rados.rados.0.smithi017.stdout:1433: done (4 left) 2024-09-21T09:43:54.353 INFO:tasks.rados.rados.0.smithi017.stdout:1434: read oid 34 snap -1 2024-09-21T09:43:54.353 INFO:tasks.rados.rados.0.smithi017.stdout:1434: expect (ObjNum 561 snap 173 seq_num 561) 2024-09-21T09:43:54.353 INFO:tasks.rados.rados.0.smithi017.stdout:1435: rollback oid 25 current snap is 173 2024-09-21T09:43:54.353 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 149 2024-09-21T09:43:54.353 INFO:tasks.rados.rados.0.smithi017.stdout:1436: rollback oid 29 current snap is 173 2024-09-21T09:43:54.353 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 29 to 171 2024-09-21T09:43:54.353 INFO:tasks.rados.rados.0.smithi017.stdout:1437: write oid 5 current snap is 173 2024-09-21T09:43:54.353 INFO:tasks.rados.rados.0.smithi017.stdout:1437: seq_num 565 ranges {695955=725807,1879385=502197,2949285=480099} 2024-09-21T09:43:54.361 INFO:tasks.rados.rados.0.smithi017.stdout:1437: writing smithi01785587-5 from 695955 to 1421762 tid 1 2024-09-21T09:43:54.367 INFO:tasks.rados.rados.0.smithi017.stdout:1437: writing smithi01785587-5 from 1879385 to 2381582 tid 2 2024-09-21T09:43:54.374 INFO:tasks.rados.rados.0.smithi017.stdout:1437: writing smithi01785587-5 from 2949285 to 3429384 tid 3 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:1438: rollback oid 39 current snap is 173 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 149 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:1429: finishing write tid 3 to smithi01785587-47 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:1429: finishing write tid 4 to smithi01785587-47 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:1429: finishing write tid 5 to smithi01785587-47 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 525 (ObjNum 562 snap 173 seq_num 562) dirty exists 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:1429: left oid 47 (ObjNum 562 snap 173 seq_num 562) 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:1430: finishing rollback tid 0 to smithi01785587-1 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:1429: done (8 left) 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:1439: rollback oid 35 current snap is 173 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 145 2024-09-21T09:43:54.375 INFO:tasks.rados.rados.0.smithi017.stdout:1430: finishing rollback tid 1 to smithi01785587-1 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:1430: finishing rollback tid 2 to smithi01785587-1 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 421 (ObjNum 458 snap 148 seq_num 458) dirty exists 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:1431: finishing write tid 1 to smithi01785587-30 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:1431: finishing write tid 2 to smithi01785587-30 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:1431: finishing write tid 3 to smithi01785587-30 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:1431: finishing write tid 4 to smithi01785587-30 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:1430: done (8 left) 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:1440: read oid 18 snap -1 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:1440: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:43:54.376 INFO:tasks.rados.rados.0.smithi017.stdout:1441: snap_remove snap 158 2024-09-21T09:43:54.604 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:54 smithi134 ceph-mon[90085]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:43:54.604 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:54 smithi134 ceph-mon[90085]: Reconfiguring daemon osd.7 on smithi134 2024-09-21T09:43:54.604 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:54 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:54.604 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:54 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:54.604 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:54 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:43:54.604 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:54 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[102913]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:43:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[102913]: Reconfiguring daemon osd.7 on smithi134 2024-09-21T09:43:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:43:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[105151]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:43:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[105151]: Reconfiguring daemon osd.7 on smithi134 2024-09-21T09:43:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi134.bwovvb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:43:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:55.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:43:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:43:54] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.51.0" 2024-09-21T09:43:55.356 INFO:tasks.rados.rados.0.smithi017.stdout:1441: done (9 left) 2024-09-21T09:43:55.356 INFO:tasks.rados.rados.0.smithi017.stdout:1442: write oid 44 current snap is 173 2024-09-21T09:43:55.356 INFO:tasks.rados.rados.0.smithi017.stdout:1431: finishing write tid 5 to smithi01785587-30 2024-09-21T09:43:55.356 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 659 (ObjNum 563 snap 173 seq_num 563) dirty exists 2024-09-21T09:43:55.356 INFO:tasks.rados.rados.0.smithi017.stdout:1431: left oid 30 (ObjNum 563 snap 173 seq_num 563) 2024-09-21T09:43:55.356 INFO:tasks.rados.rados.0.smithi017.stdout:1432: finishing write tid 1 to smithi01785587-45 2024-09-21T09:43:55.356 INFO:tasks.rados.rados.0.smithi017.stdout:1432: finishing write tid 2 to smithi01785587-45 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:1432: finishing write tid 3 to smithi01785587-45 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:1432: finishing write tid 4 to smithi01785587-45 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:1432: finishing write tid 5 to smithi01785587-45 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 507 (ObjNum 564 snap 173 seq_num 564) dirty exists 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:1432: left oid 45 (ObjNum 564 snap 173 seq_num 564) 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:1435: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:1436: finishing rollback tid 1 to smithi01785587-29 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 275 (ObjNum 49 snap 0 seq_num 2384147568) dirty dne 2024-09-21T09:43:55.357 INFO:tasks.rados.rados.0.smithi017.stdout:1434: expect (ObjNum 561 snap 173 seq_num 561) 2024-09-21T09:43:55.565 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:55 smithi134 ceph-mon[90085]: pgmap v193: 89 pgs: 89 active+clean; 227 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:43:55.565 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:55 smithi134 ceph-mon[90085]: Reconfiguring mds.foo.smithi134.bwovvb (monmap changed)... 2024-09-21T09:43:55.565 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:55 smithi134 ceph-mon[90085]: Reconfiguring daemon mds.foo.smithi134.bwovvb on smithi134 2024-09-21T09:43:55.565 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:55 smithi134 ceph-mon[90085]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:43:55.565 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:55.587 INFO:tasks.rados.rados.0.smithi017.stdout:1437: finishing write tid 1 to smithi01785587-5 2024-09-21T09:43:55.587 INFO:tasks.rados.rados.0.smithi017.stdout:1437: finishing write tid 2 to smithi01785587-5 2024-09-21T09:43:55.587 INFO:tasks.rados.rados.0.smithi017.stdout:1439: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:43:55.587 INFO:tasks.rados.rados.0.smithi017.stdout:1439: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:43:55.587 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 190 (ObjNum 129 snap 0 seq_num 2399696616) dirty dne 2024-09-21T09:43:55.587 INFO:tasks.rados.rados.0.smithi017.stdout:1438: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:43:55.587 INFO:tasks.rados.rados.0.smithi017.stdout:1437: finishing write tid 3 to smithi01785587-5 2024-09-21T09:43:55.587 INFO:tasks.rados.rados.0.smithi017.stdout:1438: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:43:55.587 INFO:tasks.rados.rados.0.smithi017.stdout:1442: seq_num 566 ranges {712762=401056,1537867=645855,2860355=652407} 2024-09-21T09:43:55.593 INFO:tasks.rados.rados.0.smithi017.stdout:1442: writing smithi01785587-44 from 712762 to 1113818 tid 1 2024-09-21T09:43:55.599 INFO:tasks.rados.rados.0.smithi017.stdout:1442: writing smithi01785587-44 from 1537867 to 2183722 tid 2 2024-09-21T09:43:55.606 INFO:tasks.rados.rados.0.smithi017.stdout:1442: writing smithi01785587-44 from 2860355 to 3512762 tid 3 2024-09-21T09:43:55.606 INFO:tasks.rados.rados.0.smithi017.stdout:1431: done (9 left) 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1432: done (8 left) 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1434: done (7 left) 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1435: done (6 left) 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1436: done (5 left) 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1439: done (4 left) 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1443: write oid 50 current snap is 173 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1437: finishing write tid 4 to smithi01785587-5 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1438: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 732 (ObjNum 459 snap 148 seq_num 459) dirty exists 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1437: finishing write tid 5 to smithi01785587-5 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 748 (ObjNum 565 snap 173 seq_num 565) dirty exists 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1437: left oid 5 (ObjNum 565 snap 173 seq_num 565) 2024-09-21T09:43:55.607 INFO:tasks.rados.rados.0.smithi017.stdout:1440: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:43:55.683 INFO:tasks.rados.rados.0.smithi017.stdout:1442: finishing write tid 1 to smithi01785587-44 2024-09-21T09:43:55.683 INFO:tasks.rados.rados.0.smithi017.stdout:1442: finishing write tid 2 to smithi01785587-44 2024-09-21T09:43:55.683 INFO:tasks.rados.rados.0.smithi017.stdout:1442: finishing write tid 3 to smithi01785587-44 2024-09-21T09:43:55.683 INFO:tasks.rados.rados.0.smithi017.stdout:1442: finishing write tid 4 to smithi01785587-44 2024-09-21T09:43:55.683 INFO:tasks.rados.rados.0.smithi017.stdout:1442: finishing write tid 5 to smithi01785587-44 2024-09-21T09:43:55.683 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 760 (ObjNum 566 snap 173 seq_num 566) dirty exists 2024-09-21T09:43:55.683 INFO:tasks.rados.rados.0.smithi017.stdout:1442: left oid 44 (ObjNum 566 snap 173 seq_num 566) 2024-09-21T09:43:55.683 INFO:tasks.rados.rados.0.smithi017.stdout:1443: seq_num 567 ranges {729569=476305,1996349=505866,3004131=525438} 2024-09-21T09:43:55.689 INFO:tasks.rados.rados.0.smithi017.stdout:1443: writing smithi01785587-50 from 729569 to 1205874 tid 1 2024-09-21T09:43:55.696 INFO:tasks.rados.rados.0.smithi017.stdout:1443: writing smithi01785587-50 from 1996349 to 2502215 tid 2 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[105151]: pgmap v193: 89 pgs: 89 active+clean; 227 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[105151]: Reconfiguring mds.foo.smithi134.bwovvb (monmap changed)... 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[105151]: Reconfiguring daemon mds.foo.smithi134.bwovvb on smithi134 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[105151]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[102913]: pgmap v193: 89 pgs: 89 active+clean; 227 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[102913]: Reconfiguring mds.foo.smithi134.bwovvb (monmap changed)... 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[102913]: Reconfiguring daemon mds.foo.smithi134.bwovvb on smithi134 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[102913]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:43:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:55.702 INFO:tasks.rados.rados.0.smithi017.stdout:1443: writing smithi01785587-50 from 3004131 to 3529569 tid 3 2024-09-21T09:43:55.702 INFO:tasks.rados.rados.0.smithi017.stdout:1437: done (4 left) 2024-09-21T09:43:55.702 INFO:tasks.rados.rados.0.smithi017.stdout:1438: done (3 left) 2024-09-21T09:43:55.702 INFO:tasks.rados.rados.0.smithi017.stdout:1440: done (2 left) 2024-09-21T09:43:55.702 INFO:tasks.rados.rados.0.smithi017.stdout:1442: done (1 left) 2024-09-21T09:43:55.702 INFO:tasks.rados.rados.0.smithi017.stdout:1444: write oid 38 current snap is 173 2024-09-21T09:43:55.703 INFO:tasks.rados.rados.0.smithi017.stdout:1443: finishing write tid 1 to smithi01785587-50 2024-09-21T09:43:55.703 INFO:tasks.rados.rados.0.smithi017.stdout:1444: seq_num 568 ranges {746376=551554,2054831=649524,3315201=231175} 2024-09-21T09:43:55.709 INFO:tasks.rados.rados.0.smithi017.stdout:1444: writing smithi01785587-38 from 746376 to 1297930 tid 1 2024-09-21T09:43:55.717 INFO:tasks.rados.rados.0.smithi017.stdout:1444: writing smithi01785587-38 from 2054831 to 2704355 tid 2 2024-09-21T09:43:55.722 INFO:tasks.rados.rados.0.smithi017.stdout:1444: writing smithi01785587-38 from 3315201 to 3546376 tid 3 2024-09-21T09:43:55.722 INFO:tasks.rados.rados.0.smithi017.stdout:1445: snap_create 2024-09-21T09:43:55.722 INFO:tasks.rados.rados.0.smithi017.stdout:1443: finishing write tid 2 to smithi01785587-50 2024-09-21T09:43:55.723 INFO:tasks.rados.rados.0.smithi017.stdout:1443: finishing write tid 3 to smithi01785587-50 2024-09-21T09:43:55.723 INFO:tasks.rados.rados.0.smithi017.stdout:1443: finishing write tid 4 to smithi01785587-50 2024-09-21T09:43:55.723 INFO:tasks.rados.rados.0.smithi017.stdout:1443: finishing write tid 5 to smithi01785587-50 2024-09-21T09:43:55.723 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 439 (ObjNum 567 snap 173 seq_num 567) dirty exists 2024-09-21T09:43:55.723 INFO:tasks.rados.rados.0.smithi017.stdout:1443: left oid 50 (ObjNum 567 snap 173 seq_num 567) 2024-09-21T09:43:55.723 INFO:tasks.rados.rados.0.smithi017.stdout:1444: finishing write tid 1 to smithi01785587-38 2024-09-21T09:43:55.723 INFO:tasks.rados.rados.0.smithi017.stdout:1444: finishing write tid 2 to smithi01785587-38 2024-09-21T09:43:55.724 INFO:tasks.rados.rados.0.smithi017.stdout:1444: finishing write tid 3 to smithi01785587-38 2024-09-21T09:43:55.724 INFO:tasks.rados.rados.0.smithi017.stdout:1444: finishing write tid 4 to smithi01785587-38 2024-09-21T09:43:55.724 INFO:tasks.rados.rados.0.smithi017.stdout:1444: finishing write tid 5 to smithi01785587-38 2024-09-21T09:43:55.725 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 444 (ObjNum 568 snap 173 seq_num 568) dirty exists 2024-09-21T09:43:55.725 INFO:tasks.rados.rados.0.smithi017.stdout:1444: left oid 38 (ObjNum 568 snap 173 seq_num 568) 2024-09-21T09:43:55.993 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:55.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:56.670 INFO:tasks.rados.rados.0.smithi017.stdout:1443: done (2 left) 2024-09-21T09:43:56.670 INFO:tasks.rados.rados.0.smithi017.stdout:1444: done (1 left) 2024-09-21T09:43:56.670 INFO:tasks.rados.rados.0.smithi017.stdout:1445: done (0 left) 2024-09-21T09:43:56.670 INFO:tasks.rados.rados.0.smithi017.stdout:1446: write oid 17 current snap is 174 2024-09-21T09:43:56.670 INFO:tasks.rados.rados.0.smithi017.stdout:1446: seq_num 569 ranges {763183=626803,1996960=793182,3226271=336912} 2024-09-21T09:43:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:1446: writing smithi01785587-17 from 763183 to 1389986 tid 1 2024-09-21T09:43:56.685 INFO:tasks.rados.rados.0.smithi017.stdout:1446: writing smithi01785587-17 from 1996960 to 2790142 tid 2 2024-09-21T09:43:56.690 INFO:tasks.rados.rados.0.smithi017.stdout:1446: writing smithi01785587-17 from 3226271 to 3563183 tid 3 2024-09-21T09:43:56.690 INFO:tasks.rados.rados.0.smithi017.stdout:1447: delete oid 4 current snap is 174 2024-09-21T09:43:56.690 INFO:tasks.rados.rados.0.smithi017.stdout:1446: finishing write tid 1 to smithi01785587-17 2024-09-21T09:43:56.690 INFO:tasks.rados.rados.0.smithi017.stdout:1446: finishing write tid 2 to smithi01785587-17 2024-09-21T09:43:56.692 INFO:tasks.rados.rados.0.smithi017.stdout:1446: finishing write tid 3 to smithi01785587-17 2024-09-21T09:43:56.693 INFO:tasks.rados.rados.0.smithi017.stdout:1446: finishing write tid 4 to smithi01785587-17 2024-09-21T09:43:56.693 INFO:tasks.rados.rados.0.smithi017.stdout:1446: finishing write tid 5 to smithi01785587-17 2024-09-21T09:43:56.693 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 516 (ObjNum 569 snap 174 seq_num 569) dirty exists 2024-09-21T09:43:56.693 INFO:tasks.rados.rados.0.smithi017.stdout:1446: left oid 17 (ObjNum 569 snap 174 seq_num 569) 2024-09-21T09:43:56.694 INFO:tasks.rados.rados.0.smithi017.stdout:1446: done (1 left) 2024-09-21T09:43:56.694 INFO:tasks.rados.rados.0.smithi017.stdout:1447: done (0 left) 2024-09-21T09:43:56.694 INFO:tasks.rados.rados.0.smithi017.stdout:1448: snap_create 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[102913]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[105151]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:56.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:56 smithi134 ceph-mon[90085]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:43:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:43:57.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:43:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:43:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:43:57.672 INFO:tasks.rados.rados.0.smithi017.stdout:1448: done (0 left) 2024-09-21T09:43:57.673 INFO:tasks.rados.rados.0.smithi017.stdout:1449: snap_create 2024-09-21T09:43:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:57 smithi017 ceph-mon[102913]: pgmap v196: 89 pgs: 89 active+clean; 224 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:43:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:57 smithi017 ceph-mon[102913]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:43:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:57 smithi017 ceph-mon[105151]: pgmap v196: 89 pgs: 89 active+clean; 224 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:43:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:57 smithi017 ceph-mon[105151]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:43:57.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:57 smithi134 ceph-mon[90085]: pgmap v196: 89 pgs: 89 active+clean; 224 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:43:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:57 smithi134 ceph-mon[90085]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:43:58.678 INFO:tasks.rados.rados.0.smithi017.stdout:1449: done (0 left) 2024-09-21T09:43:58.678 INFO:tasks.rados.rados.0.smithi017.stdout:1450: rollback oid 17 current snap is 176 2024-09-21T09:43:58.678 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 174 2024-09-21T09:43:58.678 INFO:tasks.rados.rados.0.smithi017.stdout:1451: read oid 29 snap -1 2024-09-21T09:43:58.678 INFO:tasks.rados.rados.0.smithi017.stdout:1451: expect deleted 2024-09-21T09:43:58.678 INFO:tasks.rados.rados.0.smithi017.stdout:1452: snap_create 2024-09-21T09:43:58.680 INFO:tasks.rados.rados.0.smithi017.stdout:1450: finishing rollback tid 0 to smithi01785587-17 2024-09-21T09:43:58.681 INFO:tasks.rados.rados.0.smithi017.stdout:1450: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:43:58.681 INFO:tasks.rados.rados.0.smithi017.stdout:1450: finishing rollback tid 2 to smithi01785587-17 2024-09-21T09:43:58.681 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 520 (ObjNum 569 snap 174 seq_num 569) dirty exists 2024-09-21T09:43:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:58 smithi017 ceph-mon[102913]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:43:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:58 smithi017 ceph-mon[102913]: pgmap v199: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.2 MiB/s wr, 28 op/s 2024-09-21T09:43:58.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:58 smithi017 ceph-mon[105151]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:43:58.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:58 smithi017 ceph-mon[105151]: pgmap v199: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.2 MiB/s wr, 28 op/s 2024-09-21T09:43:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:58 smithi134 ceph-mon[90085]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:43:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:58 smithi134 ceph-mon[90085]: pgmap v199: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.2 MiB/s wr, 28 op/s 2024-09-21T09:43:59.680 INFO:tasks.rados.rados.0.smithi017.stdout:1450: done (2 left) 2024-09-21T09:43:59.680 INFO:tasks.rados.rados.0.smithi017.stdout:1451: done (1 left) 2024-09-21T09:43:59.680 INFO:tasks.rados.rados.0.smithi017.stdout:1452: done (0 left) 2024-09-21T09:43:59.680 INFO:tasks.rados.rados.0.smithi017.stdout:1453: delete oid 6 current snap is 177 2024-09-21T09:43:59.687 INFO:tasks.rados.rados.0.smithi017.stdout:1453: done (0 left) 2024-09-21T09:43:59.687 INFO:tasks.rados.rados.0.smithi017.stdout:1454: rollback oid 25 current snap is 177 2024-09-21T09:43:59.687 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 172 2024-09-21T09:43:59.687 INFO:tasks.rados.rados.0.smithi017.stdout:1455: read oid 45 snap -1 2024-09-21T09:43:59.687 INFO:tasks.rados.rados.0.smithi017.stdout:1455: expect (ObjNum 564 snap 173 seq_num 564) 2024-09-21T09:43:59.688 INFO:tasks.rados.rados.0.smithi017.stdout:1456: read oid 21 snap -1 2024-09-21T09:43:59.688 INFO:tasks.rados.rados.0.smithi017.stdout:1456: expect (ObjNum 551 snap 172 seq_num 551) 2024-09-21T09:43:59.688 INFO:tasks.rados.rados.0.smithi017.stdout:1457: read oid 46 snap -1 2024-09-21T09:43:59.688 INFO:tasks.rados.rados.0.smithi017.stdout:1457: expect (ObjNum 537 snap 169 seq_num 537) 2024-09-21T09:43:59.688 INFO:tasks.rados.rados.0.smithi017.stdout:1458: rollback oid 40 current snap is 177 2024-09-21T09:43:59.688 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 149 2024-09-21T09:43:59.688 INFO:tasks.rados.rados.0.smithi017.stdout:1459: snap_remove snap 175 2024-09-21T09:43:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:43:59 smithi017 ceph-mon[102913]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T09:43:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:43:59 smithi017 ceph-mon[105151]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T09:43:59.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:43:59 smithi134 ceph-mon[90085]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T09:44:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:00.692 INFO:tasks.rados.rados.0.smithi017.stdout:1459: done (5 left) 2024-09-21T09:44:00.692 INFO:tasks.rados.rados.0.smithi017.stdout:1460: read oid 49 snap -1 2024-09-21T09:44:00.692 INFO:tasks.rados.rados.0.smithi017.stdout:1460: expect (ObjNum 447 snap 145 seq_num 447) 2024-09-21T09:44:00.692 INFO:tasks.rados.rados.0.smithi017.stdout:1461: snap_remove snap 176 2024-09-21T09:44:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:00 smithi017 ceph-mon[102913]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T09:44:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:00 smithi017 ceph-mon[102913]: pgmap v202: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-09-21T09:44:00.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:00 smithi017 ceph-mon[105151]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T09:44:00.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:00 smithi017 ceph-mon[105151]: pgmap v202: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-09-21T09:44:00.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:00 smithi134 ceph-mon[90085]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T09:44:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:00 smithi134 ceph-mon[90085]: pgmap v202: 89 pgs: 89 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-09-21T09:44:01.700 INFO:tasks.rados.rados.0.smithi017.stdout:1461: done (6 left) 2024-09-21T09:44:01.700 INFO:tasks.rados.rados.0.smithi017.stdout:1462: rollback oid 27 current snap is 177 2024-09-21T09:44:01.700 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 172 2024-09-21T09:44:01.700 INFO:tasks.rados.rados.0.smithi017.stdout:1454: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:44:01.700 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 0 (ObjNum 129 snap 0 seq_num 2399819600) dirty dne 2024-09-21T09:44:01.700 INFO:tasks.rados.rados.0.smithi017.stdout:1458: finishing rollback tid 0 to smithi01785587-40 2024-09-21T09:44:01.700 INFO:tasks.rados.rados.0.smithi017.stdout:1457: expect (ObjNum 537 snap 169 seq_num 537) 2024-09-21T09:44:01.892 INFO:tasks.rados.rados.0.smithi017.stdout:1456: expect (ObjNum 551 snap 172 seq_num 551) 2024-09-21T09:44:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:01 smithi017 ceph-mon[102913]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T09:44:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:01 smithi017 ceph-mon[105151]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T09:44:01.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:01 smithi134 ceph-mon[90085]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T09:44:02.127 INFO:tasks.rados.rados.0.smithi017.stdout:1458: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:44:02.127 INFO:tasks.rados.rados.0.smithi017.stdout:1458: finishing rollback tid 2 to smithi01785587-40 2024-09-21T09:44:02.127 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 744 (ObjNum 407 snap 126 seq_num 407) dirty exists 2024-09-21T09:44:02.127 INFO:tasks.rados.rados.0.smithi017.stdout:1455: expect (ObjNum 564 snap 173 seq_num 564) 2024-09-21T09:44:02.244 INFO:tasks.rados.rados.0.smithi017.stdout:1460: expect (ObjNum 447 snap 145 seq_num 447) 2024-09-21T09:44:02.490 INFO:tasks.rados.rados.0.smithi017.stdout:1462: finishing rollback tid 0 to smithi01785587-27 2024-09-21T09:44:02.490 INFO:tasks.rados.rados.0.smithi017.stdout:1462: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:44:02.490 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 525 (ObjNum 33 snap 0 seq_num 2399683536) dirty dne 2024-09-21T09:44:02.490 INFO:tasks.rados.rados.0.smithi017.stdout:1454: done (6 left) 2024-09-21T09:44:02.490 INFO:tasks.rados.rados.0.smithi017.stdout:1455: done (5 left) 2024-09-21T09:44:02.491 INFO:tasks.rados.rados.0.smithi017.stdout:1456: done (4 left) 2024-09-21T09:44:02.491 INFO:tasks.rados.rados.0.smithi017.stdout:1457: done (3 left) 2024-09-21T09:44:02.491 INFO:tasks.rados.rados.0.smithi017.stdout:1458: done (2 left) 2024-09-21T09:44:02.491 INFO:tasks.rados.rados.0.smithi017.stdout:1460: done (1 left) 2024-09-21T09:44:02.491 INFO:tasks.rados.rados.0.smithi017.stdout:1462: done (0 left) 2024-09-21T09:44:02.491 INFO:tasks.rados.rados.0.smithi017.stdout:1463: read oid 18 snap -1 2024-09-21T09:44:02.491 INFO:tasks.rados.rados.0.smithi017.stdout:1463: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:44:02.491 INFO:tasks.rados.rados.0.smithi017.stdout:1464: snap_remove snap 149 2024-09-21T09:44:02.706 INFO:tasks.rados.rados.0.smithi017.stdout:1464: done (1 left) 2024-09-21T09:44:02.706 INFO:tasks.rados.rados.0.smithi017.stdout:1465: delete oid 31 current snap is 177 2024-09-21T09:44:02.706 INFO:tasks.rados.rados.0.smithi017.stdout:1463: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:44:02.783 INFO:tasks.rados.rados.0.smithi017.stdout:1463: done (1 left) 2024-09-21T09:44:02.784 INFO:tasks.rados.rados.0.smithi017.stdout:1465: done (0 left) 2024-09-21T09:44:02.784 INFO:tasks.rados.rados.0.smithi017.stdout:1466: write oid 14 current snap is 177 2024-09-21T09:44:02.784 INFO:tasks.rados.rados.0.smithi017.stdout:1466: seq_num 573 ranges {430411=644152,1830888=400520,2935963=402981} 2024-09-21T09:44:02.789 INFO:tasks.rados.rados.0.smithi017.stdout:1466: writing smithi01785587-14 from 430411 to 1074563 tid 1 2024-09-21T09:44:02.795 INFO:tasks.rados.rados.0.smithi017.stdout:1466: writing smithi01785587-14 from 1830888 to 2231408 tid 2 2024-09-21T09:44:02.802 INFO:tasks.rados.rados.0.smithi017.stdout:1466: writing smithi01785587-14 from 2935963 to 3338944 tid 3 2024-09-21T09:44:02.802 INFO:tasks.rados.rados.0.smithi017.stdout:1467: read oid 30 snap -1 2024-09-21T09:44:02.802 INFO:tasks.rados.rados.0.smithi017.stdout:1467: expect (ObjNum 563 snap 173 seq_num 563) 2024-09-21T09:44:02.802 INFO:tasks.rados.rados.0.smithi017.stdout:1468: snap_create 2024-09-21T09:44:02.802 INFO:tasks.rados.rados.0.smithi017.stdout:1466: finishing write tid 1 to smithi01785587-14 2024-09-21T09:44:02.802 INFO:tasks.rados.rados.0.smithi017.stdout:1466: finishing write tid 2 to smithi01785587-14 2024-09-21T09:44:02.804 INFO:tasks.rados.rados.0.smithi017.stdout:1466: finishing write tid 3 to smithi01785587-14 2024-09-21T09:44:02.804 INFO:tasks.rados.rados.0.smithi017.stdout:1466: finishing write tid 4 to smithi01785587-14 2024-09-21T09:44:02.804 INFO:tasks.rados.rados.0.smithi017.stdout:1466: finishing write tid 5 to smithi01785587-14 2024-09-21T09:44:02.804 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 774 (ObjNum 573 snap 177 seq_num 573) dirty exists 2024-09-21T09:44:02.804 INFO:tasks.rados.rados.0.smithi017.stdout:1466: left oid 14 (ObjNum 573 snap 177 seq_num 573) 2024-09-21T09:44:02.810 INFO:tasks.rados.rados.0.smithi017.stdout:1467: expect (ObjNum 563 snap 173 seq_num 563) 2024-09-21T09:44:02.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:02 smithi134 ceph-mon[90085]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T09:44:02.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:02 smithi134 ceph-mon[90085]: pgmap v205: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 429 KiB/s wr, 10 op/s 2024-09-21T09:44:03.048 INFO:tasks.rados.rados.0.smithi017.stdout:1466: done (2 left) 2024-09-21T09:44:03.049 INFO:tasks.rados.rados.0.smithi017.stdout:1467: done (1 left) 2024-09-21T09:44:03.049 INFO:tasks.rados.rados.0.smithi017.stdout:1468: done (0 left) 2024-09-21T09:44:03.049 INFO:tasks.rados.rados.0.smithi017.stdout:1469: write oid 32 current snap is 178 2024-09-21T09:44:03.049 INFO:tasks.rados.rados.0.smithi017.stdout:1469: seq_num 574 ranges {447218=719401,1889370=660531,2963386=614253} 2024-09-21T09:44:03.054 INFO:tasks.rados.rados.0.smithi017.stdout:1469: writing smithi01785587-32 from 447218 to 1166619 tid 1 2024-09-21T09:44:03.062 INFO:tasks.rados.rados.0.smithi017.stdout:1469: writing smithi01785587-32 from 1889370 to 2549901 tid 2 2024-09-21T09:44:03.068 INFO:tasks.rados.rados.0.smithi017.stdout:1469: writing smithi01785587-32 from 2963386 to 3577639 tid 3 2024-09-21T09:44:03.068 INFO:tasks.rados.rados.0.smithi017.stdout:1470: snap_create 2024-09-21T09:44:03.068 INFO:tasks.rados.rados.0.smithi017.stdout:1469: finishing write tid 1 to smithi01785587-32 2024-09-21T09:44:03.068 INFO:tasks.rados.rados.0.smithi017.stdout:1469: finishing write tid 2 to smithi01785587-32 2024-09-21T09:44:03.071 INFO:tasks.rados.rados.0.smithi017.stdout:1469: finishing write tid 3 to smithi01785587-32 2024-09-21T09:44:03.071 INFO:tasks.rados.rados.0.smithi017.stdout:1469: finishing write tid 4 to smithi01785587-32 2024-09-21T09:44:03.071 INFO:tasks.rados.rados.0.smithi017.stdout:1469: finishing write tid 5 to smithi01785587-32 2024-09-21T09:44:03.071 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 696 (ObjNum 574 snap 178 seq_num 574) dirty exists 2024-09-21T09:44:03.071 INFO:tasks.rados.rados.0.smithi017.stdout:1469: left oid 32 (ObjNum 574 snap 178 seq_num 574) 2024-09-21T09:44:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:02 smithi017 ceph-mon[102913]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T09:44:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:02 smithi017 ceph-mon[102913]: pgmap v205: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 429 KiB/s wr, 10 op/s 2024-09-21T09:44:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:02 smithi017 ceph-mon[105151]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T09:44:03.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:02 smithi017 ceph-mon[105151]: pgmap v205: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 429 KiB/s wr, 10 op/s 2024-09-21T09:44:03.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:03 smithi134 ceph-mon[90085]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T09:44:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:03 smithi134 ceph-mon[90085]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T09:44:04.051 INFO:tasks.rados.rados.0.smithi017.stdout:1469: done (1 left) 2024-09-21T09:44:04.051 INFO:tasks.rados.rados.0.smithi017.stdout:1470: done (0 left) 2024-09-21T09:44:04.051 INFO:tasks.rados.rados.0.smithi017.stdout:1471: write oid 12 current snap is 179 2024-09-21T09:44:04.051 INFO:tasks.rados.rados.0.smithi017.stdout:1471: seq_num 575 ranges {464025=794650,1947852=404189,2990809=425525} 2024-09-21T09:44:04.058 INFO:tasks.rados.rados.0.smithi017.stdout:1471: writing smithi01785587-12 from 464025 to 1258675 tid 1 2024-09-21T09:44:04.064 INFO:tasks.rados.rados.0.smithi017.stdout:1471: writing smithi01785587-12 from 1947852 to 2352041 tid 2 2024-09-21T09:44:04.070 INFO:tasks.rados.rados.0.smithi017.stdout:1471: writing smithi01785587-12 from 2990809 to 3416334 tid 3 2024-09-21T09:44:04.070 INFO:tasks.rados.rados.0.smithi017.stdout:1472: snap_create 2024-09-21T09:44:04.070 INFO:tasks.rados.rados.0.smithi017.stdout:1471: finishing write tid 1 to smithi01785587-12 2024-09-21T09:44:04.070 INFO:tasks.rados.rados.0.smithi017.stdout:1471: finishing write tid 2 to smithi01785587-12 2024-09-21T09:44:04.072 INFO:tasks.rados.rados.0.smithi017.stdout:1471: finishing write tid 3 to smithi01785587-12 2024-09-21T09:44:04.073 INFO:tasks.rados.rados.0.smithi017.stdout:1471: finishing write tid 4 to smithi01785587-12 2024-09-21T09:44:04.073 INFO:tasks.rados.rados.0.smithi017.stdout:1471: finishing write tid 5 to smithi01785587-12 2024-09-21T09:44:04.073 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 544 (ObjNum 575 snap 179 seq_num 575) dirty exists 2024-09-21T09:44:04.073 INFO:tasks.rados.rados.0.smithi017.stdout:1471: left oid 12 (ObjNum 575 snap 179 seq_num 575) 2024-09-21T09:44:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:03 smithi017 ceph-mon[102913]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T09:44:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:03 smithi017 ceph-mon[102913]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T09:44:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:03 smithi017 ceph-mon[105151]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T09:44:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:03 smithi017 ceph-mon[105151]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T09:44:04.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:04 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:05.049 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:44:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:44:04] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-21T09:44:05.056 INFO:tasks.rados.rados.0.smithi017.stdout:1471: done (1 left) 2024-09-21T09:44:05.056 INFO:tasks.rados.rados.0.smithi017.stdout:1472: done (0 left) 2024-09-21T09:44:05.056 INFO:tasks.rados.rados.0.smithi017.stdout:1473: read oid 36 snap -1 2024-09-21T09:44:05.056 INFO:tasks.rados.rados.0.smithi017.stdout:1473: expect deleted 2024-09-21T09:44:05.057 INFO:tasks.rados.rados.0.smithi017.stdout:1474: write oid 10 current snap is 180 2024-09-21T09:44:05.057 INFO:tasks.rados.rados.0.smithi017.stdout:1474: seq_num 576 ranges {480832=469899,1606334=664200,3018232=662600} 2024-09-21T09:44:05.061 INFO:tasks.rados.rados.0.smithi017.stdout:1474: writing smithi01785587-10 from 480832 to 950731 tid 1 2024-09-21T09:44:05.069 INFO:tasks.rados.rados.0.smithi017.stdout:1474: writing smithi01785587-10 from 1606334 to 2270534 tid 2 2024-09-21T09:44:05.077 INFO:tasks.rados.rados.0.smithi017.stdout:1474: writing smithi01785587-10 from 3018232 to 3680832 tid 3 2024-09-21T09:44:05.077 INFO:tasks.rados.rados.0.smithi017.stdout:1475: rollback oid 48 current snap is 180 2024-09-21T09:44:05.077 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 174 2024-09-21T09:44:05.077 INFO:tasks.rados.rados.0.smithi017.stdout:1474: finishing write tid 1 to smithi01785587-10 2024-09-21T09:44:05.077 INFO:tasks.rados.rados.0.smithi017.stdout:1473: done (2 left) 2024-09-21T09:44:05.077 INFO:tasks.rados.rados.0.smithi017.stdout:1476: snap_remove snap 173 2024-09-21T09:44:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:05 smithi017 ceph-mon[102913]: pgmap v208: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 429 KiB/s wr, 10 op/s 2024-09-21T09:44:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:05 smithi017 ceph-mon[102913]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T09:44:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:05 smithi017 ceph-mon[105151]: pgmap v208: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 429 KiB/s wr, 10 op/s 2024-09-21T09:44:05.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:05 smithi017 ceph-mon[105151]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T09:44:05.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:05 smithi134 ceph-mon[90085]: pgmap v208: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 MiB/s rd, 429 KiB/s wr, 10 op/s 2024-09-21T09:44:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:05 smithi134 ceph-mon[90085]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T09:44:05.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:05 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:05.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:05 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:06.062 INFO:tasks.rados.rados.0.smithi017.stdout:1476: done (2 left) 2024-09-21T09:44:06.062 INFO:tasks.rados.rados.0.smithi017.stdout:1477: snap_remove snap 179 2024-09-21T09:44:06.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:06 smithi017 ceph-mon[102913]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T09:44:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:06 smithi017 ceph-mon[105151]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T09:44:06.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:06 smithi134 ceph-mon[90085]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1477: done (2 left) 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1478: read oid 47 snap -1 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1478: expect (ObjNum 562 snap 173 seq_num 562) 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1479: write oid 35 current snap is 180 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1474: finishing write tid 2 to smithi01785587-10 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1474: finishing write tid 3 to smithi01785587-10 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1474: finishing write tid 4 to smithi01785587-10 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1474: finishing write tid 5 to smithi01785587-10 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 124 (ObjNum 576 snap 180 seq_num 576) dirty exists 2024-09-21T09:44:07.064 INFO:tasks.rados.rados.0.smithi017.stdout:1474: left oid 10 (ObjNum 576 snap 180 seq_num 576) 2024-09-21T09:44:07.065 INFO:tasks.rados.rados.0.smithi017.stdout:1475: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:44:07.065 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 614 (ObjNum 49 snap 0 seq_num 2384147568) dirty dne 2024-09-21T09:44:07.065 INFO:tasks.rados.rados.0.smithi017.stdout:1479: seq_num 577 ranges {497639=545148,1548463=407858,2529302=564422,3528143=169496} 2024-09-21T09:44:07.069 INFO:tasks.rados.rados.0.smithi017.stdout:1479: writing smithi01785587-35 from 497639 to 1042787 tid 1 2024-09-21T09:44:07.075 INFO:tasks.rados.rados.0.smithi017.stdout:1479: writing smithi01785587-35 from 1548463 to 1956321 tid 2 2024-09-21T09:44:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:1479: writing smithi01785587-35 from 2529302 to 3093724 tid 3 2024-09-21T09:44:07.085 INFO:tasks.rados.rados.0.smithi017.stdout:1479: writing smithi01785587-35 from 3528143 to 3697639 tid 4 2024-09-21T09:44:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1474: done (3 left) 2024-09-21T09:44:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1475: done (2 left) 2024-09-21T09:44:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1480: read oid 32 snap -1 2024-09-21T09:44:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1480: expect (ObjNum 574 snap 178 seq_num 574) 2024-09-21T09:44:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1481: snap_create 2024-09-21T09:44:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1479: finishing write tid 1 to smithi01785587-35 2024-09-21T09:44:07.086 INFO:tasks.rados.rados.0.smithi017.stdout:1478: expect (ObjNum 562 snap 173 seq_num 562) 2024-09-21T09:44:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:07 smithi134 ceph-mon[90085]: pgmap v211: 89 pgs: 89 active+clean; 246 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 4 op/s 2024-09-21T09:44:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:07 smithi134 ceph-mon[90085]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T09:44:07.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:06 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:07.293 INFO:tasks.rados.rados.0.smithi017.stdout:1479: finishing write tid 2 to smithi01785587-35 2024-09-21T09:44:07.293 INFO:tasks.rados.rados.0.smithi017.stdout:1479: finishing write tid 3 to smithi01785587-35 2024-09-21T09:44:07.294 INFO:tasks.rados.rados.0.smithi017.stdout:1479: finishing write tid 4 to smithi01785587-35 2024-09-21T09:44:07.294 INFO:tasks.rados.rados.0.smithi017.stdout:1479: finishing write tid 5 to smithi01785587-35 2024-09-21T09:44:07.294 INFO:tasks.rados.rados.0.smithi017.stdout:1479: finishing write tid 6 to smithi01785587-35 2024-09-21T09:44:07.294 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 195 (ObjNum 577 snap 180 seq_num 577) dirty exists 2024-09-21T09:44:07.294 INFO:tasks.rados.rados.0.smithi017.stdout:1479: left oid 35 (ObjNum 577 snap 180 seq_num 577) 2024-09-21T09:44:07.294 INFO:tasks.rados.rados.0.smithi017.stdout:1480: expect (ObjNum 574 snap 178 seq_num 574) 2024-09-21T09:44:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:07 smithi017 ceph-mon[102913]: pgmap v211: 89 pgs: 89 active+clean; 246 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 4 op/s 2024-09-21T09:44:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:07 smithi017 ceph-mon[102913]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T09:44:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:07 smithi017 ceph-mon[105151]: pgmap v211: 89 pgs: 89 active+clean; 246 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 4 op/s 2024-09-21T09:44:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:07 smithi017 ceph-mon[105151]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T09:44:08.072 INFO:tasks.rados.rados.0.smithi017.stdout:1478: done (3 left) 2024-09-21T09:44:08.073 INFO:tasks.rados.rados.0.smithi017.stdout:1479: done (2 left) 2024-09-21T09:44:08.073 INFO:tasks.rados.rados.0.smithi017.stdout:1480: done (1 left) 2024-09-21T09:44:08.073 INFO:tasks.rados.rados.0.smithi017.stdout:1481: done (0 left) 2024-09-21T09:44:08.073 INFO:tasks.rados.rados.0.smithi017.stdout:1482: write oid 7 current snap is 181 2024-09-21T09:44:08.073 INFO:tasks.rados.rados.0.smithi017.stdout:1482: seq_num 578 ranges {514446=736750,1723298=667869,3073078=641368} 2024-09-21T09:44:08.079 INFO:tasks.rados.rados.0.smithi017.stdout:1482: writing smithi01785587-7 from 514446 to 1251196 tid 1 2024-09-21T09:44:08.086 INFO:tasks.rados.rados.0.smithi017.stdout:1482: writing smithi01785587-7 from 1723298 to 2391167 tid 2 2024-09-21T09:44:08.093 INFO:tasks.rados.rados.0.smithi017.stdout:1482: writing smithi01785587-7 from 3073078 to 3714446 tid 3 2024-09-21T09:44:08.094 INFO:tasks.rados.rados.0.smithi017.stdout:1483: write oid 21 current snap is 181 2024-09-21T09:44:08.094 INFO:tasks.rados.rados.0.smithi017.stdout:1482: finishing write tid 1 to smithi01785587-7 2024-09-21T09:44:08.094 INFO:tasks.rados.rados.0.smithi017.stdout:1482: finishing write tid 2 to smithi01785587-7 2024-09-21T09:44:08.094 INFO:tasks.rados.rados.0.smithi017.stdout:1483: seq_num 579 ranges {531253=411999,1381780=411527,2300501=586966,3376974=354279} 2024-09-21T09:44:08.099 INFO:tasks.rados.rados.0.smithi017.stdout:1483: writing smithi01785587-21 from 531253 to 943252 tid 1 2024-09-21T09:44:08.103 INFO:tasks.rados.rados.0.smithi017.stdout:1483: writing smithi01785587-21 from 1381780 to 1793307 tid 2 2024-09-21T09:44:08.110 INFO:tasks.rados.rados.0.smithi017.stdout:1483: writing smithi01785587-21 from 2300501 to 2887467 tid 3 2024-09-21T09:44:08.115 INFO:tasks.rados.rados.0.smithi017.stdout:1483: writing smithi01785587-21 from 3376974 to 3731253 tid 4 2024-09-21T09:44:08.115 INFO:tasks.rados.rados.0.smithi017.stdout:1484: write oid 44 current snap is 181 2024-09-21T09:44:08.115 INFO:tasks.rados.rados.0.smithi017.stdout:1482: finishing write tid 3 to smithi01785587-7 2024-09-21T09:44:08.115 INFO:tasks.rados.rados.0.smithi017.stdout:1482: finishing write tid 4 to smithi01785587-7 2024-09-21T09:44:08.115 INFO:tasks.rados.rados.0.smithi017.stdout:1482: finishing write tid 5 to smithi01785587-7 2024-09-21T09:44:08.116 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 792 (ObjNum 578 snap 181 seq_num 578) dirty exists 2024-09-21T09:44:08.116 INFO:tasks.rados.rados.0.smithi017.stdout:1482: left oid 7 (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:44:08.116 INFO:tasks.rados.rados.0.smithi017.stdout:1483: finishing write tid 1 to smithi01785587-21 2024-09-21T09:44:08.116 INFO:tasks.rados.rados.0.smithi017.stdout:1483: finishing write tid 2 to smithi01785587-21 2024-09-21T09:44:08.116 INFO:tasks.rados.rados.0.smithi017.stdout:1483: finishing write tid 3 to smithi01785587-21 2024-09-21T09:44:08.116 INFO:tasks.rados.rados.0.smithi017.stdout:1484: seq_num 580 ranges {548060=487248,1440262=671538,2844277=798238} 2024-09-21T09:44:08.120 INFO:tasks.rados.rados.0.smithi017.stdout:1484: writing smithi01785587-44 from 548060 to 1035308 tid 1 2024-09-21T09:44:08.127 INFO:tasks.rados.rados.0.smithi017.stdout:1484: writing smithi01785587-44 from 1440262 to 2111800 tid 2 2024-09-21T09:44:08.135 INFO:tasks.rados.rados.0.smithi017.stdout:1484: writing smithi01785587-44 from 2844277 to 3642515 tid 3 2024-09-21T09:44:08.135 INFO:tasks.rados.rados.0.smithi017.stdout:1482: done (2 left) 2024-09-21T09:44:08.135 INFO:tasks.rados.rados.0.smithi017.stdout:1485: delete oid 48 current snap is 181 2024-09-21T09:44:08.135 INFO:tasks.rados.rados.0.smithi017.stdout:1483: finishing write tid 4 to smithi01785587-21 2024-09-21T09:44:08.135 INFO:tasks.rados.rados.0.smithi017.stdout:1483: finishing write tid 5 to smithi01785587-21 2024-09-21T09:44:08.135 INFO:tasks.rados.rados.0.smithi017.stdout:1483: finishing write tid 6 to smithi01785587-21 2024-09-21T09:44:08.135 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 798 (ObjNum 579 snap 181 seq_num 579) dirty exists 2024-09-21T09:44:08.136 INFO:tasks.rados.rados.0.smithi017.stdout:1483: left oid 21 (ObjNum 579 snap 181 seq_num 579) 2024-09-21T09:44:08.136 INFO:tasks.rados.rados.0.smithi017.stdout:1484: finishing write tid 1 to smithi01785587-44 2024-09-21T09:44:08.136 INFO:tasks.rados.rados.0.smithi017.stdout:1484: finishing write tid 2 to smithi01785587-44 2024-09-21T09:44:08.137 INFO:tasks.rados.rados.0.smithi017.stdout:1483: done (2 left) 2024-09-21T09:44:08.137 INFO:tasks.rados.rados.0.smithi017.stdout:1485: done (1 left) 2024-09-21T09:44:08.137 INFO:tasks.rados.rados.0.smithi017.stdout:1486: write oid 39 current snap is 181 2024-09-21T09:44:08.137 INFO:tasks.rados.rados.0.smithi017.stdout:1486: seq_num 582 ranges {581674=637746,1840873=558854,3066417=537135} 2024-09-21T09:44:08.143 INFO:tasks.rados.rados.0.smithi017.stdout:1486: writing smithi01785587-39 from 581674 to 1219420 tid 1 2024-09-21T09:44:08.150 INFO:tasks.rados.rados.0.smithi017.stdout:1486: writing smithi01785587-39 from 1840873 to 2399727 tid 2 2024-09-21T09:44:08.157 INFO:tasks.rados.rados.0.smithi017.stdout:1486: writing smithi01785587-39 from 3066417 to 3603552 tid 3 2024-09-21T09:44:08.157 INFO:tasks.rados.rados.0.smithi017.stdout:1487: snap_remove snap 171 2024-09-21T09:44:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:08 smithi017 ceph-mon[102913]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T09:44:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:08 smithi017 ceph-mon[105151]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T09:44:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:08 smithi134 ceph-mon[90085]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T09:44:09.078 INFO:tasks.rados.rados.0.smithi017.stdout:1487: done (2 left) 2024-09-21T09:44:09.078 INFO:tasks.rados.rados.0.smithi017.stdout:1488: read oid 41 snap -1 2024-09-21T09:44:09.078 INFO:tasks.rados.rados.0.smithi017.stdout:1488: expect (ObjNum 540 snap 170 seq_num 540) 2024-09-21T09:44:09.078 INFO:tasks.rados.rados.0.smithi017.stdout:1489: write oid 6 current snap is 181 2024-09-21T09:44:09.078 INFO:tasks.rados.rados.0.smithi017.stdout:1484: finishing write tid 3 to smithi01785587-44 2024-09-21T09:44:09.079 INFO:tasks.rados.rados.0.smithi017.stdout:1484: finishing write tid 4 to smithi01785587-44 2024-09-21T09:44:09.079 INFO:tasks.rados.rados.0.smithi017.stdout:1489: seq_num 583 ranges {598481=712995,1899355=418865,3093840=704641} 2024-09-21T09:44:09.086 INFO:tasks.rados.rados.0.smithi017.stdout:1489: writing smithi01785587-6 from 598481 to 1311476 tid 1 2024-09-21T09:44:09.092 INFO:tasks.rados.rados.0.smithi017.stdout:1489: writing smithi01785587-6 from 1899355 to 2318220 tid 2 2024-09-21T09:44:09.101 INFO:tasks.rados.rados.0.smithi017.stdout:1489: writing smithi01785587-6 from 3093840 to 3798481 tid 3 2024-09-21T09:44:09.101 INFO:tasks.rados.rados.0.smithi017.stdout:1490: snap_create 2024-09-21T09:44:09.101 INFO:tasks.rados.rados.0.smithi017.stdout:1484: finishing write tid 5 to smithi01785587-44 2024-09-21T09:44:09.101 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 781 (ObjNum 580 snap 181 seq_num 580) dirty exists 2024-09-21T09:44:09.101 INFO:tasks.rados.rados.0.smithi017.stdout:1484: left oid 44 (ObjNum 580 snap 181 seq_num 580) 2024-09-21T09:44:09.101 INFO:tasks.rados.rados.0.smithi017.stdout:1486: finishing write tid 1 to smithi01785587-39 2024-09-21T09:44:09.101 INFO:tasks.rados.rados.0.smithi017.stdout:1486: finishing write tid 2 to smithi01785587-39 2024-09-21T09:44:09.101 INFO:tasks.rados.rados.0.smithi017.stdout:1486: finishing write tid 3 to smithi01785587-39 2024-09-21T09:44:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:1486: finishing write tid 4 to smithi01785587-39 2024-09-21T09:44:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:1486: finishing write tid 5 to smithi01785587-39 2024-09-21T09:44:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 761 (ObjNum 582 snap 181 seq_num 582) dirty exists 2024-09-21T09:44:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:1486: left oid 39 (ObjNum 582 snap 181 seq_num 582) 2024-09-21T09:44:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:1488: expect (ObjNum 540 snap 170 seq_num 540) 2024-09-21T09:44:09.248 INFO:tasks.rados.rados.0.smithi017.stdout:1489: finishing write tid 1 to smithi01785587-6 2024-09-21T09:44:09.248 INFO:tasks.rados.rados.0.smithi017.stdout:1489: finishing write tid 2 to smithi01785587-6 2024-09-21T09:44:09.248 INFO:tasks.rados.rados.0.smithi017.stdout:1489: finishing write tid 3 to smithi01785587-6 2024-09-21T09:44:09.248 INFO:tasks.rados.rados.0.smithi017.stdout:1489: finishing write tid 4 to smithi01785587-6 2024-09-21T09:44:09.248 INFO:tasks.rados.rados.0.smithi017.stdout:1489: finishing write tid 5 to smithi01785587-6 2024-09-21T09:44:09.248 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 453 (ObjNum 583 snap 181 seq_num 583) dirty exists 2024-09-21T09:44:09.249 INFO:tasks.rados.rados.0.smithi017.stdout:1489: left oid 6 (ObjNum 583 snap 181 seq_num 583) 2024-09-21T09:44:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:09 smithi017 ceph-mon[102913]: pgmap v214: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:44:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:09 smithi017 ceph-mon[102913]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T09:44:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:09 smithi017 ceph-mon[105151]: pgmap v214: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:44:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:09 smithi017 ceph-mon[105151]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T09:44:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:09 smithi134 ceph-mon[90085]: pgmap v214: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:44:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:09 smithi134 ceph-mon[90085]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T09:44:10.084 INFO:tasks.rados.rados.0.smithi017.stdout:1484: done (4 left) 2024-09-21T09:44:10.084 INFO:tasks.rados.rados.0.smithi017.stdout:1486: done (3 left) 2024-09-21T09:44:10.084 INFO:tasks.rados.rados.0.smithi017.stdout:1488: done (2 left) 2024-09-21T09:44:10.085 INFO:tasks.rados.rados.0.smithi017.stdout:1489: done (1 left) 2024-09-21T09:44:10.085 INFO:tasks.rados.rados.0.smithi017.stdout:1490: done (0 left) 2024-09-21T09:44:10.085 INFO:tasks.rados.rados.0.smithi017.stdout:1491: snap_create 2024-09-21T09:44:10.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:10 smithi017 ceph-mon[102913]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T09:44:10.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:10 smithi017 ceph-mon[105151]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T09:44:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:10 smithi134 ceph-mon[90085]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T09:44:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:11.088 INFO:tasks.rados.rados.0.smithi017.stdout:1491: done (0 left) 2024-09-21T09:44:11.088 INFO:tasks.rados.rados.0.smithi017.stdout:1492: snap_create 2024-09-21T09:44:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:11 smithi017 ceph-mon[102913]: pgmap v217: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 354 KiB/s wr, 5 op/s 2024-09-21T09:44:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:11 smithi017 ceph-mon[102913]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:44:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:11 smithi017 ceph-mon[105151]: pgmap v217: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 354 KiB/s wr, 5 op/s 2024-09-21T09:44:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:11 smithi017 ceph-mon[105151]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:44:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:11.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:11 smithi134 ceph-mon[90085]: pgmap v217: 89 pgs: 89 active+clean; 243 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s rd, 354 KiB/s wr, 5 op/s 2024-09-21T09:44:11.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:11 smithi134 ceph-mon[90085]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:44:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:12.096 INFO:tasks.rados.rados.0.smithi017.stdout:1492: done (0 left) 2024-09-21T09:44:12.096 INFO:tasks.rados.rados.0.smithi017.stdout:1493: snap_remove snap 183 2024-09-21T09:44:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:12 smithi017 ceph-mon[102913]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:44:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:12 smithi017 ceph-mon[105151]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:44:12.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:12 smithi134 ceph-mon[90085]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:44:13.051 INFO:tasks.rados.rados.0.smithi017.stdout:1493: done (0 left) 2024-09-21T09:44:13.052 INFO:tasks.rados.rados.0.smithi017.stdout:1494: snap_create 2024-09-21T09:44:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:13 smithi017 ceph-mon[102913]: pgmap v220: 89 pgs: 89 active+clean; 253 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-09-21T09:44:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:13 smithi017 ceph-mon[102913]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:44:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:13 smithi017 ceph-mon[102913]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:44:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:13 smithi017 ceph-mon[105151]: pgmap v220: 89 pgs: 89 active+clean; 253 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-09-21T09:44:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:13 smithi017 ceph-mon[105151]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:44:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:13 smithi017 ceph-mon[105151]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:44:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:13 smithi134 ceph-mon[90085]: pgmap v220: 89 pgs: 89 active+clean; 253 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-09-21T09:44:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:13 smithi134 ceph-mon[90085]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:44:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:13 smithi134 ceph-mon[90085]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:44:14.052 INFO:tasks.rados.rados.0.smithi017.stdout:1494: done (0 left) 2024-09-21T09:44:14.052 INFO:tasks.rados.rados.0.smithi017.stdout:1495: snap_remove snap 180 2024-09-21T09:44:14.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:15.056 INFO:tasks.rados.rados.0.smithi017.stdout:1495: done (0 left) 2024-09-21T09:44:15.056 INFO:tasks.rados.rados.0.smithi017.stdout:1496: rollback oid 50 current snap is 185 2024-09-21T09:44:15.056 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 50 to 174 2024-09-21T09:44:15.056 INFO:tasks.rados.rados.0.smithi017.stdout:1497: write oid 24 current snap is 185 2024-09-21T09:44:15.056 INFO:tasks.rados.rados.0.smithi017.stdout:1497: seq_num 584 ranges {615288=788244,1957837=562523,3121263=559679} 2024-09-21T09:44:15.057 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:44:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:44:14] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.51.0" 2024-09-21T09:44:15.064 INFO:tasks.rados.rados.0.smithi017.stdout:1497: writing smithi01785587-24 from 615288 to 1403532 tid 1 2024-09-21T09:44:15.071 INFO:tasks.rados.rados.0.smithi017.stdout:1497: writing smithi01785587-24 from 1957837 to 2520360 tid 2 2024-09-21T09:44:15.077 INFO:tasks.rados.rados.0.smithi017.stdout:1497: writing smithi01785587-24 from 3121263 to 3680942 tid 3 2024-09-21T09:44:15.077 INFO:tasks.rados.rados.0.smithi017.stdout:1498: write oid 49 current snap is 185 2024-09-21T09:44:15.078 INFO:tasks.rados.rados.0.smithi017.stdout:1498: seq_num 585 ranges {632095=463493,1616319=422534,2748686=487304} 2024-09-21T09:44:15.083 INFO:tasks.rados.rados.0.smithi017.stdout:1498: writing smithi01785587-49 from 632095 to 1095588 tid 1 2024-09-21T09:44:15.089 INFO:tasks.rados.rados.0.smithi017.stdout:1498: writing smithi01785587-49 from 1616319 to 2038853 tid 2 2024-09-21T09:44:15.095 INFO:tasks.rados.rados.0.smithi017.stdout:1498: writing smithi01785587-49 from 2748686 to 3235990 tid 3 2024-09-21T09:44:15.096 INFO:tasks.rados.rados.0.smithi017.stdout:1499: snap_remove snap 145 2024-09-21T09:44:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:15 smithi017 ceph-mon[102913]: pgmap v223: 89 pgs: 89 active+clean; 253 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-09-21T09:44:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:15 smithi017 ceph-mon[102913]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:44:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:15 smithi017 ceph-mon[105151]: pgmap v223: 89 pgs: 89 active+clean; 253 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-09-21T09:44:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:15 smithi017 ceph-mon[105151]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:44:15.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:15 smithi134 ceph-mon[90085]: pgmap v223: 89 pgs: 89 active+clean; 253 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-09-21T09:44:15.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:15 smithi134 ceph-mon[90085]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:44:15.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:15.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:16.065 INFO:tasks.rados.rados.0.smithi017.stdout:1499: done (3 left) 2024-09-21T09:44:16.065 INFO:tasks.rados.rados.0.smithi017.stdout:1500: delete oid 36 current snap is 185 2024-09-21T09:44:16.065 INFO:tasks.rados.rados.0.smithi017.stdout:1496: finishing rollback tid 0 to smithi01785587-50 2024-09-21T09:44:16.065 INFO:tasks.rados.rados.0.smithi017.stdout:1496: finishing rollback tid 1 to smithi01785587-50 2024-09-21T09:44:16.065 INFO:tasks.rados.rados.0.smithi017.stdout:1496: finishing rollback tid 2 to smithi01785587-50 2024-09-21T09:44:16.065 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 452 (ObjNum 567 snap 173 seq_num 567) dirty exists 2024-09-21T09:44:16.065 INFO:tasks.rados.rados.0.smithi017.stdout:1497: finishing write tid 1 to smithi01785587-24 2024-09-21T09:44:16.065 INFO:tasks.rados.rados.0.smithi017.stdout:1497: finishing write tid 2 to smithi01785587-24 2024-09-21T09:44:16.065 INFO:tasks.rados.rados.0.smithi017.stdout:1497: finishing write tid 3 to smithi01785587-24 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:1497: finishing write tid 4 to smithi01785587-24 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:1497: finishing write tid 5 to smithi01785587-24 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 457 (ObjNum 584 snap 185 seq_num 584) dirty exists 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:1497: left oid 24 (ObjNum 584 snap 185 seq_num 584) 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:1498: finishing write tid 1 to smithi01785587-49 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:1498: finishing write tid 2 to smithi01785587-49 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:1498: finishing write tid 3 to smithi01785587-49 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:1498: finishing write tid 4 to smithi01785587-49 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:1498: finishing write tid 5 to smithi01785587-49 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 813 (ObjNum 585 snap 185 seq_num 585) dirty exists 2024-09-21T09:44:16.066 INFO:tasks.rados.rados.0.smithi017.stdout:1498: left oid 49 (ObjNum 585 snap 185 seq_num 585) 2024-09-21T09:44:16.072 INFO:tasks.rados.rados.0.smithi017.stdout:1496: done (3 left) 2024-09-21T09:44:16.073 INFO:tasks.rados.rados.0.smithi017.stdout:1497: done (2 left) 2024-09-21T09:44:16.073 INFO:tasks.rados.rados.0.smithi017.stdout:1498: done (1 left) 2024-09-21T09:44:16.073 INFO:tasks.rados.rados.0.smithi017.stdout:1500: done (0 left) 2024-09-21T09:44:16.073 INFO:tasks.rados.rados.0.smithi017.stdout:1501: write oid 39 current snap is 185 2024-09-21T09:44:16.073 INFO:tasks.rados.rados.0.smithi017.stdout:1501: seq_num 587 ranges {665709=730344,2133283=426203,3203532=509848} 2024-09-21T09:44:16.080 INFO:tasks.rados.rados.0.smithi017.stdout:1501: writing smithi01785587-39 from 665709 to 1396053 tid 1 2024-09-21T09:44:16.087 INFO:tasks.rados.rados.0.smithi017.stdout:1501: writing smithi01785587-39 from 2133283 to 2559486 tid 2 2024-09-21T09:44:16.094 INFO:tasks.rados.rados.0.smithi017.stdout:1501: writing smithi01785587-39 from 3203532 to 3713380 tid 3 2024-09-21T09:44:16.094 INFO:tasks.rados.rados.0.smithi017.stdout:1502: read oid 49 snap 172 2024-09-21T09:44:16.094 INFO:tasks.rados.rados.0.smithi017.stdout:1502: expect (ObjNum 447 snap 145 seq_num 447) 2024-09-21T09:44:16.094 INFO:tasks.rados.rados.0.smithi017.stdout:1503: read oid 10 snap -1 2024-09-21T09:44:16.094 INFO:tasks.rados.rados.0.smithi017.stdout:1503: expect (ObjNum 576 snap 180 seq_num 576) 2024-09-21T09:44:16.094 INFO:tasks.rados.rados.0.smithi017.stdout:1504: write oid 17 current snap is 185 2024-09-21T09:44:16.094 INFO:tasks.rados.rados.0.smithi017.stdout:1501: finishing write tid 1 to smithi01785587-39 2024-09-21T09:44:16.094 INFO:tasks.rados.rados.0.smithi017.stdout:1504: seq_num 588 ranges {682516=405593,1791765=569861,2830955=721120} 2024-09-21T09:44:16.100 INFO:tasks.rados.rados.0.smithi017.stdout:1504: writing smithi01785587-17 from 682516 to 1088109 tid 1 2024-09-21T09:44:16.108 INFO:tasks.rados.rados.0.smithi017.stdout:1504: writing smithi01785587-17 from 1791765 to 2361626 tid 2 2024-09-21T09:44:16.115 INFO:tasks.rados.rados.0.smithi017.stdout:1504: writing smithi01785587-17 from 2830955 to 3552075 tid 3 2024-09-21T09:44:16.115 INFO:tasks.rados.rados.0.smithi017.stdout:1505: snap_remove snap 166 2024-09-21T09:44:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:16 smithi017 ceph-mon[102913]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:44:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:16 smithi017 ceph-mon[105151]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:44:16.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:16 smithi134 ceph-mon[90085]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:44:17.070 INFO:tasks.rados.rados.0.smithi017.stdout:1505: done (4 left) 2024-09-21T09:44:17.070 INFO:tasks.rados.rados.0.smithi017.stdout:1506: write oid 36 current snap is 185 2024-09-21T09:44:17.070 INFO:tasks.rados.rados.0.smithi017.stdout:1501: finishing write tid 2 to smithi01785587-39 2024-09-21T09:44:17.070 INFO:tasks.rados.rados.0.smithi017.stdout:1506: seq_num 589 ranges {699323=480842,1850247=429872,2858378=532392} 2024-09-21T09:44:17.077 INFO:tasks.rados.rados.0.smithi017.stdout:1506: writing smithi01785587-36 from 699323 to 1180165 tid 1 2024-09-21T09:44:17.083 INFO:tasks.rados.rados.0.smithi017.stdout:1506: writing smithi01785587-36 from 1850247 to 2280119 tid 2 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi017.stdout:1506: writing smithi01785587-36 from 2858378 to 3390770 tid 3 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi017.stdout:1507: delete oid 26 current snap is 185 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi017.stdout:1501: finishing write tid 3 to smithi01785587-39 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi017.stdout:1501: finishing write tid 4 to smithi01785587-39 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi017.stdout:1501: finishing write tid 5 to smithi01785587-39 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 779 (ObjNum 587 snap 185 seq_num 587) dirty exists 2024-09-21T09:44:17.091 INFO:tasks.rados.rados.0.smithi017.stdout:1501: left oid 39 (ObjNum 587 snap 185 seq_num 587) 2024-09-21T09:44:17.091 INFO:tasks.rados.rados.0.smithi017.stdout:1502: expect (ObjNum 447 snap 145 seq_num 447) 2024-09-21T09:44:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:17 smithi134 ceph-mon[90085]: pgmap v226: 89 pgs: 89 active+clean; 256 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 802 KiB/s wr, 3 op/s 2024-09-21T09:44:17.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:17 smithi134 ceph-mon[90085]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T09:44:17.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:17.340 INFO:tasks.rados.rados.0.smithi017.stdout:1503: expect (ObjNum 576 snap 180 seq_num 576) 2024-09-21T09:44:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:17 smithi017 ceph-mon[102913]: pgmap v226: 89 pgs: 89 active+clean; 256 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 802 KiB/s wr, 3 op/s 2024-09-21T09:44:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:17 smithi017 ceph-mon[102913]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T09:44:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:17 smithi017 ceph-mon[105151]: pgmap v226: 89 pgs: 89 active+clean; 256 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 802 KiB/s wr, 3 op/s 2024-09-21T09:44:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:17 smithi017 ceph-mon[105151]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T09:44:17.461 INFO:tasks.rados.rados.0.smithi017.stdout:1504: finishing write tid 1 to smithi01785587-17 2024-09-21T09:44:17.461 INFO:tasks.rados.rados.0.smithi017.stdout:1504: finishing write tid 2 to smithi01785587-17 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1504: finishing write tid 3 to smithi01785587-17 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1504: finishing write tid 4 to smithi01785587-17 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1504: finishing write tid 5 to smithi01785587-17 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 540 (ObjNum 588 snap 185 seq_num 588) dirty exists 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1504: left oid 17 (ObjNum 588 snap 185 seq_num 588) 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1501: done (5 left) 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1502: done (4 left) 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1503: done (3 left) 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1504: done (2 left) 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1507: done (1 left) 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1508: read oid 47 snap -1 2024-09-21T09:44:17.462 INFO:tasks.rados.rados.0.smithi017.stdout:1508: expect (ObjNum 562 snap 173 seq_num 562) 2024-09-21T09:44:17.463 INFO:tasks.rados.rados.0.smithi017.stdout:1509: snap_create 2024-09-21T09:44:17.463 INFO:tasks.rados.rados.0.smithi017.stdout:1506: finishing write tid 1 to smithi01785587-36 2024-09-21T09:44:17.463 INFO:tasks.rados.rados.0.smithi017.stdout:1506: finishing write tid 2 to smithi01785587-36 2024-09-21T09:44:17.463 INFO:tasks.rados.rados.0.smithi017.stdout:1506: finishing write tid 3 to smithi01785587-36 2024-09-21T09:44:17.463 INFO:tasks.rados.rados.0.smithi017.stdout:1506: finishing write tid 4 to smithi01785587-36 2024-09-21T09:44:17.463 INFO:tasks.rados.rados.0.smithi017.stdout:1506: finishing write tid 5 to smithi01785587-36 2024-09-21T09:44:17.463 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 789 (ObjNum 589 snap 185 seq_num 589) dirty exists 2024-09-21T09:44:17.463 INFO:tasks.rados.rados.0.smithi017.stdout:1506: left oid 36 (ObjNum 589 snap 185 seq_num 589) 2024-09-21T09:44:17.464 INFO:tasks.rados.rados.0.smithi017.stdout:1508: expect (ObjNum 562 snap 173 seq_num 562) 2024-09-21T09:44:18.075 INFO:tasks.rados.rados.0.smithi017.stdout:1506: done (2 left) 2024-09-21T09:44:18.075 INFO:tasks.rados.rados.0.smithi017.stdout:1508: done (1 left) 2024-09-21T09:44:18.075 INFO:tasks.rados.rados.0.smithi017.stdout:1509: done (0 left) 2024-09-21T09:44:18.075 INFO:tasks.rados.rados.0.smithi017.stdout:1510: delete oid 15 current snap is 186 2024-09-21T09:44:18.079 INFO:tasks.rados.rados.0.smithi017.stdout:1510: done (0 left) 2024-09-21T09:44:18.079 INFO:tasks.rados.rados.0.smithi017.stdout:1511: read oid 24 snap -1 2024-09-21T09:44:18.079 INFO:tasks.rados.rados.0.smithi017.stdout:1511: expect (ObjNum 584 snap 185 seq_num 584) 2024-09-21T09:44:18.079 INFO:tasks.rados.rados.0.smithi017.stdout:1512: read oid 34 snap 174 2024-09-21T09:44:18.080 INFO:tasks.rados.rados.0.smithi017.stdout:1512: expect (ObjNum 561 snap 173 seq_num 561) 2024-09-21T09:44:18.080 INFO:tasks.rados.rados.0.smithi017.stdout:1513: snap_remove snap 157 2024-09-21T09:44:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:18 smithi017 ceph-mon[102913]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T09:44:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:18 smithi017 ceph-mon[105151]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T09:44:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:18 smithi134 ceph-mon[90085]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T09:44:19.082 INFO:tasks.rados.rados.0.smithi017.stdout:1513: done (2 left) 2024-09-21T09:44:19.082 INFO:tasks.rados.rados.0.smithi017.stdout:1514: write oid 29 current snap is 186 2024-09-21T09:44:19.082 INFO:tasks.rados.rados.0.smithi017.stdout:1514: seq_num 592 ranges {749744=706589,1909340=577199,3224294=482561} 2024-09-21T09:44:19.091 INFO:tasks.rados.rados.0.smithi017.stdout:1514: writing smithi01785587-29 from 749744 to 1456333 tid 1 2024-09-21T09:44:19.097 INFO:tasks.rados.rados.0.smithi017.stdout:1514: writing smithi01785587-29 from 1909340 to 2486539 tid 2 2024-09-21T09:44:19.103 INFO:tasks.rados.rados.0.smithi017.stdout:1514: writing smithi01785587-29 from 3224294 to 3706855 tid 3 2024-09-21T09:44:19.104 INFO:tasks.rados.rados.0.smithi017.stdout:1515: read oid 31 snap -1 2024-09-21T09:44:19.104 INFO:tasks.rados.rados.0.smithi017.stdout:1515: expect deleted 2024-09-21T09:44:19.104 INFO:tasks.rados.rados.0.smithi017.stdout:1516: snap_remove snap 172 2024-09-21T09:44:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:19 smithi017 ceph-mon[105151]: pgmap v229: 89 pgs: 89 active+clean; 256 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 802 KiB/s wr, 3 op/s 2024-09-21T09:44:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:19 smithi017 ceph-mon[105151]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T09:44:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:19 smithi017 ceph-mon[102913]: pgmap v229: 89 pgs: 89 active+clean; 256 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 802 KiB/s wr, 3 op/s 2024-09-21T09:44:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:19 smithi017 ceph-mon[102913]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T09:44:19.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:19 smithi134 ceph-mon[90085]: pgmap v229: 89 pgs: 89 active+clean; 256 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 802 KiB/s wr, 3 op/s 2024-09-21T09:44:19.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:19 smithi134 ceph-mon[90085]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T09:44:20.089 INFO:tasks.rados.rados.0.smithi017.stdout:1511: expect (ObjNum 584 snap 185 seq_num 584) 2024-09-21T09:44:20.302 INFO:tasks.rados.rados.0.smithi017.stdout:1512: expect (ObjNum 561 snap 173 seq_num 561) 2024-09-21T09:44:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:20 smithi017 ceph-mon[102913]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T09:44:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:20 smithi017 ceph-mon[105151]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T09:44:20.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:20 smithi134 ceph-mon[90085]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T09:44:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:20.531 INFO:tasks.rados.rados.0.smithi017.stdout:1514: finishing write tid 1 to smithi01785587-29 2024-09-21T09:44:20.531 INFO:tasks.rados.rados.0.smithi017.stdout:1514: finishing write tid 2 to smithi01785587-29 2024-09-21T09:44:20.531 INFO:tasks.rados.rados.0.smithi017.stdout:1514: finishing write tid 3 to smithi01785587-29 2024-09-21T09:44:20.531 INFO:tasks.rados.rados.0.smithi017.stdout:1514: finishing write tid 4 to smithi01785587-29 2024-09-21T09:44:20.531 INFO:tasks.rados.rados.0.smithi017.stdout:1514: finishing write tid 5 to smithi01785587-29 2024-09-21T09:44:20.531 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 333 (ObjNum 592 snap 186 seq_num 592) dirty exists 2024-09-21T09:44:20.531 INFO:tasks.rados.rados.0.smithi017.stdout:1514: left oid 29 (ObjNum 592 snap 186 seq_num 592) 2024-09-21T09:44:20.532 INFO:tasks.rados.rados.0.smithi017.stdout:1511: done (4 left) 2024-09-21T09:44:20.532 INFO:tasks.rados.rados.0.smithi017.stdout:1512: done (3 left) 2024-09-21T09:44:20.532 INFO:tasks.rados.rados.0.smithi017.stdout:1514: done (2 left) 2024-09-21T09:44:20.532 INFO:tasks.rados.rados.0.smithi017.stdout:1515: done (1 left) 2024-09-21T09:44:20.532 INFO:tasks.rados.rados.0.smithi017.stdout:1516: done (0 left) 2024-09-21T09:44:20.532 INFO:tasks.rados.rados.0.smithi017.stdout:1517: write oid 38 current snap is 186 2024-09-21T09:44:20.532 INFO:tasks.rados.rados.0.smithi017.stdout:1517: seq_num 593 ranges {766551=498191,1684175=720857,2851717=693833} 2024-09-21T09:44:20.537 INFO:tasks.rados.rados.0.smithi017.stdout:1517: writing smithi01785587-38 from 766551 to 1264742 tid 1 2024-09-21T09:44:20.544 INFO:tasks.rados.rados.0.smithi017.stdout:1517: writing smithi01785587-38 from 1684175 to 2405032 tid 2 2024-09-21T09:44:20.550 INFO:tasks.rados.rados.0.smithi017.stdout:1517: writing smithi01785587-38 from 2851717 to 3545550 tid 3 2024-09-21T09:44:20.550 INFO:tasks.rados.rados.0.smithi017.stdout:1518: read oid 7 snap -1 2024-09-21T09:44:20.550 INFO:tasks.rados.rados.0.smithi017.stdout:1518: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:44:20.550 INFO:tasks.rados.rados.0.smithi017.stdout:1519: snap_remove snap 184 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:1519: done (2 left) 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:1520: rollback oid 42 current snap is 186 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 182 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:1517: finishing write tid 1 to smithi01785587-38 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:1517: finishing write tid 2 to smithi01785587-38 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:1517: finishing write tid 3 to smithi01785587-38 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:1517: finishing write tid 4 to smithi01785587-38 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:1517: finishing write tid 5 to smithi01785587-38 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 472 (ObjNum 593 snap 186 seq_num 593) dirty exists 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:1517: left oid 38 (ObjNum 593 snap 186 seq_num 593) 2024-09-21T09:44:21.095 INFO:tasks.rados.rados.0.smithi017.stdout:1518: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:44:21.302 INFO:tasks.rados.rados.0.smithi017.stdout:1520: finishing rollback tid 0 to smithi01785587-42 2024-09-21T09:44:21.302 INFO:tasks.rados.rados.0.smithi017.stdout:1520: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:44:21.302 INFO:tasks.rados.rados.0.smithi017.stdout:1520: finishing rollback tid 2 to smithi01785587-42 2024-09-21T09:44:21.302 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 570 (ObjNum 426 snap 133 seq_num 426) dirty exists 2024-09-21T09:44:21.303 INFO:tasks.rados.rados.0.smithi017.stdout:1517: done (2 left) 2024-09-21T09:44:21.303 INFO:tasks.rados.rados.0.smithi017.stdout:1518: done (1 left) 2024-09-21T09:44:21.303 INFO:tasks.rados.rados.0.smithi017.stdout:1520: done (0 left) 2024-09-21T09:44:21.303 INFO:tasks.rados.rados.0.smithi017.stdout:1521: write oid 9 current snap is 186 2024-09-21T09:44:21.303 INFO:tasks.rados.rados.0.smithi017.stdout:1521: seq_num 594 ranges {783358=573440,2026304=580868,3279140=621458} 2024-09-21T09:44:21.309 INFO:tasks.rados.rados.0.smithi017.stdout:1521: writing smithi01785587-9 from 783358 to 1356798 tid 1 2024-09-21T09:44:21.316 INFO:tasks.rados.rados.0.smithi017.stdout:1521: writing smithi01785587-9 from 2026304 to 2607172 tid 2 2024-09-21T09:44:21.324 INFO:tasks.rados.rados.0.smithi017.stdout:1521: writing smithi01785587-9 from 3279140 to 3900598 tid 3 2024-09-21T09:44:21.324 INFO:tasks.rados.rados.0.smithi017.stdout:1522: write oid 37 current snap is 186 2024-09-21T09:44:21.324 INFO:tasks.rados.rados.0.smithi017.stdout:1522: seq_num 595 ranges {400165=648689,1684786=315379} 2024-09-21T09:44:21.329 INFO:tasks.rados.rados.0.smithi017.stdout:1522: writing smithi01785587-37 from 400165 to 1048854 tid 1 2024-09-21T09:44:21.335 INFO:tasks.rados.rados.0.smithi017.stdout:1522: writing smithi01785587-37 from 1684786 to 2000165 tid 2 2024-09-21T09:44:21.335 INFO:tasks.rados.rados.0.smithi017.stdout:1523: write oid 30 current snap is 186 2024-09-21T09:44:21.335 INFO:tasks.rados.rados.0.smithi017.stdout:1521: finishing write tid 1 to smithi01785587-9 2024-09-21T09:44:21.335 INFO:tasks.rados.rados.0.smithi017.stdout:1523: seq_num 596 ranges {416972=723938,1743268=273704} 2024-09-21T09:44:21.341 INFO:tasks.rados.rados.0.smithi017.stdout:1523: writing smithi01785587-30 from 416972 to 1140910 tid 1 2024-09-21T09:44:21.346 INFO:tasks.rados.rados.0.smithi017.stdout:1523: writing smithi01785587-30 from 1743268 to 2016972 tid 2 2024-09-21T09:44:21.346 INFO:tasks.rados.rados.0.smithi017.stdout:1524: snap_remove snap 177 2024-09-21T09:44:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:21 smithi017 ceph-mon[105151]: pgmap v232: 89 pgs: 89 active+clean; 256 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:44:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:21 smithi017 ceph-mon[105151]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T09:44:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:21 smithi017 ceph-mon[102913]: pgmap v232: 89 pgs: 89 active+clean; 256 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:44:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:21 smithi017 ceph-mon[102913]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T09:44:21.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:21 smithi134 ceph-mon[90085]: pgmap v232: 89 pgs: 89 active+clean; 256 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:44:21.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:21 smithi134 ceph-mon[90085]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T09:44:22.101 INFO:tasks.rados.rados.0.smithi017.stdout:1524: done (3 left) 2024-09-21T09:44:22.101 INFO:tasks.rados.rados.0.smithi017.stdout:1525: delete oid 10 current snap is 186 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1521: finishing write tid 2 to smithi01785587-9 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1521: finishing write tid 3 to smithi01785587-9 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1521: finishing write tid 4 to smithi01785587-9 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1521: finishing write tid 5 to smithi01785587-9 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 299 (ObjNum 594 snap 186 seq_num 594) dirty exists 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1521: left oid 9 (ObjNum 594 snap 186 seq_num 594) 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1522: finishing write tid 1 to smithi01785587-37 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1522: finishing write tid 2 to smithi01785587-37 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1522: finishing write tid 3 to smithi01785587-37 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1522: finishing write tid 4 to smithi01785587-37 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 808 (ObjNum 595 snap 186 seq_num 595) dirty exists 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1522: left oid 37 (ObjNum 595 snap 186 seq_num 595) 2024-09-21T09:44:22.102 INFO:tasks.rados.rados.0.smithi017.stdout:1523: finishing write tid 1 to smithi01785587-30 2024-09-21T09:44:22.103 INFO:tasks.rados.rados.0.smithi017.stdout:1523: finishing write tid 2 to smithi01785587-30 2024-09-21T09:44:22.103 INFO:tasks.rados.rados.0.smithi017.stdout:1523: finishing write tid 3 to smithi01785587-30 2024-09-21T09:44:22.103 INFO:tasks.rados.rados.0.smithi017.stdout:1523: finishing write tid 4 to smithi01785587-30 2024-09-21T09:44:22.103 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 687 (ObjNum 596 snap 186 seq_num 596) dirty exists 2024-09-21T09:44:22.103 INFO:tasks.rados.rados.0.smithi017.stdout:1523: left oid 30 (ObjNum 596 snap 186 seq_num 596) 2024-09-21T09:44:22.107 INFO:tasks.rados.rados.0.smithi017.stdout:1521: done (3 left) 2024-09-21T09:44:22.107 INFO:tasks.rados.rados.0.smithi017.stdout:1522: done (2 left) 2024-09-21T09:44:22.107 INFO:tasks.rados.rados.0.smithi017.stdout:1523: done (1 left) 2024-09-21T09:44:22.107 INFO:tasks.rados.rados.0.smithi017.stdout:1525: done (0 left) 2024-09-21T09:44:22.107 INFO:tasks.rados.rados.0.smithi017.stdout:1526: read oid 40 snap -1 2024-09-21T09:44:22.107 INFO:tasks.rados.rados.0.smithi017.stdout:1526: expect (ObjNum 407 snap 126 seq_num 407) 2024-09-21T09:44:22.107 INFO:tasks.rados.rados.0.smithi017.stdout:1527: write oid 14 current snap is 186 2024-09-21T09:44:22.107 INFO:tasks.rados.rados.0.smithi017.stdout:1527: seq_num 598 ranges {450586=474436,1343879=588206} 2024-09-21T09:44:22.112 INFO:tasks.rados.rados.0.smithi017.stdout:1527: writing smithi01785587-14 from 450586 to 925022 tid 1 2024-09-21T09:44:22.118 INFO:tasks.rados.rados.0.smithi017.stdout:1527: writing smithi01785587-14 from 1343879 to 1932085 tid 2 2024-09-21T09:44:22.118 INFO:tasks.rados.rados.0.smithi017.stdout:1528: snap_create 2024-09-21T09:44:22.119 INFO:tasks.rados.rados.0.smithi017.stdout:1526: expect (ObjNum 407 snap 126 seq_num 407) 2024-09-21T09:44:22.408 INFO:tasks.rados.rados.0.smithi017.stdout:1527: finishing write tid 1 to smithi01785587-14 2024-09-21T09:44:22.408 INFO:tasks.rados.rados.0.smithi017.stdout:1527: finishing write tid 2 to smithi01785587-14 2024-09-21T09:44:22.408 INFO:tasks.rados.rados.0.smithi017.stdout:1527: finishing write tid 3 to smithi01785587-14 2024-09-21T09:44:22.408 INFO:tasks.rados.rados.0.smithi017.stdout:1527: finishing write tid 4 to smithi01785587-14 2024-09-21T09:44:22.408 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 813 (ObjNum 598 snap 186 seq_num 598) dirty exists 2024-09-21T09:44:22.408 INFO:tasks.rados.rados.0.smithi017.stdout:1527: left oid 14 (ObjNum 598 snap 186 seq_num 598) 2024-09-21T09:44:22.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:22 smithi017 ceph-mon[102913]: osdmap e437: 8 total, 8 up, 8 in 2024-09-21T09:44:22.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:22 smithi017 ceph-mon[105151]: osdmap e437: 8 total, 8 up, 8 in 2024-09-21T09:44:22.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:22 smithi134 ceph-mon[90085]: osdmap e437: 8 total, 8 up, 8 in 2024-09-21T09:44:23.054 INFO:tasks.rados.rados.0.smithi017.stdout:1526: done (2 left) 2024-09-21T09:44:23.054 INFO:tasks.rados.rados.0.smithi017.stdout:1527: done (1 left) 2024-09-21T09:44:23.055 INFO:tasks.rados.rados.0.smithi017.stdout:1528: done (0 left) 2024-09-21T09:44:23.055 INFO:tasks.rados.rados.0.smithi017.stdout:1529: write oid 26 current snap is 187 2024-09-21T09:44:23.055 INFO:tasks.rados.rados.0.smithi017.stdout:1529: seq_num 599 ranges {467393=549685,1802361=265032} 2024-09-21T09:44:23.060 INFO:tasks.rados.rados.0.smithi017.stdout:1529: writing smithi01785587-26 from 467393 to 1017078 tid 1 2024-09-21T09:44:23.066 INFO:tasks.rados.rados.0.smithi017.stdout:1529: writing smithi01785587-26 from 1802361 to 2067393 tid 2 2024-09-21T09:44:23.067 INFO:tasks.rados.rados.0.smithi017.stdout:1530: snap_remove snap 182 2024-09-21T09:44:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:23 smithi017 ceph-mon[102913]: pgmap v235: 89 pgs: 89 active+clean; 139 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-09-21T09:44:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:23 smithi017 ceph-mon[102913]: osdmap e438: 8 total, 8 up, 8 in 2024-09-21T09:44:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:23 smithi017 ceph-mon[102913]: osdmap e439: 8 total, 8 up, 8 in 2024-09-21T09:44:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:23 smithi017 ceph-mon[105151]: pgmap v235: 89 pgs: 89 active+clean; 139 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-09-21T09:44:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:23 smithi017 ceph-mon[105151]: osdmap e438: 8 total, 8 up, 8 in 2024-09-21T09:44:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:23 smithi017 ceph-mon[105151]: osdmap e439: 8 total, 8 up, 8 in 2024-09-21T09:44:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:23 smithi134 ceph-mon[90085]: pgmap v235: 89 pgs: 89 active+clean; 139 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-09-21T09:44:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:23 smithi134 ceph-mon[90085]: osdmap e438: 8 total, 8 up, 8 in 2024-09-21T09:44:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:23 smithi134 ceph-mon[90085]: osdmap e439: 8 total, 8 up, 8 in 2024-09-21T09:44:24.055 INFO:tasks.rados.rados.0.smithi017.stdout:1530: done (1 left) 2024-09-21T09:44:24.055 INFO:tasks.rados.rados.0.smithi017.stdout:1531: delete oid 22 current snap is 187 2024-09-21T09:44:24.055 INFO:tasks.rados.rados.0.smithi017.stdout:1529: finishing write tid 1 to smithi01785587-26 2024-09-21T09:44:24.055 INFO:tasks.rados.rados.0.smithi017.stdout:1529: finishing write tid 2 to smithi01785587-26 2024-09-21T09:44:24.055 INFO:tasks.rados.rados.0.smithi017.stdout:1529: finishing write tid 3 to smithi01785587-26 2024-09-21T09:44:24.055 INFO:tasks.rados.rados.0.smithi017.stdout:1529: finishing write tid 4 to smithi01785587-26 2024-09-21T09:44:24.056 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 769 (ObjNum 599 snap 187 seq_num 599) dirty exists 2024-09-21T09:44:24.056 INFO:tasks.rados.rados.0.smithi017.stdout:1529: left oid 26 (ObjNum 599 snap 187 seq_num 599) 2024-09-21T09:44:24.059 INFO:tasks.rados.rados.0.smithi017.stdout:1529: done (1 left) 2024-09-21T09:44:24.059 INFO:tasks.rados.rados.0.smithi017.stdout:1531: done (0 left) 2024-09-21T09:44:24.060 INFO:tasks.rados.rados.0.smithi017.stdout:1532: snap_create 2024-09-21T09:44:24.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:25.053 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:44:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:44:24] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.51.0" 2024-09-21T09:44:25.061 INFO:tasks.rados.rados.0.smithi017.stdout:1532: done (0 left) 2024-09-21T09:44:25.061 INFO:tasks.rados.rados.0.smithi017.stdout:1533: snap_remove snap 185 2024-09-21T09:44:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:25 smithi017 ceph-mon[102913]: pgmap v238: 89 pgs: 89 active+clean; 139 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-09-21T09:44:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:25 smithi017 ceph-mon[102913]: osdmap e440: 8 total, 8 up, 8 in 2024-09-21T09:44:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:25 smithi017 ceph-mon[105151]: pgmap v238: 89 pgs: 89 active+clean; 139 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-09-21T09:44:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:25 smithi017 ceph-mon[105151]: osdmap e440: 8 total, 8 up, 8 in 2024-09-21T09:44:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:25 smithi134 ceph-mon[90085]: pgmap v238: 89 pgs: 89 active+clean; 139 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-09-21T09:44:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:25 smithi134 ceph-mon[90085]: osdmap e440: 8 total, 8 up, 8 in 2024-09-21T09:44:25.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:25.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:26.069 INFO:tasks.rados.rados.0.smithi017.stdout:1533: done (0 left) 2024-09-21T09:44:26.069 INFO:tasks.rados.rados.0.smithi017.stdout:1534: read oid 45 snap -1 2024-09-21T09:44:26.069 INFO:tasks.rados.rados.0.smithi017.stdout:1534: expect (ObjNum 564 snap 173 seq_num 564) 2024-09-21T09:44:26.069 INFO:tasks.rados.rados.0.smithi017.stdout:1535: snap_remove snap 187 2024-09-21T09:44:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:26 smithi017 ceph-mon[102913]: osdmap e441: 8 total, 8 up, 8 in 2024-09-21T09:44:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:26 smithi017 ceph-mon[105151]: osdmap e441: 8 total, 8 up, 8 in 2024-09-21T09:44:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:26 smithi134 ceph-mon[90085]: osdmap e441: 8 total, 8 up, 8 in 2024-09-21T09:44:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:27.068 INFO:tasks.rados.rados.0.smithi017.stdout:1535: done (1 left) 2024-09-21T09:44:27.068 INFO:tasks.rados.rados.0.smithi017.stdout:1536: read oid 38 snap -1 2024-09-21T09:44:27.068 INFO:tasks.rados.rados.0.smithi017.stdout:1536: expect (ObjNum 593 snap 186 seq_num 593) 2024-09-21T09:44:27.068 INFO:tasks.rados.rados.0.smithi017.stdout:1537: snap_remove snap 178 2024-09-21T09:44:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:27 smithi134 ceph-mon[90085]: pgmap v241: 89 pgs: 89 active+clean; 146 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:44:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:27 smithi134 ceph-mon[90085]: osdmap e442: 8 total, 8 up, 8 in 2024-09-21T09:44:27.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:27 smithi017 ceph-mon[102913]: pgmap v241: 89 pgs: 89 active+clean; 146 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:44:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:27 smithi017 ceph-mon[102913]: osdmap e442: 8 total, 8 up, 8 in 2024-09-21T09:44:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:27 smithi017 ceph-mon[105151]: pgmap v241: 89 pgs: 89 active+clean; 146 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:44:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:27 smithi017 ceph-mon[105151]: osdmap e442: 8 total, 8 up, 8 in 2024-09-21T09:44:28.081 INFO:tasks.rados.rados.0.smithi017.stdout:1534: expect (ObjNum 564 snap 173 seq_num 564) 2024-09-21T09:44:28.200 INFO:tasks.rados.rados.0.smithi017.stdout:1536: expect (ObjNum 593 snap 186 seq_num 593) 2024-09-21T09:44:28.423 INFO:tasks.rados.rados.0.smithi017.stdout:1534: done (2 left) 2024-09-21T09:44:28.423 INFO:tasks.rados.rados.0.smithi017.stdout:1536: done (1 left) 2024-09-21T09:44:28.423 INFO:tasks.rados.rados.0.smithi017.stdout:1537: done (0 left) 2024-09-21T09:44:28.423 INFO:tasks.rados.rados.0.smithi017.stdout:1538: read oid 26 snap -1 2024-09-21T09:44:28.423 INFO:tasks.rados.rados.0.smithi017.stdout:1538: expect (ObjNum 599 snap 187 seq_num 599) 2024-09-21T09:44:28.423 INFO:tasks.rados.rados.0.smithi017.stdout:1539: read oid 27 snap 181 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1539: expect deleted 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1540: read oid 36 snap 181 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1540: expect deleted 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1541: read oid 18 snap 186 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1541: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1542: read oid 21 snap -1 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1542: expect (ObjNum 579 snap 181 seq_num 579) 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1543: read oid 14 snap 186 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1543: expect (ObjNum 598 snap 186 seq_num 598) 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1544: read oid 22 snap -1 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1544: expect deleted 2024-09-21T09:44:28.424 INFO:tasks.rados.rados.0.smithi017.stdout:1545: delete oid 30 current snap is 188 2024-09-21T09:44:28.428 INFO:tasks.rados.rados.0.smithi017.stdout:1543: expect (ObjNum 598 snap 186 seq_num 598) 2024-09-21T09:44:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:28 smithi017 ceph-mon[102913]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:44:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:28 smithi017 ceph-mon[105151]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:44:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:28 smithi134 ceph-mon[90085]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:44:28.583 INFO:tasks.rados.rados.0.smithi017.stdout:1542: expect (ObjNum 579 snap 181 seq_num 579) 2024-09-21T09:44:28.838 INFO:tasks.rados.rados.0.smithi017.stdout:1541: expect (ObjNum 489 snap 155 seq_num 489) 2024-09-21T09:44:28.910 INFO:tasks.rados.rados.0.smithi017.stdout:1538: expect (ObjNum 599 snap 187 seq_num 599) 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1538: done (7 left) 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1539: done (6 left) 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1540: done (5 left) 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1541: done (4 left) 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1542: done (3 left) 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1543: done (2 left) 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1544: done (1 left) 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1545: done (0 left) 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1546: read oid 10 snap -1 2024-09-21T09:44:28.981 INFO:tasks.rados.rados.0.smithi017.stdout:1546: expect deleted 2024-09-21T09:44:28.982 INFO:tasks.rados.rados.0.smithi017.stdout:1547: write oid 27 current snap is 188 2024-09-21T09:44:28.982 INFO:tasks.rados.rados.0.smithi017.stdout:1547: seq_num 602 ranges {517814=491785,1577807=540007} 2024-09-21T09:44:28.986 INFO:tasks.rados.rados.0.smithi017.stdout:1547: writing smithi01785587-27 from 517814 to 1009599 tid 1 2024-09-21T09:44:28.992 INFO:tasks.rados.rados.0.smithi017.stdout:1547: writing smithi01785587-27 from 1577807 to 2117814 tid 2 2024-09-21T09:44:28.992 INFO:tasks.rados.rados.0.smithi017.stdout:1548: read oid 7 snap -1 2024-09-21T09:44:28.992 INFO:tasks.rados.rados.0.smithi017.stdout:1548: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:44:28.992 INFO:tasks.rados.rados.0.smithi017.stdout:1549: write oid 8 current snap is 188 2024-09-21T09:44:28.992 INFO:tasks.rados.rados.0.smithi017.stdout:1549: seq_num 603 ranges {534621=567034,1636289=498332} 2024-09-21T09:44:28.998 INFO:tasks.rados.rados.0.smithi017.stdout:1549: writing smithi01785587-8 from 534621 to 1101655 tid 1 2024-09-21T09:44:29.004 INFO:tasks.rados.rados.0.smithi017.stdout:1549: writing smithi01785587-8 from 1636289 to 2134621 tid 2 2024-09-21T09:44:29.005 INFO:tasks.rados.rados.0.smithi017.stdout:1550: delete oid 30 current snap is 188 2024-09-21T09:44:29.005 INFO:tasks.rados.rados.0.smithi017.stdout:1547: finishing write tid 1 to smithi01785587-27 2024-09-21T09:44:29.005 INFO:tasks.rados.rados.0.smithi017.stdout:1547: finishing write tid 2 to smithi01785587-27 2024-09-21T09:44:29.005 INFO:tasks.rados.rados.0.smithi017.stdout:1547: finishing write tid 3 to smithi01785587-27 2024-09-21T09:44:29.005 INFO:tasks.rados.rados.0.smithi017.stdout:1547: finishing write tid 4 to smithi01785587-27 2024-09-21T09:44:29.005 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 557 (ObjNum 602 snap 188 seq_num 602) dirty exists 2024-09-21T09:44:29.005 INFO:tasks.rados.rados.0.smithi017.stdout:1547: left oid 27 (ObjNum 602 snap 188 seq_num 602) 2024-09-21T09:44:29.005 INFO:tasks.rados.rados.0.smithi017.stdout:1548: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:44:29.210 INFO:tasks.rados.rados.0.smithi017.stdout:1549: finishing write tid 1 to smithi01785587-8 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1549: finishing write tid 2 to smithi01785587-8 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1549: finishing write tid 3 to smithi01785587-8 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1549: finishing write tid 4 to smithi01785587-8 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 732 (ObjNum 603 snap 188 seq_num 603) dirty exists 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1549: left oid 8 (ObjNum 603 snap 188 seq_num 603) 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1546: done (4 left) 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1547: done (3 left) 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1548: done (2 left) 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1549: done (1 left) 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1550: done (0 left) 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1551: read oid 34 snap -1 2024-09-21T09:44:29.211 INFO:tasks.rados.rados.0.smithi017.stdout:1551: expect (ObjNum 561 snap 173 seq_num 561) 2024-09-21T09:44:29.212 INFO:tasks.rados.rados.0.smithi017.stdout:1552: read oid 20 snap -1 2024-09-21T09:44:29.212 INFO:tasks.rados.rados.0.smithi017.stdout:1552: expect deleted 2024-09-21T09:44:29.212 INFO:tasks.rados.rados.0.smithi017.stdout:1553: snap_remove snap 174 2024-09-21T09:44:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:29 smithi017 ceph-mon[102913]: pgmap v244: 89 pgs: 89 active+clean; 147 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:44:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:29 smithi017 ceph-mon[102913]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:44:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:29 smithi017 ceph-mon[105151]: pgmap v244: 89 pgs: 89 active+clean; 147 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:44:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:29 smithi017 ceph-mon[105151]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:44:29.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:29 smithi134 ceph-mon[90085]: pgmap v244: 89 pgs: 89 active+clean; 147 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:44:29.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:29 smithi134 ceph-mon[90085]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:44:30.086 INFO:tasks.rados.rados.0.smithi017.stdout:1553: done (2 left) 2024-09-21T09:44:30.087 INFO:tasks.rados.rados.0.smithi017.stdout:1554: delete oid 11 current snap is 188 2024-09-21T09:44:30.087 INFO:tasks.rados.rados.0.smithi017.stdout:1551: expect (ObjNum 561 snap 173 seq_num 561) 2024-09-21T09:44:30.319 INFO:tasks.rados.rados.0.smithi017.stdout:1551: done (2 left) 2024-09-21T09:44:30.319 INFO:tasks.rados.rados.0.smithi017.stdout:1552: done (1 left) 2024-09-21T09:44:30.319 INFO:tasks.rados.rados.0.smithi017.stdout:1554: done (0 left) 2024-09-21T09:44:30.319 INFO:tasks.rados.rados.0.smithi017.stdout:1555: write oid 29 current snap is 188 2024-09-21T09:44:30.319 INFO:tasks.rados.rados.0.smithi017.stdout:1555: seq_num 606 ranges {585042=792781,2095382=89660} 2024-09-21T09:44:30.326 INFO:tasks.rados.rados.0.smithi017.stdout:1555: writing smithi01785587-29 from 585042 to 1377823 tid 1 2024-09-21T09:44:30.331 INFO:tasks.rados.rados.0.smithi017.stdout:1555: writing smithi01785587-29 from 2095382 to 2185042 tid 2 2024-09-21T09:44:30.331 INFO:tasks.rados.rados.0.smithi017.stdout:1556: snap_remove snap 181 2024-09-21T09:44:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:31.091 INFO:tasks.rados.rados.0.smithi017.stdout:1556: done (1 left) 2024-09-21T09:44:31.091 INFO:tasks.rados.rados.0.smithi017.stdout:1557: read oid 46 snap -1 2024-09-21T09:44:31.091 INFO:tasks.rados.rados.0.smithi017.stdout:1557: expect (ObjNum 537 snap 169 seq_num 537) 2024-09-21T09:44:31.091 INFO:tasks.rados.rados.0.smithi017.stdout:1558: rollback oid 22 current snap is 188 2024-09-21T09:44:31.091 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 22 to 186 2024-09-21T09:44:31.091 INFO:tasks.rados.rados.0.smithi017.stdout:1555: finishing write tid 1 to smithi01785587-29 2024-09-21T09:44:31.091 INFO:tasks.rados.rados.0.smithi017.stdout:1555: finishing write tid 2 to smithi01785587-29 2024-09-21T09:44:31.091 INFO:tasks.rados.rados.0.smithi017.stdout:1555: finishing write tid 3 to smithi01785587-29 2024-09-21T09:44:31.091 INFO:tasks.rados.rados.0.smithi017.stdout:1555: finishing write tid 4 to smithi01785587-29 2024-09-21T09:44:31.092 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 337 (ObjNum 606 snap 188 seq_num 606) dirty exists 2024-09-21T09:44:31.092 INFO:tasks.rados.rados.0.smithi017.stdout:1555: left oid 29 (ObjNum 606 snap 188 seq_num 606) 2024-09-21T09:44:31.092 INFO:tasks.rados.rados.0.smithi017.stdout:1555: done (2 left) 2024-09-21T09:44:31.092 INFO:tasks.rados.rados.0.smithi017.stdout:1559: read oid 29 snap -1 2024-09-21T09:44:31.092 INFO:tasks.rados.rados.0.smithi017.stdout:1559: expect (ObjNum 606 snap 188 seq_num 606) 2024-09-21T09:44:31.092 INFO:tasks.rados.rados.0.smithi017.stdout:1560: read oid 12 snap -1 2024-09-21T09:44:31.092 INFO:tasks.rados.rados.0.smithi017.stdout:1560: expect (ObjNum 575 snap 179 seq_num 575) 2024-09-21T09:44:31.092 INFO:tasks.rados.rados.0.smithi017.stdout:1561: snap_create 2024-09-21T09:44:31.095 INFO:tasks.rados.rados.0.smithi017.stdout:1558: finishing rollback tid 1 to smithi01785587-22 2024-09-21T09:44:31.096 INFO:tasks.rados.rados.0.smithi017.stdout:1558: finishing rollback tid 2 to smithi01785587-22 2024-09-21T09:44:31.096 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 789 (ObjNum 505 snap 158 seq_num 505) dirty exists 2024-09-21T09:44:31.104 INFO:tasks.rados.rados.0.smithi017.stdout:1559: expect (ObjNum 606 snap 188 seq_num 606) 2024-09-21T09:44:31.210 INFO:tasks.rados.rados.0.smithi017.stdout:1560: expect (ObjNum 575 snap 179 seq_num 575) 2024-09-21T09:44:31.420 INFO:tasks.rados.rados.0.smithi017.stdout:1557: expect (ObjNum 537 snap 169 seq_num 537) 2024-09-21T09:44:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:31 smithi017 ceph-mon[102913]: pgmap v246: 89 pgs: 89 active+clean; 147 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:44:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:31 smithi017 ceph-mon[102913]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:44:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:31 smithi017 ceph-mon[105151]: pgmap v246: 89 pgs: 89 active+clean; 147 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:44:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:31 smithi017 ceph-mon[105151]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:44:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:31 smithi134 ceph-mon[90085]: pgmap v246: 89 pgs: 89 active+clean; 147 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:44:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:31 smithi134 ceph-mon[90085]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:44:32.099 INFO:tasks.rados.rados.0.smithi017.stdout:1557: done (4 left) 2024-09-21T09:44:32.099 INFO:tasks.rados.rados.0.smithi017.stdout:1558: done (3 left) 2024-09-21T09:44:32.099 INFO:tasks.rados.rados.0.smithi017.stdout:1559: done (2 left) 2024-09-21T09:44:32.099 INFO:tasks.rados.rados.0.smithi017.stdout:1560: done (1 left) 2024-09-21T09:44:32.099 INFO:tasks.rados.rados.0.smithi017.stdout:1561: done (0 left) 2024-09-21T09:44:32.099 INFO:tasks.rados.rados.0.smithi017.stdout:1562: snap_create 2024-09-21T09:44:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:32 smithi017 ceph-mon[102913]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:44:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:32 smithi017 ceph-mon[105151]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:44:32.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:32 smithi134 ceph-mon[90085]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:44:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1562: done (0 left) 2024-09-21T09:44:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1563: rollback oid 6 current snap is 190 2024-09-21T09:44:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 188 2024-09-21T09:44:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1564: write oid 32 current snap is 190 2024-09-21T09:44:33.056 INFO:tasks.rados.rados.0.smithi017.stdout:1564: seq_num 607 ranges {601849=468030,1753864=447985} 2024-09-21T09:44:33.062 INFO:tasks.rados.rados.0.smithi017.stdout:1564: writing smithi01785587-32 from 601849 to 1069879 tid 1 2024-09-21T09:44:33.069 INFO:tasks.rados.rados.0.smithi017.stdout:1564: writing smithi01785587-32 from 1753864 to 2201849 tid 2 2024-09-21T09:44:33.069 INFO:tasks.rados.rados.0.smithi017.stdout:1565: rollback oid 46 current snap is 190 2024-09-21T09:44:33.069 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 189 2024-09-21T09:44:33.069 INFO:tasks.rados.rados.0.smithi017.stdout:1563: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:44:33.070 INFO:tasks.rados.rados.0.smithi017.stdout:1566: read oid 14 snap 188 2024-09-21T09:44:33.070 INFO:tasks.rados.rados.0.smithi017.stdout:1566: expect (ObjNum 598 snap 186 seq_num 598) 2024-09-21T09:44:33.070 INFO:tasks.rados.rados.0.smithi017.stdout:1567: snap_create 2024-09-21T09:44:33.070 INFO:tasks.rados.rados.0.smithi017.stdout:1563: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:44:33.070 INFO:tasks.rados.rados.0.smithi017.stdout:1563: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:44:33.070 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 479 (ObjNum 583 snap 181 seq_num 583) dirty exists 2024-09-21T09:44:33.070 INFO:tasks.rados.rados.0.smithi017.stdout:1564: finishing write tid 1 to smithi01785587-32 2024-09-21T09:44:33.072 INFO:tasks.rados.rados.0.smithi017.stdout:1566: expect (ObjNum 598 snap 186 seq_num 598) 2024-09-21T09:44:33.227 INFO:tasks.rados.rados.0.smithi017.stdout:1564: finishing write tid 2 to smithi01785587-32 2024-09-21T09:44:33.228 INFO:tasks.rados.rados.0.smithi017.stdout:1564: finishing write tid 3 to smithi01785587-32 2024-09-21T09:44:33.228 INFO:tasks.rados.rados.0.smithi017.stdout:1564: finishing write tid 4 to smithi01785587-32 2024-09-21T09:44:33.228 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 743 (ObjNum 607 snap 190 seq_num 607) dirty exists 2024-09-21T09:44:33.228 INFO:tasks.rados.rados.0.smithi017.stdout:1564: left oid 32 (ObjNum 607 snap 190 seq_num 607) 2024-09-21T09:44:33.228 INFO:tasks.rados.rados.0.smithi017.stdout:1565: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:44:33.228 INFO:tasks.rados.rados.0.smithi017.stdout:1565: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:44:33.228 INFO:tasks.rados.rados.0.smithi017.stdout:1565: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:44:33.228 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 747 (ObjNum 537 snap 169 seq_num 537) dirty exists 2024-09-21T09:44:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:33 smithi017 ceph-mon[102913]: pgmap v249: 89 pgs: 89 active+clean; 133 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 614 KiB/s wr, 17 op/s 2024-09-21T09:44:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:33 smithi017 ceph-mon[102913]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T09:44:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:33 smithi017 ceph-mon[102913]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T09:44:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:33 smithi017 ceph-mon[105151]: pgmap v249: 89 pgs: 89 active+clean; 133 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 614 KiB/s wr, 17 op/s 2024-09-21T09:44:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:33 smithi017 ceph-mon[105151]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T09:44:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:33 smithi017 ceph-mon[105151]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T09:44:33.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:33 smithi134 ceph-mon[90085]: pgmap v249: 89 pgs: 89 active+clean; 133 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 614 KiB/s wr, 17 op/s 2024-09-21T09:44:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:33 smithi134 ceph-mon[90085]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T09:44:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:33 smithi134 ceph-mon[90085]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T09:44:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1563: done (4 left) 2024-09-21T09:44:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1564: done (3 left) 2024-09-21T09:44:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1565: done (2 left) 2024-09-21T09:44:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1566: done (1 left) 2024-09-21T09:44:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1567: done (0 left) 2024-09-21T09:44:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1568: read oid 17 snap 189 2024-09-21T09:44:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1568: expect (ObjNum 588 snap 185 seq_num 588) 2024-09-21T09:44:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1569: read oid 36 snap -1 2024-09-21T09:44:34.059 INFO:tasks.rados.rados.0.smithi017.stdout:1569: expect (ObjNum 589 snap 185 seq_num 589) 2024-09-21T09:44:34.060 INFO:tasks.rados.rados.0.smithi017.stdout:1570: snap_remove snap 190 2024-09-21T09:44:34.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:35.055 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:44:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:44:34] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-21T09:44:35.064 INFO:tasks.rados.rados.0.smithi017.stdout:1570: done (2 left) 2024-09-21T09:44:35.064 INFO:tasks.rados.rados.0.smithi017.stdout:1571: rollback oid 15 current snap is 191 2024-09-21T09:44:35.064 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 186 2024-09-21T09:44:35.064 INFO:tasks.rados.rados.0.smithi017.stdout:1569: expect (ObjNum 589 snap 185 seq_num 589) 2024-09-21T09:44:35.199 INFO:tasks.rados.rados.0.smithi017.stdout:1568: expect (ObjNum 588 snap 185 seq_num 588) 2024-09-21T09:44:35.423 INFO:tasks.rados.rados.0.smithi017.stdout:1571: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:44:35.441 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 0 (ObjNum 241 snap 0 seq_num 2399891056) dirty dne 2024-09-21T09:44:35.442 INFO:tasks.rados.rados.0.smithi017.stdout:1568: done (2 left) 2024-09-21T09:44:35.442 INFO:tasks.rados.rados.0.smithi017.stdout:1569: done (1 left) 2024-09-21T09:44:35.442 INFO:tasks.rados.rados.0.smithi017.stdout:1571: done (0 left) 2024-09-21T09:44:35.442 INFO:tasks.rados.rados.0.smithi017.stdout:1572: write oid 4 current snap is 191 2024-09-21T09:44:35.442 INFO:tasks.rados.rados.0.smithi017.stdout:1572: seq_num 608 ranges {618656=543279,1812346=406310} 2024-09-21T09:44:35.443 INFO:tasks.rados.rados.0.smithi017.stdout:1572: writing smithi01785587-4 from 618656 to 1161935 tid 1 2024-09-21T09:44:35.443 INFO:tasks.rados.rados.0.smithi017.stdout:1572: writing smithi01785587-4 from 1812346 to 2218656 tid 2 2024-09-21T09:44:35.443 INFO:tasks.rados.rados.0.smithi017.stdout:1573: write oid 42 current snap is 191 2024-09-21T09:44:35.443 INFO:tasks.rados.rados.0.smithi017.stdout:1573: seq_num 609 ranges {635463=734881,1987181=248282} 2024-09-21T09:44:35.443 INFO:tasks.rados.rados.0.smithi017.stdout:1573: writing smithi01785587-42 from 635463 to 1370344 tid 1 2024-09-21T09:44:35.448 INFO:tasks.rados.rados.0.smithi017.stdout:1573: writing smithi01785587-42 from 1987181 to 2235463 tid 2 2024-09-21T09:44:35.448 INFO:tasks.rados.rados.0.smithi017.stdout:1574: read oid 14 snap -1 2024-09-21T09:44:35.448 INFO:tasks.rados.rados.0.smithi017.stdout:1574: expect (ObjNum 598 snap 186 seq_num 598) 2024-09-21T09:44:35.448 INFO:tasks.rados.rados.0.smithi017.stdout:1575: rollback oid 20 current snap is 191 2024-09-21T09:44:35.448 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 186 2024-09-21T09:44:35.448 INFO:tasks.rados.rados.0.smithi017.stdout:1572: finishing write tid 1 to smithi01785587-4 2024-09-21T09:44:35.448 INFO:tasks.rados.rados.0.smithi017.stdout:1572: finishing write tid 2 to smithi01785587-4 2024-09-21T09:44:35.448 INFO:tasks.rados.rados.0.smithi017.stdout:1572: finishing write tid 3 to smithi01785587-4 2024-09-21T09:44:35.448 INFO:tasks.rados.rados.0.smithi017.stdout:1576: snap_remove snap 189 2024-09-21T09:44:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:35 smithi017 ceph-mon[102913]: pgmap v252: 89 pgs: 89 active+clean; 133 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 728 KiB/s wr, 20 op/s 2024-09-21T09:44:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:35 smithi017 ceph-mon[102913]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T09:44:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:35 smithi017 ceph-mon[105151]: pgmap v252: 89 pgs: 89 active+clean; 133 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 728 KiB/s wr, 20 op/s 2024-09-21T09:44:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:35 smithi017 ceph-mon[105151]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T09:44:35.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:35 smithi134 ceph-mon[90085]: pgmap v252: 89 pgs: 89 active+clean; 133 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 728 KiB/s wr, 20 op/s 2024-09-21T09:44:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:35 smithi134 ceph-mon[90085]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T09:44:35.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:35.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:36.068 INFO:tasks.rados.rados.0.smithi017.stdout:1576: done (4 left) 2024-09-21T09:44:36.068 INFO:tasks.rados.rados.0.smithi017.stdout:1577: snap_create 2024-09-21T09:44:36.068 INFO:tasks.rados.rados.0.smithi017.stdout:1572: finishing write tid 4 to smithi01785587-4 2024-09-21T09:44:36.068 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 792 (ObjNum 608 snap 191 seq_num 608) dirty exists 2024-09-21T09:44:36.068 INFO:tasks.rados.rados.0.smithi017.stdout:1572: left oid 4 (ObjNum 608 snap 191 seq_num 608) 2024-09-21T09:44:36.068 INFO:tasks.rados.rados.0.smithi017.stdout:1575: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:44:36.068 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 0 (ObjNum 225 snap 0 seq_num 4181695920) dirty dne 2024-09-21T09:44:36.068 INFO:tasks.rados.rados.0.smithi017.stdout:1574: expect (ObjNum 598 snap 186 seq_num 598) 2024-09-21T09:44:36.126 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:36 smithi017 ceph-mon[102913]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T09:44:36.127 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:36 smithi017 ceph-mon[105151]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T09:44:36.222 INFO:tasks.rados.rados.0.smithi017.stdout:1573: finishing write tid 1 to smithi01785587-42 2024-09-21T09:44:36.222 INFO:tasks.rados.rados.0.smithi017.stdout:1573: finishing write tid 2 to smithi01785587-42 2024-09-21T09:44:36.222 INFO:tasks.rados.rados.0.smithi017.stdout:1573: finishing write tid 3 to smithi01785587-42 2024-09-21T09:44:36.222 INFO:tasks.rados.rados.0.smithi017.stdout:1573: finishing write tid 4 to smithi01785587-42 2024-09-21T09:44:36.222 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 592 (ObjNum 609 snap 191 seq_num 609) dirty exists 2024-09-21T09:44:36.222 INFO:tasks.rados.rados.0.smithi017.stdout:1573: left oid 42 (ObjNum 609 snap 191 seq_num 609) 2024-09-21T09:44:36.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:36 smithi134 ceph-mon[90085]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T09:44:36.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade resume' 2024-09-21T09:44:36.806 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:37.070 INFO:tasks.rados.rados.0.smithi017.stdout:1572: done (4 left) 2024-09-21T09:44:37.070 INFO:tasks.rados.rados.0.smithi017.stdout:1573: done (3 left) 2024-09-21T09:44:37.070 INFO:tasks.rados.rados.0.smithi017.stdout:1574: done (2 left) 2024-09-21T09:44:37.070 INFO:tasks.rados.rados.0.smithi017.stdout:1575: done (1 left) 2024-09-21T09:44:37.070 INFO:tasks.rados.rados.0.smithi017.stdout:1577: done (0 left) 2024-09-21T09:44:37.070 INFO:tasks.rados.rados.0.smithi017.stdout:1578: snap_remove snap 191 2024-09-21T09:44:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:37 smithi017 ceph-mon[102913]: pgmap v255: 89 pgs: 89 active+clean; 122 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 696 KiB/s wr, 17 op/s 2024-09-21T09:44:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:37 smithi017 ceph-mon[102913]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T09:44:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:37 smithi017 ceph-mon[105151]: pgmap v255: 89 pgs: 89 active+clean; 122 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 696 KiB/s wr, 17 op/s 2024-09-21T09:44:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:37 smithi017 ceph-mon[105151]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T09:44:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:37 smithi134 ceph-mon[90085]: pgmap v255: 89 pgs: 89 active+clean; 122 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 696 KiB/s wr, 17 op/s 2024-09-21T09:44:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:37 smithi134 ceph-mon[90085]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T09:44:37.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:38.081 INFO:tasks.rados.rados.0.smithi017.stdout:1578: done (0 left) 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:1579: read oid 14 snap -1 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:1579: expect (ObjNum 598 snap 186 seq_num 598) 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:1580: read oid 5 snap -1 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:1580: expect (ObjNum 565 snap 173 seq_num 565) 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:1581: rollback oid 23 current snap is 192 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 23 to 186 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:1582: read oid 1 snap 188 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:1582: expect (ObjNum 458 snap 148 seq_num 458) 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:1583: write oid 22 current snap is 192 2024-09-21T09:44:38.082 INFO:tasks.rados.rados.0.smithi017.stdout:1583: seq_num 610 ranges {652270=410130,1529310=493867} 2024-09-21T09:44:38.087 INFO:tasks.rados.rados.0.smithi017.stdout:1583: writing smithi01785587-22 from 652270 to 1062400 tid 1 2024-09-21T09:44:38.093 INFO:tasks.rados.rados.0.smithi017.stdout:1583: writing smithi01785587-22 from 1529310 to 2023177 tid 2 2024-09-21T09:44:38.093 INFO:tasks.rados.rados.0.smithi017.stdout:1584: write oid 11 current snap is 192 2024-09-21T09:44:38.093 INFO:tasks.rados.rados.0.smithi017.stdout:1584: seq_num 611 ranges {669077=485379,1587792=681285} 2024-09-21T09:44:38.099 INFO:tasks.rados.rados.0.smithi017.stdout:1584: writing smithi01785587-11 from 669077 to 1154456 tid 1 2024-09-21T09:44:38.106 INFO:tasks.rados.rados.0.smithi017.stdout:1584: writing smithi01785587-11 from 1587792 to 2269077 tid 2 2024-09-21T09:44:38.106 INFO:tasks.rados.rados.0.smithi017.stdout:1585: write oid 27 current snap is 192 2024-09-21T09:44:38.106 INFO:tasks.rados.rados.0.smithi017.stdout:1585: seq_num 612 ranges {685884=560628,2046274=239610} 2024-09-21T09:44:38.112 INFO:tasks.rados.rados.0.smithi017.stdout:1585: writing smithi01785587-27 from 685884 to 1246512 tid 1 2024-09-21T09:44:38.118 INFO:tasks.rados.rados.0.smithi017.stdout:1585: writing smithi01785587-27 from 2046274 to 2285884 tid 2 2024-09-21T09:44:38.119 INFO:tasks.rados.rados.0.smithi017.stdout:1586: delete oid 8 current snap is 192 2024-09-21T09:44:38.119 INFO:tasks.rados.rados.0.smithi017.stdout:1579: expect (ObjNum 598 snap 186 seq_num 598) 2024-09-21T09:44:38.273 INFO:tasks.rados.rados.0.smithi017.stdout:1581: finishing rollback tid 1 to smithi01785587-23 2024-09-21T09:44:38.273 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 0 (ObjNum 177 snap 0 seq_num 2399706768) dirty dne 2024-09-21T09:44:38.273 INFO:tasks.rados.rados.0.smithi017.stdout:1580: expect (ObjNum 565 snap 173 seq_num 565) 2024-09-21T09:44:38.289 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:38 smithi017 ceph-mon[102913]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T09:44:38.290 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:38 smithi017 ceph-mon[105151]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T09:44:38.395 INFO:tasks.rados.rados.0.smithi017.stdout:1582: expect (ObjNum 458 snap 148 seq_num 458) 2024-09-21T09:44:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:38 smithi134 ceph-mon[90085]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1583: finishing write tid 1 to smithi01785587-22 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1583: finishing write tid 2 to smithi01785587-22 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1579: done (7 left) 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1580: done (6 left) 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1581: done (5 left) 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1582: done (4 left) 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1586: done (3 left) 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1587: write oid 36 current snap is 192 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1583: finishing write tid 3 to smithi01785587-22 2024-09-21T09:44:38.649 INFO:tasks.rados.rados.0.smithi017.stdout:1583: finishing write tid 4 to smithi01785587-22 2024-09-21T09:44:38.650 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 796 (ObjNum 610 snap 192 seq_num 610) dirty exists 2024-09-21T09:44:38.650 INFO:tasks.rados.rados.0.smithi017.stdout:1583: left oid 22 (ObjNum 610 snap 192 seq_num 610) 2024-09-21T09:44:38.650 INFO:tasks.rados.rados.0.smithi017.stdout:1584: finishing write tid 1 to smithi01785587-11 2024-09-21T09:44:38.650 INFO:tasks.rados.rados.0.smithi017.stdout:1584: finishing write tid 2 to smithi01785587-11 2024-09-21T09:44:38.650 INFO:tasks.rados.rados.0.smithi017.stdout:1587: seq_num 614 ranges {719498=711126,2046885=272613} 2024-09-21T09:44:38.656 INFO:tasks.rados.rados.0.smithi017.stdout:1587: writing smithi01785587-36 from 719498 to 1430624 tid 1 2024-09-21T09:44:38.662 INFO:tasks.rados.rados.0.smithi017.stdout:1587: writing smithi01785587-36 from 2046885 to 2319498 tid 2 2024-09-21T09:44:38.662 INFO:tasks.rados.rados.0.smithi017.stdout:1583: done (3 left) 2024-09-21T09:44:38.662 INFO:tasks.rados.rados.0.smithi017.stdout:1588: read oid 47 snap 188 2024-09-21T09:44:38.662 INFO:tasks.rados.rados.0.smithi017.stdout:1588: expect (ObjNum 562 snap 173 seq_num 562) 2024-09-21T09:44:38.662 INFO:tasks.rados.rados.0.smithi017.stdout:1589: delete oid 28 current snap is 192 2024-09-21T09:44:38.662 INFO:tasks.rados.rados.0.smithi017.stdout:1584: finishing write tid 3 to smithi01785587-11 2024-09-21T09:44:38.662 INFO:tasks.rados.rados.0.smithi017.stdout:1584: finishing write tid 4 to smithi01785587-11 2024-09-21T09:44:38.662 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 754 (ObjNum 611 snap 192 seq_num 611) dirty exists 2024-09-21T09:44:38.662 INFO:tasks.rados.rados.0.smithi017.stdout:1584: left oid 11 (ObjNum 611 snap 192 seq_num 611) 2024-09-21T09:44:38.663 INFO:tasks.rados.rados.0.smithi017.stdout:1585: finishing write tid 1 to smithi01785587-27 2024-09-21T09:44:38.663 INFO:tasks.rados.rados.0.smithi017.stdout:1585: finishing write tid 2 to smithi01785587-27 2024-09-21T09:44:38.663 INFO:tasks.rados.rados.0.smithi017.stdout:1585: finishing write tid 3 to smithi01785587-27 2024-09-21T09:44:38.663 INFO:tasks.rados.rados.0.smithi017.stdout:1585: finishing write tid 4 to smithi01785587-27 2024-09-21T09:44:38.663 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 567 (ObjNum 612 snap 192 seq_num 612) dirty exists 2024-09-21T09:44:38.663 INFO:tasks.rados.rados.0.smithi017.stdout:1585: left oid 27 (ObjNum 612 snap 192 seq_num 612) 2024-09-21T09:44:38.663 INFO:tasks.rados.rados.0.smithi017.stdout:1587: finishing write tid 1 to smithi01785587-36 2024-09-21T09:44:38.663 INFO:tasks.rados.rados.0.smithi017.stdout:1584: done (4 left) 2024-09-21T09:44:38.663 INFO:tasks.rados.rados.0.smithi017.stdout:1585: done (3 left) 2024-09-21T09:44:38.664 INFO:tasks.rados.rados.0.smithi017.stdout:1589: done (2 left) 2024-09-21T09:44:38.664 INFO:tasks.rados.rados.0.smithi017.stdout:1590: write oid 11 current snap is 192 2024-09-21T09:44:38.664 INFO:tasks.rados.rados.0.smithi017.stdout:1590: seq_num 616 ranges {753112=577977,1880202=472910} 2024-09-21T09:44:38.671 INFO:tasks.rados.rados.0.smithi017.stdout:1590: writing smithi01785587-11 from 753112 to 1331089 tid 1 2024-09-21T09:44:38.677 INFO:tasks.rados.rados.0.smithi017.stdout:1590: writing smithi01785587-11 from 1880202 to 2353112 tid 2 2024-09-21T09:44:38.677 INFO:tasks.rados.rados.0.smithi017.stdout:1591: snap_remove snap 186 2024-09-21T09:44:38.735 INFO:teuthology.orchestra.run.smithi017.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:44:39.087 INFO:tasks.rados.rados.0.smithi017.stdout:1591: done (3 left) 2024-09-21T09:44:39.088 INFO:tasks.rados.rados.0.smithi017.stdout:1592: delete oid 35 current snap is 192 2024-09-21T09:44:39.088 INFO:tasks.rados.rados.0.smithi017.stdout:1587: finishing write tid 2 to smithi01785587-36 2024-09-21T09:44:39.088 INFO:tasks.rados.rados.0.smithi017.stdout:1587: finishing write tid 3 to smithi01785587-36 2024-09-21T09:44:39.088 INFO:tasks.rados.rados.0.smithi017.stdout:1587: finishing write tid 4 to smithi01785587-36 2024-09-21T09:44:39.088 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 824 (ObjNum 614 snap 192 seq_num 614) dirty exists 2024-09-21T09:44:39.088 INFO:tasks.rados.rados.0.smithi017.stdout:1587: left oid 36 (ObjNum 614 snap 192 seq_num 614) 2024-09-21T09:44:39.088 INFO:tasks.rados.rados.0.smithi017.stdout:1588: expect (ObjNum 562 snap 173 seq_num 562) 2024-09-21T09:44:39.296 INFO:tasks.rados.rados.0.smithi017.stdout:1590: finishing write tid 1 to smithi01785587-11 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:1590: finishing write tid 2 to smithi01785587-11 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:1590: finishing write tid 3 to smithi01785587-11 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:1590: finishing write tid 4 to smithi01785587-11 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 759 (ObjNum 616 snap 192 seq_num 616) dirty exists 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:1590: left oid 11 (ObjNum 616 snap 192 seq_num 616) 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:1587: done (3 left) 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:1588: done (2 left) 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:1590: done (1 left) 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:1592: done (0 left) 2024-09-21T09:44:39.297 INFO:tasks.rados.rados.0.smithi017.stdout:1593: snap_remove snap 188 2024-09-21T09:44:39.356 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:39 smithi134 ceph-mon[90085]: pgmap v258: 89 pgs: 89 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 696 KiB/s wr, 20 op/s 2024-09-21T09:44:39.357 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:39 smithi134 ceph-mon[90085]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T09:44:39.357 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:39.357 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:39.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:39 smithi017 ceph-mon[102913]: pgmap v258: 89 pgs: 89 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 696 KiB/s wr, 20 op/s 2024-09-21T09:44:39.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:39 smithi017 ceph-mon[102913]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T09:44:39.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:39.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:39.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:39 smithi017 ceph-mon[105151]: pgmap v258: 89 pgs: 89 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 696 KiB/s wr, 20 op/s 2024-09-21T09:44:39.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:39 smithi017 ceph-mon[105151]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T09:44:39.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:39.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:39.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-21T09:44:40.092 INFO:tasks.rados.rados.0.smithi017.stdout:1593: done (0 left) 2024-09-21T09:44:40.092 INFO:tasks.rados.rados.0.smithi017.stdout:1594: snap_create 2024-09-21T09:44:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:40 smithi017 ceph-mon[102913]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:40 smithi017 ceph-mon[102913]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:44:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:40 smithi017 ceph-mon[102913]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T09:44:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:40 smithi017 ceph-mon[105151]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:40 smithi017 ceph-mon[105151]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:44:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:40 smithi017 ceph-mon[105151]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T09:44:40.352 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:40 smithi134 ceph-mon[90085]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:40.352 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:40 smithi134 ceph-mon[90085]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:44:40.352 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:40 smithi134 ceph-mon[90085]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T09:44:40.353 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:40.436 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:1594: done (0 left) 2024-09-21T09:44:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:1595: rollback oid 25 current snap is 193 2024-09-21T09:44:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 192 2024-09-21T09:44:41.097 INFO:tasks.rados.rados.0.smithi017.stdout:1596: rollback oid 38 current snap is 193 2024-09-21T09:44:41.097 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 192 2024-09-21T09:44:41.097 INFO:tasks.rados.rados.0.smithi017.stdout:1597: rollback oid 43 current snap is 193 2024-09-21T09:44:41.097 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 192 2024-09-21T09:44:41.097 INFO:tasks.rados.rados.0.smithi017.stdout:1598: rollback oid 6 current snap is 193 2024-09-21T09:44:41.097 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 192 2024-09-21T09:44:41.097 INFO:tasks.rados.rados.0.smithi017.stdout:1599: read oid 2 snap -1 2024-09-21T09:44:41.097 INFO:tasks.rados.rados.0.smithi017.stdout:1599: expect (ObjNum 479 snap 153 seq_num 479) 2024-09-21T09:44:41.097 INFO:tasks.rados.rados.0.smithi017.stdout:1600: delete oid 9 current snap is 193 2024-09-21T09:44:41.099 INFO:tasks.rados.rados.0.smithi017.stdout:1595: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:44:41.099 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 0 (ObjNum 113 snap 0 seq_num 2399474560) dirty dne 2024-09-21T09:44:41.102 INFO:tasks.rados.rados.0.smithi017.stdout:1596: finishing rollback tid 0 to smithi01785587-38 2024-09-21T09:44:41.102 INFO:tasks.rados.rados.0.smithi017.stdout:1596: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:44:41.103 INFO:tasks.rados.rados.0.smithi017.stdout:1599: expect (ObjNum 479 snap 153 seq_num 479) 2024-09-21T09:44:41.211 INFO:tasks.rados.rados.0.smithi017.stdout:1597: finishing rollback tid 0 to smithi01785587-43 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1597: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1597: finishing rollback tid 2 to smithi01785587-43 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 489 (ObjNum 512 snap 160 seq_num 512) dirty exists 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1598: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1598: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1598: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 493 (ObjNum 583 snap 181 seq_num 583) dirty exists 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1596: finishing rollback tid 2 to smithi01785587-38 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 514 (ObjNum 593 snap 186 seq_num 593) dirty exists 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1595: done (5 left) 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1596: done (4 left) 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1597: done (3 left) 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1598: done (2 left) 2024-09-21T09:44:41.212 INFO:tasks.rados.rados.0.smithi017.stdout:1599: done (1 left) 2024-09-21T09:44:41.213 INFO:tasks.rados.rados.0.smithi017.stdout:1600: done (0 left) 2024-09-21T09:44:41.213 INFO:tasks.rados.rados.0.smithi017.stdout:1601: write oid 34 current snap is 193 2024-09-21T09:44:41.213 INFO:tasks.rados.rados.0.smithi017.stdout:1601: seq_num 619 ranges {403533=403724,1539295=652201} 2024-09-21T09:44:41.216 INFO:tasks.rados.rados.0.smithi017.stdout:1601: writing smithi01785587-34 from 403533 to 807257 tid 1 2024-09-21T09:44:41.224 INFO:tasks.rados.rados.0.smithi017.stdout:1601: writing smithi01785587-34 from 1539295 to 2191496 tid 2 2024-09-21T09:44:41.224 INFO:tasks.rados.rados.0.smithi017.stdout:1602: rollback oid 29 current snap is 193 2024-09-21T09:44:41.224 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 29 to 192 2024-09-21T09:44:41.224 INFO:tasks.rados.rados.0.smithi017.stdout:1601: finishing write tid 1 to smithi01785587-34 2024-09-21T09:44:41.224 INFO:tasks.rados.rados.0.smithi017.stdout:1603: snap_create 2024-09-21T09:44:41.226 INFO:tasks.rados.rados.0.smithi017.stdout:1602: finishing rollback tid 0 to smithi01785587-29 2024-09-21T09:44:41.226 INFO:tasks.rados.rados.0.smithi017.stdout:1602: finishing rollback tid 1 to smithi01785587-29 2024-09-21T09:44:41.226 INFO:tasks.rados.rados.0.smithi017.stdout:1602: finishing rollback tid 2 to smithi01785587-29 2024-09-21T09:44:41.226 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 344 (ObjNum 606 snap 188 seq_num 606) dirty exists 2024-09-21T09:44:41.227 INFO:tasks.rados.rados.0.smithi017.stdout:1601: finishing write tid 2 to smithi01785587-34 2024-09-21T09:44:41.227 INFO:tasks.rados.rados.0.smithi017.stdout:1601: finishing write tid 3 to smithi01785587-34 2024-09-21T09:44:41.227 INFO:tasks.rados.rados.0.smithi017.stdout:1601: finishing write tid 4 to smithi01785587-34 2024-09-21T09:44:41.228 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 869 (ObjNum 619 snap 193 seq_num 619) dirty exists 2024-09-21T09:44:41.228 INFO:tasks.rados.rados.0.smithi017.stdout:1601: left oid 34 (ObjNum 619 snap 193 seq_num 619) 2024-09-21T09:44:41.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:41 smithi017 ceph-mon[102913]: pgmap v261: 89 pgs: 89 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2 op/s 2024-09-21T09:44:41.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:41 smithi017 ceph-mon[102913]: osdmap e455: 8 total, 8 up, 8 in 2024-09-21T09:44:41.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:41.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:41 smithi017 ceph-mon[105151]: pgmap v261: 89 pgs: 89 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2 op/s 2024-09-21T09:44:41.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:41 smithi017 ceph-mon[105151]: osdmap e455: 8 total, 8 up, 8 in 2024-09-21T09:44:41.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:41.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:41 smithi134 ceph-mon[90085]: pgmap v261: 89 pgs: 89 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2 op/s 2024-09-21T09:44:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:41 smithi134 ceph-mon[90085]: osdmap e455: 8 total, 8 up, 8 in 2024-09-21T09:44:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:42.102 INFO:tasks.rados.rados.0.smithi017.stdout:1601: done (2 left) 2024-09-21T09:44:42.102 INFO:tasks.rados.rados.0.smithi017.stdout:1602: done (1 left) 2024-09-21T09:44:42.102 INFO:tasks.rados.rados.0.smithi017.stdout:1603: done (0 left) 2024-09-21T09:44:42.103 INFO:tasks.rados.rados.0.smithi017.stdout:1604: read oid 25 snap -1 2024-09-21T09:44:42.103 INFO:tasks.rados.rados.0.smithi017.stdout:1604: expect deleted 2024-09-21T09:44:42.103 INFO:tasks.rados.rados.0.smithi017.stdout:1605: snap_create 2024-09-21T09:44:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:42 smithi017 ceph-mon[102913]: osdmap e456: 8 total, 8 up, 8 in 2024-09-21T09:44:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:42 smithi017 ceph-mon[105151]: osdmap e456: 8 total, 8 up, 8 in 2024-09-21T09:44:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:42 smithi134 ceph-mon[90085]: osdmap e456: 8 total, 8 up, 8 in 2024-09-21T09:44:42.533 INFO:teuthology.orchestra.run.smithi017.stdout:wait for half of osds to upgrade 2024-09-21T09:44:43.059 INFO:tasks.rados.rados.0.smithi017.stdout:1604: done (1 left) 2024-09-21T09:44:43.060 INFO:tasks.rados.rados.0.smithi017.stdout:1605: done (0 left) 2024-09-21T09:44:43.060 INFO:tasks.rados.rados.0.smithi017.stdout:1606: delete oid 41 current snap is 195 2024-09-21T09:44:43.064 INFO:tasks.rados.rados.0.smithi017.stdout:1606: done (0 left) 2024-09-21T09:44:43.064 INFO:tasks.rados.rados.0.smithi017.stdout:1607: write oid 41 current snap is 195 2024-09-21T09:44:43.064 INFO:tasks.rados.rados.0.smithi017.stdout:1607: seq_num 621 ranges {437147=554222,1656259=655870} 2024-09-21T09:44:43.069 INFO:tasks.rados.rados.0.smithi017.stdout:1607: writing smithi01785587-41 from 437147 to 991369 tid 1 2024-09-21T09:44:43.077 INFO:tasks.rados.rados.0.smithi017.stdout:1607: writing smithi01785587-41 from 1656259 to 2312129 tid 2 2024-09-21T09:44:43.077 INFO:tasks.rados.rados.0.smithi017.stdout:1608: rollback oid 43 current snap is 195 2024-09-21T09:44:43.077 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 194 2024-09-21T09:44:43.077 INFO:tasks.rados.rados.0.smithi017.stdout:1607: finishing write tid 1 to smithi01785587-41 2024-09-21T09:44:43.077 INFO:tasks.rados.rados.0.smithi017.stdout:1609: snap_create 2024-09-21T09:44:43.081 INFO:tasks.rados.rados.0.smithi017.stdout:1607: finishing write tid 2 to smithi01785587-41 2024-09-21T09:44:43.081 INFO:tasks.rados.rados.0.smithi017.stdout:1607: finishing write tid 3 to smithi01785587-41 2024-09-21T09:44:43.081 INFO:tasks.rados.rados.0.smithi017.stdout:1607: finishing write tid 4 to smithi01785587-41 2024-09-21T09:44:43.081 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 275 (ObjNum 621 snap 195 seq_num 621) dirty exists 2024-09-21T09:44:43.081 INFO:tasks.rados.rados.0.smithi017.stdout:1607: left oid 41 (ObjNum 621 snap 195 seq_num 621) 2024-09-21T09:44:43.083 INFO:tasks.rados.rados.0.smithi017.stdout:1608: finishing rollback tid 0 to smithi01785587-43 2024-09-21T09:44:43.083 INFO:tasks.rados.rados.0.smithi017.stdout:1608: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:44:43.083 INFO:tasks.rados.rados.0.smithi017.stdout:1608: finishing rollback tid 2 to smithi01785587-43 2024-09-21T09:44:43.083 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 497 (ObjNum 512 snap 160 seq_num 512) dirty exists 2024-09-21T09:44:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[105151]: pgmap v264: 89 pgs: 89 active+clean; 99 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:44:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[105151]: osdmap e457: 8 total, 8 up, 8 in 2024-09-21T09:44:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[105151]: osdmap e458: 8 total, 8 up, 8 in 2024-09-21T09:44:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[102913]: pgmap v264: 89 pgs: 89 active+clean; 99 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:44:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[102913]: osdmap e457: 8 total, 8 up, 8 in 2024-09-21T09:44:43.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:43.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:43.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:43 smithi017 ceph-mon[102913]: osdmap e458: 8 total, 8 up, 8 in 2024-09-21T09:44:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:43 smithi134 ceph-mon[90085]: pgmap v264: 89 pgs: 89 active+clean; 99 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:44:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:43 smithi134 ceph-mon[90085]: osdmap e457: 8 total, 8 up, 8 in 2024-09-21T09:44:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:43 smithi134 ceph-mon[90085]: osdmap e458: 8 total, 8 up, 8 in 2024-09-21T09:44:43.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-09-21T09:44:44.059 INFO:tasks.rados.rados.0.smithi017.stdout:1607: done (2 left) 2024-09-21T09:44:44.059 INFO:tasks.rados.rados.0.smithi017.stdout:1608: done (1 left) 2024-09-21T09:44:44.059 INFO:tasks.rados.rados.0.smithi017.stdout:1609: done (0 left) 2024-09-21T09:44:44.059 INFO:tasks.rados.rados.0.smithi017.stdout:1610: write oid 37 current snap is 196 2024-09-21T09:44:44.059 INFO:tasks.rados.rados.0.smithi017.stdout:1610: seq_num 622 ranges {453954=629471,1598388=515881} 2024-09-21T09:44:44.064 INFO:tasks.rados.rados.0.smithi017.stdout:1610: writing smithi01785587-37 from 453954 to 1083425 tid 1 2024-09-21T09:44:44.071 INFO:tasks.rados.rados.0.smithi017.stdout:1610: writing smithi01785587-37 from 1598388 to 2114269 tid 2 2024-09-21T09:44:44.071 INFO:tasks.rados.rados.0.smithi017.stdout:1611: snap_remove snap 193 2024-09-21T09:44:44.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:44 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:44.549 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:44.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:44:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:44:44] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-21T09:44:45.064 INFO:tasks.rados.rados.0.smithi017.stdout:1611: done (1 left) 2024-09-21T09:44:45.064 INFO:tasks.rados.rados.0.smithi017.stdout:1612: read oid 48 snap -1 2024-09-21T09:44:45.064 INFO:tasks.rados.rados.0.smithi017.stdout:1612: expect deleted 2024-09-21T09:44:45.064 INFO:tasks.rados.rados.0.smithi017.stdout:1613: read oid 19 snap -1 2024-09-21T09:44:45.064 INFO:tasks.rados.rados.0.smithi017.stdout:1613: expect deleted 2024-09-21T09:44:45.064 INFO:tasks.rados.rados.0.smithi017.stdout:1614: snap_remove snap 192 2024-09-21T09:44:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:45 smithi017 ceph-mon[105151]: pgmap v267: 89 pgs: 89 active+clean; 99 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:44:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:45 smithi017 ceph-mon[105151]: osdmap e459: 8 total, 8 up, 8 in 2024-09-21T09:44:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:45 smithi017 ceph-mon[102913]: pgmap v267: 89 pgs: 89 active+clean; 99 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:44:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:45 smithi017 ceph-mon[102913]: osdmap e459: 8 total, 8 up, 8 in 2024-09-21T09:44:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:45 smithi134 ceph-mon[90085]: pgmap v267: 89 pgs: 89 active+clean; 99 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:44:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:45 smithi134 ceph-mon[90085]: osdmap e459: 8 total, 8 up, 8 in 2024-09-21T09:44:45.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:45.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:45 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:46.069 INFO:tasks.rados.rados.0.smithi017.stdout:1614: done (3 left) 2024-09-21T09:44:46.069 INFO:tasks.rados.rados.0.smithi017.stdout:1615: write oid 41 current snap is 196 2024-09-21T09:44:46.069 INFO:tasks.rados.rados.0.smithi017.stdout:1610: finishing write tid 1 to smithi01785587-37 2024-09-21T09:44:46.069 INFO:tasks.rados.rados.0.smithi017.stdout:1615: seq_num 623 ranges {470761=704720,1656870=659539} 2024-09-21T09:44:46.075 INFO:tasks.rados.rados.0.smithi017.stdout:1615: writing smithi01785587-41 from 470761 to 1175481 tid 1 2024-09-21T09:44:46.083 INFO:tasks.rados.rados.0.smithi017.stdout:1615: writing smithi01785587-41 from 1656870 to 2316409 tid 2 2024-09-21T09:44:46.083 INFO:tasks.rados.rados.0.smithi017.stdout:1616: snap_remove snap 195 2024-09-21T09:44:46.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:46 smithi017 ceph-mon[102913]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:44:46.146 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:46 smithi017 ceph-mon[105151]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:44:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:46 smithi134 ceph-mon[90085]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1616: done (4 left) 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1617: read oid 1 snap -1 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1617: expect (ObjNum 458 snap 148 seq_num 458) 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1618: snap_create 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1610: finishing write tid 2 to smithi01785587-37 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1610: finishing write tid 3 to smithi01785587-37 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1610: finishing write tid 4 to smithi01785587-37 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 832 (ObjNum 622 snap 196 seq_num 622) dirty exists 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1610: left oid 37 (ObjNum 622 snap 196 seq_num 622) 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1615: finishing write tid 1 to smithi01785587-41 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1615: finishing write tid 2 to smithi01785587-41 2024-09-21T09:44:47.075 INFO:tasks.rados.rados.0.smithi017.stdout:1615: finishing write tid 3 to smithi01785587-41 2024-09-21T09:44:47.076 INFO:tasks.rados.rados.0.smithi017.stdout:1615: finishing write tid 4 to smithi01785587-41 2024-09-21T09:44:47.076 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 283 (ObjNum 623 snap 196 seq_num 623) dirty exists 2024-09-21T09:44:47.076 INFO:tasks.rados.rados.0.smithi017.stdout:1615: left oid 41 (ObjNum 623 snap 196 seq_num 623) 2024-09-21T09:44:47.089 INFO:tasks.rados.rados.0.smithi017.stdout:1617: expect (ObjNum 458 snap 148 seq_num 458) 2024-09-21T09:44:47.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi017\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi017\", job=\"node\", machine=\"x86_64\", nodename=\"smithi017\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:47 smithi134 ceph-mon[90085]: pgmap v270: 89 pgs: 89 active+clean; 104 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 833 KiB/s wr, 9 op/s 2024-09-21T09:44:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:47 smithi134 ceph-mon[90085]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:44:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:47 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2166927612' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:47.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:47.283 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:44:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[105151]: pgmap v270: 89 pgs: 89 active+clean; 104 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 833 KiB/s wr, 9 op/s 2024-09-21T09:44:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[105151]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:44:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2166927612' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[102913]: pgmap v270: 89 pgs: 89 active+clean; 104 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 833 KiB/s wr, 9 op/s 2024-09-21T09:44:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[102913]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:44:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:47.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:47.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2166927612' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:47.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:47.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:47.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:48.061 INFO:tasks.rados.rados.0.smithi017.stdout:1610: done (5 left) 2024-09-21T09:44:48.061 INFO:tasks.rados.rados.0.smithi017.stdout:1612: done (4 left) 2024-09-21T09:44:48.061 INFO:tasks.rados.rados.0.smithi017.stdout:1613: done (3 left) 2024-09-21T09:44:48.061 INFO:tasks.rados.rados.0.smithi017.stdout:1615: done (2 left) 2024-09-21T09:44:48.062 INFO:tasks.rados.rados.0.smithi017.stdout:1617: done (1 left) 2024-09-21T09:44:48.062 INFO:tasks.rados.rados.0.smithi017.stdout:1618: done (0 left) 2024-09-21T09:44:48.062 INFO:tasks.rados.rados.0.smithi017.stdout:1619: snap_remove snap 194 2024-09-21T09:44:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:44:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: Upgrade: Setting container_image for all mon 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: Upgrade: Setting container_image for all crash 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: Upgrade: osd.0 is safe to restart 2024-09-21T09:44:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[105151]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: Upgrade: Setting container_image for all mon 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:44:48.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:44:48.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:44:48.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: Upgrade: Setting container_image for all crash 2024-09-21T09:44:48.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:48.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:44:48.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:44:48.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: Upgrade: osd.0 is safe to restart 2024-09-21T09:44:48.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:48 smithi017 ceph-mon[102913]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:44:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:44:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: Upgrade: Setting container_image for all mon 2024-09-21T09:44:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:44:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: Upgrade: Setting container_image for all crash 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: Upgrade: osd.0 is safe to restart 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:48 smithi134 ceph-mon[90085]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:44:49.061 INFO:tasks.rados.rados.0.smithi017.stdout:1619: done (0 left) 2024-09-21T09:44:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1620: read oid 34 snap -1 2024-09-21T09:44:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1620: expect (ObjNum 619 snap 193 seq_num 619) 2024-09-21T09:44:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1621: read oid 35 snap -1 2024-09-21T09:44:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1621: expect deleted 2024-09-21T09:44:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1622: read oid 30 snap -1 2024-09-21T09:44:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1622: expect deleted 2024-09-21T09:44:49.062 INFO:tasks.rados.rados.0.smithi017.stdout:1623: delete oid 14 current snap is 197 2024-09-21T09:44:49.066 INFO:tasks.rados.rados.0.smithi017.stdout:1621: done (3 left) 2024-09-21T09:44:49.066 INFO:tasks.rados.rados.0.smithi017.stdout:1622: done (2 left) 2024-09-21T09:44:49.066 INFO:tasks.rados.rados.0.smithi017.stdout:1623: done (1 left) 2024-09-21T09:44:49.066 INFO:tasks.rados.rados.0.smithi017.stdout:1624: delete oid 26 current snap is 197 2024-09-21T09:44:49.069 INFO:tasks.rados.rados.0.smithi017.stdout:1620: expect (ObjNum 619 snap 193 seq_num 619) 2024-09-21T09:44:49.203 INFO:tasks.rados.rados.0.smithi017.stdout:1620: done (1 left) 2024-09-21T09:44:49.203 INFO:tasks.rados.rados.0.smithi017.stdout:1624: done (0 left) 2024-09-21T09:44:49.203 INFO:tasks.rados.rados.0.smithi017.stdout:1625: read oid 13 snap -1 2024-09-21T09:44:49.203 INFO:tasks.rados.rados.0.smithi017.stdout:1625: expect deleted 2024-09-21T09:44:49.203 INFO:tasks.rados.rados.0.smithi017.stdout:1626: read oid 46 snap -1 2024-09-21T09:44:49.203 INFO:tasks.rados.rados.0.smithi017.stdout:1626: expect (ObjNum 537 snap 169 seq_num 537) 2024-09-21T09:44:49.204 INFO:tasks.rados.rados.0.smithi017.stdout:1627: write oid 39 current snap is 197 2024-09-21T09:44:49.204 INFO:tasks.rados.rados.0.smithi017.stdout:1627: seq_num 626 ranges {521182=646820,1948669=523219} 2024-09-21T09:44:49.209 INFO:tasks.rados.rados.0.smithi017.stdout:1627: writing smithi01785587-39 from 521182 to 1168002 tid 1 2024-09-21T09:44:49.217 INFO:tasks.rados.rados.0.smithi017.stdout:1627: writing smithi01785587-39 from 1948669 to 2471888 tid 2 2024-09-21T09:44:49.217 INFO:tasks.rados.rados.0.smithi017.stdout:1628: delete oid 22 current snap is 197 2024-09-21T09:44:49.217 INFO:tasks.rados.rados.0.smithi017.stdout:1626: expect (ObjNum 537 snap 169 seq_num 537) 2024-09-21T09:44:49.344 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[102913]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:49.344 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[102913]: pgmap v273: 89 pgs: 89 active+clean; 104 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 833 KiB/s wr, 9 op/s 2024-09-21T09:44:49.344 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:49.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:44:49.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:49.345 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[102913]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:44:49.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[105151]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:49.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[105151]: pgmap v273: 89 pgs: 89 active+clean; 104 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 833 KiB/s wr, 9 op/s 2024-09-21T09:44:49.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:49.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:44:49.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:49.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:49 smithi017 ceph-mon[105151]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:44:49.411 INFO:tasks.rados.rados.0.smithi017.stdout:1627: finishing write tid 1 to smithi01785587-39 2024-09-21T09:44:49.411 INFO:tasks.rados.rados.0.smithi017.stdout:1627: finishing write tid 2 to smithi01785587-39 2024-09-21T09:44:49.411 INFO:tasks.rados.rados.0.smithi017.stdout:1625: done (3 left) 2024-09-21T09:44:49.411 INFO:tasks.rados.rados.0.smithi017.stdout:1626: done (2 left) 2024-09-21T09:44:49.411 INFO:tasks.rados.rados.0.smithi017.stdout:1628: done (1 left) 2024-09-21T09:44:49.411 INFO:tasks.rados.rados.0.smithi017.stdout:1629: read oid 7 snap 196 2024-09-21T09:44:49.411 INFO:tasks.rados.rados.0.smithi017.stdout:1629: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:44:49.411 INFO:tasks.rados.rados.0.smithi017.stdout:1630: read oid 16 snap -1 2024-09-21T09:44:49.412 INFO:tasks.rados.rados.0.smithi017.stdout:1630: expect deleted 2024-09-21T09:44:49.412 INFO:tasks.rados.rados.0.smithi017.stdout:1631: snap_create 2024-09-21T09:44:49.412 INFO:tasks.rados.rados.0.smithi017.stdout:1627: finishing write tid 3 to smithi01785587-39 2024-09-21T09:44:49.412 INFO:tasks.rados.rados.0.smithi017.stdout:1627: finishing write tid 4 to smithi01785587-39 2024-09-21T09:44:49.412 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 844 (ObjNum 626 snap 197 seq_num 626) dirty exists 2024-09-21T09:44:49.412 INFO:tasks.rados.rados.0.smithi017.stdout:1627: left oid 39 (ObjNum 626 snap 197 seq_num 626) 2024-09-21T09:44:49.412 INFO:tasks.rados.rados.0.smithi017.stdout:1629: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:44:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:49 smithi134 ceph-mon[90085]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:49 smithi134 ceph-mon[90085]: pgmap v273: 89 pgs: 89 active+clean; 104 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 833 KiB/s wr, 9 op/s 2024-09-21T09:44:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:44:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:49 smithi134 ceph-mon[90085]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:44:50.176 INFO:tasks.rados.rados.0.smithi017.stdout:1627: done (3 left) 2024-09-21T09:44:50.177 INFO:tasks.rados.rados.0.smithi017.stdout:1629: done (2 left) 2024-09-21T09:44:50.177 INFO:tasks.rados.rados.0.smithi017.stdout:1630: done (1 left) 2024-09-21T09:44:50.177 INFO:tasks.rados.rados.0.smithi017.stdout:1631: done (0 left) 2024-09-21T09:44:50.177 INFO:tasks.rados.rados.0.smithi017.stdout:1632: read oid 48 snap -1 2024-09-21T09:44:50.177 INFO:tasks.rados.rados.0.smithi017.stdout:1632: expect deleted 2024-09-21T09:44:50.177 INFO:tasks.rados.rados.0.smithi017.stdout:1633: write oid 22 current snap is 198 2024-09-21T09:44:50.177 INFO:tasks.rados.rados.0.smithi017.stdout:1633: seq_num 628 ranges {554796=797318,1949280=526888} 2024-09-21T09:44:50.184 INFO:tasks.rados.rados.0.smithi017.stdout:1633: writing smithi01785587-22 from 554796 to 1352114 tid 1 2024-09-21T09:44:50.187 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:44:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:1633: writing smithi01785587-22 from 1949280 to 2476168 tid 2 2024-09-21T09:44:50.191 INFO:tasks.rados.rados.0.smithi017.stdout:1634: snap_remove snap 196 2024-09-21T09:44:50.274 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:50 smithi017 ceph-mon[102913]: Upgrade: Updating osd.0 2024-09-21T09:44:50.274 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:50 smithi017 ceph-mon[102913]: Deploying daemon osd.0 on smithi017 2024-09-21T09:44:50.274 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:50 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1570603999' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:50.275 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:50 smithi017 ceph-mon[105151]: Upgrade: Updating osd.0 2024-09-21T09:44:50.275 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:50 smithi017 ceph-mon[105151]: Deploying daemon osd.0 on smithi017 2024-09-21T09:44:50.275 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:50 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1570603999' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:50 smithi134 ceph-mon[90085]: Upgrade: Updating osd.0 2024-09-21T09:44:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:50 smithi134 ceph-mon[90085]: Deploying daemon osd.0 on smithi017 2024-09-21T09:44:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:50 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1570603999' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:51.183 INFO:tasks.rados.rados.0.smithi017.stdout:1634: done (2 left) 2024-09-21T09:44:51.183 INFO:tasks.rados.rados.0.smithi017.stdout:1635: read oid 28 snap -1 2024-09-21T09:44:51.183 INFO:tasks.rados.rados.0.smithi017.stdout:1635: expect deleted 2024-09-21T09:44:51.183 INFO:tasks.rados.rados.0.smithi017.stdout:1636: snap_remove snap 197 2024-09-21T09:44:51.265 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-mon[102913]: pgmap v276: 89 pgs: 89 active+clean; 104 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:44:51.265 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-mon[102913]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:51.265 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-mon[102913]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:44:51.266 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-mon[102913]: from='client.34329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:51.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-mon[105151]: pgmap v276: 89 pgs: 89 active+clean; 104 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:44:51.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-mon[105151]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:51.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-mon[105151]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:44:51.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-mon[105151]: from='client.34329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:51 smithi134 ceph-mon[90085]: pgmap v276: 89 pgs: 89 active+clean; 104 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:44:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:51 smithi134 ceph-mon[90085]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:51 smithi134 ceph-mon[90085]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:44:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:51 smithi134 ceph-mon[90085]: from='client.34329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:51.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:51 smithi017 systemd[1]: Stopping Ceph osd.0 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:44:51.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[45076]: 2024-09-21T09:44:51.860+0000 7f45483ac640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:44:51.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[45076]: 2024-09-21T09:44:51.860+0000 7f45483ac640 -1 osd.0 466 *** Got signal Terminated *** 2024-09-21T09:44:51.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[45076]: 2024-09-21T09:44:51.860+0000 7f45483ac640 -1 osd.0 466 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:44:52.194 INFO:tasks.rados.rados.0.smithi017.stdout:1636: done (3 left) 2024-09-21T09:44:52.195 INFO:tasks.rados.rados.0.smithi017.stdout:1637: delete oid 41 current snap is 198 2024-09-21T09:44:52.195 INFO:tasks.rados.rados.0.smithi017.stdout:1633: finishing write tid 1 to smithi01785587-22 2024-09-21T09:44:52.195 INFO:tasks.rados.rados.0.smithi017.stdout:1633: finishing write tid 2 to smithi01785587-22 2024-09-21T09:44:52.195 INFO:tasks.rados.rados.0.smithi017.stdout:1633: finishing write tid 3 to smithi01785587-22 2024-09-21T09:44:52.195 INFO:tasks.rados.rados.0.smithi017.stdout:1633: finishing write tid 4 to smithi01785587-22 2024-09-21T09:44:52.195 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 808 (ObjNum 628 snap 198 seq_num 628) dirty exists 2024-09-21T09:44:52.195 INFO:tasks.rados.rados.0.smithi017.stdout:1633: left oid 22 (ObjNum 628 snap 198 seq_num 628) 2024-09-21T09:44:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:52 smithi017 ceph-mon[102913]: osdmap e466: 8 total, 8 up, 8 in 2024-09-21T09:44:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:52 smithi017 ceph-mon[102913]: osd.0 marked itself down and dead 2024-09-21T09:44:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:52 smithi017 ceph-mon[105151]: osdmap e466: 8 total, 8 up, 8 in 2024-09-21T09:44:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:52 smithi017 ceph-mon[105151]: osd.0 marked itself down and dead 2024-09-21T09:44:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:52 smithi134 ceph-mon[90085]: osdmap e466: 8 total, 8 up, 8 in 2024-09-21T09:44:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:52 smithi134 ceph-mon[90085]: osd.0 marked itself down and dead 2024-09-21T09:44:53.020 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:52 smithi017 podman[113473]: 2024-09-21 09:44:52.721746661 +0000 UTC m=+0.948082598 container died 958bb6e5f55de22a7ebc1f1f39303397c8c4b0a5d0b7d52f947316d7069ae7fa (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T09:44:53.067 INFO:tasks.rados.rados.0.smithi017.stdout:1632: done (3 left) 2024-09-21T09:44:53.067 INFO:tasks.rados.rados.0.smithi017.stdout:1633: done (2 left) 2024-09-21T09:44:53.067 INFO:tasks.rados.rados.0.smithi017.stdout:1635: done (1 left) 2024-09-21T09:44:53.068 INFO:tasks.rados.rados.0.smithi017.stdout:1637: done (0 left) 2024-09-21T09:44:53.068 INFO:tasks.rados.rados.0.smithi017.stdout:1638: read oid 37 snap -1 2024-09-21T09:44:53.068 INFO:tasks.rados.rados.0.smithi017.stdout:1638: expect (ObjNum 622 snap 196 seq_num 622) 2024-09-21T09:44:53.068 INFO:tasks.rados.rados.0.smithi017.stdout:1639: snap_create 2024-09-21T09:44:53.073 INFO:tasks.rados.rados.0.smithi017.stdout:1638: expect (ObjNum 622 snap 196 seq_num 622) 2024-09-21T09:44:53.102 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:44:53.362 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:53 smithi017 podman[113473]: 2024-09-21 09:44:53.109747226 +0000 UTC m=+1.336083154 container cleanup 958bb6e5f55de22a7ebc1f1f39303397c8c4b0a5d0b7d52f947316d7069ae7fa (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T09:44:53.362 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:53 smithi017 bash[113473]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0 2024-09-21T09:44:53.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[102913]: pgmap v279: 89 pgs: 89 active+clean; 97 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 943 KiB/s wr, 13 op/s 2024-09-21T09:44:53.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[102913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:44:53.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[102913]: osdmap e467: 8 total, 7 up, 8 in 2024-09-21T09:44:53.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2400796423' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:53.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[102913]: osdmap e468: 8 total, 7 up, 8 in 2024-09-21T09:44:53.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[105151]: pgmap v279: 89 pgs: 89 active+clean; 97 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 943 KiB/s wr, 13 op/s 2024-09-21T09:44:53.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[105151]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:44:53.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[105151]: osdmap e467: 8 total, 7 up, 8 in 2024-09-21T09:44:53.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2400796423' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:53.364 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:53 smithi017 ceph-mon[105151]: osdmap e468: 8 total, 7 up, 8 in 2024-09-21T09:44:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:53 smithi134 ceph-mon[90085]: pgmap v279: 89 pgs: 89 active+clean; 97 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 943 KiB/s wr, 13 op/s 2024-09-21T09:44:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:53 smithi134 ceph-mon[90085]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:44:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:53 smithi134 ceph-mon[90085]: osdmap e467: 8 total, 7 up, 8 in 2024-09-21T09:44:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:53 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2400796423' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:53 smithi134 ceph-mon[90085]: osdmap e468: 8 total, 7 up, 8 in 2024-09-21T09:44:53.690 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:53 smithi017 podman[113516]: 2024-09-21 09:44:53.361071107 +0000 UTC m=+0.637471562 container remove 958bb6e5f55de22a7ebc1f1f39303397c8c4b0a5d0b7d52f947316d7069ae7fa (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T09:44:53.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:53 smithi017 podman[113659]: 2024-09-21 09:44:53.592644348 +0000 UTC m=+0.016712834 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:44:53.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:53 smithi017 podman[113659]: 2024-09-21 09:44:53.712124701 +0000 UTC m=+0.136193187 container create 94dc92578c2743bdfc6913dd5aa705daf5d5239986ef2933cc4384b59c44bee9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:44:53.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:53 smithi017 podman[113659]: 2024-09-21 09:44:53.931332753 +0000 UTC m=+0.355401246 container init 94dc92578c2743bdfc6913dd5aa705daf5d5239986ef2933cc4384b59c44bee9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T09:44:53.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:53 smithi017 podman[113659]: 2024-09-21 09:44:53.936719043 +0000 UTC m=+0.360787528 container start 94dc92578c2743bdfc6913dd5aa705daf5d5239986ef2933cc4384b59c44bee9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T09:44:54.061 INFO:tasks.rados.rados.0.smithi017.stdout:1638: done (1 left) 2024-09-21T09:44:54.062 INFO:tasks.rados.rados.0.smithi017.stdout:1639: done (0 left) 2024-09-21T09:44:54.062 INFO:tasks.rados.rados.0.smithi017.stdout:1640: rollback oid 21 current snap is 199 2024-09-21T09:44:54.062 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 198 2024-09-21T09:44:54.062 INFO:tasks.rados.rados.0.smithi017.stdout:1641: rollback oid 48 current snap is 199 2024-09-21T09:44:54.062 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 198 2024-09-21T09:44:54.062 INFO:tasks.rados.rados.0.smithi017.stdout:1642: snap_create 2024-09-21T09:44:54.063 INFO:tasks.rados.rados.0.smithi017.stdout:1641: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:44:54.064 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 0 (ObjNum 65 snap 0 seq_num 1) dirty dne 2024-09-21T09:44:54.065 INFO:tasks.rados.rados.0.smithi017.stdout:1640: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:44:54.066 INFO:tasks.rados.rados.0.smithi017.stdout:1640: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:44:54.066 INFO:tasks.rados.rados.0.smithi017.stdout:1640: finishing rollback tid 2 to smithi01785587-21 2024-09-21T09:44:54.066 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 875 (ObjNum 579 snap 181 seq_num 579) dirty exists 2024-09-21T09:44:54.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:54 smithi017 ceph-mon[105151]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:54.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:54 smithi017 ceph-mon[105151]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:54.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:54 smithi017 ceph-mon[105151]: osdmap e469: 8 total, 7 up, 8 in 2024-09-21T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:54 smithi017 ceph-mon[102913]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:54 smithi017 ceph-mon[102913]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:54.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:54 smithi017 ceph-mon[102913]: osdmap e469: 8 total, 7 up, 8 in 2024-09-21T09:44:54.429 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:54 smithi017 podman[113659]: 2024-09-21 09:44:54.031085942 +0000 UTC m=+0.455154434 container attach 94dc92578c2743bdfc6913dd5aa705daf5d5239986ef2933cc4384b59c44bee9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T09:44:54.429 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:54 smithi017 podman[113659]: 2024-09-21 09:44:54.140100247 +0000 UTC m=+0.564168732 container died 94dc92578c2743bdfc6913dd5aa705daf5d5239986ef2933cc4384b59c44bee9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T09:44:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:54 smithi134 ceph-mon[90085]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:54 smithi134 ceph-mon[90085]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:54 smithi134 ceph-mon[90085]: osdmap e469: 8 total, 7 up, 8 in 2024-09-21T09:44:54.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:54 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:55.064 INFO:tasks.rados.rados.0.smithi017.stdout:1640: done (2 left) 2024-09-21T09:44:55.064 INFO:tasks.rados.rados.0.smithi017.stdout:1641: done (1 left) 2024-09-21T09:44:55.064 INFO:tasks.rados.rados.0.smithi017.stdout:1642: done (0 left) 2024-09-21T09:44:55.064 INFO:tasks.rados.rados.0.smithi017.stdout:1643: write oid 40 current snap is 200 2024-09-21T09:44:55.065 INFO:tasks.rados.rados.0.smithi017.stdout:1643: seq_num 630 ranges {588410=547816,1666244=414204,2484603=103807} 2024-09-21T09:44:55.070 INFO:tasks.rados.rados.0.smithi017.stdout:1643: writing smithi01785587-40 from 588410 to 1136226 tid 1 2024-09-21T09:44:55.075 INFO:tasks.rados.rados.0.smithi017.stdout:1643: writing smithi01785587-40 from 1666244 to 2080448 tid 2 2024-09-21T09:44:55.078 INFO:tasks.rados.rados.0.smithi017.stdout:1643: writing smithi01785587-40 from 2484603 to 2588410 tid 3 2024-09-21T09:44:55.079 INFO:tasks.rados.rados.0.smithi017.stdout:1644: rollback oid 8 current snap is 200 2024-09-21T09:44:55.079 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 198 2024-09-21T09:44:55.079 INFO:tasks.rados.rados.0.smithi017.stdout:1643: finishing write tid 1 to smithi01785587-40 2024-09-21T09:44:55.079 INFO:tasks.rados.rados.0.smithi017.stdout:1643: finishing write tid 2 to smithi01785587-40 2024-09-21T09:44:55.079 INFO:tasks.rados.rados.0.smithi017.stdout:1645: write oid 47 current snap is 200 2024-09-21T09:44:55.079 INFO:tasks.rados.rados.0.smithi017.stdout:1645: seq_num 631 ranges {605217=739418,2124726=480491} 2024-09-21T09:44:55.086 INFO:tasks.rados.rados.0.smithi017.stdout:1645: writing smithi01785587-47 from 605217 to 1344635 tid 1 2024-09-21T09:44:55.093 INFO:tasks.rados.rados.0.smithi017.stdout:1645: writing smithi01785587-47 from 2124726 to 2605217 tid 2 2024-09-21T09:44:55.093 INFO:tasks.rados.rados.0.smithi017.stdout:1646: read oid 7 snap -1 2024-09-21T09:44:55.093 INFO:tasks.rados.rados.0.smithi017.stdout:1646: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1647: write oid 13 current snap is 200 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1644: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 0 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1643: finishing write tid 3 to smithi01785587-40 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1643: finishing write tid 4 to smithi01785587-40 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1643: finishing write tid 5 to smithi01785587-40 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 851 (ObjNum 630 snap 200 seq_num 630) dirty exists 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1643: left oid 40 (ObjNum 630 snap 200 seq_num 630) 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1645: finishing write tid 1 to smithi01785587-47 2024-09-21T09:44:55.094 INFO:tasks.rados.rados.0.smithi017.stdout:1647: seq_num 632 ranges {622024=414667,1783208=417873} 2024-09-21T09:44:55.099 INFO:tasks.rados.rados.0.smithi017.stdout:1647: writing smithi01785587-13 from 622024 to 1036691 tid 1 2024-09-21T09:44:55.106 INFO:tasks.rados.rados.0.smithi017.stdout:1647: writing smithi01785587-13 from 1783208 to 2201081 tid 2 2024-09-21T09:44:55.106 INFO:tasks.rados.rados.0.smithi017.stdout:1643: done (4 left) 2024-09-21T09:44:55.106 INFO:tasks.rados.rados.0.smithi017.stdout:1644: done (3 left) 2024-09-21T09:44:55.106 INFO:tasks.rados.rados.0.smithi017.stdout:1648: write oid 42 current snap is 200 2024-09-21T09:44:55.106 INFO:tasks.rados.rados.0.smithi017.stdout:1648: seq_num 633 ranges {638831=489916,1841690=677884} 2024-09-21T09:44:55.112 INFO:tasks.rados.rados.0.smithi017.stdout:1648: writing smithi01785587-42 from 638831 to 1128747 tid 1 2024-09-21T09:44:55.119 INFO:tasks.rados.rados.0.smithi017.stdout:1648: writing smithi01785587-42 from 1841690 to 2519574 tid 2 2024-09-21T09:44:55.119 INFO:tasks.rados.rados.0.smithi017.stdout:1649: read oid 35 snap -1 2024-09-21T09:44:55.119 INFO:tasks.rados.rados.0.smithi017.stdout:1649: expect deleted 2024-09-21T09:44:55.120 INFO:tasks.rados.rados.0.smithi017.stdout:1650: snap_remove snap 199 2024-09-21T09:44:55.181 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:44:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:44:54] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.51.0" 2024-09-21T09:44:55.181 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:54 smithi017 podman[113701]: 2024-09-21 09:44:54.73513757 +0000 UTC m=+0.589209218 container remove 94dc92578c2743bdfc6913dd5aa705daf5d5239986ef2933cc4384b59c44bee9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T09:44:55.181 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:54 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T09:44:55.181 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:54 smithi017 systemd[1]: Stopped Ceph osd.0 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:44:55.181 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:54 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.0.service: Consumed 12.484s CPU time. 2024-09-21T09:44:55.181 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:54 smithi017 systemd[1]: Starting Ceph osd.0 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:44:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:55 smithi017 ceph-mon[102913]: pgmap v282: 89 pgs: 13 stale+active+clean, 76 active+clean; 97 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 943 KiB/s wr, 13 op/s 2024-09-21T09:44:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:55 smithi017 ceph-mon[102913]: osdmap e470: 8 total, 7 up, 8 in 2024-09-21T09:44:55.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:55 smithi017 ceph-mon[105151]: pgmap v282: 89 pgs: 13 stale+active+clean, 76 active+clean; 97 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 943 KiB/s wr, 13 op/s 2024-09-21T09:44:55.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:55 smithi017 ceph-mon[105151]: osdmap e470: 8 total, 7 up, 8 in 2024-09-21T09:44:55.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:55 smithi017 podman[113809]: 2024-09-21 09:44:55.083195928 +0000 UTC m=+0.018910870 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:44:55.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:55 smithi017 podman[113809]: 2024-09-21 09:44:55.261173197 +0000 UTC m=+0.196888169 container create d6052bd4723f296870df57d356e319f435bd3029a55020afd4fd2b061ac30d28 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:44:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:55 smithi134 ceph-mon[90085]: pgmap v282: 89 pgs: 13 stale+active+clean, 76 active+clean; 97 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 943 KiB/s wr, 13 op/s 2024-09-21T09:44:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:55 smithi134 ceph-mon[90085]: osdmap e470: 8 total, 7 up, 8 in 2024-09-21T09:44:55.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:55 smithi017 podman[113809]: 2024-09-21 09:44:55.460669554 +0000 UTC m=+0.396384480 container init d6052bd4723f296870df57d356e319f435bd3029a55020afd4fd2b061ac30d28 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T09:44:55.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:55 smithi017 podman[113809]: 2024-09-21 09:44:55.466039123 +0000 UTC m=+0.401754050 container start d6052bd4723f296870df57d356e319f435bd3029a55020afd4fd2b061ac30d28 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T09:44:55.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:55 smithi017 podman[113809]: 2024-09-21 09:44:55.555435693 +0000 UTC m=+0.491150628 container attach d6052bd4723f296870df57d356e319f435bd3029a55020afd4fd2b061ac30d28 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T09:44:55.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:55 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate[113824]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:44:55.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:55 smithi017 bash[113809]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:44:55.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:55 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate[113824]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:44:55.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:55 smithi017 bash[113809]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:44:55.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:55.994 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:55 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:56.024 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:44:56.066 INFO:tasks.rados.rados.0.smithi017.stdout:1650: done (5 left) 2024-09-21T09:44:56.066 INFO:tasks.rados.rados.0.smithi017.stdout:1651: rollback oid 9 current snap is 200 2024-09-21T09:44:56.066 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 9 to 198 2024-09-21T09:44:56.066 INFO:tasks.rados.rados.0.smithi017.stdout:1646: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:44:56.290 INFO:tasks.rados.rados.0.smithi017.stdout:1645: finishing write tid 2 to smithi01785587-47 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1645: finishing write tid 3 to smithi01785587-47 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1645: finishing write tid 4 to smithi01785587-47 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 602 (ObjNum 631 snap 200 seq_num 631) dirty exists 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1645: left oid 47 (ObjNum 631 snap 200 seq_num 631) 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1647: finishing write tid 1 to smithi01785587-13 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1645: done (5 left) 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1646: done (4 left) 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1652: read oid 34 snap -1 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1652: expect (ObjNum 619 snap 193 seq_num 619) 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1653: snap_create 2024-09-21T09:44:56.291 INFO:tasks.rados.rados.0.smithi017.stdout:1647: finishing write tid 2 to smithi01785587-13 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:1647: finishing write tid 3 to smithi01785587-13 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:1647: finishing write tid 4 to smithi01785587-13 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 707 (ObjNum 632 snap 200 seq_num 632) dirty exists 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:1647: left oid 13 (ObjNum 632 snap 200 seq_num 632) 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:1648: finishing write tid 1 to smithi01785587-42 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:1648: finishing write tid 2 to smithi01785587-42 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:1648: finishing write tid 3 to smithi01785587-42 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:1648: finishing write tid 4 to smithi01785587-42 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 606 (ObjNum 633 snap 200 seq_num 633) dirty exists 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:1648: left oid 42 (ObjNum 633 snap 200 seq_num 633) 2024-09-21T09:44:56.292 INFO:tasks.rados.rados.0.smithi017.stdout:1651: finishing rollback tid 1 to smithi01785587-9 2024-09-21T09:44:56.293 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 0 (ObjNum 177 snap 0 seq_num 2399799264) dirty dne 2024-09-21T09:44:56.294 INFO:tasks.rados.rados.0.smithi017.stdout:1652: expect (ObjNum 619 snap 193 seq_num 619) 2024-09-21T09:44:56.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:56.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:56.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2073527552' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:56.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-mon[102913]: osdmap e471: 8 total, 7 up, 8 in 2024-09-21T09:44:56.391 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:56.391 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:56.391 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2073527552' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:56.391 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-mon[105151]: osdmap e471: 8 total, 7 up, 8 in 2024-09-21T09:44:56.391 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate[113824]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:44:56.391 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 bash[113809]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:44:56.391 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate[113824]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:44:56.391 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 bash[113809]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:44:56.391 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate[113824]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:44:56.391 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 bash[113809]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:44:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:56 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2073527552' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:56 smithi134 ceph-mon[90085]: osdmap e471: 8 total, 7 up, 8 in 2024-09-21T09:44:56.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate[113824]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:44:56.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 bash[113809]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:44:56.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate[113824]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T09:44:56.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 bash[113809]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T09:44:56.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate[113824]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:44:56.701 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 bash[113809]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:44:56.701 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate[113824]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T09:44:56.701 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 bash[113809]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T09:44:56.701 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:56 smithi017 podman[114048]: 2024-09-21 09:44:56.447452197 +0000 UTC m=+0.017941662 container died d6052bd4723f296870df57d356e319f435bd3029a55020afd4fd2b061ac30d28 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T09:44:57.240 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:44:56 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:44:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi134\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi134\", job=\"node\", machine=\"x86_64\", nodename=\"smithi134\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:44:57.250 INFO:tasks.rados.rados.0.smithi017.stdout:1647: done (5 left) 2024-09-21T09:44:57.250 INFO:tasks.rados.rados.0.smithi017.stdout:1648: done (4 left) 2024-09-21T09:44:57.251 INFO:tasks.rados.rados.0.smithi017.stdout:1649: done (3 left) 2024-09-21T09:44:57.251 INFO:tasks.rados.rados.0.smithi017.stdout:1651: done (2 left) 2024-09-21T09:44:57.251 INFO:tasks.rados.rados.0.smithi017.stdout:1652: done (1 left) 2024-09-21T09:44:57.251 INFO:tasks.rados.rados.0.smithi017.stdout:1653: done (0 left) 2024-09-21T09:44:57.251 INFO:tasks.rados.rados.0.smithi017.stdout:1654: delete oid 27 current snap is 201 2024-09-21T09:44:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:1654: done (0 left) 2024-09-21T09:44:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:1655: delete oid 15 current snap is 201 2024-09-21T09:44:57.256 INFO:tasks.rados.rados.0.smithi017.stdout:1655: done (0 left) 2024-09-21T09:44:57.256 INFO:tasks.rados.rados.0.smithi017.stdout:1656: snap_remove snap 200 2024-09-21T09:44:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:57 smithi017 ceph-mon[102913]: pgmap v285: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 27/183 objects degraded (14.754%) 2024-09-21T09:44:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:57 smithi017 ceph-mon[102913]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:57 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 27/183 objects degraded (14.754%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:44:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:57 smithi017 ceph-mon[102913]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:57 smithi017 ceph-mon[105151]: pgmap v285: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 27/183 objects degraded (14.754%) 2024-09-21T09:44:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:57 smithi017 ceph-mon[105151]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:57 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 27/183 objects degraded (14.754%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:44:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:57 smithi017 ceph-mon[105151]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:57.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:57 smithi017 podman[114048]: 2024-09-21 09:44:57.159370477 +0000 UTC m=+0.729859980 container remove d6052bd4723f296870df57d356e319f435bd3029a55020afd4fd2b061ac30d28 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:44:57.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:57 smithi017 podman[114108]: 2024-09-21 09:44:57.282636014 +0000 UTC m=+0.017840404 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:44:57.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:57 smithi017 podman[114108]: 2024-09-21 09:44:57.385195531 +0000 UTC m=+0.120399906 container create 68ddc8c6bab9fda157e6fbb6414dc80ef7af1ef689ce76d50d3709d23f9500a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-62da54b, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:44:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:57 smithi134 ceph-mon[90085]: pgmap v285: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 27/183 objects degraded (14.754%) 2024-09-21T09:44:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:57 smithi134 ceph-mon[90085]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:57 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 27/183 objects degraded (14.754%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:44:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:57 smithi134 ceph-mon[90085]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:57.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:57 smithi017 podman[114108]: 2024-09-21 09:44:57.646105355 +0000 UTC m=+0.381309731 container init 68ddc8c6bab9fda157e6fbb6414dc80ef7af1ef689ce76d50d3709d23f9500a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-62da54b, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T09:44:57.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:57 smithi017 podman[114108]: 2024-09-21 09:44:57.651430872 +0000 UTC m=+0.386635247 container start 68ddc8c6bab9fda157e6fbb6414dc80ef7af1ef689ce76d50d3709d23f9500a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T09:44:57.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:57 smithi017 ceph-osd[114146]: -- 172.21.15.17:0/3426324319 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55abc1ccda40 con 0x55abc2a9c400 2024-09-21T09:44:57.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:57 smithi017 bash[114108]: 68ddc8c6bab9fda157e6fbb6414dc80ef7af1ef689ce76d50d3709d23f9500a7 2024-09-21T09:44:57.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:57 smithi017 systemd[1]: Started Ceph osd.0 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:44:58.063 INFO:tasks.rados.rados.0.smithi017.stdout:1656: done (0 left) 2024-09-21T09:44:58.063 INFO:tasks.rados.rados.0.smithi017.stdout:1657: read oid 37 snap 198 2024-09-21T09:44:58.063 INFO:tasks.rados.rados.0.smithi017.stdout:1657: expect (ObjNum 622 snap 196 seq_num 622) 2024-09-21T09:44:58.063 INFO:tasks.rados.rados.0.smithi017.stdout:1658: rollback oid 14 current snap is 201 2024-09-21T09:44:58.063 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 198 2024-09-21T09:44:58.063 INFO:tasks.rados.rados.0.smithi017.stdout:1659: write oid 30 current snap is 201 2024-09-21T09:44:58.063 INFO:tasks.rados.rados.0.smithi017.stdout:1659: seq_num 636 ranges {689252=715663,1900783=425211} 2024-09-21T09:44:58.070 INFO:tasks.rados.rados.0.smithi017.stdout:1659: writing smithi01785587-30 from 689252 to 1404915 tid 1 2024-09-21T09:44:58.076 INFO:tasks.rados.rados.0.smithi017.stdout:1659: writing smithi01785587-30 from 1900783 to 2325994 tid 2 2024-09-21T09:44:58.076 INFO:tasks.rados.rados.0.smithi017.stdout:1660: delete oid 9 current snap is 201 2024-09-21T09:44:58.076 INFO:tasks.rados.rados.0.smithi017.stdout:1658: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:44:58.076 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 0 (ObjNum 129 snap 0 seq_num 2399831120) dirty dne 2024-09-21T09:44:58.076 INFO:tasks.rados.rados.0.smithi017.stdout:1657: expect (ObjNum 622 snap 196 seq_num 622) 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1659: finishing write tid 1 to smithi01785587-30 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1659: finishing write tid 2 to smithi01785587-30 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1659: finishing write tid 3 to smithi01785587-30 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1659: finishing write tid 4 to smithi01785587-30 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 710 (ObjNum 636 snap 201 seq_num 636) dirty exists 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1659: left oid 30 (ObjNum 636 snap 201 seq_num 636) 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1657: done (3 left) 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1658: done (2 left) 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1659: done (1 left) 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1660: done (0 left) 2024-09-21T09:44:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:1661: snap_create 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[102913]: osdmap e472: 8 total, 7 up, 8 in 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[102913]: osdmap e473: 8 total, 7 up, 8 in 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[105151]: osdmap e472: 8 total, 7 up, 8 in 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:58.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:58 smithi017 ceph-mon[105151]: osdmap e473: 8 total, 7 up, 8 in 2024-09-21T09:44:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:58 smithi134 ceph-mon[90085]: osdmap e472: 8 total, 7 up, 8 in 2024-09-21T09:44:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:58 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:58 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:44:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:58 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:58 smithi134 ceph-mon[90085]: osdmap e473: 8 total, 7 up, 8 in 2024-09-21T09:44:58.979 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:44:59.068 INFO:tasks.rados.rados.0.smithi017.stdout:1661: done (0 left) 2024-09-21T09:44:59.068 INFO:tasks.rados.rados.0.smithi017.stdout:1662: write oid 33 current snap is 202 2024-09-21T09:44:59.068 INFO:tasks.rados.rados.0.smithi017.stdout:1662: seq_num 638 ranges {722866=466161,1617747=428880,2587634=135232} 2024-09-21T09:44:59.074 INFO:tasks.rados.rados.0.smithi017.stdout:1662: writing smithi01785587-33 from 722866 to 1189027 tid 1 2024-09-21T09:44:59.079 INFO:tasks.rados.rados.0.smithi017.stdout:1662: writing smithi01785587-33 from 1617747 to 2046627 tid 2 2024-09-21T09:44:59.083 INFO:tasks.rados.rados.0.smithi017.stdout:1662: writing smithi01785587-33 from 2587634 to 2722866 tid 3 2024-09-21T09:44:59.083 INFO:tasks.rados.rados.0.smithi017.stdout:1663: delete oid 15 current snap is 202 2024-09-21T09:44:59.083 INFO:tasks.rados.rados.0.smithi017.stdout:1662: finishing write tid 1 to smithi01785587-33 2024-09-21T09:44:59.083 INFO:tasks.rados.rados.0.smithi017.stdout:1662: finishing write tid 2 to smithi01785587-33 2024-09-21T09:44:59.084 INFO:tasks.rados.rados.0.smithi017.stdout:1663: done (1 left) 2024-09-21T09:44:59.085 INFO:tasks.rados.rados.0.smithi017.stdout:1664: write oid 49 current snap is 202 2024-09-21T09:44:59.085 INFO:tasks.rados.rados.0.smithi017.stdout:1664: seq_num 640 ranges {756480=733012,2134711=432549} 2024-09-21T09:44:59.093 INFO:tasks.rados.rados.0.smithi017.stdout:1664: writing smithi01785587-49 from 756480 to 1489492 tid 1 2024-09-21T09:44:59.099 INFO:tasks.rados.rados.0.smithi017.stdout:1664: writing smithi01785587-49 from 2134711 to 2567260 tid 2 2024-09-21T09:44:59.099 INFO:tasks.rados.rados.0.smithi017.stdout:1665: read oid 37 snap -1 2024-09-21T09:44:59.099 INFO:tasks.rados.rados.0.smithi017.stdout:1665: expect (ObjNum 622 snap 196 seq_num 622) 2024-09-21T09:44:59.099 INFO:tasks.rados.rados.0.smithi017.stdout:1666: write oid 50 current snap is 202 2024-09-21T09:44:59.099 INFO:tasks.rados.rados.0.smithi017.stdout:1662: finishing write tid 3 to smithi01785587-33 2024-09-21T09:44:59.099 INFO:tasks.rados.rados.0.smithi017.stdout:1666: seq_num 641 ranges {773287=408261,1793193=576207} 2024-09-21T09:44:59.105 INFO:tasks.rados.rados.0.smithi017.stdout:1666: writing smithi01785587-50 from 773287 to 1181548 tid 1 2024-09-21T09:44:59.112 INFO:tasks.rados.rados.0.smithi017.stdout:1666: writing smithi01785587-50 from 1793193 to 2369400 tid 2 2024-09-21T09:44:59.112 INFO:tasks.rados.rados.0.smithi017.stdout:1667: delete oid 2 current snap is 202 2024-09-21T09:44:59.112 INFO:tasks.rados.rados.0.smithi017.stdout:1662: finishing write tid 4 to smithi01785587-33 2024-09-21T09:44:59.113 INFO:tasks.rados.rados.0.smithi017.stdout:1662: finishing write tid 5 to smithi01785587-33 2024-09-21T09:44:59.113 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 816 (ObjNum 638 snap 202 seq_num 638) dirty exists 2024-09-21T09:44:59.113 INFO:tasks.rados.rados.0.smithi017.stdout:1662: left oid 33 (ObjNum 638 snap 202 seq_num 638) 2024-09-21T09:44:59.113 INFO:tasks.rados.rados.0.smithi017.stdout:1664: finishing write tid 1 to smithi01785587-49 2024-09-21T09:44:59.113 INFO:tasks.rados.rados.0.smithi017.stdout:1665: expect (ObjNum 622 snap 196 seq_num 622) 2024-09-21T09:44:59.230 INFO:tasks.rados.rados.0.smithi017.stdout:1664: finishing write tid 2 to smithi01785587-49 2024-09-21T09:44:59.230 INFO:tasks.rados.rados.0.smithi017.stdout:1664: finishing write tid 3 to smithi01785587-49 2024-09-21T09:44:59.230 INFO:tasks.rados.rados.0.smithi017.stdout:1664: finishing write tid 4 to smithi01785587-49 2024-09-21T09:44:59.230 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 879 (ObjNum 640 snap 202 seq_num 640) dirty exists 2024-09-21T09:44:59.230 INFO:tasks.rados.rados.0.smithi017.stdout:1664: left oid 49 (ObjNum 640 snap 202 seq_num 640) 2024-09-21T09:44:59.230 INFO:tasks.rados.rados.0.smithi017.stdout:1666: finishing write tid 1 to smithi01785587-50 2024-09-21T09:44:59.230 INFO:tasks.rados.rados.0.smithi017.stdout:1662: done (4 left) 2024-09-21T09:44:59.230 INFO:tasks.rados.rados.0.smithi017.stdout:1664: done (3 left) 2024-09-21T09:44:59.230 INFO:tasks.rados.rados.0.smithi017.stdout:1665: done (2 left) 2024-09-21T09:44:59.231 INFO:tasks.rados.rados.0.smithi017.stdout:1667: done (1 left) 2024-09-21T09:44:59.231 INFO:tasks.rados.rados.0.smithi017.stdout:1668: snap_remove snap 201 2024-09-21T09:44:59.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:44:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[114123]: 2024-09-21T09:44:59.313+0000 7f702b7e9640 -1 Falling back to public interface 2024-09-21T09:44:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:59 smithi017 ceph-mon[102913]: pgmap v288: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 27/183 objects degraded (14.754%) 2024-09-21T09:44:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:59 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/105765333' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:44:59 smithi017 ceph-mon[102913]: osdmap e474: 8 total, 7 up, 8 in 2024-09-21T09:44:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:59 smithi017 ceph-mon[105151]: pgmap v288: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 27/183 objects degraded (14.754%) 2024-09-21T09:44:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:59 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/105765333' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:44:59 smithi017 ceph-mon[105151]: osdmap e474: 8 total, 7 up, 8 in 2024-09-21T09:44:59.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:59 smithi134 ceph-mon[90085]: pgmap v288: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 27/183 objects degraded (14.754%) 2024-09-21T09:44:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:59 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/105765333' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:44:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:44:59 smithi134 ceph-mon[90085]: osdmap e474: 8 total, 7 up, 8 in 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:1668: done (1 left) 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:1669: rollback oid 19 current snap is 202 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 19 to 198 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:1666: finishing write tid 2 to smithi01785587-50 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:1666: finishing write tid 3 to smithi01785587-50 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:1666: finishing write tid 4 to smithi01785587-50 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 520 (ObjNum 641 snap 202 seq_num 641) dirty exists 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:1666: left oid 50 (ObjNum 641 snap 202 seq_num 641) 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:1666: done (1 left) 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:1670: read oid 11 snap -1 2024-09-21T09:45:00.070 INFO:tasks.rados.rados.0.smithi017.stdout:1670: expect (ObjNum 616 snap 192 seq_num 616) 2024-09-21T09:45:00.071 INFO:tasks.rados.rados.0.smithi017.stdout:1671: delete oid 44 current snap is 202 2024-09-21T09:45:00.072 INFO:tasks.rados.rados.0.smithi017.stdout:1671: done (2 left) 2024-09-21T09:45:00.073 INFO:tasks.rados.rados.0.smithi017.stdout:1672: snap_create 2024-09-21T09:45:00.074 INFO:tasks.rados.rados.0.smithi017.stdout:1669: finishing rollback tid 1 to smithi01785587-19 2024-09-21T09:45:00.074 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 0 (ObjNum 65 snap 0 seq_num 1) dirty dne 2024-09-21T09:45:00.076 INFO:tasks.rados.rados.0.smithi017.stdout:1670: expect (ObjNum 616 snap 192 seq_num 616) 2024-09-21T09:45:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:00 smithi017 ceph-mon[102913]: from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:00 smithi017 ceph-mon[102913]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:00 smithi017 ceph-mon[102913]: osdmap e475: 8 total, 7 up, 8 in 2024-09-21T09:45:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:00 smithi017 ceph-mon[105151]: from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:00 smithi017 ceph-mon[105151]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:00 smithi017 ceph-mon[105151]: osdmap e475: 8 total, 7 up, 8 in 2024-09-21T09:45:00.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:00 smithi134 ceph-mon[90085]: from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:00 smithi134 ceph-mon[90085]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:00 smithi134 ceph-mon[90085]: osdmap e475: 8 total, 7 up, 8 in 2024-09-21T09:45:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:45:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:45:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:01.070 INFO:tasks.rados.rados.0.smithi017.stdout:1669: done (2 left) 2024-09-21T09:45:01.070 INFO:tasks.rados.rados.0.smithi017.stdout:1670: done (1 left) 2024-09-21T09:45:01.070 INFO:tasks.rados.rados.0.smithi017.stdout:1672: done (0 left) 2024-09-21T09:45:01.070 INFO:tasks.rados.rados.0.smithi017.stdout:1673: snap_create 2024-09-21T09:45:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:01 smithi017 ceph-mon[102913]: pgmap v291: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 27/183 objects degraded (14.754%) 2024-09-21T09:45:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:01 smithi017 ceph-mon[102913]: osdmap e476: 8 total, 7 up, 8 in 2024-09-21T09:45:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:01 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:01 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:01 smithi017 ceph-mon[105151]: pgmap v291: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 27/183 objects degraded (14.754%) 2024-09-21T09:45:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:01 smithi017 ceph-mon[105151]: osdmap e476: 8 total, 7 up, 8 in 2024-09-21T09:45:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:01 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:01 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:01.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:01 smithi134 ceph-mon[90085]: pgmap v291: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 93 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 27/183 objects degraded (14.754%) 2024-09-21T09:45:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:01 smithi134 ceph-mon[90085]: osdmap e476: 8 total, 7 up, 8 in 2024-09-21T09:45:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:01 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:01 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:01.925 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:02.071 INFO:tasks.rados.rados.0.smithi017.stdout:1673: done (0 left) 2024-09-21T09:45:02.071 INFO:tasks.rados.rados.0.smithi017.stdout:1674: snap_remove snap 198 2024-09-21T09:45:02.491 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3996832873' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:02.491 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:02 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:02.491 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:02 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:02.491 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:02 smithi017 ceph-mon[102913]: osdmap e477: 8 total, 7 up, 8 in 2024-09-21T09:45:02.491 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3996832873' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:02.492 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:02 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:02.492 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:02 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:02.492 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:02 smithi017 ceph-mon[105151]: osdmap e477: 8 total, 7 up, 8 in 2024-09-21T09:45:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3996832873' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:02 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:02 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:02 smithi134 ceph-mon[90085]: osdmap e477: 8 total, 7 up, 8 in 2024-09-21T09:45:03.073 INFO:tasks.rados.rados.0.smithi017.stdout:1674: done (0 left) 2024-09-21T09:45:03.073 INFO:tasks.rados.rados.0.smithi017.stdout:1675: snap_remove snap 202 2024-09-21T09:45:03.200 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:45:02 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[114123]: 2024-09-21T09:45:02.931+0000 7f702b7e9640 -1 osd.0 466 log_to_monitors true 2024-09-21T09:45:03.580 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[114123]: 2024-09-21T09:45:03.359+0000 7f701f58b640 -1 osd.0 466 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:45:03.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[102913]: pgmap v294: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 100 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 14 op/s; 31/204 objects degraded (15.196%) 2024-09-21T09:45:03.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[102913]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:03.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[102913]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:03.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 31/204 objects degraded (15.196%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:03.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[102913]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:45:03.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[102913]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:45:03.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[102913]: osdmap e478: 8 total, 7 up, 8 in 2024-09-21T09:45:03.581 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[102913]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:45:03.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[105151]: pgmap v294: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 100 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 14 op/s; 31/204 objects degraded (15.196%) 2024-09-21T09:45:03.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[105151]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:03.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[105151]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:03.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 31/204 objects degraded (15.196%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:03.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[105151]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:45:03.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[105151]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:45:03.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[105151]: osdmap e478: 8 total, 7 up, 8 in 2024-09-21T09:45:03.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:03 smithi017 ceph-mon[105151]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:45:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:03 smithi134 ceph-mon[90085]: pgmap v294: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 100 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 14 op/s; 31/204 objects degraded (15.196%) 2024-09-21T09:45:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:03 smithi134 ceph-mon[90085]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:03 smithi134 ceph-mon[90085]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:03 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 31/204 objects degraded (15.196%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:03 smithi134 ceph-mon[90085]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:45:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:03 smithi134 ceph-mon[90085]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:45:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:03 smithi134 ceph-mon[90085]: osdmap e478: 8 total, 7 up, 8 in 2024-09-21T09:45:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:03 smithi134 ceph-mon[90085]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:45:04.079 INFO:tasks.rados.rados.0.smithi017.stdout:1675: done (0 left) 2024-09-21T09:45:04.080 INFO:tasks.rados.rados.0.smithi017.stdout:1676: write oid 25 current snap is 204 2024-09-21T09:45:04.080 INFO:tasks.rados.rados.0.smithi017.stdout:1676: seq_num 644 ranges {423708=634008,1852286=439887,2752172=71536} 2024-09-21T09:45:04.086 INFO:tasks.rados.rados.0.smithi017.stdout:1676: writing smithi01785587-25 from 423708 to 1057716 tid 1 2024-09-21T09:45:04.095 INFO:tasks.rados.rados.0.smithi017.stdout:1676: writing smithi01785587-25 from 1852286 to 2292173 tid 2 2024-09-21T09:45:04.098 INFO:tasks.rados.rados.0.smithi017.stdout:1676: writing smithi01785587-25 from 2752172 to 2823708 tid 3 2024-09-21T09:45:04.099 INFO:tasks.rados.rados.0.smithi017.stdout:1676: finishing write tid 1 to smithi01785587-25 2024-09-21T09:45:04.099 INFO:tasks.rados.rados.0.smithi017.stdout:1676: finishing write tid 2 to smithi01785587-25 2024-09-21T09:45:04.099 INFO:tasks.rados.rados.0.smithi017.stdout:1677: write oid 31 current snap is 204 2024-09-21T09:45:04.099 INFO:tasks.rados.rados.0.smithi017.stdout:1677: seq_num 645 ranges {440515=709257,1910768=583545} 2024-09-21T09:45:04.105 INFO:tasks.rados.rados.0.smithi017.stdout:1677: writing smithi01785587-31 from 440515 to 1149772 tid 1 2024-09-21T09:45:04.112 INFO:tasks.rados.rados.0.smithi017.stdout:1677: writing smithi01785587-31 from 1910768 to 2494313 tid 2 2024-09-21T09:45:04.112 INFO:tasks.rados.rados.0.smithi017.stdout:1678: snap_create 2024-09-21T09:45:04.113 INFO:tasks.rados.rados.0.smithi017.stdout:1676: finishing write tid 3 to smithi01785587-25 2024-09-21T09:45:04.113 INFO:tasks.rados.rados.0.smithi017.stdout:1676: finishing write tid 4 to smithi01785587-25 2024-09-21T09:45:04.113 INFO:tasks.rados.rados.0.smithi017.stdout:1676: finishing write tid 5 to smithi01785587-25 2024-09-21T09:45:04.113 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 350 (ObjNum 644 snap 204 seq_num 644) dirty exists 2024-09-21T09:45:04.113 INFO:tasks.rados.rados.0.smithi017.stdout:1676: left oid 25 (ObjNum 644 snap 204 seq_num 644) 2024-09-21T09:45:04.113 INFO:tasks.rados.rados.0.smithi017.stdout:1677: finishing write tid 1 to smithi01785587-31 2024-09-21T09:45:04.115 INFO:tasks.rados.rados.0.smithi017.stdout:1677: finishing write tid 2 to smithi01785587-31 2024-09-21T09:45:04.115 INFO:tasks.rados.rados.0.smithi017.stdout:1677: finishing write tid 3 to smithi01785587-31 2024-09-21T09:45:04.115 INFO:tasks.rados.rados.0.smithi017.stdout:1677: finishing write tid 4 to smithi01785587-31 2024-09-21T09:45:04.115 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 838 (ObjNum 645 snap 204 seq_num 645) dirty exists 2024-09-21T09:45:04.116 INFO:tasks.rados.rados.0.smithi017.stdout:1677: left oid 31 (ObjNum 645 snap 204 seq_num 645) 2024-09-21T09:45:04.627 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[102913]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' 2024-09-21T09:45:04.627 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[102913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:45:04.627 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[102913]: osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120] boot 2024-09-21T09:45:04.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[102913]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:45:04.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:45:04.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[105151]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' 2024-09-21T09:45:04.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[105151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:45:04.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[105151]: osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120] boot 2024-09-21T09:45:04.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[105151]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:45:04.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:45:04.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:04 smithi134 ceph-mon[90085]: from='osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120]' entity='osd.0' 2024-09-21T09:45:04.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:04 smithi134 ceph-mon[90085]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:45:04.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:04 smithi134 ceph-mon[90085]: osd.0 [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120] boot 2024-09-21T09:45:04.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:04 smithi134 ceph-mon[90085]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:45:04.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:04 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:45:04.831 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:04.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:45:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:45:04] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.51.0" 2024-09-21T09:45:05.078 INFO:tasks.rados.rados.0.smithi017.stdout:1676: done (2 left) 2024-09-21T09:45:05.078 INFO:tasks.rados.rados.0.smithi017.stdout:1677: done (1 left) 2024-09-21T09:45:05.078 INFO:tasks.rados.rados.0.smithi017.stdout:1678: done (0 left) 2024-09-21T09:45:05.078 INFO:tasks.rados.rados.0.smithi017.stdout:1679: snap_remove snap 203 2024-09-21T09:45:05.353 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:45:05.120+0000 7f3131349640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:45:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: pgmap v297: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 100 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 14 op/s; 31/204 objects degraded (15.196%) 2024-09-21T09:45:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1848500371' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: pgmap v297: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 100 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 14 op/s; 31/204 objects degraded (15.196%) 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1848500371' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:05.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:05.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:05.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:05.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:45:05.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:05.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:05.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:05.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:05.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: pgmap v297: 89 pgs: 21 active+undersized, 13 active+undersized+degraded, 55 active+clean; 100 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 14 op/s; 31/204 objects degraded (15.196%) 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1848500371' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:05.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:05.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:45:06.078 INFO:tasks.rados.rados.0.smithi017.stdout:1679: done (0 left) 2024-09-21T09:45:06.078 INFO:tasks.rados.rados.0.smithi017.stdout:1680: delete oid 40 current snap is 205 2024-09-21T09:45:06.084 INFO:tasks.rados.rados.0.smithi017.stdout:1680: done (0 left) 2024-09-21T09:45:06.084 INFO:tasks.rados.rados.0.smithi017.stdout:1681: read oid 22 snap -1 2024-09-21T09:45:06.085 INFO:tasks.rados.rados.0.smithi017.stdout:1681: expect (ObjNum 628 snap 198 seq_num 628) 2024-09-21T09:45:06.085 INFO:tasks.rados.rados.0.smithi017.stdout:1682: write oid 35 current snap is 205 2024-09-21T09:45:06.085 INFO:tasks.rados.rados.0.smithi017.stdout:1682: seq_num 647 ranges {474129=576108,1627732=587214,2718088=156041} 2024-09-21T09:45:06.089 INFO:tasks.rados.rados.0.smithi017.stdout:1682: writing smithi01785587-35 from 474129 to 1050237 tid 1 2024-09-21T09:45:06.097 INFO:tasks.rados.rados.0.smithi017.stdout:1682: writing smithi01785587-35 from 1627732 to 2214946 tid 2 2024-09-21T09:45:06.101 INFO:tasks.rados.rados.0.smithi017.stdout:1682: writing smithi01785587-35 from 2718088 to 2874129 tid 3 2024-09-21T09:45:06.101 INFO:tasks.rados.rados.0.smithi017.stdout:1683: delete oid 20 current snap is 205 2024-09-21T09:45:06.101 INFO:tasks.rados.rados.0.smithi017.stdout:1681: expect (ObjNum 628 snap 198 seq_num 628) 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:1682: finishing write tid 1 to smithi01785587-35 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:1682: finishing write tid 2 to smithi01785587-35 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:1682: finishing write tid 3 to smithi01785587-35 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:1682: finishing write tid 4 to smithi01785587-35 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:1682: finishing write tid 5 to smithi01785587-35 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 207 (ObjNum 647 snap 205 seq_num 647) dirty exists 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:1682: left oid 35 (ObjNum 647 snap 205 seq_num 647) 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:1681: done (2 left) 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:1682: done (1 left) 2024-09-21T09:45:06.184 INFO:tasks.rados.rados.0.smithi017.stdout:1683: done (0 left) 2024-09-21T09:45:06.185 INFO:tasks.rados.rados.0.smithi017.stdout:1684: write oid 12 current snap is 205 2024-09-21T09:45:06.185 INFO:tasks.rados.rados.0.smithi017.stdout:1684: seq_num 649 ranges {507743=726606,1744696=590883,2772934=134809} 2024-09-21T09:45:06.190 INFO:tasks.rados.rados.0.smithi017.stdout:1684: writing smithi01785587-12 from 507743 to 1234349 tid 1 2024-09-21T09:45:06.196 INFO:tasks.rados.rados.0.smithi017.stdout:1684: writing smithi01785587-12 from 1744696 to 2335579 tid 2 2024-09-21T09:45:06.200 INFO:tasks.rados.rados.0.smithi017.stdout:1684: writing smithi01785587-12 from 2772934 to 2907743 tid 3 2024-09-21T09:45:06.200 INFO:tasks.rados.rados.0.smithi017.stdout:1685: read oid 9 snap -1 2024-09-21T09:45:06.200 INFO:tasks.rados.rados.0.smithi017.stdout:1685: expect deleted 2024-09-21T09:45:06.200 INFO:tasks.rados.rados.0.smithi017.stdout:1686: write oid 23 current snap is 205 2024-09-21T09:45:06.200 INFO:tasks.rados.rados.0.smithi017.stdout:1684: finishing write tid 1 to smithi01785587-12 2024-09-21T09:45:06.200 INFO:tasks.rados.rados.0.smithi017.stdout:1684: finishing write tid 2 to smithi01785587-12 2024-09-21T09:45:06.200 INFO:tasks.rados.rados.0.smithi017.stdout:1686: seq_num 650 ranges {524550=401855,1403178=450894,2516710=407840} 2024-09-21T09:45:06.205 INFO:tasks.rados.rados.0.smithi017.stdout:1686: writing smithi01785587-23 from 524550 to 926405 tid 1 2024-09-21T09:45:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:1686: writing smithi01785587-23 from 1403178 to 1854072 tid 2 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1686: writing smithi01785587-23 from 2516710 to 2924550 tid 3 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1687: read oid 19 snap -1 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1687: expect deleted 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1688: delete oid 21 current snap is 205 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1684: finishing write tid 3 to smithi01785587-12 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1684: finishing write tid 4 to smithi01785587-12 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1684: finishing write tid 5 to smithi01785587-12 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 619 (ObjNum 649 snap 205 seq_num 649) dirty exists 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1684: left oid 12 (ObjNum 649 snap 205 seq_num 649) 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1686: finishing write tid 1 to smithi01785587-23 2024-09-21T09:45:06.217 INFO:tasks.rados.rados.0.smithi017.stdout:1686: finishing write tid 2 to smithi01785587-23 2024-09-21T09:45:06.219 INFO:tasks.rados.rados.0.smithi017.stdout:1684: done (4 left) 2024-09-21T09:45:06.219 INFO:tasks.rados.rados.0.smithi017.stdout:1685: done (3 left) 2024-09-21T09:45:06.219 INFO:tasks.rados.rados.0.smithi017.stdout:1687: done (2 left) 2024-09-21T09:45:06.219 INFO:tasks.rados.rados.0.smithi017.stdout:1688: done (1 left) 2024-09-21T09:45:06.219 INFO:tasks.rados.rados.0.smithi017.stdout:1689: write oid 1 current snap is 205 2024-09-21T09:45:06.219 INFO:tasks.rados.rados.0.smithi017.stdout:1689: seq_num 652 ranges {558164=552353,1803789=454563,2855203=102961} 2024-09-21T09:45:06.226 INFO:tasks.rados.rados.0.smithi017.stdout:1689: writing smithi01785587-1 from 558164 to 1110517 tid 1 2024-09-21T09:45:06.231 INFO:tasks.rados.rados.0.smithi017.stdout:1689: writing smithi01785587-1 from 1803789 to 2258352 tid 2 2024-09-21T09:45:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:1689: writing smithi01785587-1 from 2855203 to 2958164 tid 3 2024-09-21T09:45:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:1690: delete oid 37 current snap is 205 2024-09-21T09:45:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:1686: finishing write tid 3 to smithi01785587-23 2024-09-21T09:45:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:1686: finishing write tid 4 to smithi01785587-23 2024-09-21T09:45:06.236 INFO:tasks.rados.rados.0.smithi017.stdout:1686: finishing write tid 5 to smithi01785587-23 2024-09-21T09:45:06.236 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 775 (ObjNum 650 snap 205 seq_num 650) dirty exists 2024-09-21T09:45:06.236 INFO:tasks.rados.rados.0.smithi017.stdout:1686: left oid 23 (ObjNum 650 snap 205 seq_num 650) 2024-09-21T09:45:06.236 INFO:tasks.rados.rados.0.smithi017.stdout:1689: finishing write tid 1 to smithi01785587-1 2024-09-21T09:45:06.236 INFO:tasks.rados.rados.0.smithi017.stdout:1689: finishing write tid 2 to smithi01785587-1 2024-09-21T09:45:06.237 INFO:tasks.rados.rados.0.smithi017.stdout:1689: finishing write tid 3 to smithi01785587-1 2024-09-21T09:45:06.237 INFO:tasks.rados.rados.0.smithi017.stdout:1686: done (2 left) 2024-09-21T09:45:06.237 INFO:tasks.rados.rados.0.smithi017.stdout:1690: done (1 left) 2024-09-21T09:45:06.237 INFO:tasks.rados.rados.0.smithi017.stdout:1691: read oid 19 snap -1 2024-09-21T09:45:06.237 INFO:tasks.rados.rados.0.smithi017.stdout:1691: expect deleted 2024-09-21T09:45:06.237 INFO:tasks.rados.rados.0.smithi017.stdout:1692: snap_create 2024-09-21T09:45:06.237 INFO:tasks.rados.rados.0.smithi017.stdout:1689: finishing write tid 4 to smithi01785587-1 2024-09-21T09:45:06.238 INFO:tasks.rados.rados.0.smithi017.stdout:1689: finishing write tid 5 to smithi01785587-1 2024-09-21T09:45:06.238 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 510 (ObjNum 652 snap 205 seq_num 652) dirty exists 2024-09-21T09:45:06.238 INFO:tasks.rados.rados.0.smithi017.stdout:1689: left oid 1 (ObjNum 652 snap 205 seq_num 652) 2024-09-21T09:45:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[102913]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[102913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:45:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[102913]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:45:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[102913]: from='client.34425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[102913]: osdmap e481: 8 total, 8 up, 8 in 2024-09-21T09:45:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[105151]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[105151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:45:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[105151]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:45:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[105151]: from='client.34425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:06.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:06 smithi017 ceph-mon[105151]: osdmap e481: 8 total, 8 up, 8 in 2024-09-21T09:45:06.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:06 smithi134 ceph-mon[90085]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:06 smithi134 ceph-mon[90085]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:45:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:06 smithi134 ceph-mon[90085]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:45:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:06 smithi134 ceph-mon[90085]: from='client.34425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:06 smithi134 ceph-mon[90085]: osdmap e481: 8 total, 8 up, 8 in 2024-09-21T09:45:07.080 INFO:tasks.rados.rados.0.smithi017.stdout:1689: done (2 left) 2024-09-21T09:45:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:1691: done (1 left) 2024-09-21T09:45:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:1692: done (0 left) 2024-09-21T09:45:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:1693: read oid 48 snap 205 2024-09-21T09:45:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:1693: expect deleted 2024-09-21T09:45:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:1694: rollback oid 3 current snap is 206 2024-09-21T09:45:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 205 2024-09-21T09:45:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:1695: read oid 15 snap -1 2024-09-21T09:45:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:1695: expect deleted 2024-09-21T09:45:07.081 INFO:tasks.rados.rados.0.smithi017.stdout:1696: delete oid 42 current snap is 206 2024-09-21T09:45:07.084 INFO:tasks.rados.rados.0.smithi017.stdout:1694: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:45:07.084 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:45:07.084 INFO:tasks.rados.rados.0.smithi017.stdout:1693: done (3 left) 2024-09-21T09:45:07.085 INFO:tasks.rados.rados.0.smithi017.stdout:1694: done (2 left) 2024-09-21T09:45:07.085 INFO:tasks.rados.rados.0.smithi017.stdout:1695: done (1 left) 2024-09-21T09:45:07.085 INFO:tasks.rados.rados.0.smithi017.stdout:1696: done (0 left) 2024-09-21T09:45:07.085 INFO:tasks.rados.rados.0.smithi017.stdout:1697: snap_remove snap 204 2024-09-21T09:45:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:07 smithi017 ceph-mon[105151]: pgmap v300: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 596 KiB/s wr, 2 op/s; 12/183 objects degraded (6.557%); 1.1 MiB/s, 1 objects/s recovering 2024-09-21T09:45:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:07 smithi017 ceph-mon[105151]: osdmap e482: 8 total, 8 up, 8 in 2024-09-21T09:45:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:07 smithi017 ceph-mon[102913]: pgmap v300: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 596 KiB/s wr, 2 op/s; 12/183 objects degraded (6.557%); 1.1 MiB/s, 1 objects/s recovering 2024-09-21T09:45:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:07 smithi017 ceph-mon[102913]: osdmap e482: 8 total, 8 up, 8 in 2024-09-21T09:45:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:07 smithi134 ceph-mon[90085]: pgmap v300: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 596 KiB/s wr, 2 op/s; 12/183 objects degraded (6.557%); 1.1 MiB/s, 1 objects/s recovering 2024-09-21T09:45:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:07 smithi134 ceph-mon[90085]: osdmap e482: 8 total, 8 up, 8 in 2024-09-21T09:45:07.766 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:08.081 INFO:tasks.rados.rados.0.smithi017.stdout:1697: done (0 left) 2024-09-21T09:45:08.081 INFO:tasks.rados.rados.0.smithi017.stdout:1698: read oid 48 snap 205 2024-09-21T09:45:08.081 INFO:tasks.rados.rados.0.smithi017.stdout:1698: expect deleted 2024-09-21T09:45:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:1699: read oid 31 snap -1 2024-09-21T09:45:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:1699: expect (ObjNum 645 snap 204 seq_num 645) 2024-09-21T09:45:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:1700: rollback oid 7 current snap is 206 2024-09-21T09:45:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 205 2024-09-21T09:45:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:1701: snap_create 2024-09-21T09:45:08.085 INFO:tasks.rados.rados.0.smithi017.stdout:1700: finishing rollback tid 0 to smithi01785587-7 2024-09-21T09:45:08.085 INFO:tasks.rados.rados.0.smithi017.stdout:1700: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:45:08.085 INFO:tasks.rados.rados.0.smithi017.stdout:1700: finishing rollback tid 2 to smithi01785587-7 2024-09-21T09:45:08.085 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 893 (ObjNum 578 snap 181 seq_num 578) dirty exists 2024-09-21T09:45:08.087 INFO:tasks.rados.rados.0.smithi017.stdout:1699: expect (ObjNum 645 snap 204 seq_num 645) 2024-09-21T09:45:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:08 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1939265115' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:08 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 12/183 objects degraded (6.557%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:08 smithi017 ceph-mon[102913]: osdmap e483: 8 total, 8 up, 8 in 2024-09-21T09:45:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:08 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1939265115' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:08 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 12/183 objects degraded (6.557%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:08 smithi017 ceph-mon[105151]: osdmap e483: 8 total, 8 up, 8 in 2024-09-21T09:45:08.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:08 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1939265115' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:08 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 12/183 objects degraded (6.557%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:08 smithi134 ceph-mon[90085]: osdmap e483: 8 total, 8 up, 8 in 2024-09-21T09:45:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:1698: done (3 left) 2024-09-21T09:45:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:1699: done (2 left) 2024-09-21T09:45:09.084 INFO:tasks.rados.rados.0.smithi017.stdout:1700: done (1 left) 2024-09-21T09:45:09.084 INFO:tasks.rados.rados.0.smithi017.stdout:1701: done (0 left) 2024-09-21T09:45:09.084 INFO:tasks.rados.rados.0.smithi017.stdout:1702: write oid 47 current snap is 207 2024-09-21T09:45:09.084 INFO:tasks.rados.rados.0.smithi017.stdout:1702: seq_num 655 ranges {608585=494453,1579235=601890,2821119=187466} 2024-09-21T09:45:09.089 INFO:tasks.rados.rados.0.smithi017.stdout:1702: writing smithi01785587-47 from 608585 to 1103038 tid 1 2024-09-21T09:45:09.095 INFO:tasks.rados.rados.0.smithi017.stdout:1702: writing smithi01785587-47 from 1579235 to 2181125 tid 2 2024-09-21T09:45:09.100 INFO:tasks.rados.rados.0.smithi017.stdout:1702: writing smithi01785587-47 from 2821119 to 3008585 tid 3 2024-09-21T09:45:09.100 INFO:tasks.rados.rados.0.smithi017.stdout:1703: snap_create 2024-09-21T09:45:09.100 INFO:tasks.rados.rados.0.smithi017.stdout:1702: finishing write tid 1 to smithi01785587-47 2024-09-21T09:45:09.100 INFO:tasks.rados.rados.0.smithi017.stdout:1702: finishing write tid 2 to smithi01785587-47 2024-09-21T09:45:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:1702: finishing write tid 3 to smithi01785587-47 2024-09-21T09:45:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:1702: finishing write tid 4 to smithi01785587-47 2024-09-21T09:45:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:1702: finishing write tid 5 to smithi01785587-47 2024-09-21T09:45:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 630 (ObjNum 655 snap 207 seq_num 655) dirty exists 2024-09-21T09:45:09.102 INFO:tasks.rados.rados.0.smithi017.stdout:1702: left oid 47 (ObjNum 655 snap 207 seq_num 655) 2024-09-21T09:45:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:09 smithi017 ceph-mon[102913]: from='client.34437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:09 smithi017 ceph-mon[102913]: pgmap v303: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 596 KiB/s wr, 2 op/s; 12/183 objects degraded (6.557%); 1.1 MiB/s, 1 objects/s recovering 2024-09-21T09:45:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:09 smithi017 ceph-mon[102913]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:09 smithi017 ceph-mon[102913]: osdmap e484: 8 total, 8 up, 8 in 2024-09-21T09:45:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:09 smithi017 ceph-mon[105151]: from='client.34437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:09 smithi017 ceph-mon[105151]: pgmap v303: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 596 KiB/s wr, 2 op/s; 12/183 objects degraded (6.557%); 1.1 MiB/s, 1 objects/s recovering 2024-09-21T09:45:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:09 smithi017 ceph-mon[105151]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:09 smithi017 ceph-mon[105151]: osdmap e484: 8 total, 8 up, 8 in 2024-09-21T09:45:09.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:09 smithi134 ceph-mon[90085]: from='client.34437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:09 smithi134 ceph-mon[90085]: pgmap v303: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 596 KiB/s wr, 2 op/s; 12/183 objects degraded (6.557%); 1.1 MiB/s, 1 objects/s recovering 2024-09-21T09:45:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:09 smithi134 ceph-mon[90085]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:09 smithi134 ceph-mon[90085]: osdmap e484: 8 total, 8 up, 8 in 2024-09-21T09:45:10.086 INFO:tasks.rados.rados.0.smithi017.stdout:1702: done (1 left) 2024-09-21T09:45:10.086 INFO:tasks.rados.rados.0.smithi017.stdout:1703: done (0 left) 2024-09-21T09:45:10.086 INFO:tasks.rados.rados.0.smithi017.stdout:1704: write oid 37 current snap is 208 2024-09-21T09:45:10.086 INFO:tasks.rados.rados.0.smithi017.stdout:1704: seq_num 656 ranges {625392=569702,1637717=745548,2848542=176850} 2024-09-21T09:45:10.092 INFO:tasks.rados.rados.0.smithi017.stdout:1704: writing smithi01785587-37 from 625392 to 1195094 tid 1 2024-09-21T09:45:10.099 INFO:tasks.rados.rados.0.smithi017.stdout:1704: writing smithi01785587-37 from 1637717 to 2383265 tid 2 2024-09-21T09:45:10.103 INFO:tasks.rados.rados.0.smithi017.stdout:1704: writing smithi01785587-37 from 2848542 to 3025392 tid 3 2024-09-21T09:45:10.103 INFO:tasks.rados.rados.0.smithi017.stdout:1705: snap_remove snap 205 2024-09-21T09:45:10.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:45:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:45:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:10.669 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:11.090 INFO:tasks.rados.rados.0.smithi017.stdout:1705: done (1 left) 2024-09-21T09:45:11.090 INFO:tasks.rados.rados.0.smithi017.stdout:1706: write oid 38 current snap is 208 2024-09-21T09:45:11.090 INFO:tasks.rados.rados.0.smithi017.stdout:1704: finishing write tid 1 to smithi01785587-37 2024-09-21T09:45:11.090 INFO:tasks.rados.rados.0.smithi017.stdout:1706: seq_num 657 ranges {642199=644951,1696199=605559,2992318=49881} 2024-09-21T09:45:11.097 INFO:tasks.rados.rados.0.smithi017.stdout:1706: writing smithi01785587-38 from 642199 to 1287150 tid 1 2024-09-21T09:45:11.103 INFO:tasks.rados.rados.0.smithi017.stdout:1706: writing smithi01785587-38 from 1696199 to 2301758 tid 2 2024-09-21T09:45:11.108 INFO:tasks.rados.rados.0.smithi017.stdout:1706: writing smithi01785587-38 from 2992318 to 3042199 tid 3 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1707: read oid 9 snap -1 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1707: expect deleted 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1708: read oid 4 snap -1 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1708: expect (ObjNum 608 snap 191 seq_num 608) 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1709: delete oid 47 current snap is 208 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1704: finishing write tid 2 to smithi01785587-37 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1704: finishing write tid 3 to smithi01785587-37 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1704: finishing write tid 4 to smithi01785587-37 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1704: finishing write tid 5 to smithi01785587-37 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 865 (ObjNum 656 snap 208 seq_num 656) dirty exists 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1704: left oid 37 (ObjNum 656 snap 208 seq_num 656) 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1706: finishing write tid 1 to smithi01785587-38 2024-09-21T09:45:11.109 INFO:tasks.rados.rados.0.smithi017.stdout:1706: finishing write tid 2 to smithi01785587-38 2024-09-21T09:45:11.110 INFO:tasks.rados.rados.0.smithi017.stdout:1704: done (4 left) 2024-09-21T09:45:11.110 INFO:tasks.rados.rados.0.smithi017.stdout:1707: done (3 left) 2024-09-21T09:45:11.110 INFO:tasks.rados.rados.0.smithi017.stdout:1709: done (2 left) 2024-09-21T09:45:11.110 INFO:tasks.rados.rados.0.smithi017.stdout:1710: write oid 12 current snap is 208 2024-09-21T09:45:11.110 INFO:tasks.rados.rados.0.smithi017.stdout:1710: seq_num 659 ranges {675813=795449,2096810=609228} 2024-09-21T09:45:11.118 INFO:tasks.rados.rados.0.smithi017.stdout:1710: writing smithi01785587-12 from 675813 to 1471262 tid 1 2024-09-21T09:45:11.126 INFO:tasks.rados.rados.0.smithi017.stdout:1710: writing smithi01785587-12 from 2096810 to 2706038 tid 2 2024-09-21T09:45:11.126 INFO:tasks.rados.rados.0.smithi017.stdout:1711: delete oid 45 current snap is 208 2024-09-21T09:45:11.126 INFO:tasks.rados.rados.0.smithi017.stdout:1706: finishing write tid 3 to smithi01785587-38 2024-09-21T09:45:11.126 INFO:tasks.rados.rados.0.smithi017.stdout:1706: finishing write tid 4 to smithi01785587-38 2024-09-21T09:45:11.126 INFO:tasks.rados.rados.0.smithi017.stdout:1706: finishing write tid 5 to smithi01785587-38 2024-09-21T09:45:11.126 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 527 (ObjNum 657 snap 208 seq_num 657) dirty exists 2024-09-21T09:45:11.126 INFO:tasks.rados.rados.0.smithi017.stdout:1706: left oid 38 (ObjNum 657 snap 208 seq_num 657) 2024-09-21T09:45:11.126 INFO:tasks.rados.rados.0.smithi017.stdout:1708: expect (ObjNum 608 snap 191 seq_num 608) 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1710: finishing write tid 1 to smithi01785587-12 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1710: finishing write tid 2 to smithi01785587-12 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1710: finishing write tid 3 to smithi01785587-12 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1710: finishing write tid 4 to smithi01785587-12 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 640 (ObjNum 659 snap 208 seq_num 659) dirty exists 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1710: left oid 12 (ObjNum 659 snap 208 seq_num 659) 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1706: done (3 left) 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1708: done (2 left) 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1710: done (1 left) 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1711: done (0 left) 2024-09-21T09:45:11.201 INFO:tasks.rados.rados.0.smithi017.stdout:1712: snap_create 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[102913]: pgmap v306: 89 pgs: 89 active+clean; 90 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[102913]: osdmap e485: 8 total, 8 up, 8 in 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2131967172' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/183 objects degraded (6.557%), 4 pgs degraded) 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[105151]: pgmap v306: 89 pgs: 89 active+clean; 90 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[105151]: osdmap e485: 8 total, 8 up, 8 in 2024-09-21T09:45:11.231 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:11.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:11.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2131967172' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:11.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/183 objects degraded (6.557%), 4 pgs degraded) 2024-09-21T09:45:11.232 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:11 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T09:45:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:11 smithi134 ceph-mon[90085]: pgmap v306: 89 pgs: 89 active+clean; 90 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-21T09:45:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:11 smithi134 ceph-mon[90085]: osdmap e485: 8 total, 8 up, 8 in 2024-09-21T09:45:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:11 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2131967172' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:11.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:11 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/183 objects degraded (6.557%), 4 pgs degraded) 2024-09-21T09:45:11.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:11 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T09:45:12.096 INFO:tasks.rados.rados.0.smithi017.stdout:1712: done (0 left) 2024-09-21T09:45:12.096 INFO:tasks.rados.rados.0.smithi017.stdout:1713: snap_remove snap 207 2024-09-21T09:45:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:12 smithi017 ceph-mon[102913]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:12 smithi017 ceph-mon[102913]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:12 smithi017 ceph-mon[102913]: osdmap e486: 8 total, 8 up, 8 in 2024-09-21T09:45:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:12 smithi017 ceph-mon[105151]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:12 smithi017 ceph-mon[105151]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:12 smithi017 ceph-mon[105151]: osdmap e486: 8 total, 8 up, 8 in 2024-09-21T09:45:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:12 smithi134 ceph-mon[90085]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:12 smithi134 ceph-mon[90085]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:12 smithi134 ceph-mon[90085]: osdmap e486: 8 total, 8 up, 8 in 2024-09-21T09:45:13.065 INFO:tasks.rados.rados.0.smithi017.stdout:1713: done (0 left) 2024-09-21T09:45:13.065 INFO:tasks.rados.rados.0.smithi017.stdout:1714: delete oid 29 current snap is 209 2024-09-21T09:45:13.070 INFO:tasks.rados.rados.0.smithi017.stdout:1714: done (0 left) 2024-09-21T09:45:13.070 INFO:tasks.rados.rados.0.smithi017.stdout:1715: rollback oid 42 current snap is 209 2024-09-21T09:45:13.070 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 206 2024-09-21T09:45:13.070 INFO:tasks.rados.rados.0.smithi017.stdout:1716: write oid 17 current snap is 209 2024-09-21T09:45:13.070 INFO:tasks.rados.rados.0.smithi017.stdout:1716: seq_num 662 ranges {726234=737549,1988609=756555} 2024-09-21T09:45:13.079 INFO:tasks.rados.rados.0.smithi017.stdout:1716: writing smithi01785587-17 from 726234 to 1463783 tid 1 2024-09-21T09:45:13.087 INFO:tasks.rados.rados.0.smithi017.stdout:1716: writing smithi01785587-17 from 1988609 to 2745164 tid 2 2024-09-21T09:45:13.087 INFO:tasks.rados.rados.0.smithi017.stdout:1717: write oid 23 current snap is 209 2024-09-21T09:45:13.087 INFO:tasks.rados.rados.0.smithi017.stdout:1715: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:45:13.087 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:45:13.087 INFO:tasks.rados.rados.0.smithi017.stdout:1716: finishing write tid 1 to smithi01785587-17 2024-09-21T09:45:13.087 INFO:tasks.rados.rados.0.smithi017.stdout:1717: seq_num 663 ranges {743041=412798,1930738=616566,3040503=102538} 2024-09-21T09:45:13.094 INFO:tasks.rados.rados.0.smithi017.stdout:1717: writing smithi01785587-23 from 743041 to 1155839 tid 1 2024-09-21T09:45:13.102 INFO:tasks.rados.rados.0.smithi017.stdout:1717: writing smithi01785587-23 from 1930738 to 2547304 tid 2 2024-09-21T09:45:13.106 INFO:tasks.rados.rados.0.smithi017.stdout:1717: writing smithi01785587-23 from 3040503 to 3143041 tid 3 2024-09-21T09:45:13.106 INFO:tasks.rados.rados.0.smithi017.stdout:1715: done (2 left) 2024-09-21T09:45:13.106 INFO:tasks.rados.rados.0.smithi017.stdout:1718: write oid 47 current snap is 209 2024-09-21T09:45:13.106 INFO:tasks.rados.rados.0.smithi017.stdout:1716: finishing write tid 2 to smithi01785587-17 2024-09-21T09:45:13.106 INFO:tasks.rados.rados.0.smithi017.stdout:1716: finishing write tid 3 to smithi01785587-17 2024-09-21T09:45:13.106 INFO:tasks.rados.rados.0.smithi017.stdout:1718: seq_num 664 ranges {759848=488047,1989220=760224} 2024-09-21T09:45:13.112 INFO:tasks.rados.rados.0.smithi017.stdout:1718: writing smithi01785587-47 from 759848 to 1247895 tid 1 2024-09-21T09:45:13.121 INFO:tasks.rados.rados.0.smithi017.stdout:1718: writing smithi01785587-47 from 1989220 to 2749444 tid 2 2024-09-21T09:45:13.121 INFO:tasks.rados.rados.0.smithi017.stdout:1719: read oid 43 snap -1 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:1719: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:1720: read oid 12 snap -1 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:1720: expect (ObjNum 659 snap 208 seq_num 659) 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:1721: write oid 3 current snap is 209 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:1716: finishing write tid 4 to smithi01785587-17 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 581 (ObjNum 662 snap 209 seq_num 662) dirty exists 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:1716: left oid 17 (ObjNum 662 snap 209 seq_num 662) 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:1717: finishing write tid 1 to smithi01785587-23 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:1717: finishing write tid 2 to smithi01785587-23 2024-09-21T09:45:13.122 INFO:tasks.rados.rados.0.smithi017.stdout:1721: seq_num 665 ranges {776655=563296,2047702=620235,3095349=81306} 2024-09-21T09:45:13.129 INFO:tasks.rados.rados.0.smithi017.stdout:1721: writing smithi01785587-3 from 776655 to 1339951 tid 1 2024-09-21T09:45:13.137 INFO:tasks.rados.rados.0.smithi017.stdout:1721: writing smithi01785587-3 from 2047702 to 2667937 tid 2 2024-09-21T09:45:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:1721: writing smithi01785587-3 from 3095349 to 3176655 tid 3 2024-09-21T09:45:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:1716: done (5 left) 2024-09-21T09:45:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:1722: snap_remove snap 208 2024-09-21T09:45:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:13 smithi017 ceph-mon[102913]: pgmap v309: 89 pgs: 89 active+clean; 92 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.5 MiB/s wr, 13 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-21T09:45:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:13 smithi017 ceph-mon[102913]: osdmap e487: 8 total, 8 up, 8 in 2024-09-21T09:45:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:13 smithi017 ceph-mon[102913]: osdmap e488: 8 total, 8 up, 8 in 2024-09-21T09:45:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:13 smithi017 ceph-mon[105151]: pgmap v309: 89 pgs: 89 active+clean; 92 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.5 MiB/s wr, 13 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-21T09:45:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:13 smithi017 ceph-mon[105151]: osdmap e487: 8 total, 8 up, 8 in 2024-09-21T09:45:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:13 smithi017 ceph-mon[105151]: osdmap e488: 8 total, 8 up, 8 in 2024-09-21T09:45:13.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:13 smithi134 ceph-mon[90085]: pgmap v309: 89 pgs: 89 active+clean; 92 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.5 MiB/s wr, 13 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-21T09:45:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:13 smithi134 ceph-mon[90085]: osdmap e487: 8 total, 8 up, 8 in 2024-09-21T09:45:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:13 smithi134 ceph-mon[90085]: osdmap e488: 8 total, 8 up, 8 in 2024-09-21T09:45:13.590 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:14.101 INFO:tasks.rados.rados.0.smithi017.stdout:1722: done (5 left) 2024-09-21T09:45:14.101 INFO:tasks.rados.rados.0.smithi017.stdout:1723: snap_remove snap 206 2024-09-21T09:45:14.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/784587358' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:14.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/784587358' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:14.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/784587358' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1723: done (5 left) 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1724: rollback oid 13 current snap is 209 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1724: done (5 left) 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1725: read oid 44 snap -1 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1725: expect deleted 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1726: delete oid 41 current snap is 209 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1717: finishing write tid 3 to smithi01785587-23 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1717: finishing write tid 4 to smithi01785587-23 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1717: finishing write tid 5 to smithi01785587-23 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 780 (ObjNum 663 snap 209 seq_num 663) dirty exists 2024-09-21T09:45:15.105 INFO:tasks.rados.rados.0.smithi017.stdout:1717: left oid 23 (ObjNum 663 snap 209 seq_num 663) 2024-09-21T09:45:15.106 INFO:tasks.rados.rados.0.smithi017.stdout:1718: finishing write tid 1 to smithi01785587-47 2024-09-21T09:45:15.106 INFO:tasks.rados.rados.0.smithi017.stdout:1718: finishing write tid 2 to smithi01785587-47 2024-09-21T09:45:15.106 INFO:tasks.rados.rados.0.smithi017.stdout:1718: finishing write tid 3 to smithi01785587-47 2024-09-21T09:45:15.106 INFO:tasks.rados.rados.0.smithi017.stdout:1718: finishing write tid 4 to smithi01785587-47 2024-09-21T09:45:15.106 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 647 (ObjNum 664 snap 209 seq_num 664) dirty exists 2024-09-21T09:45:15.106 INFO:tasks.rados.rados.0.smithi017.stdout:1718: left oid 47 (ObjNum 664 snap 209 seq_num 664) 2024-09-21T09:45:15.106 INFO:tasks.rados.rados.0.smithi017.stdout:1720: expect (ObjNum 659 snap 208 seq_num 659) 2024-09-21T09:45:15.106 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:45:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:45:14] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.51.0" 2024-09-21T09:45:15.328 INFO:tasks.rados.rados.0.smithi017.stdout:1719: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:45:15.439 INFO:tasks.rados.rados.0.smithi017.stdout:1721: finishing write tid 1 to smithi01785587-3 2024-09-21T09:45:15.439 INFO:tasks.rados.rados.0.smithi017.stdout:1721: finishing write tid 2 to smithi01785587-3 2024-09-21T09:45:15.439 INFO:tasks.rados.rados.0.smithi017.stdout:1717: done (6 left) 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1718: done (5 left) 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1719: done (4 left) 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1720: done (3 left) 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1726: done (2 left) 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1727: read oid 36 snap -1 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1727: expect (ObjNum 614 snap 192 seq_num 614) 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1728: read oid 10 snap -1 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1728: expect deleted 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1729: read oid 38 snap -1 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1729: expect (ObjNum 657 snap 208 seq_num 657) 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1730: write oid 14 current snap is 209 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1721: finishing write tid 3 to smithi01785587-3 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1721: finishing write tid 4 to smithi01785587-3 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:1721: finishing write tid 5 to smithi01785587-3 2024-09-21T09:45:15.440 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 843 (ObjNum 665 snap 209 seq_num 665) dirty exists 2024-09-21T09:45:15.441 INFO:tasks.rados.rados.0.smithi017.stdout:1721: left oid 3 (ObjNum 665 snap 209 seq_num 665) 2024-09-21T09:45:15.441 INFO:tasks.rados.rados.0.smithi017.stdout:1730: seq_num 667 ranges {410269=713794,1648313=507551,2917489=292780} 2024-09-21T09:45:15.446 INFO:tasks.rados.rados.0.smithi017.stdout:1730: writing smithi01785587-14 from 410269 to 1124063 tid 1 2024-09-21T09:45:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:15 smithi017 ceph-mon[102913]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:15 smithi017 ceph-mon[102913]: pgmap v312: 89 pgs: 89 active+clean; 91 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2024-09-21T09:45:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:15 smithi017 ceph-mon[102913]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:15 smithi017 ceph-mon[102913]: osdmap e489: 8 total, 8 up, 8 in 2024-09-21T09:45:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:15 smithi017 ceph-mon[105151]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:15 smithi017 ceph-mon[105151]: pgmap v312: 89 pgs: 89 active+clean; 91 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2024-09-21T09:45:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:15 smithi017 ceph-mon[105151]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:15 smithi017 ceph-mon[105151]: osdmap e489: 8 total, 8 up, 8 in 2024-09-21T09:45:15.452 INFO:tasks.rados.rados.0.smithi017.stdout:1730: writing smithi01785587-14 from 1648313 to 2155864 tid 2 2024-09-21T09:45:15.458 INFO:tasks.rados.rados.0.smithi017.stdout:1730: writing smithi01785587-14 from 2917489 to 3210269 tid 3 2024-09-21T09:45:15.458 INFO:tasks.rados.rados.0.smithi017.stdout:1721: done (5 left) 2024-09-21T09:45:15.458 INFO:tasks.rados.rados.0.smithi017.stdout:1731: read oid 19 snap -1 2024-09-21T09:45:15.458 INFO:tasks.rados.rados.0.smithi017.stdout:1731: expect deleted 2024-09-21T09:45:15.458 INFO:tasks.rados.rados.0.smithi017.stdout:1732: delete oid 50 current snap is 209 2024-09-21T09:45:15.459 INFO:tasks.rados.rados.0.smithi017.stdout:1727: expect (ObjNum 614 snap 192 seq_num 614) 2024-09-21T09:45:15.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:15 smithi134 ceph-mon[90085]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:15 smithi134 ceph-mon[90085]: pgmap v312: 89 pgs: 89 active+clean; 91 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2024-09-21T09:45:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:15 smithi134 ceph-mon[90085]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:15 smithi134 ceph-mon[90085]: osdmap e489: 8 total, 8 up, 8 in 2024-09-21T09:45:15.567 INFO:tasks.rados.rados.0.smithi017.stdout:1729: expect (ObjNum 657 snap 208 seq_num 657) 2024-09-21T09:45:15.795 INFO:tasks.rados.rados.0.smithi017.stdout:1730: finishing write tid 1 to smithi01785587-14 2024-09-21T09:45:15.795 INFO:tasks.rados.rados.0.smithi017.stdout:1730: finishing write tid 2 to smithi01785587-14 2024-09-21T09:45:15.795 INFO:tasks.rados.rados.0.smithi017.stdout:1730: finishing write tid 3 to smithi01785587-14 2024-09-21T09:45:15.795 INFO:tasks.rados.rados.0.smithi017.stdout:1730: finishing write tid 4 to smithi01785587-14 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:1725: done (6 left) 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:1727: done (5 left) 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:1728: done (4 left) 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:1729: done (3 left) 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:1731: done (2 left) 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:1732: done (1 left) 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:1733: delete oid 4 current snap is 209 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:1730: finishing write tid 5 to smithi01785587-14 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 847 (ObjNum 667 snap 209 seq_num 667) dirty exists 2024-09-21T09:45:15.796 INFO:tasks.rados.rados.0.smithi017.stdout:1730: left oid 14 (ObjNum 667 snap 209 seq_num 667) 2024-09-21T09:45:15.797 INFO:tasks.rados.rados.0.smithi017.stdout:1730: done (1 left) 2024-09-21T09:45:15.797 INFO:tasks.rados.rados.0.smithi017.stdout:1733: done (0 left) 2024-09-21T09:45:15.797 INFO:tasks.rados.rados.0.smithi017.stdout:1734: delete oid 27 current snap is 209 2024-09-21T09:45:15.798 INFO:tasks.rados.rados.0.smithi017.stdout:1734: done (0 left) 2024-09-21T09:45:15.799 INFO:tasks.rados.rados.0.smithi017.stdout:1735: read oid 13 snap -1 2024-09-21T09:45:15.799 INFO:tasks.rados.rados.0.smithi017.stdout:1735: expect (ObjNum 632 snap 200 seq_num 632) 2024-09-21T09:45:15.799 INFO:tasks.rados.rados.0.smithi017.stdout:1736: snap_create 2024-09-21T09:45:15.802 INFO:tasks.rados.rados.0.smithi017.stdout:1735: expect (ObjNum 632 snap 200 seq_num 632) 2024-09-21T09:45:16.277 INFO:tasks.rados.rados.0.smithi017.stdout:1735: done (1 left) 2024-09-21T09:45:16.278 INFO:tasks.rados.rados.0.smithi017.stdout:1736: done (0 left) 2024-09-21T09:45:16.278 INFO:tasks.rados.rados.0.smithi017.stdout:1737: snap_create 2024-09-21T09:45:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:16 smithi017 ceph-mon[105151]: osdmap e490: 8 total, 8 up, 8 in 2024-09-21T09:45:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:16 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:16.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:16 smithi017 ceph-mon[102913]: osdmap e490: 8 total, 8 up, 8 in 2024-09-21T09:45:16.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:16 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:16.476 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:16.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:16 smithi134 ceph-mon[90085]: osdmap e490: 8 total, 8 up, 8 in 2024-09-21T09:45:16.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:16 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:17.278 INFO:tasks.rados.rados.0.smithi017.stdout:1737: done (0 left) 2024-09-21T09:45:17.278 INFO:tasks.rados.rados.0.smithi017.stdout:1738: write oid 50 current snap is 211 2024-09-21T09:45:17.278 INFO:tasks.rados.rados.0.smithi017.stdout:1738: seq_num 671 ranges {477497=731143,1998594=514889,3143534=133963} 2024-09-21T09:45:17.284 INFO:tasks.rados.rados.0.smithi017.stdout:1738: writing smithi01785587-50 from 477497 to 1208640 tid 1 2024-09-21T09:45:17.292 INFO:tasks.rados.rados.0.smithi017.stdout:1738: writing smithi01785587-50 from 1998594 to 2513483 tid 2 2024-09-21T09:45:17.297 INFO:tasks.rados.rados.0.smithi017.stdout:1738: writing smithi01785587-50 from 3143534 to 3277497 tid 3 2024-09-21T09:45:17.297 INFO:tasks.rados.rados.0.smithi017.stdout:1739: read oid 40 snap -1 2024-09-21T09:45:17.297 INFO:tasks.rados.rados.0.smithi017.stdout:1739: expect deleted 2024-09-21T09:45:17.297 INFO:tasks.rados.rados.0.smithi017.stdout:1740: snap_create 2024-09-21T09:45:17.297 INFO:tasks.rados.rados.0.smithi017.stdout:1738: finishing write tid 1 to smithi01785587-50 2024-09-21T09:45:17.297 INFO:tasks.rados.rados.0.smithi017.stdout:1738: finishing write tid 2 to smithi01785587-50 2024-09-21T09:45:17.299 INFO:tasks.rados.rados.0.smithi017.stdout:1738: finishing write tid 3 to smithi01785587-50 2024-09-21T09:45:17.299 INFO:tasks.rados.rados.0.smithi017.stdout:1738: finishing write tid 4 to smithi01785587-50 2024-09-21T09:45:17.299 INFO:tasks.rados.rados.0.smithi017.stdout:1738: finishing write tid 5 to smithi01785587-50 2024-09-21T09:45:17.299 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 536 (ObjNum 671 snap 211 seq_num 671) dirty exists 2024-09-21T09:45:17.299 INFO:tasks.rados.rados.0.smithi017.stdout:1738: left oid 50 (ObjNum 671 snap 211 seq_num 671) 2024-09-21T09:45:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:17 smithi017 ceph-mon[102913]: pgmap v315: 89 pgs: 89 active+clean; 86 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.6 MiB/s wr, 22 op/s 2024-09-21T09:45:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/772556152' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:17 smithi017 ceph-mon[102913]: osdmap e491: 8 total, 8 up, 8 in 2024-09-21T09:45:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:17 smithi017 ceph-mon[102913]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:17 smithi017 ceph-mon[105151]: pgmap v315: 89 pgs: 89 active+clean; 86 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.6 MiB/s wr, 22 op/s 2024-09-21T09:45:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/772556152' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:17 smithi017 ceph-mon[105151]: osdmap e491: 8 total, 8 up, 8 in 2024-09-21T09:45:17.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:17 smithi017 ceph-mon[105151]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:17 smithi134 ceph-mon[90085]: pgmap v315: 89 pgs: 89 active+clean; 86 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.6 MiB/s wr, 22 op/s 2024-09-21T09:45:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/772556152' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:17 smithi134 ceph-mon[90085]: osdmap e491: 8 total, 8 up, 8 in 2024-09-21T09:45:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:17 smithi134 ceph-mon[90085]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.068 INFO:tasks.rados.rados.0.smithi017.stdout:1738: done (2 left) 2024-09-21T09:45:18.068 INFO:tasks.rados.rados.0.smithi017.stdout:1739: done (1 left) 2024-09-21T09:45:18.068 INFO:tasks.rados.rados.0.smithi017.stdout:1740: done (0 left) 2024-09-21T09:45:18.069 INFO:tasks.rados.rados.0.smithi017.stdout:1741: read oid 25 snap -1 2024-09-21T09:45:18.069 INFO:tasks.rados.rados.0.smithi017.stdout:1741: expect (ObjNum 644 snap 204 seq_num 644) 2024-09-21T09:45:18.069 INFO:tasks.rados.rados.0.smithi017.stdout:1742: read oid 1 snap -1 2024-09-21T09:45:18.069 INFO:tasks.rados.rados.0.smithi017.stdout:1742: expect (ObjNum 652 snap 205 seq_num 652) 2024-09-21T09:45:18.069 INFO:tasks.rados.rados.0.smithi017.stdout:1743: write oid 29 current snap is 212 2024-09-21T09:45:18.069 INFO:tasks.rados.rados.0.smithi017.stdout:1743: seq_num 672 ranges {494304=406392,1540723=774900,2770957=523347} 2024-09-21T09:45:18.074 INFO:tasks.rados.rados.0.smithi017.stdout:1743: writing smithi01785587-29 from 494304 to 900696 tid 1 2024-09-21T09:45:18.082 INFO:tasks.rados.rados.0.smithi017.stdout:1743: writing smithi01785587-29 from 1540723 to 2315623 tid 2 2024-09-21T09:45:18.088 INFO:tasks.rados.rados.0.smithi017.stdout:1743: writing smithi01785587-29 from 2770957 to 3294304 tid 3 2024-09-21T09:45:18.088 INFO:tasks.rados.rados.0.smithi017.stdout:1744: delete oid 3 current snap is 212 2024-09-21T09:45:18.088 INFO:tasks.rados.rados.0.smithi017.stdout:1742: expect (ObjNum 652 snap 205 seq_num 652) 2024-09-21T09:45:18.338 INFO:tasks.rados.rados.0.smithi017.stdout:1741: expect (ObjNum 644 snap 204 seq_num 644) 2024-09-21T09:45:18.434 INFO:tasks.rados.rados.0.smithi017.stdout:1743: finishing write tid 1 to smithi01785587-29 2024-09-21T09:45:18.434 INFO:tasks.rados.rados.0.smithi017.stdout:1743: finishing write tid 2 to smithi01785587-29 2024-09-21T09:45:18.434 INFO:tasks.rados.rados.0.smithi017.stdout:1743: finishing write tid 3 to smithi01785587-29 2024-09-21T09:45:18.434 INFO:tasks.rados.rados.0.smithi017.stdout:1743: finishing write tid 4 to smithi01785587-29 2024-09-21T09:45:18.434 INFO:tasks.rados.rados.0.smithi017.stdout:1743: finishing write tid 5 to smithi01785587-29 2024-09-21T09:45:18.434 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 361 (ObjNum 672 snap 212 seq_num 672) dirty exists 2024-09-21T09:45:18.434 INFO:tasks.rados.rados.0.smithi017.stdout:1743: left oid 29 (ObjNum 672 snap 212 seq_num 672) 2024-09-21T09:45:18.435 INFO:tasks.rados.rados.0.smithi017.stdout:1741: done (3 left) 2024-09-21T09:45:18.435 INFO:tasks.rados.rados.0.smithi017.stdout:1742: done (2 left) 2024-09-21T09:45:18.435 INFO:tasks.rados.rados.0.smithi017.stdout:1743: done (1 left) 2024-09-21T09:45:18.435 INFO:tasks.rados.rados.0.smithi017.stdout:1744: done (0 left) 2024-09-21T09:45:18.435 INFO:tasks.rados.rados.0.smithi017.stdout:1745: delete oid 44 current snap is 212 2024-09-21T09:45:18.435 INFO:tasks.rados.rados.0.smithi017.stdout:1745: done (0 left) 2024-09-21T09:45:18.436 INFO:tasks.rados.rados.0.smithi017.stdout:1746: write oid 25 current snap is 212 2024-09-21T09:45:18.436 INFO:tasks.rados.rados.0.smithi017.stdout:1746: seq_num 675 ranges {544725=632139,1716169=522227,2736873=512491} 2024-09-21T09:45:18.442 INFO:tasks.rados.rados.0.smithi017.stdout:1746: writing smithi01785587-25 from 544725 to 1176864 tid 1 2024-09-21T09:45:18.448 INFO:tasks.rados.rados.0.smithi017.stdout:1746: writing smithi01785587-25 from 1716169 to 2238396 tid 2 2024-09-21T09:45:18.453 INFO:tasks.rados.rados.0.smithi017.stdout:1746: writing smithi01785587-25 from 2736873 to 3249364 tid 3 2024-09-21T09:45:18.453 INFO:tasks.rados.rados.0.smithi017.stdout:1747: snap_remove snap 211 2024-09-21T09:45:18.453 INFO:tasks.rados.rados.0.smithi017.stdout:1746: finishing write tid 1 to smithi01785587-25 2024-09-21T09:45:18.453 INFO:tasks.rados.rados.0.smithi017.stdout:1746: finishing write tid 2 to smithi01785587-25 2024-09-21T09:45:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:18 smithi017 ceph-mon[105151]: from='client.34497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:18 smithi017 ceph-mon[105151]: osdmap e492: 8 total, 8 up, 8 in 2024-09-21T09:45:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:18 smithi017 ceph-mon[105151]: osdmap e493: 8 total, 8 up, 8 in 2024-09-21T09:45:18.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:18 smithi017 ceph-mon[102913]: from='client.34497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:18 smithi017 ceph-mon[102913]: osdmap e492: 8 total, 8 up, 8 in 2024-09-21T09:45:18.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:18 smithi017 ceph-mon[102913]: osdmap e493: 8 total, 8 up, 8 in 2024-09-21T09:45:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:18 smithi134 ceph-mon[90085]: from='client.34497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:18 smithi134 ceph-mon[90085]: osdmap e492: 8 total, 8 up, 8 in 2024-09-21T09:45:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:18 smithi134 ceph-mon[90085]: osdmap e493: 8 total, 8 up, 8 in 2024-09-21T09:45:19.289 INFO:tasks.rados.rados.0.smithi017.stdout:1747: done (1 left) 2024-09-21T09:45:19.289 INFO:tasks.rados.rados.0.smithi017.stdout:1748: snap_remove snap 210 2024-09-21T09:45:19.370 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:19 smithi017 ceph-mon[105151]: pgmap v318: 89 pgs: 89 active+clean; 86 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-21T09:45:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:19 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3644674050' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:19 smithi017 ceph-mon[102913]: pgmap v318: 89 pgs: 89 active+clean; 86 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-21T09:45:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:19 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3644674050' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:19.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:19 smithi134 ceph-mon[90085]: pgmap v318: 89 pgs: 89 active+clean; 86 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-21T09:45:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:19 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3644674050' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:20.295 INFO:tasks.rados.rados.0.smithi017.stdout:1748: done (1 left) 2024-09-21T09:45:20.295 INFO:tasks.rados.rados.0.smithi017.stdout:1749: snap_remove snap 209 2024-09-21T09:45:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:20 smithi134 ceph-mon[90085]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:45:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:20 smithi134 ceph-mon[90085]: from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:20 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:45:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:45:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:45:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:20.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:20 smithi017 ceph-mon[102913]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:45:20.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:20 smithi017 ceph-mon[102913]: from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:20.580 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:45:20.580 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:20 smithi017 ceph-mon[105151]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:45:20.580 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:20 smithi017 ceph-mon[105151]: from='client.34509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:20.580 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:45:21.295 INFO:tasks.rados.rados.0.smithi017.stdout:1749: done (1 left) 2024-09-21T09:45:21.295 INFO:tasks.rados.rados.0.smithi017.stdout:1750: write oid 38 current snap is 212 2024-09-21T09:45:21.295 INFO:tasks.rados.rados.0.smithi017.stdout:1750: seq_num 676 ranges {561532=707388,2058298=782238} 2024-09-21T09:45:21.302 INFO:tasks.rados.rados.0.smithi017.stdout:1750: writing smithi01785587-38 from 561532 to 1268920 tid 1 2024-09-21T09:45:21.312 INFO:tasks.rados.rados.0.smithi017.stdout:1750: writing smithi01785587-38 from 2058298 to 2840536 tid 2 2024-09-21T09:45:21.312 INFO:tasks.rados.rados.0.smithi017.stdout:1751: rollback oid 29 current snap is 212 2024-09-21T09:45:21.312 INFO:tasks.rados.rados.0.smithi017.stdout:1751: done (2 left) 2024-09-21T09:45:21.312 INFO:tasks.rados.rados.0.smithi017.stdout:1752: write oid 34 current snap is 212 2024-09-21T09:45:21.312 INFO:tasks.rados.rados.0.smithi017.stdout:1746: finishing write tid 3 to smithi01785587-25 2024-09-21T09:45:21.312 INFO:tasks.rados.rados.0.smithi017.stdout:1746: finishing write tid 4 to smithi01785587-25 2024-09-21T09:45:21.312 INFO:tasks.rados.rados.0.smithi017.stdout:1752: seq_num 677 ranges {578339=498990,1833133=525896,2791719=586620} 2024-09-21T09:45:21.317 INFO:tasks.rados.rados.0.smithi017.stdout:1752: writing smithi01785587-34 from 578339 to 1077329 tid 1 2024-09-21T09:45:21.325 INFO:tasks.rados.rados.0.smithi017.stdout:1752: writing smithi01785587-34 from 1833133 to 2359029 tid 2 2024-09-21T09:45:21.331 INFO:tasks.rados.rados.0.smithi017.stdout:1752: writing smithi01785587-34 from 2791719 to 3378339 tid 3 2024-09-21T09:45:21.331 INFO:tasks.rados.rados.0.smithi017.stdout:1753: read oid 35 snap -1 2024-09-21T09:45:21.331 INFO:tasks.rados.rados.0.smithi017.stdout:1753: expect (ObjNum 647 snap 205 seq_num 647) 2024-09-21T09:45:21.331 INFO:tasks.rados.rados.0.smithi017.stdout:1754: write oid 15 current snap is 212 2024-09-21T09:45:21.332 INFO:tasks.rados.rados.0.smithi017.stdout:1746: finishing write tid 5 to smithi01785587-25 2024-09-21T09:45:21.332 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 366 (ObjNum 675 snap 212 seq_num 675) dirty exists 2024-09-21T09:45:21.332 INFO:tasks.rados.rados.0.smithi017.stdout:1746: left oid 25 (ObjNum 675 snap 212 seq_num 675) 2024-09-21T09:45:21.332 INFO:tasks.rados.rados.0.smithi017.stdout:1750: finishing write tid 1 to smithi01785587-38 2024-09-21T09:45:21.332 INFO:tasks.rados.rados.0.smithi017.stdout:1754: seq_num 678 ranges {595146=574239,1891615=669554,3219142=176004} 2024-09-21T09:45:21.338 INFO:tasks.rados.rados.0.smithi017.stdout:1754: writing smithi01785587-15 from 595146 to 1169385 tid 1 2024-09-21T09:45:21.345 INFO:tasks.rados.rados.0.smithi017.stdout:1754: writing smithi01785587-15 from 1891615 to 2561169 tid 2 2024-09-21T09:45:21.350 INFO:tasks.rados.rados.0.smithi017.stdout:1754: writing smithi01785587-15 from 3219142 to 3395146 tid 3 2024-09-21T09:45:21.351 INFO:tasks.rados.rados.0.smithi017.stdout:1746: done (4 left) 2024-09-21T09:45:21.351 INFO:tasks.rados.rados.0.smithi017.stdout:1755: write oid 18 current snap is 212 2024-09-21T09:45:21.351 INFO:tasks.rados.rados.0.smithi017.stdout:1750: finishing write tid 2 to smithi01785587-38 2024-09-21T09:45:21.351 INFO:tasks.rados.rados.0.smithi017.stdout:1750: finishing write tid 3 to smithi01785587-38 2024-09-21T09:45:21.351 INFO:tasks.rados.rados.0.smithi017.stdout:1750: finishing write tid 4 to smithi01785587-38 2024-09-21T09:45:21.351 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 539 (ObjNum 676 snap 212 seq_num 676) dirty exists 2024-09-21T09:45:21.351 INFO:tasks.rados.rados.0.smithi017.stdout:1750: left oid 38 (ObjNum 676 snap 212 seq_num 676) 2024-09-21T09:45:21.351 INFO:tasks.rados.rados.0.smithi017.stdout:1752: finishing write tid 1 to smithi01785587-34 2024-09-21T09:45:21.351 INFO:tasks.rados.rados.0.smithi017.stdout:1755: seq_num 679 ranges {611953=649488,1950097=529565,3246565=165388} 2024-09-21T09:45:21.357 INFO:tasks.rados.rados.0.smithi017.stdout:1755: writing smithi01785587-18 from 611953 to 1261441 tid 1 2024-09-21T09:45:21.364 INFO:tasks.rados.rados.0.smithi017.stdout:1755: writing smithi01785587-18 from 1950097 to 2479662 tid 2 2024-09-21T09:45:21.369 INFO:tasks.rados.rados.0.smithi017.stdout:1755: writing smithi01785587-18 from 3246565 to 3411953 tid 3 2024-09-21T09:45:21.369 INFO:tasks.rados.rados.0.smithi017.stdout:1750: done (4 left) 2024-09-21T09:45:21.369 INFO:tasks.rados.rados.0.smithi017.stdout:1756: snap_create 2024-09-21T09:45:21.369 INFO:tasks.rados.rados.0.smithi017.stdout:1752: finishing write tid 2 to smithi01785587-34 2024-09-21T09:45:21.370 INFO:tasks.rados.rados.0.smithi017.stdout:1752: finishing write tid 3 to smithi01785587-34 2024-09-21T09:45:21.370 INFO:tasks.rados.rados.0.smithi017.stdout:1752: finishing write tid 4 to smithi01785587-34 2024-09-21T09:45:21.370 INFO:tasks.rados.rados.0.smithi017.stdout:1752: finishing write tid 5 to smithi01785587-34 2024-09-21T09:45:21.370 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 899 (ObjNum 677 snap 212 seq_num 677) dirty exists 2024-09-21T09:45:21.370 INFO:tasks.rados.rados.0.smithi017.stdout:1752: left oid 34 (ObjNum 677 snap 212 seq_num 677) 2024-09-21T09:45:21.370 INFO:tasks.rados.rados.0.smithi017.stdout:1753: expect (ObjNum 647 snap 205 seq_num 647) 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:1754: finishing write tid 1 to smithi01785587-15 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:1754: finishing write tid 2 to smithi01785587-15 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:1754: finishing write tid 3 to smithi01785587-15 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:1754: finishing write tid 4 to smithi01785587-15 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:1754: finishing write tid 5 to smithi01785587-15 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 295 (ObjNum 678 snap 212 seq_num 678) dirty exists 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:1754: left oid 15 (ObjNum 678 snap 212 seq_num 678) 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:1755: finishing write tid 1 to smithi01785587-18 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:1755: finishing write tid 2 to smithi01785587-18 2024-09-21T09:45:21.465 INFO:tasks.rados.rados.0.smithi017.stdout:1755: finishing write tid 3 to smithi01785587-18 2024-09-21T09:45:21.466 INFO:tasks.rados.rados.0.smithi017.stdout:1755: finishing write tid 4 to smithi01785587-18 2024-09-21T09:45:21.466 INFO:tasks.rados.rados.0.smithi017.stdout:1755: finishing write tid 5 to smithi01785587-18 2024-09-21T09:45:21.466 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 123 (ObjNum 679 snap 212 seq_num 679) dirty exists 2024-09-21T09:45:21.466 INFO:tasks.rados.rados.0.smithi017.stdout:1755: left oid 18 (ObjNum 679 snap 212 seq_num 679) 2024-09-21T09:45:21.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[102913]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:21.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[102913]: pgmap v321: 89 pgs: 89 active+clean; 81 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1 op/s 2024-09-21T09:45:21.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[102913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:45:21.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[102913]: Upgrade: osd.1 is safe to restart 2024-09-21T09:45:21.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[102913]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:45:21.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[105151]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:21.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[105151]: pgmap v321: 89 pgs: 89 active+clean; 81 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1 op/s 2024-09-21T09:45:21.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[105151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:45:21.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[105151]: Upgrade: osd.1 is safe to restart 2024-09-21T09:45:21.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:21 smithi017 ceph-mon[105151]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:45:21.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:21 smithi134 ceph-mon[90085]: from='client.34515 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:21 smithi134 ceph-mon[90085]: pgmap v321: 89 pgs: 89 active+clean; 81 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1 op/s 2024-09-21T09:45:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:21 smithi134 ceph-mon[90085]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:45:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:21 smithi134 ceph-mon[90085]: Upgrade: osd.1 is safe to restart 2024-09-21T09:45:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:21 smithi134 ceph-mon[90085]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:45:22.271 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:22.309 INFO:tasks.rados.rados.0.smithi017.stdout:1752: done (4 left) 2024-09-21T09:45:22.309 INFO:tasks.rados.rados.0.smithi017.stdout:1753: done (3 left) 2024-09-21T09:45:22.309 INFO:tasks.rados.rados.0.smithi017.stdout:1754: done (2 left) 2024-09-21T09:45:22.309 INFO:tasks.rados.rados.0.smithi017.stdout:1755: done (1 left) 2024-09-21T09:45:22.309 INFO:tasks.rados.rados.0.smithi017.stdout:1756: done (0 left) 2024-09-21T09:45:22.309 INFO:tasks.rados.rados.0.smithi017.stdout:1757: snap_remove snap 212 2024-09-21T09:45:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[102913]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:45:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:45:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3165881635' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[105151]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:45:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:45:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:22 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3165881635' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:22 smithi134 ceph-mon[90085]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:45:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:45:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:22 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3165881635' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:23.069 INFO:tasks.rados.rados.0.smithi017.stdout:1757: done (0 left) 2024-09-21T09:45:23.069 INFO:tasks.rados.rados.0.smithi017.stdout:1758: snap_create 2024-09-21T09:45:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[102913]: Upgrade: Updating osd.1 2024-09-21T09:45:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[102913]: Deploying daemon osd.1 on smithi017 2024-09-21T09:45:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[102913]: pgmap v324: 89 pgs: 89 active+clean; 89 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:45:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[102913]: from='client.34527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[102913]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:45:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[102913]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:45:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[105151]: Upgrade: Updating osd.1 2024-09-21T09:45:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[105151]: Deploying daemon osd.1 on smithi017 2024-09-21T09:45:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[105151]: pgmap v324: 89 pgs: 89 active+clean; 89 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:45:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[105151]: from='client.34527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[105151]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:45:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:23 smithi017 ceph-mon[105151]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:45:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:23 smithi134 ceph-mon[90085]: Upgrade: Updating osd.1 2024-09-21T09:45:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:23 smithi134 ceph-mon[90085]: Deploying daemon osd.1 on smithi017 2024-09-21T09:45:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:23 smithi134 ceph-mon[90085]: pgmap v324: 89 pgs: 89 active+clean; 89 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:45:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:23 smithi134 ceph-mon[90085]: from='client.34527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:23 smithi134 ceph-mon[90085]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:45:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:23 smithi134 ceph-mon[90085]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:45:24.069 INFO:tasks.rados.rados.0.smithi017.stdout:1758: done (0 left) 2024-09-21T09:45:24.069 INFO:tasks.rados.rados.0.smithi017.stdout:1759: read oid 19 snap -1 2024-09-21T09:45:24.070 INFO:tasks.rados.rados.0.smithi017.stdout:1759: expect deleted 2024-09-21T09:45:24.070 INFO:tasks.rados.rados.0.smithi017.stdout:1760: snap_create 2024-09-21T09:45:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:24 smithi017 ceph-mon[105151]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:24 smithi017 ceph-mon[105151]: osdmap e499: 8 total, 8 up, 8 in 2024-09-21T09:45:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:24 smithi017 ceph-mon[102913]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:24.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:24 smithi017 ceph-mon[102913]: osdmap e499: 8 total, 8 up, 8 in 2024-09-21T09:45:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:24 smithi134 ceph-mon[90085]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:24 smithi134 ceph-mon[90085]: osdmap e499: 8 total, 8 up, 8 in 2024-09-21T09:45:24.844 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:45:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:45:24] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.51.0" 2024-09-21T09:45:25.070 INFO:tasks.rados.rados.0.smithi017.stdout:1759: done (1 left) 2024-09-21T09:45:25.071 INFO:tasks.rados.rados.0.smithi017.stdout:1760: done (0 left) 2024-09-21T09:45:25.071 INFO:tasks.rados.rados.0.smithi017.stdout:1761: delete oid 42 current snap is 215 2024-09-21T09:45:25.074 INFO:tasks.rados.rados.0.smithi017.stdout:1761: done (0 left) 2024-09-21T09:45:25.074 INFO:tasks.rados.rados.0.smithi017.stdout:1762: rollback oid 48 current snap is 215 2024-09-21T09:45:25.074 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 214 2024-09-21T09:45:25.074 INFO:tasks.rados.rados.0.smithi017.stdout:1763: write oid 24 current snap is 215 2024-09-21T09:45:25.074 INFO:tasks.rados.rados.0.smithi017.stdout:1763: seq_num 681 ranges {645567=799986,1950708=533234,3185058=260509} 2024-09-21T09:45:25.084 INFO:tasks.rados.rados.0.smithi017.stdout:1763: writing smithi01785587-24 from 645567 to 1445553 tid 1 2024-09-21T09:45:25.091 INFO:tasks.rados.rados.0.smithi017.stdout:1763: writing smithi01785587-24 from 1950708 to 2483942 tid 2 2024-09-21T09:45:25.097 INFO:tasks.rados.rados.0.smithi017.stdout:1763: writing smithi01785587-24 from 3185058 to 3445567 tid 3 2024-09-21T09:45:25.097 INFO:tasks.rados.rados.0.smithi017.stdout:1764: delete oid 11 current snap is 215 2024-09-21T09:45:25.098 INFO:tasks.rados.rados.0.smithi017.stdout:1762: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:45:25.098 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 0 (ObjNum 129 snap 0 seq_num 2399871360) dirty dne 2024-09-21T09:45:25.098 INFO:tasks.rados.rados.0.smithi017.stdout:1763: finishing write tid 1 to smithi01785587-24 2024-09-21T09:45:25.098 INFO:tasks.rados.rados.0.smithi017.stdout:1763: finishing write tid 2 to smithi01785587-24 2024-09-21T09:45:25.101 INFO:tasks.rados.rados.0.smithi017.stdout:1762: done (2 left) 2024-09-21T09:45:25.101 INFO:tasks.rados.rados.0.smithi017.stdout:1764: done (1 left) 2024-09-21T09:45:25.101 INFO:tasks.rados.rados.0.smithi017.stdout:1765: read oid 33 snap 214 2024-09-21T09:45:25.101 INFO:tasks.rados.rados.0.smithi017.stdout:1765: expect (ObjNum 638 snap 202 seq_num 638) 2024-09-21T09:45:25.101 INFO:tasks.rados.rados.0.smithi017.stdout:1766: write oid 31 current snap is 215 2024-09-21T09:45:25.101 INFO:tasks.rados.rados.0.smithi017.stdout:1763: finishing write tid 3 to smithi01785587-24 2024-09-21T09:45:25.101 INFO:tasks.rados.rados.0.smithi017.stdout:1763: finishing write tid 4 to smithi01785587-24 2024-09-21T09:45:25.102 INFO:tasks.rados.rados.0.smithi017.stdout:1766: seq_num 683 ranges {679181=550484,1667672=536903,2839904=435373} 2024-09-21T09:45:25.107 INFO:tasks.rados.rados.0.smithi017.stdout:1766: writing smithi01785587-31 from 679181 to 1229665 tid 1 2024-09-21T09:45:25.113 INFO:tasks.rados.rados.0.smithi017.stdout:1766: writing smithi01785587-31 from 1667672 to 2204575 tid 2 2024-09-21T09:45:25.119 INFO:tasks.rados.rados.0.smithi017.stdout:1766: writing smithi01785587-31 from 2839904 to 3275277 tid 3 2024-09-21T09:45:25.119 INFO:tasks.rados.rados.0.smithi017.stdout:1767: snap_create 2024-09-21T09:45:25.119 INFO:tasks.rados.rados.0.smithi017.stdout:1763: finishing write tid 5 to smithi01785587-24 2024-09-21T09:45:25.119 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 544 (ObjNum 681 snap 215 seq_num 681) dirty exists 2024-09-21T09:45:25.119 INFO:tasks.rados.rados.0.smithi017.stdout:1763: left oid 24 (ObjNum 681 snap 215 seq_num 681) 2024-09-21T09:45:25.119 INFO:tasks.rados.rados.0.smithi017.stdout:1765: expect (ObjNum 638 snap 202 seq_num 638) 2024-09-21T09:45:25.124 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:24 smithi017 systemd[1]: Stopping Ceph osd.1 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:45:25.213 INFO:tasks.rados.rados.0.smithi017.stdout:1766: finishing write tid 1 to smithi01785587-31 2024-09-21T09:45:25.213 INFO:tasks.rados.rados.0.smithi017.stdout:1766: finishing write tid 2 to smithi01785587-31 2024-09-21T09:45:25.213 INFO:tasks.rados.rados.0.smithi017.stdout:1766: finishing write tid 3 to smithi01785587-31 2024-09-21T09:45:25.213 INFO:tasks.rados.rados.0.smithi017.stdout:1766: finishing write tid 4 to smithi01785587-31 2024-09-21T09:45:25.213 INFO:tasks.rados.rados.0.smithi017.stdout:1766: finishing write tid 5 to smithi01785587-31 2024-09-21T09:45:25.214 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 861 (ObjNum 683 snap 215 seq_num 683) dirty exists 2024-09-21T09:45:25.214 INFO:tasks.rados.rados.0.smithi017.stdout:1766: left oid 31 (ObjNum 683 snap 215 seq_num 683) 2024-09-21T09:45:25.234 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:25.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[51310]: 2024-09-21T09:45:25.122+0000 7fd9b8930640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:45:25.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[51310]: 2024-09-21T09:45:25.122+0000 7fd9b8930640 -1 osd.1 500 *** Got signal Terminated *** 2024-09-21T09:45:25.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[51310]: 2024-09-21T09:45:25.122+0000 7fd9b8930640 -1 osd.1 500 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:45:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[102913]: pgmap v327: 89 pgs: 89 active+clean; 89 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:45:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2486852473' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[102913]: osdmap e500: 8 total, 8 up, 8 in 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[102913]: osd.1 marked itself down and dead 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[105151]: pgmap v327: 89 pgs: 89 active+clean; 89 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2486852473' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[105151]: osdmap e500: 8 total, 8 up, 8 in 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[105151]: osd.1 marked itself down and dead 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:25 smithi134 ceph-mon[90085]: pgmap v327: 89 pgs: 89 active+clean; 89 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:45:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:25 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2486852473' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:25 smithi134 ceph-mon[90085]: osdmap e500: 8 total, 8 up, 8 in 2024-09-21T09:45:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:25 smithi134 ceph-mon[90085]: osd.1 marked itself down and dead 2024-09-21T09:45:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:26.075 INFO:tasks.rados.rados.0.smithi017.stdout:1763: done (3 left) 2024-09-21T09:45:26.075 INFO:tasks.rados.rados.0.smithi017.stdout:1765: done (2 left) 2024-09-21T09:45:26.075 INFO:tasks.rados.rados.0.smithi017.stdout:1766: done (1 left) 2024-09-21T09:45:26.075 INFO:tasks.rados.rados.0.smithi017.stdout:1767: done (0 left) 2024-09-21T09:45:26.075 INFO:tasks.rados.rados.0.smithi017.stdout:1768: rollback oid 26 current snap is 216 2024-09-21T09:45:26.075 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 26 to 213 2024-09-21T09:45:26.075 INFO:tasks.rados.rados.0.smithi017.stdout:1769: read oid 24 snap -1 2024-09-21T09:45:26.075 INFO:tasks.rados.rados.0.smithi017.stdout:1769: expect (ObjNum 681 snap 215 seq_num 681) 2024-09-21T09:45:26.076 INFO:tasks.rados.rados.0.smithi017.stdout:1770: delete oid 31 current snap is 216 2024-09-21T09:45:26.085 INFO:tasks.rados.rados.0.smithi017.stdout:1768: finishing rollback tid 1 to smithi01785587-26 2024-09-21T09:45:26.085 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 0 (ObjNum 129 snap 0 seq_num 2399754728) dirty dne 2024-09-21T09:45:26.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:26 smithi017 ceph-mon[102913]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:26.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:26 smithi017 ceph-mon[102913]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:26.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:26 smithi017 ceph-mon[102913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:45:26.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:26 smithi017 ceph-mon[102913]: osdmap e501: 8 total, 7 up, 8 in 2024-09-21T09:45:26.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:26 smithi017 ceph-mon[105151]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:26.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:26 smithi017 ceph-mon[105151]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:26.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:26 smithi017 ceph-mon[105151]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:45:26.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:26 smithi017 ceph-mon[105151]: osdmap e501: 8 total, 7 up, 8 in 2024-09-21T09:45:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:26 smithi134 ceph-mon[90085]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:26 smithi134 ceph-mon[90085]: from='client.44414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:26 smithi134 ceph-mon[90085]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:45:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:26 smithi134 ceph-mon[90085]: osdmap e501: 8 total, 7 up, 8 in 2024-09-21T09:45:26.929 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:26 smithi017 podman[118824]: 2024-09-21 09:45:26.60522975 +0000 UTC m=+1.585803249 container died 5e9c6f25d8a59c6389c5c58023bce22d2983ec32bafd8ec99f9f02627fa1f9aa (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T09:45:27.088 INFO:tasks.rados.rados.0.smithi017.stdout:1769: expect (ObjNum 681 snap 215 seq_num 681) 2024-09-21T09:45:27.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:26 smithi017 podman[118824]: 2024-09-21 09:45:26.927876128 +0000 UTC m=+1.908449626 container cleanup 5e9c6f25d8a59c6389c5c58023bce22d2983ec32bafd8ec99f9f02627fa1f9aa (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T09:45:27.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:26 smithi017 bash[118824]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1 2024-09-21T09:45:27.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:27 smithi017 podman[118871]: 2024-09-21 09:45:27.112308518 +0000 UTC m=+0.507091539 container remove 5e9c6f25d8a59c6389c5c58023bce22d2983ec32bafd8ec99f9f02627fa1f9aa (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T09:45:27.349 INFO:tasks.rados.rados.0.smithi017.stdout:1768: done (2 left) 2024-09-21T09:45:27.349 INFO:tasks.rados.rados.0.smithi017.stdout:1769: done (1 left) 2024-09-21T09:45:27.350 INFO:tasks.rados.rados.0.smithi017.stdout:1770: done (0 left) 2024-09-21T09:45:27.350 INFO:tasks.rados.rados.0.smithi017.stdout:1771: rollback oid 40 current snap is 216 2024-09-21T09:45:27.350 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 213 2024-09-21T09:45:27.350 INFO:tasks.rados.rados.0.smithi017.stdout:1772: read oid 30 snap -1 2024-09-21T09:45:27.350 INFO:tasks.rados.rados.0.smithi017.stdout:1772: expect (ObjNum 636 snap 201 seq_num 636) 2024-09-21T09:45:27.350 INFO:tasks.rados.rados.0.smithi017.stdout:1773: snap_create 2024-09-21T09:45:27.351 INFO:tasks.rados.rados.0.smithi017.stdout:1771: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:45:27.351 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:45:27.356 INFO:tasks.rados.rados.0.smithi017.stdout:1772: expect (ObjNum 636 snap 201 seq_num 636) 2024-09-21T09:45:27.582 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:27 smithi017 ceph-mon[102913]: pgmap v330: 89 pgs: 89 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:45:27.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:27 smithi017 ceph-mon[102913]: osdmap e502: 8 total, 7 up, 8 in 2024-09-21T09:45:27.583 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:27 smithi017 ceph-mon[105151]: pgmap v330: 89 pgs: 89 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:45:27.583 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:27 smithi017 ceph-mon[105151]: osdmap e502: 8 total, 7 up, 8 in 2024-09-21T09:45:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:27 smithi134 ceph-mon[90085]: pgmap v330: 89 pgs: 89 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:45:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:27 smithi134 ceph-mon[90085]: osdmap e502: 8 total, 7 up, 8 in 2024-09-21T09:45:27.851 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:27 smithi017 podman[118965]: 2024-09-21 09:45:27.484187836 +0000 UTC m=+0.016731834 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:45:27.851 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:27 smithi017 podman[118965]: 2024-09-21 09:45:27.596582422 +0000 UTC m=+0.129126407 container create 1418596f32d77e71e1c2514af99e94d6e583dcc11c8319633e05464c01b4b2e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T09:45:28.071 INFO:tasks.rados.rados.0.smithi017.stdout:1771: done (2 left) 2024-09-21T09:45:28.072 INFO:tasks.rados.rados.0.smithi017.stdout:1772: done (1 left) 2024-09-21T09:45:28.072 INFO:tasks.rados.rados.0.smithi017.stdout:1773: done (0 left) 2024-09-21T09:45:28.072 INFO:tasks.rados.rados.0.smithi017.stdout:1774: write oid 35 current snap is 217 2024-09-21T09:45:28.072 INFO:tasks.rados.rados.0.smithi017.stdout:1774: seq_num 685 ranges {712795=417335,1784636=540572,2894750=457917} 2024-09-21T09:45:28.078 INFO:tasks.rados.rados.0.smithi017.stdout:1774: writing smithi01785587-35 from 712795 to 1130130 tid 1 2024-09-21T09:45:28.084 INFO:tasks.rados.rados.0.smithi017.stdout:1774: writing smithi01785587-35 from 1784636 to 2325208 tid 2 2024-09-21T09:45:28.090 INFO:tasks.rados.rados.0.smithi017.stdout:1774: writing smithi01785587-35 from 2894750 to 3352667 tid 3 2024-09-21T09:45:28.090 INFO:tasks.rados.rados.0.smithi017.stdout:1775: write oid 48 current snap is 217 2024-09-21T09:45:28.090 INFO:tasks.rados.rados.0.smithi017.stdout:1774: finishing write tid 1 to smithi01785587-35 2024-09-21T09:45:28.090 INFO:tasks.rados.rados.0.smithi017.stdout:1775: seq_num 686 ranges {729602=492584,1843118=684230,3322173=207429} 2024-09-21T09:45:28.097 INFO:tasks.rados.rados.0.smithi017.stdout:1775: writing smithi01785587-48 from 729602 to 1222186 tid 1 2024-09-21T09:45:28.104 INFO:tasks.rados.rados.0.smithi017.stdout:1775: writing smithi01785587-48 from 1843118 to 2527348 tid 2 2024-09-21T09:45:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:1775: writing smithi01785587-48 from 3322173 to 3529602 tid 3 2024-09-21T09:45:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:1776: rollback oid 20 current snap is 217 2024-09-21T09:45:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 213 2024-09-21T09:45:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:1774: finishing write tid 2 to smithi01785587-35 2024-09-21T09:45:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:1774: finishing write tid 3 to smithi01785587-35 2024-09-21T09:45:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:1774: finishing write tid 4 to smithi01785587-35 2024-09-21T09:45:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:1774: finishing write tid 5 to smithi01785587-35 2024-09-21T09:45:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 212 (ObjNum 685 snap 217 seq_num 685) dirty exists 2024-09-21T09:45:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:1774: left oid 35 (ObjNum 685 snap 217 seq_num 685) 2024-09-21T09:45:28.111 INFO:tasks.rados.rados.0.smithi017.stdout:1775: finishing write tid 1 to smithi01785587-48 2024-09-21T09:45:28.111 INFO:tasks.rados.rados.0.smithi017.stdout:1774: done (2 left) 2024-09-21T09:45:28.111 INFO:tasks.rados.rados.0.smithi017.stdout:1777: read oid 35 snap -1 2024-09-21T09:45:28.111 INFO:tasks.rados.rados.0.smithi017.stdout:1777: expect (ObjNum 685 snap 217 seq_num 685) 2024-09-21T09:45:28.111 INFO:tasks.rados.rados.0.smithi017.stdout:1778: snap_create 2024-09-21T09:45:28.111 INFO:tasks.rados.rados.0.smithi017.stdout:1775: finishing write tid 2 to smithi01785587-48 2024-09-21T09:45:28.111 INFO:tasks.rados.rados.0.smithi017.stdout:1776: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:45:28.111 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 0 (ObjNum 129 snap 0 seq_num 2399839440) dirty dne 2024-09-21T09:45:28.112 INFO:tasks.rados.rados.0.smithi017.stdout:1775: finishing write tid 3 to smithi01785587-48 2024-09-21T09:45:28.112 INFO:tasks.rados.rados.0.smithi017.stdout:1775: finishing write tid 4 to smithi01785587-48 2024-09-21T09:45:28.113 INFO:tasks.rados.rados.0.smithi017.stdout:1775: finishing write tid 5 to smithi01785587-48 2024-09-21T09:45:28.113 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 715 (ObjNum 686 snap 217 seq_num 686) dirty exists 2024-09-21T09:45:28.113 INFO:tasks.rados.rados.0.smithi017.stdout:1775: left oid 48 (ObjNum 686 snap 217 seq_num 686) 2024-09-21T09:45:28.119 INFO:tasks.rados.rados.0.smithi017.stdout:1777: expect (ObjNum 685 snap 217 seq_num 685) 2024-09-21T09:45:28.139 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:27 smithi017 podman[118965]: 2024-09-21 09:45:27.850412103 +0000 UTC m=+0.382956089 container init 1418596f32d77e71e1c2514af99e94d6e583dcc11c8319633e05464c01b4b2e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, RELEASE=main-62da54b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T09:45:28.139 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:27 smithi017 podman[118965]: 2024-09-21 09:45:27.855923185 +0000 UTC m=+0.388467183 container start 1418596f32d77e71e1c2514af99e94d6e583dcc11c8319633e05464c01b4b2e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, ceph=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T09:45:28.139 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:27 smithi017 podman[118965]: 2024-09-21 09:45:27.932740451 +0000 UTC m=+0.465284439 container attach 1418596f32d77e71e1c2514af99e94d6e583dcc11c8319633e05464c01b4b2e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T09:45:28.139 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:28 smithi017 podman[118965]: 2024-09-21 09:45:28.061932006 +0000 UTC m=+0.594476007 container died 1418596f32d77e71e1c2514af99e94d6e583dcc11c8319633e05464c01b4b2e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T09:45:28.139 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:28.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:28 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4166927710' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:28.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:28 smithi017 ceph-mon[102913]: osdmap e503: 8 total, 7 up, 8 in 2024-09-21T09:45:28.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:28 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4166927710' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:28.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:28 smithi017 ceph-mon[105151]: osdmap e503: 8 total, 7 up, 8 in 2024-09-21T09:45:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:28 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4166927710' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:28 smithi134 ceph-mon[90085]: osdmap e503: 8 total, 7 up, 8 in 2024-09-21T09:45:29.071 INFO:tasks.rados.rados.0.smithi017.stdout:1775: done (3 left) 2024-09-21T09:45:29.071 INFO:tasks.rados.rados.0.smithi017.stdout:1776: done (2 left) 2024-09-21T09:45:29.071 INFO:tasks.rados.rados.0.smithi017.stdout:1777: done (1 left) 2024-09-21T09:45:29.071 INFO:tasks.rados.rados.0.smithi017.stdout:1778: done (0 left) 2024-09-21T09:45:29.071 INFO:tasks.rados.rados.0.smithi017.stdout:1779: delete oid 48 current snap is 218 2024-09-21T09:45:29.073 INFO:tasks.rados.rados.0.smithi017.stdout:1779: done (0 left) 2024-09-21T09:45:29.073 INFO:tasks.rados.rados.0.smithi017.stdout:1780: write oid 37 current snap is 218 2024-09-21T09:45:29.074 INFO:tasks.rados.rados.0.smithi017.stdout:1780: seq_num 688 ranges {763216=643082,1843729=687899,3260666=302550} 2024-09-21T09:45:29.082 INFO:tasks.rados.rados.0.smithi017.stdout:1780: writing smithi01785587-37 from 763216 to 1406298 tid 1 2024-09-21T09:45:29.090 INFO:tasks.rados.rados.0.smithi017.stdout:1780: writing smithi01785587-37 from 1843729 to 2531628 tid 2 2024-09-21T09:45:29.096 INFO:tasks.rados.rados.0.smithi017.stdout:1780: writing smithi01785587-37 from 3260666 to 3563216 tid 3 2024-09-21T09:45:29.096 INFO:tasks.rados.rados.0.smithi017.stdout:1781: snap_create 2024-09-21T09:45:29.096 INFO:tasks.rados.rados.0.smithi017.stdout:1780: finishing write tid 1 to smithi01785587-37 2024-09-21T09:45:29.097 INFO:tasks.rados.rados.0.smithi017.stdout:1780: finishing write tid 2 to smithi01785587-37 2024-09-21T09:45:29.099 INFO:tasks.rados.rados.0.smithi017.stdout:1780: finishing write tid 3 to smithi01785587-37 2024-09-21T09:45:29.099 INFO:tasks.rados.rados.0.smithi017.stdout:1780: finishing write tid 4 to smithi01785587-37 2024-09-21T09:45:29.099 INFO:tasks.rados.rados.0.smithi017.stdout:1780: finishing write tid 5 to smithi01785587-37 2024-09-21T09:45:29.099 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 871 (ObjNum 688 snap 218 seq_num 688) dirty exists 2024-09-21T09:45:29.099 INFO:tasks.rados.rados.0.smithi017.stdout:1780: left oid 37 (ObjNum 688 snap 218 seq_num 688) 2024-09-21T09:45:29.178 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:28 smithi017 podman[119054]: 2024-09-21 09:45:28.736507936 +0000 UTC m=+0.666626320 container remove 1418596f32d77e71e1c2514af99e94d6e583dcc11c8319633e05464c01b4b2e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, RELEASE=main-62da54b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:45:29.178 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:28 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T09:45:29.178 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:28 smithi017 systemd[1]: Stopped Ceph osd.1 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:45:29.178 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:28 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.1.service: Consumed 11.381s CPU time. 2024-09-21T09:45:29.178 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:28 smithi017 systemd[1]: Starting Ceph osd.1 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:45:29.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-mon[102913]: pgmap v333: 89 pgs: 10 stale+active+clean, 79 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:45:29.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-mon[102913]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:29.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-mon[102913]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:29.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-mon[102913]: osdmap e504: 8 total, 7 up, 8 in 2024-09-21T09:45:29.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-mon[105151]: pgmap v333: 89 pgs: 10 stale+active+clean, 79 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:45:29.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-mon[105151]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:29.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-mon[105151]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:29.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-mon[105151]: osdmap e504: 8 total, 7 up, 8 in 2024-09-21T09:45:29.442 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:29 smithi017 podman[119186]: 2024-09-21 09:45:29.081050339 +0000 UTC m=+0.020013491 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:45:29.442 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:29 smithi017 podman[119186]: 2024-09-21 09:45:29.195899587 +0000 UTC m=+0.134862708 container create 6ed96670b5604615fb025a1d61e0a9f5c9c17174911a52b89a6ffe9c9027c8cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T09:45:29.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:29 smithi017 podman[119186]: 2024-09-21 09:45:29.440346836 +0000 UTC m=+0.379309958 container init 6ed96670b5604615fb025a1d61e0a9f5c9c17174911a52b89a6ffe9c9027c8cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:45:29.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:29 smithi017 podman[119186]: 2024-09-21 09:45:29.445984246 +0000 UTC m=+0.384947374 container start 6ed96670b5604615fb025a1d61e0a9f5c9c17174911a52b89a6ffe9c9027c8cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-62da54b) 2024-09-21T09:45:29.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:29 smithi017 podman[119186]: 2024-09-21 09:45:29.5403093 +0000 UTC m=+0.479272442 container attach 6ed96670b5604615fb025a1d61e0a9f5c9c17174911a52b89a6ffe9c9027c8cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-21T09:45:29.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[119201]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:45:29.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:29 smithi017 bash[119186]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:45:29.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:29 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[119201]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:45:29.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:29 smithi017 bash[119186]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:45:29.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:29 smithi134 ceph-mon[90085]: pgmap v333: 89 pgs: 10 stale+active+clean, 79 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-09-21T09:45:29.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:29 smithi134 ceph-mon[90085]: from='client.44426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:29 smithi134 ceph-mon[90085]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:29 smithi134 ceph-mon[90085]: osdmap e504: 8 total, 7 up, 8 in 2024-09-21T09:45:30.072 INFO:tasks.rados.rados.0.smithi017.stdout:1780: done (1 left) 2024-09-21T09:45:30.072 INFO:tasks.rados.rados.0.smithi017.stdout:1781: done (0 left) 2024-09-21T09:45:30.073 INFO:tasks.rados.rados.0.smithi017.stdout:1782: delete oid 20 current snap is 219 2024-09-21T09:45:30.076 INFO:tasks.rados.rados.0.smithi017.stdout:1782: done (0 left) 2024-09-21T09:45:30.076 INFO:tasks.rados.rados.0.smithi017.stdout:1783: write oid 16 current snap is 219 2024-09-21T09:45:30.076 INFO:tasks.rados.rados.0.smithi017.stdout:1783: seq_num 690 ranges {796830=793580,2360693=691568} 2024-09-21T09:45:30.084 INFO:tasks.rados.rados.0.smithi017.stdout:1783: writing smithi01785587-16 from 796830 to 1590410 tid 1 2024-09-21T09:45:30.094 INFO:tasks.rados.rados.0.smithi017.stdout:1783: writing smithi01785587-16 from 2360693 to 3052261 tid 2 2024-09-21T09:45:30.094 INFO:tasks.rados.rados.0.smithi017.stdout:1784: read oid 11 snap -1 2024-09-21T09:45:30.094 INFO:tasks.rados.rados.0.smithi017.stdout:1784: expect deleted 2024-09-21T09:45:30.094 INFO:tasks.rados.rados.0.smithi017.stdout:1785: read oid 3 snap -1 2024-09-21T09:45:30.094 INFO:tasks.rados.rados.0.smithi017.stdout:1785: expect deleted 2024-09-21T09:45:30.094 INFO:tasks.rados.rados.0.smithi017.stdout:1786: rollback oid 28 current snap is 219 2024-09-21T09:45:30.094 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 28 to 217 2024-09-21T09:45:30.094 INFO:tasks.rados.rados.0.smithi017.stdout:1783: finishing write tid 1 to smithi01785587-16 2024-09-21T09:45:30.095 INFO:tasks.rados.rados.0.smithi017.stdout:1787: write oid 26 current snap is 219 2024-09-21T09:45:30.095 INFO:tasks.rados.rados.0.smithi017.stdout:1787: seq_num 691 ranges {413637=468829,1619175=435226,2542935=758255} 2024-09-21T09:45:30.099 INFO:tasks.rados.rados.0.smithi017.stdout:1787: writing smithi01785587-26 from 413637 to 882466 tid 1 2024-09-21T09:45:30.105 INFO:tasks.rados.rados.0.smithi017.stdout:1787: writing smithi01785587-26 from 1619175 to 2054401 tid 2 2024-09-21T09:45:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1787: writing smithi01785587-26 from 2542935 to 3301190 tid 3 2024-09-21T09:45:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1788: rollback oid 45 current snap is 219 2024-09-21T09:45:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 45 to 215 2024-09-21T09:45:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1783: finishing write tid 2 to smithi01785587-16 2024-09-21T09:45:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1783: finishing write tid 3 to smithi01785587-16 2024-09-21T09:45:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:1783: finishing write tid 4 to smithi01785587-16 2024-09-21T09:45:30.113 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 821 (ObjNum 690 snap 219 seq_num 690) dirty exists 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1783: left oid 16 (ObjNum 690 snap 219 seq_num 690) 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1786: finishing rollback tid 1 to smithi01785587-28 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 0 (ObjNum 81 snap 0 seq_num 2) dirty dne 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1787: finishing write tid 1 to smithi01785587-26 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1783: done (5 left) 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1784: done (4 left) 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1785: done (3 left) 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1786: done (2 left) 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1789: write oid 5 current snap is 219 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1787: finishing write tid 2 to smithi01785587-26 2024-09-21T09:45:30.114 INFO:tasks.rados.rados.0.smithi017.stdout:1789: seq_num 692 ranges {430444=660431,1677657=695237,2970358=569527} 2024-09-21T09:45:30.118 INFO:tasks.rados.rados.0.smithi017.stdout:1789: writing smithi01785587-5 from 430444 to 1090875 tid 1 2024-09-21T09:45:30.125 INFO:tasks.rados.rados.0.smithi017.stdout:1789: writing smithi01785587-5 from 1677657 to 2372894 tid 2 2024-09-21T09:45:30.132 INFO:tasks.rados.rados.0.smithi017.stdout:1789: writing smithi01785587-5 from 2970358 to 3539885 tid 3 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:1790: read oid 37 snap -1 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:1790: expect (ObjNum 688 snap 218 seq_num 688) 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:1791: rollback oid 12 current snap is 219 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:1788: finishing rollback tid 1 to smithi01785587-45 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 0 (ObjNum 49 snap 0 seq_num 0) dirty dne 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:1787: finishing write tid 3 to smithi01785587-26 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:1787: finishing write tid 4 to smithi01785587-26 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 12 to 215 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:1787: finishing write tid 5 to smithi01785587-26 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 826 (ObjNum 691 snap 219 seq_num 691) dirty exists 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:1787: left oid 26 (ObjNum 691 snap 219 seq_num 691) 2024-09-21T09:45:30.133 INFO:tasks.rados.rados.0.smithi017.stdout:1789: finishing write tid 1 to smithi01785587-5 2024-09-21T09:45:30.134 INFO:tasks.rados.rados.0.smithi017.stdout:1789: finishing write tid 2 to smithi01785587-5 2024-09-21T09:45:30.134 INFO:tasks.rados.rados.0.smithi017.stdout:1787: done (4 left) 2024-09-21T09:45:30.134 INFO:tasks.rados.rados.0.smithi017.stdout:1788: done (3 left) 2024-09-21T09:45:30.134 INFO:tasks.rados.rados.0.smithi017.stdout:1792: read oid 7 snap -1 2024-09-21T09:45:30.134 INFO:tasks.rados.rados.0.smithi017.stdout:1792: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:45:30.134 INFO:tasks.rados.rados.0.smithi017.stdout:1793: snap_remove snap 213 2024-09-21T09:45:30.357 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[119201]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:45:30.358 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 bash[119186]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:45:30.358 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[119201]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:45:30.358 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 bash[119186]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:45:30.358 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[119201]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:45:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:45:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:45:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:30.687 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 bash[119186]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:45:30.687 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[119201]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:45:30.687 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 bash[119186]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:45:30.687 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[119201]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T09:45:30.687 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 bash[119186]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T09:45:30.687 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[119201]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:45:30.687 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 bash[119186]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:45:30.687 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[119201]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T09:45:30.688 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 bash[119186]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T09:45:30.688 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 podman[119186]: 2024-09-21 09:45:30.385316159 +0000 UTC m=+1.324279294 container died 6ed96670b5604615fb025a1d61e0a9f5c9c17174911a52b89a6ffe9c9027c8cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:45:31.052 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:1793: done (4 left) 2024-09-21T09:45:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:1794: snap_create 2024-09-21T09:45:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:1789: finishing write tid 3 to smithi01785587-5 2024-09-21T09:45:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:1789: finishing write tid 4 to smithi01785587-5 2024-09-21T09:45:31.079 INFO:tasks.rados.rados.0.smithi017.stdout:1789: finishing write tid 5 to smithi01785587-5 2024-09-21T09:45:31.080 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 868 (ObjNum 692 snap 219 seq_num 692) dirty exists 2024-09-21T09:45:31.080 INFO:tasks.rados.rados.0.smithi017.stdout:1789: left oid 5 (ObjNum 692 snap 219 seq_num 692) 2024-09-21T09:45:31.080 INFO:tasks.rados.rados.0.smithi017.stdout:1792: expect (ObjNum 578 snap 181 seq_num 578) 2024-09-21T09:45:31.229 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:31 smithi017 ceph-mon[102913]: pgmap v336: 89 pgs: 3 active+undersized, 9 stale+active+clean, 2 active+undersized+degraded, 75 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/180 objects degraded (2.222%) 2024-09-21T09:45:31.229 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:31 smithi017 ceph-mon[102913]: osdmap e505: 8 total, 7 up, 8 in 2024-09-21T09:45:31.229 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:31 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 4/180 objects degraded (2.222%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:31.229 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:31 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4226048210' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:31.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:31 smithi017 ceph-mon[105151]: pgmap v336: 89 pgs: 3 active+undersized, 9 stale+active+clean, 2 active+undersized+degraded, 75 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/180 objects degraded (2.222%) 2024-09-21T09:45:31.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:31 smithi017 ceph-mon[105151]: osdmap e505: 8 total, 7 up, 8 in 2024-09-21T09:45:31.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:31 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 4/180 objects degraded (2.222%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:31.230 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:31 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4226048210' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:31.230 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:30 smithi017 podman[119362]: 2024-09-21 09:45:30.977312333 +0000 UTC m=+0.585690297 container remove 6ed96670b5604615fb025a1d61e0a9f5c9c17174911a52b89a6ffe9c9027c8cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T09:45:31.230 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:31 smithi017 podman[119475]: 2024-09-21 09:45:31.105762135 +0000 UTC m=+0.020696105 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:45:31.230 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:31 smithi017 podman[119475]: 2024-09-21 09:45:31.228344802 +0000 UTC m=+0.143278769 container create 228e704d86811dd16f9d70a40fd4d555b290ff83fde7842e8cfcda4090f4493f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T09:45:31.288 INFO:tasks.rados.rados.0.smithi017.stdout:1791: finishing rollback tid 0 to smithi01785587-12 2024-09-21T09:45:31.288 INFO:tasks.rados.rados.0.smithi017.stdout:1791: finishing rollback tid 1 to smithi01785587-12 2024-09-21T09:45:31.289 INFO:tasks.rados.rados.0.smithi017.stdout:1791: finishing rollback tid 2 to smithi01785587-12 2024-09-21T09:45:31.289 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 656 (ObjNum 659 snap 208 seq_num 659) dirty exists 2024-09-21T09:45:31.289 INFO:tasks.rados.rados.0.smithi017.stdout:1790: expect (ObjNum 688 snap 218 seq_num 688) 2024-09-21T09:45:31.484 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:31 smithi017 podman[119475]: 2024-09-21 09:45:31.45572435 +0000 UTC m=+0.370658312 container init 228e704d86811dd16f9d70a40fd4d555b290ff83fde7842e8cfcda4090f4493f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:45:31.484 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:31 smithi017 podman[119475]: 2024-09-21 09:45:31.460312928 +0000 UTC m=+0.375246890 container start 228e704d86811dd16f9d70a40fd4d555b290ff83fde7842e8cfcda4090f4493f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b) 2024-09-21T09:45:31.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:31 smithi134 ceph-mon[90085]: pgmap v336: 89 pgs: 3 active+undersized, 9 stale+active+clean, 2 active+undersized+degraded, 75 active+clean; 93 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/180 objects degraded (2.222%) 2024-09-21T09:45:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:31 smithi134 ceph-mon[90085]: osdmap e505: 8 total, 7 up, 8 in 2024-09-21T09:45:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:31 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 4/180 objects degraded (2.222%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:31 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4226048210' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:31.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:31 smithi017 ceph-osd[119535]: -- 172.21.15.17:0/2441579635 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559c83669a40 con 0x559c84438000 2024-09-21T09:45:31.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:31 smithi017 bash[119475]: 228e704d86811dd16f9d70a40fd4d555b290ff83fde7842e8cfcda4090f4493f 2024-09-21T09:45:31.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:31 smithi017 systemd[1]: Started Ceph osd.1 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:45:32.083 INFO:tasks.rados.rados.0.smithi017.stdout:1789: done (4 left) 2024-09-21T09:45:32.083 INFO:tasks.rados.rados.0.smithi017.stdout:1790: done (3 left) 2024-09-21T09:45:32.083 INFO:tasks.rados.rados.0.smithi017.stdout:1791: done (2 left) 2024-09-21T09:45:32.083 INFO:tasks.rados.rados.0.smithi017.stdout:1792: done (1 left) 2024-09-21T09:45:32.083 INFO:tasks.rados.rados.0.smithi017.stdout:1794: done (0 left) 2024-09-21T09:45:32.083 INFO:tasks.rados.rados.0.smithi017.stdout:1795: read oid 43 snap 219 2024-09-21T09:45:32.083 INFO:tasks.rados.rados.0.smithi017.stdout:1795: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:45:32.083 INFO:tasks.rados.rados.0.smithi017.stdout:1796: write oid 48 current snap is 220 2024-09-21T09:45:32.083 INFO:tasks.rados.rados.0.smithi017.stdout:1796: seq_num 693 ranges {447251=735680,1736139=438895,2597781=780799} 2024-09-21T09:45:32.089 INFO:tasks.rados.rados.0.smithi017.stdout:1796: writing smithi01785587-48 from 447251 to 1182931 tid 1 2024-09-21T09:45:32.094 INFO:tasks.rados.rados.0.smithi017.stdout:1796: writing smithi01785587-48 from 1736139 to 2175034 tid 2 2024-09-21T09:45:32.101 INFO:tasks.rados.rados.0.smithi017.stdout:1796: writing smithi01785587-48 from 2597781 to 3378580 tid 3 2024-09-21T09:45:32.101 INFO:tasks.rados.rados.0.smithi017.stdout:1797: write oid 7 current snap is 220 2024-09-21T09:45:32.101 INFO:tasks.rados.rados.0.smithi017.stdout:1797: seq_num 694 ranges {464058=410929,1394621=698906,2625204=592071,3656107=7951} 2024-09-21T09:45:32.106 INFO:tasks.rados.rados.0.smithi017.stdout:1797: writing smithi01785587-7 from 464058 to 874987 tid 1 2024-09-21T09:45:32.113 INFO:tasks.rados.rados.0.smithi017.stdout:1797: writing smithi01785587-7 from 1394621 to 2093527 tid 2 2024-09-21T09:45:32.119 INFO:tasks.rados.rados.0.smithi017.stdout:1797: writing smithi01785587-7 from 2625204 to 3217275 tid 3 2024-09-21T09:45:32.121 INFO:tasks.rados.rados.0.smithi017.stdout:1797: writing smithi01785587-7 from 3656107 to 3664058 tid 4 2024-09-21T09:45:32.122 INFO:tasks.rados.rados.0.smithi017.stdout:1798: snap_create 2024-09-21T09:45:32.122 INFO:tasks.rados.rados.0.smithi017.stdout:1795: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:45:32.233 INFO:tasks.rados.rados.0.smithi017.stdout:1796: finishing write tid 1 to smithi01785587-48 2024-09-21T09:45:32.233 INFO:tasks.rados.rados.0.smithi017.stdout:1796: finishing write tid 2 to smithi01785587-48 2024-09-21T09:45:32.233 INFO:tasks.rados.rados.0.smithi017.stdout:1796: finishing write tid 3 to smithi01785587-48 2024-09-21T09:45:32.233 INFO:tasks.rados.rados.0.smithi017.stdout:1796: finishing write tid 4 to smithi01785587-48 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:1796: finishing write tid 5 to smithi01785587-48 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 721 (ObjNum 693 snap 220 seq_num 693) dirty exists 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:1796: left oid 48 (ObjNum 693 snap 220 seq_num 693) 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:1797: finishing write tid 1 to smithi01785587-7 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:1797: finishing write tid 2 to smithi01785587-7 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:1797: finishing write tid 3 to smithi01785587-7 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:1797: finishing write tid 4 to smithi01785587-7 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:1797: finishing write tid 5 to smithi01785587-7 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:1797: finishing write tid 6 to smithi01785587-7 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 905 (ObjNum 694 snap 220 seq_num 694) dirty exists 2024-09-21T09:45:32.234 INFO:tasks.rados.rados.0.smithi017.stdout:1797: left oid 7 (ObjNum 694 snap 220 seq_num 694) 2024-09-21T09:45:32.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[105151]: from='client.44444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:32.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[105151]: osdmap e506: 8 total, 7 up, 8 in 2024-09-21T09:45:32.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[105151]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:32.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:32.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:32.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:32.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[102913]: from='client.44444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:32.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[102913]: osdmap e506: 8 total, 7 up, 8 in 2024-09-21T09:45:32.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[102913]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:32.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:32.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:32.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:32 smithi134 ceph-mon[90085]: from='client.44444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:32 smithi134 ceph-mon[90085]: osdmap e506: 8 total, 7 up, 8 in 2024-09-21T09:45:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:32 smithi134 ceph-mon[90085]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:33.071 INFO:tasks.rados.rados.0.smithi017.stdout:1795: done (3 left) 2024-09-21T09:45:33.071 INFO:tasks.rados.rados.0.smithi017.stdout:1796: done (2 left) 2024-09-21T09:45:33.072 INFO:tasks.rados.rados.0.smithi017.stdout:1797: done (1 left) 2024-09-21T09:45:33.072 INFO:tasks.rados.rados.0.smithi017.stdout:1798: done (0 left) 2024-09-21T09:45:33.072 INFO:tasks.rados.rados.0.smithi017.stdout:1799: rollback oid 29 current snap is 221 2024-09-21T09:45:33.072 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 29 to 220 2024-09-21T09:45:33.072 INFO:tasks.rados.rados.0.smithi017.stdout:1800: write oid 18 current snap is 221 2024-09-21T09:45:33.072 INFO:tasks.rados.rados.0.smithi017.stdout:1800: seq_num 695 ranges {480865=486178,1453103=442564,2652627=519696} 2024-09-21T09:45:33.076 INFO:tasks.rados.rados.0.smithi017.stdout:1800: writing smithi01785587-18 from 480865 to 967043 tid 1 2024-09-21T09:45:33.082 INFO:tasks.rados.rados.0.smithi017.stdout:1800: writing smithi01785587-18 from 1453103 to 1895667 tid 2 2024-09-21T09:45:33.090 INFO:tasks.rados.rados.0.smithi017.stdout:1800: writing smithi01785587-18 from 2652627 to 3172323 tid 3 2024-09-21T09:45:33.090 INFO:tasks.rados.rados.0.smithi017.stdout:1801: write oid 34 current snap is 221 2024-09-21T09:45:33.090 INFO:tasks.rados.rados.0.smithi017.stdout:1799: finishing rollback tid 0 to smithi01785587-29 2024-09-21T09:45:33.090 INFO:tasks.rados.rados.0.smithi017.stdout:1801: seq_num 696 ranges {497672=561427,1795232=702575,2963697=730968} 2024-09-21T09:45:33.096 INFO:tasks.rados.rados.0.smithi017.stdout:1801: writing smithi01785587-34 from 497672 to 1059099 tid 1 2024-09-21T09:45:33.103 INFO:tasks.rados.rados.0.smithi017.stdout:1801: writing smithi01785587-34 from 1795232 to 2497807 tid 2 2024-09-21T09:45:33.111 INFO:tasks.rados.rados.0.smithi017.stdout:1801: writing smithi01785587-34 from 2963697 to 3694665 tid 3 2024-09-21T09:45:33.111 INFO:tasks.rados.rados.0.smithi017.stdout:1802: write oid 38 current snap is 221 2024-09-21T09:45:33.111 INFO:tasks.rados.rados.0.smithi017.stdout:1799: finishing rollback tid 1 to smithi01785587-29 2024-09-21T09:45:33.111 INFO:tasks.rados.rados.0.smithi017.stdout:1799: finishing rollback tid 2 to smithi01785587-29 2024-09-21T09:45:33.111 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 378 (ObjNum 672 snap 212 seq_num 672) dirty exists 2024-09-21T09:45:33.111 INFO:tasks.rados.rados.0.smithi017.stdout:1800: finishing write tid 1 to smithi01785587-18 2024-09-21T09:45:33.111 INFO:tasks.rados.rados.0.smithi017.stdout:1800: finishing write tid 2 to smithi01785587-18 2024-09-21T09:45:33.111 INFO:tasks.rados.rados.0.smithi017.stdout:1802: seq_num 697 ranges {514479=636676,1853714=446233,2991120=542240} 2024-09-21T09:45:33.117 INFO:tasks.rados.rados.0.smithi017.stdout:1802: writing smithi01785587-38 from 514479 to 1151155 tid 1 2024-09-21T09:45:33.123 INFO:tasks.rados.rados.0.smithi017.stdout:1802: writing smithi01785587-38 from 1853714 to 2299947 tid 2 2024-09-21T09:45:33.130 INFO:tasks.rados.rados.0.smithi017.stdout:1802: writing smithi01785587-38 from 2991120 to 3533360 tid 3 2024-09-21T09:45:33.130 INFO:tasks.rados.rados.0.smithi017.stdout:1799: done (3 left) 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1803: write oid 37 current snap is 221 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1800: finishing write tid 3 to smithi01785587-18 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1800: finishing write tid 4 to smithi01785587-18 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1800: finishing write tid 5 to smithi01785587-18 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 128 (ObjNum 695 snap 221 seq_num 695) dirty exists 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1800: left oid 18 (ObjNum 695 snap 221 seq_num 695) 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1801: finishing write tid 1 to smithi01785587-34 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1801: finishing write tid 2 to smithi01785587-34 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1801: finishing write tid 3 to smithi01785587-34 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1801: finishing write tid 4 to smithi01785587-34 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1801: finishing write tid 5 to smithi01785587-34 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 910 (ObjNum 696 snap 221 seq_num 696) dirty exists 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1801: left oid 34 (ObjNum 696 snap 221 seq_num 696) 2024-09-21T09:45:33.131 INFO:tasks.rados.rados.0.smithi017.stdout:1802: finishing write tid 1 to smithi01785587-38 2024-09-21T09:45:33.132 INFO:tasks.rados.rados.0.smithi017.stdout:1802: finishing write tid 2 to smithi01785587-38 2024-09-21T09:45:33.132 INFO:tasks.rados.rados.0.smithi017.stdout:1803: seq_num 698 ranges {531286=711925,1912196=706244,3018543=712743} 2024-09-21T09:45:33.137 INFO:tasks.rados.rados.0.smithi017.stdout:1803: writing smithi01785587-37 from 531286 to 1243211 tid 1 2024-09-21T09:45:33.139 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:33 smithi017 ceph-mon[102913]: pgmap v339: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 107 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.9 MiB/s wr, 15 op/s; 18/204 objects degraded (8.824%) 2024-09-21T09:45:33.139 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:33 smithi017 ceph-mon[102913]: osdmap e507: 8 total, 7 up, 8 in 2024-09-21T09:45:33.139 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:33 smithi017 ceph-mon[102913]: osdmap e508: 8 total, 7 up, 8 in 2024-09-21T09:45:33.139 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:32 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[119509]: 2024-09-21T09:45:32.870+0000 7f03a8631640 -1 Falling back to public interface 2024-09-21T09:45:33.139 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:33 smithi017 ceph-mon[105151]: pgmap v339: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 107 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.9 MiB/s wr, 15 op/s; 18/204 objects degraded (8.824%) 2024-09-21T09:45:33.139 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:33 smithi017 ceph-mon[105151]: osdmap e507: 8 total, 7 up, 8 in 2024-09-21T09:45:33.139 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:33 smithi017 ceph-mon[105151]: osdmap e508: 8 total, 7 up, 8 in 2024-09-21T09:45:33.144 INFO:tasks.rados.rados.0.smithi017.stdout:1803: writing smithi01785587-37 from 1912196 to 2618440 tid 2 2024-09-21T09:45:33.151 INFO:tasks.rados.rados.0.smithi017.stdout:1803: writing smithi01785587-37 from 3018543 to 3731286 tid 3 2024-09-21T09:45:33.151 INFO:tasks.rados.rados.0.smithi017.stdout:1800: done (3 left) 2024-09-21T09:45:33.151 INFO:tasks.rados.rados.0.smithi017.stdout:1801: done (2 left) 2024-09-21T09:45:33.152 INFO:tasks.rados.rados.0.smithi017.stdout:1804: snap_create 2024-09-21T09:45:33.152 INFO:tasks.rados.rados.0.smithi017.stdout:1802: finishing write tid 3 to smithi01785587-38 2024-09-21T09:45:33.152 INFO:tasks.rados.rados.0.smithi017.stdout:1802: finishing write tid 4 to smithi01785587-38 2024-09-21T09:45:33.152 INFO:tasks.rados.rados.0.smithi017.stdout:1802: finishing write tid 5 to smithi01785587-38 2024-09-21T09:45:33.152 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 549 (ObjNum 697 snap 221 seq_num 697) dirty exists 2024-09-21T09:45:33.152 INFO:tasks.rados.rados.0.smithi017.stdout:1802: left oid 38 (ObjNum 697 snap 221 seq_num 697) 2024-09-21T09:45:33.152 INFO:tasks.rados.rados.0.smithi017.stdout:1803: finishing write tid 1 to smithi01785587-37 2024-09-21T09:45:33.152 INFO:tasks.rados.rados.0.smithi017.stdout:1803: finishing write tid 2 to smithi01785587-37 2024-09-21T09:45:33.154 INFO:tasks.rados.rados.0.smithi017.stdout:1803: finishing write tid 3 to smithi01785587-37 2024-09-21T09:45:33.154 INFO:tasks.rados.rados.0.smithi017.stdout:1803: finishing write tid 4 to smithi01785587-37 2024-09-21T09:45:33.155 INFO:tasks.rados.rados.0.smithi017.stdout:1803: finishing write tid 5 to smithi01785587-37 2024-09-21T09:45:33.155 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 876 (ObjNum 698 snap 221 seq_num 698) dirty exists 2024-09-21T09:45:33.155 INFO:tasks.rados.rados.0.smithi017.stdout:1803: left oid 37 (ObjNum 698 snap 221 seq_num 698) 2024-09-21T09:45:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:33 smithi134 ceph-mon[90085]: pgmap v339: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 107 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.9 MiB/s wr, 15 op/s; 18/204 objects degraded (8.824%) 2024-09-21T09:45:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:33 smithi134 ceph-mon[90085]: osdmap e507: 8 total, 7 up, 8 in 2024-09-21T09:45:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:33 smithi134 ceph-mon[90085]: osdmap e508: 8 total, 7 up, 8 in 2024-09-21T09:45:33.982 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:34.089 INFO:tasks.rados.rados.0.smithi017.stdout:1802: done (2 left) 2024-09-21T09:45:34.089 INFO:tasks.rados.rados.0.smithi017.stdout:1803: done (1 left) 2024-09-21T09:45:34.089 INFO:tasks.rados.rados.0.smithi017.stdout:1804: done (0 left) 2024-09-21T09:45:34.089 INFO:tasks.rados.rados.0.smithi017.stdout:1805: snap_create 2024-09-21T09:45:34.177 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3212881759' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:34.177 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3212881759' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:34.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3212881759' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:35.092 INFO:tasks.rados.rados.0.smithi017.stdout:1805: done (0 left) 2024-09-21T09:45:35.092 INFO:tasks.rados.rados.0.smithi017.stdout:1806: read oid 48 snap 221 2024-09-21T09:45:35.092 INFO:tasks.rados.rados.0.smithi017.stdout:1806: expect (ObjNum 693 snap 220 seq_num 693) 2024-09-21T09:45:35.092 INFO:tasks.rados.rados.0.smithi017.stdout:1807: read oid 5 snap 222 2024-09-21T09:45:35.092 INFO:tasks.rados.rados.0.smithi017.stdout:1807: expect (ObjNum 692 snap 219 seq_num 692) 2024-09-21T09:45:35.092 INFO:tasks.rados.rados.0.smithi017.stdout:1808: snap_create 2024-09-21T09:45:35.092 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:45:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:45:34] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-09-21T09:45:35.099 INFO:tasks.rados.rados.0.smithi017.stdout:1807: expect (ObjNum 692 snap 219 seq_num 692) 2024-09-21T09:45:35.276 INFO:tasks.rados.rados.0.smithi017.stdout:1806: expect (ObjNum 693 snap 220 seq_num 693) 2024-09-21T09:45:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:35 smithi017 ceph-mon[102913]: pgmap v342: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 109 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 18 op/s; 19/207 objects degraded (9.179%) 2024-09-21T09:45:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:35 smithi017 ceph-mon[102913]: from='client.44465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:35 smithi017 ceph-mon[102913]: osdmap e509: 8 total, 7 up, 8 in 2024-09-21T09:45:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:35 smithi017 ceph-mon[102913]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:35 smithi017 ceph-mon[105151]: pgmap v342: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 109 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 18 op/s; 19/207 objects degraded (9.179%) 2024-09-21T09:45:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:35 smithi017 ceph-mon[105151]: from='client.44465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:35 smithi017 ceph-mon[105151]: osdmap e509: 8 total, 7 up, 8 in 2024-09-21T09:45:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:35 smithi017 ceph-mon[105151]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:35.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:35 smithi134 ceph-mon[90085]: pgmap v342: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 109 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 18 op/s; 19/207 objects degraded (9.179%) 2024-09-21T09:45:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:35 smithi134 ceph-mon[90085]: from='client.44465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:35 smithi134 ceph-mon[90085]: osdmap e509: 8 total, 7 up, 8 in 2024-09-21T09:45:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:35 smithi134 ceph-mon[90085]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:36.100 INFO:tasks.rados.rados.0.smithi017.stdout:1806: done (2 left) 2024-09-21T09:45:36.100 INFO:tasks.rados.rados.0.smithi017.stdout:1807: done (1 left) 2024-09-21T09:45:36.100 INFO:tasks.rados.rados.0.smithi017.stdout:1808: done (0 left) 2024-09-21T09:45:36.100 INFO:tasks.rados.rados.0.smithi017.stdout:1809: snap_create 2024-09-21T09:45:36.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[102913]: osdmap e510: 8 total, 7 up, 8 in 2024-09-21T09:45:36.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[105151]: osdmap e510: 8 total, 7 up, 8 in 2024-09-21T09:45:36.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:36 smithi134 ceph-mon[90085]: osdmap e510: 8 total, 7 up, 8 in 2024-09-21T09:45:36.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:36.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:36 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[119509]: 2024-09-21T09:45:36.500+0000 7f03a8631640 -1 osd.1 500 log_to_monitors true 2024-09-21T09:45:36.894 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:37.103 INFO:tasks.rados.rados.0.smithi017.stdout:1809: done (0 left) 2024-09-21T09:45:37.103 INFO:tasks.rados.rados.0.smithi017.stdout:1810: read oid 31 snap 219 2024-09-21T09:45:37.103 INFO:tasks.rados.rados.0.smithi017.stdout:1810: expect deleted 2024-09-21T09:45:37.103 INFO:tasks.rados.rados.0.smithi017.stdout:1811: rollback oid 3 current snap is 225 2024-09-21T09:45:37.103 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 215 2024-09-21T09:45:37.103 INFO:tasks.rados.rados.0.smithi017.stdout:1812: rollback oid 37 current snap is 225 2024-09-21T09:45:37.103 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 223 2024-09-21T09:45:37.103 INFO:tasks.rados.rados.0.smithi017.stdout:1813: write oid 34 current snap is 225 2024-09-21T09:45:37.104 INFO:tasks.rados.rados.0.smithi017.stdout:1813: seq_num 699 ranges {548093=787174,1970678=449902,3045966=681137} 2024-09-21T09:45:37.111 INFO:tasks.rados.rados.0.smithi017.stdout:1813: writing smithi01785587-34 from 548093 to 1335267 tid 1 2024-09-21T09:45:37.119 INFO:tasks.rados.rados.0.smithi017.stdout:1813: writing smithi01785587-34 from 1970678 to 2420580 tid 2 2024-09-21T09:45:37.127 INFO:tasks.rados.rados.0.smithi017.stdout:1813: writing smithi01785587-34 from 3045966 to 3727103 tid 3 2024-09-21T09:45:37.127 INFO:tasks.rados.rados.0.smithi017.stdout:1814: snap_create 2024-09-21T09:45:37.127 INFO:tasks.rados.rados.0.smithi017.stdout:1811: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:45:37.127 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:45:37.127 INFO:tasks.rados.rados.0.smithi017.stdout:1812: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:45:37.127 INFO:tasks.rados.rados.0.smithi017.stdout:1812: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:45:37.127 INFO:tasks.rados.rados.0.smithi017.stdout:1812: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:45:37.127 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 880 (ObjNum 698 snap 221 seq_num 698) dirty exists 2024-09-21T09:45:37.127 INFO:tasks.rados.rados.0.smithi017.stdout:1813: finishing write tid 1 to smithi01785587-34 2024-09-21T09:45:37.128 INFO:tasks.rados.rados.0.smithi017.stdout:1813: finishing write tid 2 to smithi01785587-34 2024-09-21T09:45:37.131 INFO:tasks.rados.rados.0.smithi017.stdout:1813: finishing write tid 3 to smithi01785587-34 2024-09-21T09:45:37.131 INFO:tasks.rados.rados.0.smithi017.stdout:1813: finishing write tid 4 to smithi01785587-34 2024-09-21T09:45:37.131 INFO:tasks.rados.rados.0.smithi017.stdout:1813: finishing write tid 5 to smithi01785587-34 2024-09-21T09:45:37.131 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 915 (ObjNum 699 snap 225 seq_num 699) dirty exists 2024-09-21T09:45:37.131 INFO:tasks.rados.rados.0.smithi017.stdout:1813: left oid 34 (ObjNum 699 snap 225 seq_num 699) 2024-09-21T09:45:37.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[119509]: 2024-09-21T09:45:37.128+0000 7f03a03db640 -1 osd.1 500 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:45:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[105151]: pgmap v345: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 119 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 18 op/s; 20/222 objects degraded (9.009%) 2024-09-21T09:45:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[105151]: osdmap e511: 8 total, 7 up, 8 in 2024-09-21T09:45:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[105151]: from='osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:45:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/616504540' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 20/222 objects degraded (9.009%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[102913]: pgmap v345: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 119 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 18 op/s; 20/222 objects degraded (9.009%) 2024-09-21T09:45:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[102913]: osdmap e511: 8 total, 7 up, 8 in 2024-09-21T09:45:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[102913]: from='osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:45:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/616504540' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:37.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:37 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 20/222 objects degraded (9.009%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:37 smithi134 ceph-mon[90085]: pgmap v345: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 119 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 18 op/s; 20/222 objects degraded (9.009%) 2024-09-21T09:45:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:37 smithi134 ceph-mon[90085]: osdmap e511: 8 total, 7 up, 8 in 2024-09-21T09:45:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:37 smithi134 ceph-mon[90085]: from='osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:45:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:37 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/616504540' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:37 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 20/222 objects degraded (9.009%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:45:38.118 INFO:tasks.rados.rados.0.smithi017.stdout:1810: done (4 left) 2024-09-21T09:45:38.118 INFO:tasks.rados.rados.0.smithi017.stdout:1811: done (3 left) 2024-09-21T09:45:38.118 INFO:tasks.rados.rados.0.smithi017.stdout:1812: done (2 left) 2024-09-21T09:45:38.118 INFO:tasks.rados.rados.0.smithi017.stdout:1813: done (1 left) 2024-09-21T09:45:38.118 INFO:tasks.rados.rados.0.smithi017.stdout:1814: done (0 left) 2024-09-21T09:45:38.118 INFO:tasks.rados.rados.0.smithi017.stdout:1815: rollback oid 49 current snap is 226 2024-09-21T09:45:38.118 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 219 2024-09-21T09:45:38.119 INFO:tasks.rados.rados.0.smithi017.stdout:1816: snap_create 2024-09-21T09:45:38.123 INFO:tasks.rados.rados.0.smithi017.stdout:1815: finishing rollback tid 0 to smithi01785587-49 2024-09-21T09:45:38.123 INFO:tasks.rados.rados.0.smithi017.stdout:1815: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:45:38.123 INFO:tasks.rados.rados.0.smithi017.stdout:1815: finishing rollback tid 2 to smithi01785587-49 2024-09-21T09:45:38.123 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 919 (ObjNum 640 snap 202 seq_num 640) dirty exists 2024-09-21T09:45:38.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[102913]: from='client.34623 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:38.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[102913]: from='osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:45:38.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[102913]: osdmap e512: 8 total, 7 up, 8 in 2024-09-21T09:45:38.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[102913]: from='osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:45:38.366 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[102913]: from='client.34629 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:38.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[105151]: from='client.34623 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:38.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[105151]: from='osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:45:38.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[105151]: osdmap e512: 8 total, 7 up, 8 in 2024-09-21T09:45:38.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[105151]: from='osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:45:38.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:38 smithi017 ceph-mon[105151]: from='client.34629 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:38 smithi134 ceph-mon[90085]: from='client.34623 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:38 smithi134 ceph-mon[90085]: from='osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:45:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:38 smithi134 ceph-mon[90085]: osdmap e512: 8 total, 7 up, 8 in 2024-09-21T09:45:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:38 smithi134 ceph-mon[90085]: from='osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:45:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:38 smithi134 ceph-mon[90085]: from='client.34629 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:39.124 INFO:tasks.rados.rados.0.smithi017.stdout:1815: done (1 left) 2024-09-21T09:45:39.124 INFO:tasks.rados.rados.0.smithi017.stdout:1816: done (0 left) 2024-09-21T09:45:39.124 INFO:tasks.rados.rados.0.smithi017.stdout:1817: delete oid 18 current snap is 227 2024-09-21T09:45:39.134 INFO:tasks.rados.rados.0.smithi017.stdout:1817: done (0 left) 2024-09-21T09:45:39.134 INFO:tasks.rados.rados.0.smithi017.stdout:1818: write oid 38 current snap is 227 2024-09-21T09:45:39.134 INFO:tasks.rados.rados.0.smithi017.stdout:1818: seq_num 701 ranges {581707=654025,1687642=453571,2700812=703681} 2024-09-21T09:45:39.137 INFO:tasks.rados.rados.0.smithi017.stdout:1818: writing smithi01785587-38 from 581707 to 1235732 tid 1 2024-09-21T09:45:39.143 INFO:tasks.rados.rados.0.smithi017.stdout:1818: writing smithi01785587-38 from 1687642 to 2141213 tid 2 2024-09-21T09:45:39.150 INFO:tasks.rados.rados.0.smithi017.stdout:1818: writing smithi01785587-38 from 2700812 to 3404493 tid 3 2024-09-21T09:45:39.151 INFO:tasks.rados.rados.0.smithi017.stdout:1819: snap_create 2024-09-21T09:45:39.161 INFO:tasks.rados.rados.0.smithi017.stdout:1818: finishing write tid 1 to smithi01785587-38 2024-09-21T09:45:39.161 INFO:tasks.rados.rados.0.smithi017.stdout:1818: finishing write tid 2 to smithi01785587-38 2024-09-21T09:45:39.162 INFO:tasks.rados.rados.0.smithi017.stdout:1818: finishing write tid 3 to smithi01785587-38 2024-09-21T09:45:39.162 INFO:tasks.rados.rados.0.smithi017.stdout:1818: finishing write tid 4 to smithi01785587-38 2024-09-21T09:45:39.162 INFO:tasks.rados.rados.0.smithi017.stdout:1818: finishing write tid 5 to smithi01785587-38 2024-09-21T09:45:39.162 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 554 (ObjNum 701 snap 227 seq_num 701) dirty exists 2024-09-21T09:45:39.162 INFO:tasks.rados.rados.0.smithi017.stdout:1818: left oid 38 (ObjNum 701 snap 227 seq_num 701) 2024-09-21T09:45:39.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:45:39.263+0000 7f3131349640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:45:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[102913]: pgmap v348: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 119 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 14 op/s; 20/222 objects degraded (9.009%) 2024-09-21T09:45:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[102913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:45:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[102913]: osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230] boot 2024-09-21T09:45:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[102913]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T09:45:39.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:45:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[105151]: pgmap v348: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 119 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 14 op/s; 20/222 objects degraded (9.009%) 2024-09-21T09:45:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[105151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:45:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[105151]: osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230] boot 2024-09-21T09:45:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[105151]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T09:45:39.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:45:39.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:39 smithi134 ceph-mon[90085]: pgmap v348: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 119 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 14 op/s; 20/222 objects degraded (9.009%) 2024-09-21T09:45:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:39 smithi134 ceph-mon[90085]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:45:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:39 smithi134 ceph-mon[90085]: osd.1 [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230] boot 2024-09-21T09:45:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:39 smithi134 ceph-mon[90085]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T09:45:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:45:39.811 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:40.156 INFO:tasks.rados.rados.0.smithi017.stdout:1818: done (1 left) 2024-09-21T09:45:40.157 INFO:tasks.rados.rados.0.smithi017.stdout:1819: done (0 left) 2024-09-21T09:45:40.157 INFO:tasks.rados.rados.0.smithi017.stdout:1820: snap_remove snap 222 2024-09-21T09:45:40.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T09:45:40.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:40.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:40.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:40.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1222096327' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:40.391 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:45:40.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1222096327' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:45:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T09:45:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:40.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:40.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:40.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:40.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:45:40.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:45:40.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:45:40.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1222096327' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:41.158 INFO:tasks.rados.rados.0.smithi017.stdout:1820: done (0 left) 2024-09-21T09:45:41.158 INFO:tasks.rados.rados.0.smithi017.stdout:1821: read oid 4 snap -1 2024-09-21T09:45:41.158 INFO:tasks.rados.rados.0.smithi017.stdout:1821: expect deleted 2024-09-21T09:45:41.158 INFO:tasks.rados.rados.0.smithi017.stdout:1822: snap_remove snap 218 2024-09-21T09:45:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[102913]: from='client.34641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[102913]: pgmap v351: 89 pgs: 4 peering, 27 active+undersized, 6 active+undersized+degraded, 52 active+clean; 121 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 521 KiB/s wr, 1 op/s; 15/228 objects degraded (6.579%) 2024-09-21T09:45:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[102913]: from='client.34647 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[102913]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[105151]: from='client.34641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[105151]: pgmap v351: 89 pgs: 4 peering, 27 active+undersized, 6 active+undersized+degraded, 52 active+clean; 121 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 521 KiB/s wr, 1 op/s; 15/228 objects degraded (6.579%) 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[105151]: from='client.34647 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[105151]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:41 smithi134 ceph-mon[90085]: from='client.34641 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:41 smithi134 ceph-mon[90085]: pgmap v351: 89 pgs: 4 peering, 27 active+undersized, 6 active+undersized+degraded, 52 active+clean; 121 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 521 KiB/s wr, 1 op/s; 15/228 objects degraded (6.579%) 2024-09-21T09:45:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:41 smithi134 ceph-mon[90085]: from='client.34647 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:41 smithi134 ceph-mon[90085]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T09:45:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:42.169 INFO:tasks.rados.rados.0.smithi017.stdout:1821: done (1 left) 2024-09-21T09:45:42.169 INFO:tasks.rados.rados.0.smithi017.stdout:1822: done (0 left) 2024-09-21T09:45:42.169 INFO:tasks.rados.rados.0.smithi017.stdout:1823: delete oid 16 current snap is 228 2024-09-21T09:45:42.172 INFO:tasks.rados.rados.0.smithi017.stdout:1823: done (0 left) 2024-09-21T09:45:42.172 INFO:tasks.rados.rados.0.smithi017.stdout:1824: read oid 11 snap -1 2024-09-21T09:45:42.172 INFO:tasks.rados.rados.0.smithi017.stdout:1824: expect deleted 2024-09-21T09:45:42.172 INFO:tasks.rados.rados.0.smithi017.stdout:1825: rollback oid 39 current snap is 228 2024-09-21T09:45:42.172 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 214 2024-09-21T09:45:42.173 INFO:tasks.rados.rados.0.smithi017.stdout:1826: write oid 36 current snap is 228 2024-09-21T09:45:42.173 INFO:tasks.rados.rados.0.smithi017.stdout:1826: seq_num 703 ranges {615321=404523,1804606=457240,2755658=726225} 2024-09-21T09:45:42.179 INFO:tasks.rados.rados.0.smithi017.stdout:1826: writing smithi01785587-36 from 615321 to 1019844 tid 1 2024-09-21T09:45:42.186 INFO:tasks.rados.rados.0.smithi017.stdout:1826: writing smithi01785587-36 from 1804606 to 2261846 tid 2 2024-09-21T09:45:42.192 INFO:tasks.rados.rados.0.smithi017.stdout:1826: writing smithi01785587-36 from 2755658 to 3481883 tid 3 2024-09-21T09:45:42.192 INFO:tasks.rados.rados.0.smithi017.stdout:1827: write oid 9 current snap is 228 2024-09-21T09:45:42.193 INFO:tasks.rados.rados.0.smithi017.stdout:1827: seq_num 704 ranges {632128=479772,1746735=600898,3066728=653850} 2024-09-21T09:45:42.199 INFO:tasks.rados.rados.0.smithi017.stdout:1827: writing smithi01785587-9 from 632128 to 1111900 tid 1 2024-09-21T09:45:42.206 INFO:tasks.rados.rados.0.smithi017.stdout:1827: writing smithi01785587-9 from 1746735 to 2347633 tid 2 2024-09-21T09:45:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:1827: writing smithi01785587-9 from 3066728 to 3720578 tid 3 2024-09-21T09:45:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:1828: read oid 20 snap -1 2024-09-21T09:45:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:1828: expect deleted 2024-09-21T09:45:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:1829: snap_remove snap 224 2024-09-21T09:45:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:42 smithi017 ceph-mon[102913]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T09:45:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:42 smithi017 ceph-mon[105151]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T09:45:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:42 smithi134 ceph-mon[90085]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T09:45:42.718 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:43.174 INFO:tasks.rados.rados.0.smithi017.stdout:1829: done (5 left) 2024-09-21T09:45:43.175 INFO:tasks.rados.rados.0.smithi017.stdout:1830: write oid 44 current snap is 228 2024-09-21T09:45:43.175 INFO:tasks.rados.rados.0.smithi017.stdout:1830: seq_num 705 ranges {648935=555021,1805217=460909,2694151=465122,3817442=31493} 2024-09-21T09:45:43.181 INFO:tasks.rados.rados.0.smithi017.stdout:1830: writing smithi01785587-44 from 648935 to 1203956 tid 1 2024-09-21T09:45:43.187 INFO:tasks.rados.rados.0.smithi017.stdout:1830: writing smithi01785587-44 from 1805217 to 2266126 tid 2 2024-09-21T09:45:43.192 INFO:tasks.rados.rados.0.smithi017.stdout:1830: writing smithi01785587-44 from 2694151 to 3159273 tid 3 2024-09-21T09:45:43.196 INFO:tasks.rados.rados.0.smithi017.stdout:1830: writing smithi01785587-44 from 3817442 to 3848935 tid 4 2024-09-21T09:45:43.196 INFO:tasks.rados.rados.0.smithi017.stdout:1831: rollback oid 33 current snap is 228 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 215 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1825: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1825: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1825: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 890 (ObjNum 626 snap 197 seq_num 626) dirty exists 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1826: finishing write tid 1 to smithi01785587-36 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1824: done (6 left) 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1825: done (5 left) 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1832: delete oid 8 current snap is 228 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1826: finishing write tid 2 to smithi01785587-36 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1826: finishing write tid 3 to smithi01785587-36 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1826: finishing write tid 4 to smithi01785587-36 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1826: finishing write tid 5 to smithi01785587-36 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 895 (ObjNum 703 snap 228 seq_num 703) dirty exists 2024-09-21T09:45:43.197 INFO:tasks.rados.rados.0.smithi017.stdout:1826: left oid 36 (ObjNum 703 snap 228 seq_num 703) 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:1827: finishing write tid 1 to smithi01785587-9 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:1827: finishing write tid 2 to smithi01785587-9 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:1827: finishing write tid 3 to smithi01785587-9 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:1827: finishing write tid 4 to smithi01785587-9 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:1827: finishing write tid 5 to smithi01785587-9 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 325 (ObjNum 704 snap 228 seq_num 704) dirty exists 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:1827: left oid 9 (ObjNum 704 snap 228 seq_num 704) 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:1830: finishing write tid 1 to smithi01785587-44 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:1830: finishing write tid 2 to smithi01785587-44 2024-09-21T09:45:43.198 INFO:tasks.rados.rados.0.smithi017.stdout:1830: finishing write tid 3 to smithi01785587-44 2024-09-21T09:45:43.199 INFO:tasks.rados.rados.0.smithi017.stdout:1830: finishing write tid 4 to smithi01785587-44 2024-09-21T09:45:43.199 INFO:tasks.rados.rados.0.smithi017.stdout:1826: done (5 left) 2024-09-21T09:45:43.199 INFO:tasks.rados.rados.0.smithi017.stdout:1827: done (4 left) 2024-09-21T09:45:43.199 INFO:tasks.rados.rados.0.smithi017.stdout:1828: done (3 left) 2024-09-21T09:45:43.199 INFO:tasks.rados.rados.0.smithi017.stdout:1832: done (2 left) 2024-09-21T09:45:43.199 INFO:tasks.rados.rados.0.smithi017.stdout:1833: write oid 40 current snap is 228 2024-09-21T09:45:43.199 INFO:tasks.rados.rados.0.smithi017.stdout:1833: seq_num 707 ranges {682549=705519,1922181=464578,3148997=487666} 2024-09-21T09:45:43.205 INFO:tasks.rados.rados.0.smithi017.stdout:1833: writing smithi01785587-40 from 682549 to 1388068 tid 1 2024-09-21T09:45:43.211 INFO:tasks.rados.rados.0.smithi017.stdout:1833: writing smithi01785587-40 from 1922181 to 2386759 tid 2 2024-09-21T09:45:43.218 INFO:tasks.rados.rados.0.smithi017.stdout:1833: writing smithi01785587-40 from 3148997 to 3636663 tid 3 2024-09-21T09:45:43.218 INFO:tasks.rados.rados.0.smithi017.stdout:1834: write oid 16 current snap is 228 2024-09-21T09:45:43.218 INFO:tasks.rados.rados.0.smithi017.stdout:1830: finishing write tid 5 to smithi01785587-44 2024-09-21T09:45:43.218 INFO:tasks.rados.rados.0.smithi017.stdout:1834: seq_num 708 ranges {699356=497121,1980663=608236,3176420=415291} 2024-09-21T09:45:43.224 INFO:tasks.rados.rados.0.smithi017.stdout:1834: writing smithi01785587-16 from 699356 to 1196477 tid 1 2024-09-21T09:45:43.232 INFO:tasks.rados.rados.0.smithi017.stdout:1834: writing smithi01785587-16 from 1980663 to 2588899 tid 2 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:1834: writing smithi01785587-16 from 3176420 to 3591711 tid 3 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:1835: rollback oid 21 current snap is 228 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 217 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:1830: finishing write tid 6 to smithi01785587-44 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 880 (ObjNum 705 snap 228 seq_num 705) dirty exists 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:1830: left oid 44 (ObjNum 705 snap 228 seq_num 705) 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:1830: done (4 left) 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:1836: rollback oid 4 current snap is 228 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 4 to 220 2024-09-21T09:45:43.239 INFO:tasks.rados.rados.0.smithi017.stdout:1831: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1831: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1831: finishing rollback tid 2 to smithi01785587-33 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 834 (ObjNum 638 snap 202 seq_num 638) dirty exists 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1833: finishing write tid 1 to smithi01785587-40 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1833: finishing write tid 2 to smithi01785587-40 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1833: finishing write tid 3 to smithi01785587-40 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1833: finishing write tid 4 to smithi01785587-40 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1833: finishing write tid 5 to smithi01785587-40 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 905 (ObjNum 707 snap 228 seq_num 707) dirty exists 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1833: left oid 40 (ObjNum 707 snap 228 seq_num 707) 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1834: finishing write tid 1 to smithi01785587-16 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1834: finishing write tid 2 to smithi01785587-16 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1831: done (4 left) 2024-09-21T09:45:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:1833: done (3 left) 2024-09-21T09:45:43.241 INFO:tasks.rados.rados.0.smithi017.stdout:1837: snap_create 2024-09-21T09:45:43.241 INFO:tasks.rados.rados.0.smithi017.stdout:1835: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:45:43.241 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 0 (ObjNum 33 snap 0 seq_num 2384147832) dirty dne 2024-09-21T09:45:43.241 INFO:tasks.rados.rados.0.smithi017.stdout:1834: finishing write tid 3 to smithi01785587-16 2024-09-21T09:45:43.241 INFO:tasks.rados.rados.0.smithi017.stdout:1834: finishing write tid 4 to smithi01785587-16 2024-09-21T09:45:43.242 INFO:tasks.rados.rados.0.smithi017.stdout:1834: finishing write tid 5 to smithi01785587-16 2024-09-21T09:45:43.242 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 838 (ObjNum 708 snap 228 seq_num 708) dirty exists 2024-09-21T09:45:43.242 INFO:tasks.rados.rados.0.smithi017.stdout:1834: left oid 16 (ObjNum 708 snap 228 seq_num 708) 2024-09-21T09:45:43.242 INFO:tasks.rados.rados.0.smithi017.stdout:1836: finishing rollback tid 1 to smithi01785587-4 2024-09-21T09:45:43.242 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 0 (ObjNum 193 snap 0 seq_num 2399808592) dirty dne 2024-09-21T09:45:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[105151]: pgmap v354: 89 pgs: 4 peering, 10 active+undersized, 75 active+clean; 125 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.4 MiB/s wr, 5 op/s; 1.9 MiB/s, 1 objects/s recovering 2024-09-21T09:45:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/228 objects degraded (6.579%), 6 pgs degraded) 2024-09-21T09:45:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T09:45:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[105151]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T09:45:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3850331904' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[102913]: pgmap v354: 89 pgs: 4 peering, 10 active+undersized, 75 active+clean; 125 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.4 MiB/s wr, 5 op/s; 1.9 MiB/s, 1 objects/s recovering 2024-09-21T09:45:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/228 objects degraded (6.579%), 6 pgs degraded) 2024-09-21T09:45:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T09:45:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[102913]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T09:45:43.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:43 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3850331904' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:43.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:43 smithi134 ceph-mon[90085]: pgmap v354: 89 pgs: 4 peering, 10 active+undersized, 75 active+clean; 125 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.4 MiB/s wr, 5 op/s; 1.9 MiB/s, 1 objects/s recovering 2024-09-21T09:45:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:43 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/228 objects degraded (6.579%), 6 pgs degraded) 2024-09-21T09:45:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:43 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T09:45:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:43 smithi134 ceph-mon[90085]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T09:45:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:43 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3850331904' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:44.178 INFO:tasks.rados.rados.0.smithi017.stdout:1834: done (3 left) 2024-09-21T09:45:44.178 INFO:tasks.rados.rados.0.smithi017.stdout:1835: done (2 left) 2024-09-21T09:45:44.178 INFO:tasks.rados.rados.0.smithi017.stdout:1836: done (1 left) 2024-09-21T09:45:44.178 INFO:tasks.rados.rados.0.smithi017.stdout:1837: done (0 left) 2024-09-21T09:45:44.178 INFO:tasks.rados.rados.0.smithi017.stdout:1838: snap_create 2024-09-21T09:45:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:44 smithi017 ceph-mon[102913]: from='client.44522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:44 smithi017 ceph-mon[102913]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:44 smithi017 ceph-mon[102913]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T09:45:44.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:44 smithi017 ceph-mon[105151]: from='client.44522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:44 smithi017 ceph-mon[105151]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:44 smithi017 ceph-mon[105151]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T09:45:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:44 smithi134 ceph-mon[90085]: from='client.44522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:44 smithi134 ceph-mon[90085]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:44.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:44 smithi134 ceph-mon[90085]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T09:45:45.178 INFO:tasks.rados.rados.0.smithi017.stdout:1838: done (0 left) 2024-09-21T09:45:45.179 INFO:tasks.rados.rados.0.smithi017.stdout:1839: snap_create 2024-09-21T09:45:45.180 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:45:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:45:44] "GET /metrics HTTP/1.1" 200 36285 "" "Prometheus/2.51.0" 2024-09-21T09:45:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:45 smithi017 ceph-mon[102913]: pgmap v357: 89 pgs: 89 active+clean; 127 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 5 op/s; 3.1 MiB/s, 2 objects/s recovering 2024-09-21T09:45:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:45 smithi017 ceph-mon[102913]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T09:45:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:45 smithi017 ceph-mon[105151]: pgmap v357: 89 pgs: 89 active+clean; 127 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 5 op/s; 3.1 MiB/s, 2 objects/s recovering 2024-09-21T09:45:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:45 smithi017 ceph-mon[105151]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T09:45:45.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:45 smithi134 ceph-mon[90085]: pgmap v357: 89 pgs: 89 active+clean; 127 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 5 op/s; 3.1 MiB/s, 2 objects/s recovering 2024-09-21T09:45:45.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:45 smithi134 ceph-mon[90085]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T09:45:45.623 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:46.189 INFO:tasks.rados.rados.0.smithi017.stdout:1839: done (0 left) 2024-09-21T09:45:46.189 INFO:tasks.rados.rados.0.smithi017.stdout:1840: read oid 26 snap 229 2024-09-21T09:45:46.189 INFO:tasks.rados.rados.0.smithi017.stdout:1840: expect (ObjNum 691 snap 219 seq_num 691) 2024-09-21T09:45:46.190 INFO:tasks.rados.rados.0.smithi017.stdout:1841: write oid 30 current snap is 231 2024-09-21T09:45:46.190 INFO:tasks.rados.rados.0.smithi017.stdout:1841: seq_num 709 ranges {716163=572370,2039145=468247,3203843=626563} 2024-09-21T09:45:46.196 INFO:tasks.rados.rados.0.smithi017.stdout:1841: writing smithi01785587-30 from 716163 to 1288533 tid 1 2024-09-21T09:45:46.203 INFO:tasks.rados.rados.0.smithi017.stdout:1841: writing smithi01785587-30 from 2039145 to 2507392 tid 2 2024-09-21T09:45:46.210 INFO:tasks.rados.rados.0.smithi017.stdout:1841: writing smithi01785587-30 from 3203843 to 3830406 tid 3 2024-09-21T09:45:46.210 INFO:tasks.rados.rados.0.smithi017.stdout:1842: snap_create 2024-09-21T09:45:46.210 INFO:tasks.rados.rados.0.smithi017.stdout:1840: expect (ObjNum 691 snap 219 seq_num 691) 2024-09-21T09:45:46.331 INFO:tasks.rados.rados.0.smithi017.stdout:1841: finishing write tid 1 to smithi01785587-30 2024-09-21T09:45:46.331 INFO:tasks.rados.rados.0.smithi017.stdout:1841: finishing write tid 2 to smithi01785587-30 2024-09-21T09:45:46.331 INFO:tasks.rados.rados.0.smithi017.stdout:1841: finishing write tid 3 to smithi01785587-30 2024-09-21T09:45:46.331 INFO:tasks.rados.rados.0.smithi017.stdout:1841: finishing write tid 4 to smithi01785587-30 2024-09-21T09:45:46.332 INFO:tasks.rados.rados.0.smithi017.stdout:1841: finishing write tid 5 to smithi01785587-30 2024-09-21T09:45:46.332 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 726 (ObjNum 709 snap 231 seq_num 709) dirty exists 2024-09-21T09:45:46.332 INFO:tasks.rados.rados.0.smithi017.stdout:1841: left oid 30 (ObjNum 709 snap 231 seq_num 709) 2024-09-21T09:45:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:46 smithi017 ceph-mon[102913]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T09:45:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2173583122' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:46 smithi017 ceph-mon[105151]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T09:45:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2173583122' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:46.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:46 smithi134 ceph-mon[90085]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T09:45:46.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2173583122' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:47.196 INFO:tasks.rados.rados.0.smithi017.stdout:1840: done (2 left) 2024-09-21T09:45:47.196 INFO:tasks.rados.rados.0.smithi017.stdout:1841: done (1 left) 2024-09-21T09:45:47.197 INFO:tasks.rados.rados.0.smithi017.stdout:1842: done (0 left) 2024-09-21T09:45:47.197 INFO:tasks.rados.rados.0.smithi017.stdout:1843: delete oid 30 current snap is 232 2024-09-21T09:45:47.202 INFO:tasks.rados.rados.0.smithi017.stdout:1843: done (0 left) 2024-09-21T09:45:47.202 INFO:tasks.rados.rados.0.smithi017.stdout:1844: read oid 24 snap -1 2024-09-21T09:45:47.202 INFO:tasks.rados.rados.0.smithi017.stdout:1844: expect (ObjNum 681 snap 215 seq_num 681) 2024-09-21T09:45:47.202 INFO:tasks.rados.rados.0.smithi017.stdout:1845: read oid 19 snap 225 2024-09-21T09:45:47.202 INFO:tasks.rados.rados.0.smithi017.stdout:1845: expect deleted 2024-09-21T09:45:47.202 INFO:tasks.rados.rados.0.smithi017.stdout:1846: snap_create 2024-09-21T09:45:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:1844: expect (ObjNum 681 snap 215 seq_num 681) 2024-09-21T09:45:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:47 smithi017 ceph-mon[105151]: from='client.34677 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:47 smithi017 ceph-mon[105151]: pgmap v360: 89 pgs: 89 active+clean; 135 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 6 op/s; 1.3 MiB/s, 0 objects/s recovering 2024-09-21T09:45:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:47 smithi017 ceph-mon[105151]: from='client.34683 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:47 smithi017 ceph-mon[105151]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T09:45:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:47 smithi017 ceph-mon[102913]: from='client.34677 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:47 smithi017 ceph-mon[102913]: pgmap v360: 89 pgs: 89 active+clean; 135 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 6 op/s; 1.3 MiB/s, 0 objects/s recovering 2024-09-21T09:45:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:47 smithi017 ceph-mon[102913]: from='client.34683 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:47.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:47 smithi017 ceph-mon[102913]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T09:45:47.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:47 smithi134 ceph-mon[90085]: from='client.34677 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:47 smithi134 ceph-mon[90085]: pgmap v360: 89 pgs: 89 active+clean; 135 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 6 op/s; 1.3 MiB/s, 0 objects/s recovering 2024-09-21T09:45:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:47 smithi134 ceph-mon[90085]: from='client.34683 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:47 smithi134 ceph-mon[90085]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T09:45:48.076 INFO:tasks.rados.rados.0.smithi017.stdout:1844: done (2 left) 2024-09-21T09:45:48.076 INFO:tasks.rados.rados.0.smithi017.stdout:1845: done (1 left) 2024-09-21T09:45:48.076 INFO:tasks.rados.rados.0.smithi017.stdout:1846: done (0 left) 2024-09-21T09:45:48.076 INFO:tasks.rados.rados.0.smithi017.stdout:1847: write oid 21 current snap is 233 2024-09-21T09:45:48.076 INFO:tasks.rados.rados.0.smithi017.stdout:1847: seq_num 711 ranges {749777=722868,2156109=471916,3258689=649107} 2024-09-21T09:45:48.083 INFO:tasks.rados.rados.0.smithi017.stdout:1847: writing smithi01785587-21 from 749777 to 1472645 tid 1 2024-09-21T09:45:48.090 INFO:tasks.rados.rados.0.smithi017.stdout:1847: writing smithi01785587-21 from 2156109 to 2628025 tid 2 2024-09-21T09:45:48.097 INFO:tasks.rados.rados.0.smithi017.stdout:1847: writing smithi01785587-21 from 3258689 to 3907796 tid 3 2024-09-21T09:45:48.097 INFO:tasks.rados.rados.0.smithi017.stdout:1848: write oid 30 current snap is 233 2024-09-21T09:45:48.098 INFO:tasks.rados.rados.0.smithi017.stdout:1848: seq_num 712 ranges {766584=798117,2098238=615574,3169759=460379} 2024-09-21T09:45:48.106 INFO:tasks.rados.rados.0.smithi017.stdout:1848: writing smithi01785587-30 from 766584 to 1564701 tid 1 2024-09-21T09:45:48.112 INFO:tasks.rados.rados.0.smithi017.stdout:1848: writing smithi01785587-30 from 2098238 to 2713812 tid 2 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:1848: writing smithi01785587-30 from 3169759 to 3630138 tid 3 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:1849: delete oid 4 current snap is 233 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:1847: finishing write tid 1 to smithi01785587-21 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:1847: finishing write tid 2 to smithi01785587-21 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:1847: finishing write tid 3 to smithi01785587-21 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:1847: finishing write tid 4 to smithi01785587-21 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:1847: finishing write tid 5 to smithi01785587-21 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 937 (ObjNum 711 snap 233 seq_num 711) dirty exists 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:1847: left oid 21 (ObjNum 711 snap 233 seq_num 711) 2024-09-21T09:45:48.118 INFO:tasks.rados.rados.0.smithi017.stdout:1848: finishing write tid 1 to smithi01785587-30 2024-09-21T09:45:48.119 INFO:tasks.rados.rados.0.smithi017.stdout:1848: finishing write tid 2 to smithi01785587-30 2024-09-21T09:45:48.119 INFO:tasks.rados.rados.0.smithi017.stdout:1847: done (2 left) 2024-09-21T09:45:48.120 INFO:tasks.rados.rados.0.smithi017.stdout:1849: done (1 left) 2024-09-21T09:45:48.120 INFO:tasks.rados.rados.0.smithi017.stdout:1850: snap_create 2024-09-21T09:45:48.120 INFO:tasks.rados.rados.0.smithi017.stdout:1848: finishing write tid 3 to smithi01785587-30 2024-09-21T09:45:48.120 INFO:tasks.rados.rados.0.smithi017.stdout:1848: finishing write tid 4 to smithi01785587-30 2024-09-21T09:45:48.121 INFO:tasks.rados.rados.0.smithi017.stdout:1848: finishing write tid 5 to smithi01785587-30 2024-09-21T09:45:48.121 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 732 (ObjNum 712 snap 233 seq_num 712) dirty exists 2024-09-21T09:45:48.121 INFO:tasks.rados.rados.0.smithi017.stdout:1848: left oid 30 (ObjNum 712 snap 233 seq_num 712) 2024-09-21T09:45:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:48 smithi017 ceph-mon[105151]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T09:45:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:48 smithi017 ceph-mon[105151]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T09:45:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:48 smithi017 ceph-mon[102913]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T09:45:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:48 smithi017 ceph-mon[102913]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T09:45:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:48 smithi134 ceph-mon[90085]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T09:45:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:48 smithi134 ceph-mon[90085]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T09:45:48.531 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:49.078 INFO:tasks.rados.rados.0.smithi017.stdout:1848: done (1 left) 2024-09-21T09:45:49.078 INFO:tasks.rados.rados.0.smithi017.stdout:1850: done (0 left) 2024-09-21T09:45:49.078 INFO:tasks.rados.rados.0.smithi017.stdout:1851: read oid 7 snap -1 2024-09-21T09:45:49.078 INFO:tasks.rados.rados.0.smithi017.stdout:1851: expect (ObjNum 694 snap 220 seq_num 694) 2024-09-21T09:45:49.078 INFO:tasks.rados.rados.0.smithi017.stdout:1852: snap_remove snap 216 2024-09-21T09:45:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:49 smithi017 ceph-mon[102913]: pgmap v363: 89 pgs: 89 active+clean; 138 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:45:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:49 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/874039424' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:49 smithi017 ceph-mon[102913]: from='client.34695 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:49 smithi017 ceph-mon[102913]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T09:45:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:49 smithi017 ceph-mon[105151]: pgmap v363: 89 pgs: 89 active+clean; 138 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:45:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:49 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/874039424' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:49 smithi017 ceph-mon[105151]: from='client.34695 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:49 smithi017 ceph-mon[105151]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T09:45:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:49 smithi134 ceph-mon[90085]: pgmap v363: 89 pgs: 89 active+clean; 138 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:45:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:49 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/874039424' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:49 smithi134 ceph-mon[90085]: from='client.34695 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:49 smithi134 ceph-mon[90085]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T09:45:50.081 INFO:tasks.rados.rados.0.smithi017.stdout:1852: done (1 left) 2024-09-21T09:45:50.081 INFO:tasks.rados.rados.0.smithi017.stdout:1853: snap_create 2024-09-21T09:45:50.081 INFO:tasks.rados.rados.0.smithi017.stdout:1851: expect (ObjNum 694 snap 220 seq_num 694) 2024-09-21T09:45:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:50 smithi134 ceph-mon[90085]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:50 smithi134 ceph-mon[90085]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:45:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:45:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:45:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:45:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:50 smithi017 ceph-mon[105151]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:50.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:50 smithi017 ceph-mon[105151]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:45:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:50 smithi017 ceph-mon[102913]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:50 smithi017 ceph-mon[102913]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:45:51.082 INFO:tasks.rados.rados.0.smithi017.stdout:1851: done (1 left) 2024-09-21T09:45:51.082 INFO:tasks.rados.rados.0.smithi017.stdout:1853: done (0 left) 2024-09-21T09:45:51.082 INFO:tasks.rados.rados.0.smithi017.stdout:1854: read oid 38 snap -1 2024-09-21T09:45:51.082 INFO:tasks.rados.rados.0.smithi017.stdout:1854: expect (ObjNum 701 snap 227 seq_num 701) 2024-09-21T09:45:51.082 INFO:tasks.rados.rados.0.smithi017.stdout:1855: snap_create 2024-09-21T09:45:51.087 INFO:tasks.rados.rados.0.smithi017.stdout:1854: expect (ObjNum 701 snap 227 seq_num 701) 2024-09-21T09:45:51.470 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:51 smithi017 ceph-mon[102913]: pgmap v366: 89 pgs: 89 active+clean; 142 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 829 KiB/s wr, 2 op/s 2024-09-21T09:45:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:51 smithi017 ceph-mon[102913]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:45:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:51 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3572624283' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:51 smithi017 ceph-mon[105151]: pgmap v366: 89 pgs: 89 active+clean; 142 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 829 KiB/s wr, 2 op/s 2024-09-21T09:45:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:51 smithi017 ceph-mon[105151]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:45:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:51 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3572624283' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:51.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:51 smithi134 ceph-mon[90085]: pgmap v366: 89 pgs: 89 active+clean; 142 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 829 KiB/s wr, 2 op/s 2024-09-21T09:45:51.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:51 smithi134 ceph-mon[90085]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:45:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:51 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3572624283' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:52.082 INFO:tasks.rados.rados.0.smithi017.stdout:1854: done (1 left) 2024-09-21T09:45:52.082 INFO:tasks.rados.rados.0.smithi017.stdout:1855: done (0 left) 2024-09-21T09:45:52.082 INFO:tasks.rados.rados.0.smithi017.stdout:1856: snap_create 2024-09-21T09:45:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:52 smithi017 ceph-mon[102913]: from='client.34713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:52 smithi017 ceph-mon[102913]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:45:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:52 smithi017 ceph-mon[105151]: from='client.34713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:52 smithi017 ceph-mon[105151]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:45:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:52 smithi134 ceph-mon[90085]: from='client.34713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:52 smithi134 ceph-mon[90085]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:45:53.079 INFO:tasks.rados.rados.0.smithi017.stdout:1856: done (0 left) 2024-09-21T09:45:53.079 INFO:tasks.rados.rados.0.smithi017.stdout:1857: write oid 5 current snap is 237 2024-09-21T09:45:53.079 INFO:tasks.rados.rados.0.smithi017.stdout:1857: seq_num 714 ranges {400198=548615,1415202=584996} 2024-09-21T09:45:53.084 INFO:tasks.rados.rados.0.smithi017.stdout:1857: writing smithi01785587-5 from 400198 to 948813 tid 1 2024-09-21T09:45:53.091 INFO:tasks.rados.rados.0.smithi017.stdout:1857: writing smithi01785587-5 from 1415202 to 2000198 tid 2 2024-09-21T09:45:53.091 INFO:tasks.rados.rados.0.smithi017.stdout:1858: rollback oid 4 current snap is 237 2024-09-21T09:45:53.091 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 4 to 215 2024-09-21T09:45:53.091 INFO:tasks.rados.rados.0.smithi017.stdout:1857: finishing write tid 1 to smithi01785587-5 2024-09-21T09:45:53.091 INFO:tasks.rados.rados.0.smithi017.stdout:1859: snap_create 2024-09-21T09:45:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:1858: finishing rollback tid 1 to smithi01785587-4 2024-09-21T09:45:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 0 (ObjNum 225 snap 0 seq_num 2399651088) dirty dne 2024-09-21T09:45:53.095 INFO:tasks.rados.rados.0.smithi017.stdout:1857: finishing write tid 2 to smithi01785587-5 2024-09-21T09:45:53.096 INFO:tasks.rados.rados.0.smithi017.stdout:1857: finishing write tid 3 to smithi01785587-5 2024-09-21T09:45:53.096 INFO:tasks.rados.rados.0.smithi017.stdout:1857: finishing write tid 4 to smithi01785587-5 2024-09-21T09:45:53.096 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 886 (ObjNum 714 snap 237 seq_num 714) dirty exists 2024-09-21T09:45:53.096 INFO:tasks.rados.rados.0.smithi017.stdout:1857: left oid 5 (ObjNum 714 snap 237 seq_num 714) 2024-09-21T09:45:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:53 smithi017 ceph-mon[102913]: from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:53 smithi017 ceph-mon[102913]: pgmap v369: 89 pgs: 89 active+clean; 152 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-21T09:45:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:53 smithi017 ceph-mon[102913]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:45:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:53 smithi017 ceph-mon[105151]: from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:53 smithi017 ceph-mon[105151]: pgmap v369: 89 pgs: 89 active+clean; 152 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-21T09:45:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:53 smithi017 ceph-mon[105151]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:45:53.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:53 smithi134 ceph-mon[90085]: from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:53 smithi134 ceph-mon[90085]: pgmap v369: 89 pgs: 89 active+clean; 152 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-21T09:45:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:53 smithi134 ceph-mon[90085]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:45:54.084 INFO:tasks.rados.rados.0.smithi017.stdout:1857: done (2 left) 2024-09-21T09:45:54.084 INFO:tasks.rados.rados.0.smithi017.stdout:1858: done (1 left) 2024-09-21T09:45:54.084 INFO:tasks.rados.rados.0.smithi017.stdout:1859: done (0 left) 2024-09-21T09:45:54.084 INFO:tasks.rados.rados.0.smithi017.stdout:1860: read oid 32 snap -1 2024-09-21T09:45:54.084 INFO:tasks.rados.rados.0.smithi017.stdout:1860: expect (ObjNum 607 snap 190 seq_num 607) 2024-09-21T09:45:54.084 INFO:tasks.rados.rados.0.smithi017.stdout:1861: snap_create 2024-09-21T09:45:54.088 INFO:tasks.rados.rados.0.smithi017.stdout:1860: expect (ObjNum 607 snap 190 seq_num 607) 2024-09-21T09:45:54.372 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:54.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:54 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3245082443' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:54.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:54 smithi017 ceph-mon[102913]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:45:54.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:54 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:45:54.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:54 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3245082443' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:54.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:54 smithi017 ceph-mon[105151]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:45:54.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:54 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:45:54.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:54 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3245082443' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:54 smithi134 ceph-mon[90085]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:45:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:54 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:45:54.910 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:45:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:45:54] "GET /metrics HTTP/1.1" 200 36285 "" "Prometheus/2.51.0" 2024-09-21T09:45:55.084 INFO:tasks.rados.rados.0.smithi017.stdout:1860: done (1 left) 2024-09-21T09:45:55.085 INFO:tasks.rados.rados.0.smithi017.stdout:1861: done (0 left) 2024-09-21T09:45:55.085 INFO:tasks.rados.rados.0.smithi017.stdout:1862: snap_create 2024-09-21T09:45:55.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[102913]: pgmap v372: 89 pgs: 89 active+clean; 152 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-21T09:45:55.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[102913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:45:55.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[102913]: Upgrade: osd.2 is safe to restart 2024-09-21T09:45:55.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[102913]: from='client.44594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:55.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[102913]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:45:55.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:55.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:55.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[105151]: pgmap v372: 89 pgs: 89 active+clean; 152 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-21T09:45:55.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[105151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:45:55.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[105151]: Upgrade: osd.2 is safe to restart 2024-09-21T09:45:55.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[105151]: from='client.44594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:55.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[105151]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:45:55.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:55.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:55 smithi134 ceph-mon[90085]: pgmap v372: 89 pgs: 89 active+clean; 152 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-09-21T09:45:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:55 smithi134 ceph-mon[90085]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:45:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:55 smithi134 ceph-mon[90085]: Upgrade: osd.2 is safe to restart 2024-09-21T09:45:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:55 smithi134 ceph-mon[90085]: from='client.44594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:55 smithi134 ceph-mon[90085]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:45:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:56.086 INFO:tasks.rados.rados.0.smithi017.stdout:1862: done (0 left) 2024-09-21T09:45:56.086 INFO:tasks.rados.rados.0.smithi017.stdout:1863: write oid 39 current snap is 240 2024-09-21T09:45:56.086 INFO:tasks.rados.rados.0.smithi017.stdout:1863: seq_num 715 ranges {417005=740217,1590037=426968} 2024-09-21T09:45:56.092 INFO:tasks.rados.rados.0.smithi017.stdout:1863: writing smithi01785587-39 from 417005 to 1157222 tid 1 2024-09-21T09:45:56.098 INFO:tasks.rados.rados.0.smithi017.stdout:1863: writing smithi01785587-39 from 1590037 to 2017005 tid 2 2024-09-21T09:45:56.098 INFO:tasks.rados.rados.0.smithi017.stdout:1864: read oid 32 snap -1 2024-09-21T09:45:56.098 INFO:tasks.rados.rados.0.smithi017.stdout:1864: expect (ObjNum 607 snap 190 seq_num 607) 2024-09-21T09:45:56.098 INFO:tasks.rados.rados.0.smithi017.stdout:1865: snap_create 2024-09-21T09:45:56.100 INFO:tasks.rados.rados.0.smithi017.stdout:1863: finishing write tid 1 to smithi01785587-39 2024-09-21T09:45:56.101 INFO:tasks.rados.rados.0.smithi017.stdout:1863: finishing write tid 2 to smithi01785587-39 2024-09-21T09:45:56.101 INFO:tasks.rados.rados.0.smithi017.stdout:1863: finishing write tid 3 to smithi01785587-39 2024-09-21T09:45:56.101 INFO:tasks.rados.rados.0.smithi017.stdout:1863: finishing write tid 4 to smithi01785587-39 2024-09-21T09:45:56.101 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 915 (ObjNum 715 snap 240 seq_num 715) dirty exists 2024-09-21T09:45:56.101 INFO:tasks.rados.rados.0.smithi017.stdout:1863: left oid 39 (ObjNum 715 snap 240 seq_num 715) 2024-09-21T09:45:56.103 INFO:tasks.rados.rados.0.smithi017.stdout:1864: expect (ObjNum 607 snap 190 seq_num 607) 2024-09-21T09:45:56.593 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[102913]: from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:56.593 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:56.593 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:45:56.593 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:56.593 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[102913]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:45:56.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[105151]: from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:56.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:56.593 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:45:56.594 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:56.594 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:56 smithi017 ceph-mon[105151]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:45:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:56 smithi134 ceph-mon[90085]: from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:45:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:45:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:56 smithi134 ceph-mon[90085]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:45:57.088 INFO:tasks.rados.rados.0.smithi017.stdout:1863: done (2 left) 2024-09-21T09:45:57.088 INFO:tasks.rados.rados.0.smithi017.stdout:1864: done (1 left) 2024-09-21T09:45:57.088 INFO:tasks.rados.rados.0.smithi017.stdout:1865: done (0 left) 2024-09-21T09:45:57.089 INFO:tasks.rados.rados.0.smithi017.stdout:1866: snap_create 2024-09-21T09:45:57.272 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:45:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[102913]: pgmap v375: 89 pgs: 89 active+clean; 153 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 277 KiB/s wr, 4 op/s 2024-09-21T09:45:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[102913]: Upgrade: Updating osd.2 2024-09-21T09:45:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[102913]: Deploying daemon osd.2 on smithi017 2024-09-21T09:45:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/21751852' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[102913]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:45:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[105151]: pgmap v375: 89 pgs: 89 active+clean; 153 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 277 KiB/s wr, 4 op/s 2024-09-21T09:45:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[105151]: Upgrade: Updating osd.2 2024-09-21T09:45:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[105151]: Deploying daemon osd.2 on smithi017 2024-09-21T09:45:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/21751852' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:57 smithi017 ceph-mon[105151]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:45:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:57 smithi134 ceph-mon[90085]: pgmap v375: 89 pgs: 89 active+clean; 153 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 277 KiB/s wr, 4 op/s 2024-09-21T09:45:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:57 smithi134 ceph-mon[90085]: Upgrade: Updating osd.2 2024-09-21T09:45:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:57 smithi134 ceph-mon[90085]: Deploying daemon osd.2 on smithi017 2024-09-21T09:45:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/21751852' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:45:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:57 smithi134 ceph-mon[90085]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:45:58.080 INFO:tasks.rados.rados.0.smithi017.stdout:1866: done (0 left) 2024-09-21T09:45:58.080 INFO:tasks.rados.rados.0.smithi017.stdout:1867: read oid 35 snap -1 2024-09-21T09:45:58.080 INFO:tasks.rados.rados.0.smithi017.stdout:1867: expect (ObjNum 685 snap 217 seq_num 685) 2024-09-21T09:45:58.080 INFO:tasks.rados.rados.0.smithi017.stdout:1868: snap_remove snap 241 2024-09-21T09:45:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:58 smithi017 ceph-mon[102913]: from='client.34749 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:58 smithi017 ceph-mon[102913]: osdmap e533: 8 total, 8 up, 8 in 2024-09-21T09:45:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:58 smithi017 ceph-mon[105151]: from='client.34749 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:58 smithi017 ceph-mon[105151]: osdmap e533: 8 total, 8 up, 8 in 2024-09-21T09:45:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:58 smithi134 ceph-mon[90085]: from='client.34749 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:58 smithi134 ceph-mon[90085]: osdmap e533: 8 total, 8 up, 8 in 2024-09-21T09:45:59.081 INFO:tasks.rados.rados.0.smithi017.stdout:1868: done (1 left) 2024-09-21T09:45:59.082 INFO:tasks.rados.rados.0.smithi017.stdout:1869: write oid 25 current snap is 242 2024-09-21T09:45:59.082 INFO:tasks.rados.rados.0.smithi017.stdout:1869: seq_num 716 ranges {433812=415466,1648519=385293} 2024-09-21T09:45:59.086 INFO:tasks.rados.rados.0.smithi017.stdout:1869: writing smithi01785587-25 from 433812 to 849278 tid 1 2024-09-21T09:45:59.092 INFO:tasks.rados.rados.0.smithi017.stdout:1869: writing smithi01785587-25 from 1648519 to 2033812 tid 2 2024-09-21T09:45:59.092 INFO:tasks.rados.rados.0.smithi017.stdout:1870: snap_remove snap 214 2024-09-21T09:45:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:59 smithi017 ceph-mon[105151]: from='client.34755 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:59 smithi017 ceph-mon[105151]: pgmap v378: 89 pgs: 89 active+clean; 153 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 277 KiB/s wr, 4 op/s 2024-09-21T09:45:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:45:59 smithi017 ceph-mon[105151]: osdmap e534: 8 total, 8 up, 8 in 2024-09-21T09:45:59.451 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:45:59 smithi017 systemd[1]: Stopping Ceph osd.2 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:45:59.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:59 smithi017 ceph-mon[102913]: from='client.34755 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:59.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:59 smithi017 ceph-mon[102913]: pgmap v378: 89 pgs: 89 active+clean; 153 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 277 KiB/s wr, 4 op/s 2024-09-21T09:45:59.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:45:59 smithi017 ceph-mon[102913]: osdmap e534: 8 total, 8 up, 8 in 2024-09-21T09:45:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:59 smithi134 ceph-mon[90085]: from='client.34755 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:59 smithi134 ceph-mon[90085]: pgmap v378: 89 pgs: 89 active+clean; 153 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 277 KiB/s wr, 4 op/s 2024-09-21T09:45:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:45:59 smithi134 ceph-mon[90085]: osdmap e534: 8 total, 8 up, 8 in 2024-09-21T09:45:59.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:45:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[57700]: 2024-09-21T09:45:59.459+0000 7f0b849e7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:45:59.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:45:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[57700]: 2024-09-21T09:45:59.459+0000 7f0b849e7640 -1 osd.2 534 *** Got signal Terminated *** 2024-09-21T09:45:59.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:45:59 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[57700]: 2024-09-21T09:45:59.459+0000 7f0b849e7640 -1 osd.2 534 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:46:00.083 INFO:tasks.rados.rados.0.smithi017.stdout:1870: done (2 left) 2024-09-21T09:46:00.083 INFO:tasks.rados.rados.0.smithi017.stdout:1871: write oid 2 current snap is 242 2024-09-21T09:46:00.083 INFO:tasks.rados.rados.0.smithi017.stdout:1867: expect (ObjNum 685 snap 217 seq_num 685) 2024-09-21T09:46:00.199 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:1869: finishing write tid 1 to smithi01785587-25 2024-09-21T09:46:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:1869: finishing write tid 2 to smithi01785587-25 2024-09-21T09:46:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:1869: finishing write tid 3 to smithi01785587-25 2024-09-21T09:46:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:1869: finishing write tid 4 to smithi01785587-25 2024-09-21T09:46:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 386 (ObjNum 716 snap 242 seq_num 716) dirty exists 2024-09-21T09:46:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:1869: left oid 25 (ObjNum 716 snap 242 seq_num 716) 2024-09-21T09:46:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:1871: seq_num 717 ranges {450619=490715,1590648=459971} 2024-09-21T09:46:00.260 INFO:tasks.rados.rados.0.smithi017.stdout:1871: writing smithi01785587-2 from 450619 to 941334 tid 1 2024-09-21T09:46:00.268 INFO:tasks.rados.rados.0.smithi017.stdout:1871: writing smithi01785587-2 from 1590648 to 2050619 tid 2 2024-09-21T09:46:00.268 INFO:tasks.rados.rados.0.smithi017.stdout:1867: done (2 left) 2024-09-21T09:46:00.268 INFO:tasks.rados.rados.0.smithi017.stdout:1869: done (1 left) 2024-09-21T09:46:00.268 INFO:tasks.rados.rados.0.smithi017.stdout:1872: read oid 10 snap -1 2024-09-21T09:46:00.268 INFO:tasks.rados.rados.0.smithi017.stdout:1872: expect deleted 2024-09-21T09:46:00.268 INFO:tasks.rados.rados.0.smithi017.stdout:1873: snap_remove snap 223 2024-09-21T09:46:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:00 smithi017 ceph-mon[102913]: osd.2 marked itself down and dead 2024-09-21T09:46:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:00 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3043673750' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:00 smithi017 ceph-mon[102913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:46:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:00 smithi017 ceph-mon[102913]: osdmap e535: 8 total, 7 up, 8 in 2024-09-21T09:46:00.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:00 smithi017 ceph-mon[105151]: osd.2 marked itself down and dead 2024-09-21T09:46:00.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:00 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3043673750' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:00.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:00 smithi017 ceph-mon[105151]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:46:00.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:00 smithi017 ceph-mon[105151]: osdmap e535: 8 total, 7 up, 8 in 2024-09-21T09:46:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:00 smithi134 ceph-mon[90085]: osd.2 marked itself down and dead 2024-09-21T09:46:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:00 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3043673750' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:00 smithi134 ceph-mon[90085]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:46:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:00 smithi134 ceph-mon[90085]: osdmap e535: 8 total, 7 up, 8 in 2024-09-21T09:46:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:46:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:46:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:00.918 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:00 smithi017 podman[124470]: 2024-09-21 09:46:00.622530841 +0000 UTC m=+1.327032586 container died b87d82e4dc7450568108a049bcb9c3fbefcb1448b633eed503f455433991100a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T09:46:01.085 INFO:tasks.rados.rados.0.smithi017.stdout:1871: finishing write tid 1 to smithi01785587-2 2024-09-21T09:46:01.085 INFO:tasks.rados.rados.0.smithi017.stdout:1871: finishing write tid 2 to smithi01785587-2 2024-09-21T09:46:01.085 INFO:tasks.rados.rados.0.smithi017.stdout:1871: finishing write tid 3 to smithi01785587-2 2024-09-21T09:46:01.085 INFO:tasks.rados.rados.0.smithi017.stdout:1871: finishing write tid 4 to smithi01785587-2 2024-09-21T09:46:01.085 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 328 (ObjNum 717 snap 242 seq_num 717) dirty exists 2024-09-21T09:46:01.085 INFO:tasks.rados.rados.0.smithi017.stdout:1871: left oid 2 (ObjNum 717 snap 242 seq_num 717) 2024-09-21T09:46:01.085 INFO:tasks.rados.rados.0.smithi017.stdout:1871: done (2 left) 2024-09-21T09:46:01.086 INFO:tasks.rados.rados.0.smithi017.stdout:1873: done (1 left) 2024-09-21T09:46:01.086 INFO:tasks.rados.rados.0.smithi017.stdout:1874: snap_create 2024-09-21T09:46:01.200 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:01 smithi017 podman[124470]: 2024-09-21 09:46:01.006439308 +0000 UTC m=+1.710941040 container cleanup b87d82e4dc7450568108a049bcb9c3fbefcb1448b633eed503f455433991100a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T09:46:01.200 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:01 smithi017 bash[124470]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2 2024-09-21T09:46:01.637 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:01 smithi017 podman[124563]: 2024-09-21 09:46:01.299924563 +0000 UTC m=+0.680400968 container remove b87d82e4dc7450568108a049bcb9c3fbefcb1448b633eed503f455433991100a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:46:01.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:01 smithi017 ceph-mon[102913]: pgmap v381: 89 pgs: 89 active+clean; 154 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 285 KiB/s wr, 1 op/s 2024-09-21T09:46:01.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:01 smithi017 ceph-mon[102913]: from='client.44630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:01.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:01 smithi017 ceph-mon[102913]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:01.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:01 smithi017 ceph-mon[102913]: osdmap e536: 8 total, 7 up, 8 in 2024-09-21T09:46:01.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:01 smithi017 ceph-mon[105151]: pgmap v381: 89 pgs: 89 active+clean; 154 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 285 KiB/s wr, 1 op/s 2024-09-21T09:46:01.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:01 smithi017 ceph-mon[105151]: from='client.44630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:01.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:01 smithi017 ceph-mon[105151]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:01.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:01 smithi017 ceph-mon[105151]: osdmap e536: 8 total, 7 up, 8 in 2024-09-21T09:46:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:01 smithi134 ceph-mon[90085]: pgmap v381: 89 pgs: 89 active+clean; 154 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 285 KiB/s wr, 1 op/s 2024-09-21T09:46:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:01 smithi134 ceph-mon[90085]: from='client.44630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:01 smithi134 ceph-mon[90085]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:01 smithi134 ceph-mon[90085]: osdmap e536: 8 total, 7 up, 8 in 2024-09-21T09:46:01.896 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:01 smithi017 podman[124657]: 2024-09-21 09:46:01.538553795 +0000 UTC m=+0.016663587 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:01.896 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:01 smithi017 podman[124657]: 2024-09-21 09:46:01.679127326 +0000 UTC m=+0.157237146 container create a55a2d64859018881b4b75b3908d59b4d0181a550a3d7f18803ded209413a2f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T09:46:02.086 INFO:tasks.rados.rados.0.smithi017.stdout:1872: done (1 left) 2024-09-21T09:46:02.086 INFO:tasks.rados.rados.0.smithi017.stdout:1874: done (0 left) 2024-09-21T09:46:02.086 INFO:tasks.rados.rados.0.smithi017.stdout:1875: rollback oid 36 current snap is 243 2024-09-21T09:46:02.086 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 36 to 235 2024-09-21T09:46:02.086 INFO:tasks.rados.rados.0.smithi017.stdout:1876: read oid 34 snap -1 2024-09-21T09:46:02.087 INFO:tasks.rados.rados.0.smithi017.stdout:1876: expect (ObjNum 699 snap 225 seq_num 699) 2024-09-21T09:46:02.087 INFO:tasks.rados.rados.0.smithi017.stdout:1877: rollback oid 46 current snap is 243 2024-09-21T09:46:02.087 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 217 2024-09-21T09:46:02.087 INFO:tasks.rados.rados.0.smithi017.stdout:1878: rollback oid 8 current snap is 243 2024-09-21T09:46:02.087 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 215 2024-09-21T09:46:02.087 INFO:tasks.rados.rados.0.smithi017.stdout:1879: read oid 14 snap -1 2024-09-21T09:46:02.087 INFO:tasks.rados.rados.0.smithi017.stdout:1879: expect (ObjNum 667 snap 209 seq_num 667) 2024-09-21T09:46:02.087 INFO:tasks.rados.rados.0.smithi017.stdout:1880: snap_remove snap 225 2024-09-21T09:46:02.200 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:01 smithi017 podman[124657]: 2024-09-21 09:46:01.894944632 +0000 UTC m=+0.373054426 container init a55a2d64859018881b4b75b3908d59b4d0181a550a3d7f18803ded209413a2f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T09:46:02.200 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:01 smithi017 podman[124657]: 2024-09-21 09:46:01.899908535 +0000 UTC m=+0.378018314 container start a55a2d64859018881b4b75b3908d59b4d0181a550a3d7f18803ded209413a2f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b) 2024-09-21T09:46:02.200 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:01 smithi017 podman[124657]: 2024-09-21 09:46:01.986471596 +0000 UTC m=+0.464581381 container attach a55a2d64859018881b4b75b3908d59b4d0181a550a3d7f18803ded209413a2f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T09:46:02.200 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:02 smithi017 podman[124657]: 2024-09-21 09:46:02.111597475 +0000 UTC m=+0.589707261 container died a55a2d64859018881b4b75b3908d59b4d0181a550a3d7f18803ded209413a2f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-21T09:46:02.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:02 smithi017 podman[124700]: 2024-09-21 09:46:02.681938823 +0000 UTC m=+0.562683856 container remove a55a2d64859018881b4b75b3908d59b4d0181a550a3d7f18803ded209413a2f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T09:46:02.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:02 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T09:46:02.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:02 smithi017 systemd[1]: Stopped Ceph osd.2 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:46:02.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:02 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.2.service: Consumed 13.483s CPU time. 2024-09-21T09:46:02.951 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:02 smithi017 systemd[1]: Starting Ceph osd.2 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:46:03.092 INFO:tasks.rados.rados.0.smithi017.stdout:1880: done (5 left) 2024-09-21T09:46:03.092 INFO:tasks.rados.rados.0.smithi017.stdout:1881: write oid 6 current snap is 243 2024-09-21T09:46:03.092 INFO:tasks.rados.rados.0.smithi017.stdout:1881: seq_num 718 ranges {467426=565964,1649130=418296} 2024-09-21T09:46:03.097 INFO:tasks.rados.rados.0.smithi017.stdout:1881: writing smithi01785587-6 from 467426 to 1033390 tid 1 2024-09-21T09:46:03.104 INFO:tasks.rados.rados.0.smithi017.stdout:1881: writing smithi01785587-6 from 1649130 to 2067426 tid 2 2024-09-21T09:46:03.104 INFO:tasks.rados.rados.0.smithi017.stdout:1882: write oid 21 current snap is 243 2024-09-21T09:46:03.104 INFO:tasks.rados.rados.0.smithi017.stdout:1877: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:46:03.104 INFO:tasks.rados.rados.0.smithi017.stdout:1877: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:46:03.104 INFO:tasks.rados.rados.0.smithi017.stdout:1877: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:46:03.104 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 791 (ObjNum 537 snap 169 seq_num 537) dirty exists 2024-09-21T09:46:03.104 INFO:tasks.rados.rados.0.smithi017.stdout:1878: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:46:03.104 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 0 (ObjNum 129 snap 0 seq_num 2399605920) dirty dne 2024-09-21T09:46:03.104 INFO:tasks.rados.rados.0.smithi017.stdout:1879: expect (ObjNum 667 snap 209 seq_num 667) 2024-09-21T09:46:03.157 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:03.240 INFO:tasks.rados.rados.0.smithi017.stdout:1875: finishing rollback tid 0 to smithi01785587-36 2024-09-21T09:46:03.240 INFO:tasks.rados.rados.0.smithi017.stdout:1875: finishing rollback tid 1 to smithi01785587-36 2024-09-21T09:46:03.240 INFO:tasks.rados.rados.0.smithi017.stdout:1882: seq_num 719 ranges {484233=641213,1707612=376621} 2024-09-21T09:46:03.246 INFO:tasks.rados.rados.0.smithi017.stdout:1882: writing smithi01785587-21 from 484233 to 1125446 tid 1 2024-09-21T09:46:03.252 INFO:tasks.rados.rados.0.smithi017.stdout:1882: writing smithi01785587-21 from 1707612 to 2084233 tid 2 2024-09-21T09:46:03.252 INFO:tasks.rados.rados.0.smithi017.stdout:1877: done (6 left) 2024-09-21T09:46:03.252 INFO:tasks.rados.rados.0.smithi017.stdout:1878: done (5 left) 2024-09-21T09:46:03.253 INFO:tasks.rados.rados.0.smithi017.stdout:1879: done (4 left) 2024-09-21T09:46:03.253 INFO:tasks.rados.rados.0.smithi017.stdout:1883: read oid 13 snap -1 2024-09-21T09:46:03.253 INFO:tasks.rados.rados.0.smithi017.stdout:1883: expect (ObjNum 632 snap 200 seq_num 632) 2024-09-21T09:46:03.253 INFO:tasks.rados.rados.0.smithi017.stdout:1884: write oid 2 current snap is 243 2024-09-21T09:46:03.253 INFO:tasks.rados.rados.0.smithi017.stdout:1875: finishing rollback tid 2 to smithi01785587-36 2024-09-21T09:46:03.253 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 931 (ObjNum 703 snap 228 seq_num 703) dirty exists 2024-09-21T09:46:03.253 INFO:tasks.rados.rados.0.smithi017.stdout:1876: expect (ObjNum 699 snap 225 seq_num 699) 2024-09-21T09:46:03.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-mon[102913]: pgmap v384: 89 pgs: 17 peering, 3 stale+active+clean, 69 active+clean; 154 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 481 KiB/s wr, 2 op/s 2024-09-21T09:46:03.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-mon[102913]: osdmap e537: 8 total, 7 up, 8 in 2024-09-21T09:46:03.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-mon[102913]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:46:03.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2906354149' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:03.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-mon[105151]: pgmap v384: 89 pgs: 17 peering, 3 stale+active+clean, 69 active+clean; 154 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 481 KiB/s wr, 2 op/s 2024-09-21T09:46:03.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-mon[105151]: osdmap e537: 8 total, 7 up, 8 in 2024-09-21T09:46:03.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-mon[105151]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:46:03.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2906354149' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:03.430 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:03 smithi017 podman[124848]: 2024-09-21 09:46:03.048484563 +0000 UTC m=+0.016228082 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:03.430 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:03 smithi017 podman[124848]: 2024-09-21 09:46:03.150147189 +0000 UTC m=+0.117890698 container create b1a5a71c3f4c730dd2d760b70b24e34e4580a9039769f2ef4f7a3cc7b5eb04bb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:46:03.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:03 smithi134 ceph-mon[90085]: pgmap v384: 89 pgs: 17 peering, 3 stale+active+clean, 69 active+clean; 154 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 481 KiB/s wr, 2 op/s 2024-09-21T09:46:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:03 smithi134 ceph-mon[90085]: osdmap e537: 8 total, 7 up, 8 in 2024-09-21T09:46:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:03 smithi134 ceph-mon[90085]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:46:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:03 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2906354149' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:03.558 INFO:tasks.rados.rados.0.smithi017.stdout:1881: finishing write tid 1 to smithi01785587-6 2024-09-21T09:46:03.558 INFO:tasks.rados.rados.0.smithi017.stdout:1881: finishing write tid 2 to smithi01785587-6 2024-09-21T09:46:03.558 INFO:tasks.rados.rados.0.smithi017.stdout:1884: seq_num 720 ranges {501040=716462,1766094=334946} 2024-09-21T09:46:03.564 INFO:tasks.rados.rados.0.smithi017.stdout:1884: writing smithi01785587-2 from 501040 to 1217502 tid 1 2024-09-21T09:46:03.570 INFO:tasks.rados.rados.0.smithi017.stdout:1884: writing smithi01785587-2 from 1766094 to 2101040 tid 2 2024-09-21T09:46:03.570 INFO:tasks.rados.rados.0.smithi017.stdout:1875: done (5 left) 2024-09-21T09:46:03.570 INFO:tasks.rados.rados.0.smithi017.stdout:1876: done (4 left) 2024-09-21T09:46:03.570 INFO:tasks.rados.rados.0.smithi017.stdout:1885: read oid 7 snap -1 2024-09-21T09:46:03.570 INFO:tasks.rados.rados.0.smithi017.stdout:1885: expect (ObjNum 694 snap 220 seq_num 694) 2024-09-21T09:46:03.570 INFO:tasks.rados.rados.0.smithi017.stdout:1886: snap_remove snap 221 2024-09-21T09:46:03.700 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:03 smithi017 podman[124848]: 2024-09-21 09:46:03.427715719 +0000 UTC m=+0.395459239 container init b1a5a71c3f4c730dd2d760b70b24e34e4580a9039769f2ef4f7a3cc7b5eb04bb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T09:46:03.700 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:03 smithi017 podman[124848]: 2024-09-21 09:46:03.434207884 +0000 UTC m=+0.401951397 container start b1a5a71c3f4c730dd2d760b70b24e34e4580a9039769f2ef4f7a3cc7b5eb04bb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T09:46:03.700 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:03 smithi017 podman[124848]: 2024-09-21 09:46:03.544408111 +0000 UTC m=+0.512151635 container attach b1a5a71c3f4c730dd2d760b70b24e34e4580a9039769f2ef4f7a3cc7b5eb04bb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate, RELEASE=main-62da54b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:46:03.700 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate[124887]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:46:03.701 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:03 smithi017 bash[124848]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:46:03.701 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:03 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate[124887]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:46:03.701 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:03 smithi017 bash[124848]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:46:04.096 INFO:tasks.rados.rados.0.smithi017.stdout:1886: done (5 left) 2024-09-21T09:46:04.096 INFO:tasks.rados.rados.0.smithi017.stdout:1887: snap_remove snap 240 2024-09-21T09:46:04.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-mon[102913]: osdmap e538: 8 total, 7 up, 8 in 2024-09-21T09:46:04.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-mon[102913]: from='client.44648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:04.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-mon[102913]: from='client.34791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:04.347 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-mon[105151]: osdmap e538: 8 total, 7 up, 8 in 2024-09-21T09:46:04.347 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-mon[105151]: from='client.44648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:04.347 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-mon[105151]: from='client.34791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:04.348 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate[124887]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:46:04.348 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 bash[124848]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:46:04.348 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate[124887]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:46:04.348 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 bash[124848]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:46:04.348 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate[124887]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:46:04.348 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 bash[124848]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:46:04.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:04 smithi134 ceph-mon[90085]: osdmap e538: 8 total, 7 up, 8 in 2024-09-21T09:46:04.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:04 smithi134 ceph-mon[90085]: from='client.44648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:04.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:04 smithi134 ceph-mon[90085]: from='client.34791 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:04.682 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate[124887]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:46:04.682 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 bash[124848]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:46:04.683 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate[124887]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T09:46:04.683 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 bash[124848]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T09:46:04.683 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate[124887]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:46:04.683 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 bash[124848]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:46:04.683 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate[124887]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T09:46:04.683 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 bash[124848]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T09:46:04.683 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 podman[124848]: 2024-09-21 09:46:04.376912097 +0000 UTC m=+1.344655607 container died b1a5a71c3f4c730dd2d760b70b24e34e4580a9039769f2ef4f7a3cc7b5eb04bb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T09:46:04.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:46:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:46:04] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.51.0" 2024-09-21T09:46:05.098 INFO:tasks.rados.rados.0.smithi017.stdout:1887: done (5 left) 2024-09-21T09:46:05.099 INFO:tasks.rados.rados.0.smithi017.stdout:1888: snap_remove snap 227 2024-09-21T09:46:05.225 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:05 smithi017 ceph-mon[102913]: pgmap v387: 89 pgs: 7 active+undersized, 17 peering, 2 active+undersized+degraded, 63 active+clean; 156 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 428 KiB/s wr, 2 op/s; 8/282 objects degraded (2.837%) 2024-09-21T09:46:05.225 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:05 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 8/282 objects degraded (2.837%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:05.226 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:05 smithi017 ceph-mon[102913]: osdmap e539: 8 total, 7 up, 8 in 2024-09-21T09:46:05.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:05 smithi017 ceph-mon[105151]: pgmap v387: 89 pgs: 7 active+undersized, 17 peering, 2 active+undersized+degraded, 63 active+clean; 156 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 428 KiB/s wr, 2 op/s; 8/282 objects degraded (2.837%) 2024-09-21T09:46:05.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:05 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 8/282 objects degraded (2.837%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:05.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:05 smithi017 ceph-mon[105151]: osdmap e539: 8 total, 7 up, 8 in 2024-09-21T09:46:05.226 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:04 smithi017 podman[125037]: 2024-09-21 09:46:04.973410242 +0000 UTC m=+0.586727275 container remove b1a5a71c3f4c730dd2d760b70b24e34e4580a9039769f2ef4f7a3cc7b5eb04bb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:46:05.226 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:05 smithi017 podman[125102]: 2024-09-21 09:46:05.088456461 +0000 UTC m=+0.016620524 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:05.481 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:05 smithi017 podman[125102]: 2024-09-21 09:46:05.224340134 +0000 UTC m=+0.152504194 container create f98642c0dccf82c1d91dc8c52471560903a50f90e833c2404f0333d7f67078c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:46:05.482 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:05 smithi017 podman[125102]: 2024-09-21 09:46:05.451897172 +0000 UTC m=+0.380061250 container init f98642c0dccf82c1d91dc8c52471560903a50f90e833c2404f0333d7f67078c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:46:05.482 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:05 smithi017 podman[125102]: 2024-09-21 09:46:05.457754765 +0000 UTC m=+0.385918819 container start f98642c0dccf82c1d91dc8c52471560903a50f90e833c2404f0333d7f67078c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-62da54b, io.buildah.version=1.37.2) 2024-09-21T09:46:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:05 smithi134 ceph-mon[90085]: pgmap v387: 89 pgs: 7 active+undersized, 17 peering, 2 active+undersized+degraded, 63 active+clean; 156 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 428 KiB/s wr, 2 op/s; 8/282 objects degraded (2.837%) 2024-09-21T09:46:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:05 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 8/282 objects degraded (2.837%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:05 smithi134 ceph-mon[90085]: osdmap e539: 8 total, 7 up, 8 in 2024-09-21T09:46:05.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:05 smithi017 ceph-osd[125141]: -- 172.21.15.17:0/4230630937 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5610e0e3da40 con 0x5610e1c0c000 2024-09-21T09:46:05.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:05 smithi017 bash[125102]: f98642c0dccf82c1d91dc8c52471560903a50f90e833c2404f0333d7f67078c3 2024-09-21T09:46:05.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:05 smithi017 systemd[1]: Started Ceph osd.2 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:46:06.106 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:1888: done (5 left) 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:1889: rollback oid 48 current snap is 243 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 231 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:1890: rollback oid 37 current snap is 243 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 236 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:1891: rollback oid 42 current snap is 243 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 217 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:1892: read oid 46 snap -1 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:1892: expect (ObjNum 537 snap 169 seq_num 537) 2024-09-21T09:46:06.111 INFO:tasks.rados.rados.0.smithi017.stdout:1893: read oid 41 snap -1 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1893: expect deleted 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1894: read oid 28 snap -1 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1894: expect deleted 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1895: snap_create 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1881: finishing write tid 3 to smithi01785587-6 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1881: finishing write tid 4 to smithi01785587-6 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 516 (ObjNum 718 snap 243 seq_num 718) dirty exists 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1881: left oid 6 (ObjNum 718 snap 243 seq_num 718) 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1882: finishing write tid 1 to smithi01785587-21 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1882: finishing write tid 2 to smithi01785587-21 2024-09-21T09:46:06.112 INFO:tasks.rados.rados.0.smithi017.stdout:1883: expect (ObjNum 632 snap 200 seq_num 632) 2024-09-21T09:46:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:1882: finishing write tid 3 to smithi01785587-21 2024-09-21T09:46:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:1882: finishing write tid 4 to smithi01785587-21 2024-09-21T09:46:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 947 (ObjNum 719 snap 243 seq_num 719) dirty exists 2024-09-21T09:46:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:1882: left oid 21 (ObjNum 719 snap 243 seq_num 719) 2024-09-21T09:46:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:1884: finishing write tid 1 to smithi01785587-2 2024-09-21T09:46:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:1884: finishing write tid 2 to smithi01785587-2 2024-09-21T09:46:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:1884: finishing write tid 3 to smithi01785587-2 2024-09-21T09:46:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:1884: finishing write tid 4 to smithi01785587-2 2024-09-21T09:46:06.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 332 (ObjNum 720 snap 243 seq_num 720) dirty exists 2024-09-21T09:46:06.211 INFO:tasks.rados.rados.0.smithi017.stdout:1884: left oid 2 (ObjNum 720 snap 243 seq_num 720) 2024-09-21T09:46:06.211 INFO:tasks.rados.rados.0.smithi017.stdout:1885: expect (ObjNum 694 snap 220 seq_num 694) 2024-09-21T09:46:06.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[102913]: osdmap e540: 8 total, 7 up, 8 in 2024-09-21T09:46:06.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:06.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:06.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:06.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1974024487' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:06.362 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[105151]: osdmap e540: 8 total, 7 up, 8 in 2024-09-21T09:46:06.362 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:06.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:06.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:06.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:06 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1974024487' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:06.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:06 smithi134 ceph-mon[90085]: osdmap e540: 8 total, 7 up, 8 in 2024-09-21T09:46:06.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:06 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:06.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:06 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:06.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:06 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:06.437 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:06 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1974024487' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:1891: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 0 (ObjNum 289 snap 0 seq_num 1540944015) dirty dne 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:1889: finishing rollback tid 0 to smithi01785587-48 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:1889: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:1889: finishing rollback tid 2 to smithi01785587-48 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 748 (ObjNum 693 snap 220 seq_num 693) dirty exists 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:1890: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:1890: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:1890: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:46:06.458 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 951 (ObjNum 698 snap 221 seq_num 698) dirty exists 2024-09-21T09:46:06.459 INFO:tasks.rados.rados.0.smithi017.stdout:1892: expect (ObjNum 537 snap 169 seq_num 537) 2024-09-21T09:46:07.115 INFO:tasks.rados.rados.0.smithi017.stdout:1881: done (11 left) 2024-09-21T09:46:07.115 INFO:tasks.rados.rados.0.smithi017.stdout:1882: done (10 left) 2024-09-21T09:46:07.115 INFO:tasks.rados.rados.0.smithi017.stdout:1883: done (9 left) 2024-09-21T09:46:07.115 INFO:tasks.rados.rados.0.smithi017.stdout:1884: done (8 left) 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1885: done (7 left) 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1889: done (6 left) 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1890: done (5 left) 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1891: done (4 left) 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1892: done (3 left) 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1893: done (2 left) 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1894: done (1 left) 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1895: done (0 left) 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1896: rollback oid 11 current snap is 244 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 243 2024-09-21T09:46:07.116 INFO:tasks.rados.rados.0.smithi017.stdout:1897: snap_remove snap 233 2024-09-21T09:46:07.427 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[125118]: 2024-09-21T09:46:07.337+0000 7f3b879c2640 -1 Falling back to public interface 2024-09-21T09:46:07.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[102913]: pgmap v390: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 722 KiB/s wr, 11 op/s; 31/291 objects degraded (10.653%) 2024-09-21T09:46:07.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[102913]: from='client.44669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:07.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[102913]: osdmap e541: 8 total, 7 up, 8 in 2024-09-21T09:46:07.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[102913]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:07.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[102913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-21T09:46:07.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[105151]: pgmap v390: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 722 KiB/s wr, 11 op/s; 31/291 objects degraded (10.653%) 2024-09-21T09:46:07.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[105151]: from='client.44669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:07.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[105151]: osdmap e541: 8 total, 7 up, 8 in 2024-09-21T09:46:07.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[105151]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:07.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:07 smithi017 ceph-mon[105151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-21T09:46:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:07 smithi134 ceph-mon[90085]: pgmap v390: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 722 KiB/s wr, 11 op/s; 31/291 objects degraded (10.653%) 2024-09-21T09:46:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:07 smithi134 ceph-mon[90085]: from='client.44669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:07 smithi134 ceph-mon[90085]: osdmap e541: 8 total, 7 up, 8 in 2024-09-21T09:46:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:07 smithi134 ceph-mon[90085]: from='client.44675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:07 smithi134 ceph-mon[90085]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-21T09:46:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:1897: done (1 left) 2024-09-21T09:46:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:1898: write oid 41 current snap is 244 2024-09-21T09:46:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:1896: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:46:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 784 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:46:08.082 INFO:tasks.rados.rados.0.smithi017.stdout:1898: seq_num 721 ranges {517847=791711,2108223=9624} 2024-09-21T09:46:08.089 INFO:tasks.rados.rados.0.smithi017.stdout:1898: writing smithi01785587-41 from 517847 to 1309558 tid 1 2024-09-21T09:46:08.094 INFO:tasks.rados.rados.0.smithi017.stdout:1898: writing smithi01785587-41 from 2108223 to 2117847 tid 2 2024-09-21T09:46:08.094 INFO:tasks.rados.rados.0.smithi017.stdout:1896: done (1 left) 2024-09-21T09:46:08.094 INFO:tasks.rados.rados.0.smithi017.stdout:1899: snap_remove snap 243 2024-09-21T09:46:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:08 smithi017 ceph-mon[102913]: osdmap e542: 8 total, 7 up, 8 in 2024-09-21T09:46:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:08 smithi017 ceph-mon[102913]: osdmap e543: 8 total, 7 up, 8 in 2024-09-21T09:46:08.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:08 smithi017 ceph-mon[105151]: osdmap e542: 8 total, 7 up, 8 in 2024-09-21T09:46:08.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:08 smithi017 ceph-mon[105151]: osdmap e543: 8 total, 7 up, 8 in 2024-09-21T09:46:08.243 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:08 smithi134 ceph-mon[90085]: osdmap e542: 8 total, 7 up, 8 in 2024-09-21T09:46:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:08 smithi134 ceph-mon[90085]: osdmap e543: 8 total, 7 up, 8 in 2024-09-21T09:46:09.015 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:1899: done (1 left) 2024-09-21T09:46:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:1900: delete oid 40 current snap is 244 2024-09-21T09:46:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:1898: finishing write tid 1 to smithi01785587-41 2024-09-21T09:46:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:1898: finishing write tid 2 to smithi01785587-41 2024-09-21T09:46:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:1898: finishing write tid 3 to smithi01785587-41 2024-09-21T09:46:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:1898: finishing write tid 4 to smithi01785587-41 2024-09-21T09:46:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 298 (ObjNum 721 snap 244 seq_num 721) dirty exists 2024-09-21T09:46:09.083 INFO:tasks.rados.rados.0.smithi017.stdout:1898: left oid 41 (ObjNum 721 snap 244 seq_num 721) 2024-09-21T09:46:09.087 INFO:tasks.rados.rados.0.smithi017.stdout:1898: done (1 left) 2024-09-21T09:46:09.087 INFO:tasks.rados.rados.0.smithi017.stdout:1900: done (0 left) 2024-09-21T09:46:09.087 INFO:tasks.rados.rados.0.smithi017.stdout:1901: snap_create 2024-09-21T09:46:09.433 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:09 smithi134 ceph-mon[90085]: pgmap v393: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 158 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 746 KiB/s wr, 12 op/s; 31/294 objects degraded (10.544%) 2024-09-21T09:46:09.434 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:09 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2948464742' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:09.434 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:09 smithi134 ceph-mon[90085]: osdmap e544: 8 total, 7 up, 8 in 2024-09-21T09:46:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:09 smithi017 ceph-mon[102913]: pgmap v393: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 158 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 746 KiB/s wr, 12 op/s; 31/294 objects degraded (10.544%) 2024-09-21T09:46:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:09 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2948464742' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:09 smithi017 ceph-mon[102913]: osdmap e544: 8 total, 7 up, 8 in 2024-09-21T09:46:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:09 smithi017 ceph-mon[105151]: pgmap v393: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 158 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 746 KiB/s wr, 12 op/s; 31/294 objects degraded (10.544%) 2024-09-21T09:46:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:09 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2948464742' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:09 smithi017 ceph-mon[105151]: osdmap e544: 8 total, 7 up, 8 in 2024-09-21T09:46:10.084 INFO:tasks.rados.rados.0.smithi017.stdout:1901: done (0 left) 2024-09-21T09:46:10.084 INFO:tasks.rados.rados.0.smithi017.stdout:1902: write oid 7 current snap is 245 2024-09-21T09:46:10.084 INFO:tasks.rados.rados.0.smithi017.stdout:1902: seq_num 723 ranges {551461=658562,1941540=209921} 2024-09-21T09:46:10.090 INFO:tasks.rados.rados.0.smithi017.stdout:1902: writing smithi01785587-7 from 551461 to 1210023 tid 1 2024-09-21T09:46:10.096 INFO:tasks.rados.rados.0.smithi017.stdout:1902: writing smithi01785587-7 from 1941540 to 2151461 tid 2 2024-09-21T09:46:10.096 INFO:tasks.rados.rados.0.smithi017.stdout:1903: rollback oid 45 current snap is 245 2024-09-21T09:46:10.096 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 45 to 220 2024-09-21T09:46:10.096 INFO:tasks.rados.rados.0.smithi017.stdout:1904: read oid 44 snap -1 2024-09-21T09:46:10.096 INFO:tasks.rados.rados.0.smithi017.stdout:1904: expect (ObjNum 705 snap 228 seq_num 705) 2024-09-21T09:46:10.096 INFO:tasks.rados.rados.0.smithi017.stdout:1905: write oid 9 current snap is 245 2024-09-21T09:46:10.097 INFO:tasks.rados.rados.0.smithi017.stdout:1905: seq_num 724 ranges {568268=733811,2000022=168246} 2024-09-21T09:46:10.103 INFO:tasks.rados.rados.0.smithi017.stdout:1905: writing smithi01785587-9 from 568268 to 1302079 tid 1 2024-09-21T09:46:10.108 INFO:tasks.rados.rados.0.smithi017.stdout:1905: writing smithi01785587-9 from 2000022 to 2168268 tid 2 2024-09-21T09:46:10.109 INFO:tasks.rados.rados.0.smithi017.stdout:1906: write oid 3 current snap is 245 2024-09-21T09:46:10.109 INFO:tasks.rados.rados.0.smithi017.stdout:1902: finishing write tid 1 to smithi01785587-7 2024-09-21T09:46:10.109 INFO:tasks.rados.rados.0.smithi017.stdout:1906: seq_num 725 ranges {585075=409060,1542151=642924} 2024-09-21T09:46:10.114 INFO:tasks.rados.rados.0.smithi017.stdout:1906: writing smithi01785587-3 from 585075 to 994135 tid 1 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1906: writing smithi01785587-3 from 1542151 to 2185075 tid 2 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1907: read oid 15 snap -1 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1907: expect (ObjNum 678 snap 212 seq_num 678) 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1908: rollback oid 47 current snap is 245 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 47 to 239 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1902: finishing write tid 2 to smithi01785587-7 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1902: finishing write tid 3 to smithi01785587-7 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1902: finishing write tid 4 to smithi01785587-7 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 951 (ObjNum 723 snap 245 seq_num 723) dirty exists 2024-09-21T09:46:10.121 INFO:tasks.rados.rados.0.smithi017.stdout:1902: left oid 7 (ObjNum 723 snap 245 seq_num 723) 2024-09-21T09:46:10.122 INFO:tasks.rados.rados.0.smithi017.stdout:1904: expect (ObjNum 705 snap 228 seq_num 705) 2024-09-21T09:46:10.256 INFO:tasks.rados.rados.0.smithi017.stdout:1903: finishing rollback tid 1 to smithi01785587-45 2024-09-21T09:46:10.257 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:46:10.257 INFO:tasks.rados.rados.0.smithi017.stdout:1905: finishing write tid 1 to smithi01785587-9 2024-09-21T09:46:10.257 INFO:tasks.rados.rados.0.smithi017.stdout:1902: done (6 left) 2024-09-21T09:46:10.257 INFO:tasks.rados.rados.0.smithi017.stdout:1903: done (5 left) 2024-09-21T09:46:10.257 INFO:tasks.rados.rados.0.smithi017.stdout:1904: done (4 left) 2024-09-21T09:46:10.257 INFO:tasks.rados.rados.0.smithi017.stdout:1909: write oid 22 current snap is 245 2024-09-21T09:46:10.257 INFO:tasks.rados.rados.0.smithi017.stdout:1905: finishing write tid 2 to smithi01785587-9 2024-09-21T09:46:10.257 INFO:tasks.rados.rados.0.smithi017.stdout:1905: finishing write tid 3 to smithi01785587-9 2024-09-21T09:46:10.257 INFO:tasks.rados.rados.0.smithi017.stdout:1909: seq_num 726 ranges {601882=484309,1600633=524904} 2024-09-21T09:46:10.262 INFO:tasks.rados.rados.0.smithi017.stdout:1909: writing smithi01785587-22 from 601882 to 1086191 tid 1 2024-09-21T09:46:10.269 INFO:tasks.rados.rados.0.smithi017.stdout:1909: writing smithi01785587-22 from 1600633 to 2125537 tid 2 2024-09-21T09:46:10.269 INFO:tasks.rados.rados.0.smithi017.stdout:1910: write oid 2 current snap is 245 2024-09-21T09:46:10.269 INFO:tasks.rados.rados.0.smithi017.stdout:1905: finishing write tid 4 to smithi01785587-9 2024-09-21T09:46:10.269 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 336 (ObjNum 724 snap 245 seq_num 724) dirty exists 2024-09-21T09:46:10.269 INFO:tasks.rados.rados.0.smithi017.stdout:1905: left oid 9 (ObjNum 724 snap 245 seq_num 724) 2024-09-21T09:46:10.269 INFO:tasks.rados.rados.0.smithi017.stdout:1906: finishing write tid 1 to smithi01785587-3 2024-09-21T09:46:10.269 INFO:tasks.rados.rados.0.smithi017.stdout:1910: seq_num 727 ranges {618689=559558,1659115=559574} 2024-09-21T09:46:10.275 INFO:tasks.rados.rados.0.smithi017.stdout:1910: writing smithi01785587-2 from 618689 to 1178247 tid 1 2024-09-21T09:46:10.281 INFO:tasks.rados.rados.0.smithi017.stdout:1910: writing smithi01785587-2 from 1659115 to 2218689 tid 2 2024-09-21T09:46:10.281 INFO:tasks.rados.rados.0.smithi017.stdout:1905: done (5 left) 2024-09-21T09:46:10.281 INFO:tasks.rados.rados.0.smithi017.stdout:1911: delete oid 1 current snap is 245 2024-09-21T09:46:10.281 INFO:tasks.rados.rados.0.smithi017.stdout:1906: finishing write tid 2 to smithi01785587-3 2024-09-21T09:46:10.282 INFO:tasks.rados.rados.0.smithi017.stdout:1906: finishing write tid 3 to smithi01785587-3 2024-09-21T09:46:10.282 INFO:tasks.rados.rados.0.smithi017.stdout:1906: finishing write tid 4 to smithi01785587-3 2024-09-21T09:46:10.282 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 903 (ObjNum 725 snap 245 seq_num 725) dirty exists 2024-09-21T09:46:10.282 INFO:tasks.rados.rados.0.smithi017.stdout:1906: left oid 3 (ObjNum 725 snap 245 seq_num 725) 2024-09-21T09:46:10.282 INFO:tasks.rados.rados.0.smithi017.stdout:1908: finishing rollback tid 0 to smithi01785587-47 2024-09-21T09:46:10.282 INFO:tasks.rados.rados.0.smithi017.stdout:1908: finishing rollback tid 1 to smithi01785587-47 2024-09-21T09:46:10.282 INFO:tasks.rados.rados.0.smithi017.stdout:1908: finishing rollback tid 2 to smithi01785587-47 2024-09-21T09:46:10.282 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 667 (ObjNum 664 snap 209 seq_num 664) dirty exists 2024-09-21T09:46:10.282 INFO:tasks.rados.rados.0.smithi017.stdout:1907: expect (ObjNum 678 snap 212 seq_num 678) 2024-09-21T09:46:10.401 INFO:tasks.rados.rados.0.smithi017.stdout:1909: finishing write tid 1 to smithi01785587-22 2024-09-21T09:46:10.401 INFO:tasks.rados.rados.0.smithi017.stdout:1909: finishing write tid 2 to smithi01785587-22 2024-09-21T09:46:10.401 INFO:tasks.rados.rados.0.smithi017.stdout:1909: finishing write tid 3 to smithi01785587-22 2024-09-21T09:46:10.401 INFO:tasks.rados.rados.0.smithi017.stdout:1909: finishing write tid 4 to smithi01785587-22 2024-09-21T09:46:10.401 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 865 (ObjNum 726 snap 245 seq_num 726) dirty exists 2024-09-21T09:46:10.401 INFO:tasks.rados.rados.0.smithi017.stdout:1909: left oid 22 (ObjNum 726 snap 245 seq_num 726) 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1910: finishing write tid 1 to smithi01785587-2 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1910: finishing write tid 2 to smithi01785587-2 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1906: done (5 left) 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1907: done (4 left) 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1908: done (3 left) 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1909: done (2 left) 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1911: done (1 left) 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1912: rollback oid 6 current snap is 245 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 229 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1910: finishing write tid 3 to smithi01785587-2 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:1910: finishing write tid 4 to smithi01785587-2 2024-09-21T09:46:10.402 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 340 (ObjNum 727 snap 245 seq_num 727) dirty exists 2024-09-21T09:46:10.403 INFO:tasks.rados.rados.0.smithi017.stdout:1910: left oid 2 (ObjNum 727 snap 245 seq_num 727) 2024-09-21T09:46:10.403 INFO:tasks.rados.rados.0.smithi017.stdout:1910: done (1 left) 2024-09-21T09:46:10.403 INFO:tasks.rados.rados.0.smithi017.stdout:1913: snap_remove snap 242 2024-09-21T09:46:10.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[102913]: from='client.44687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:10.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[102913]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:10.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[102913]: osdmap e545: 8 total, 7 up, 8 in 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[105151]: from='client.44687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[105151]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-mon[105151]: osdmap e545: 8 total, 7 up, 8 in 2024-09-21T09:46:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:46:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-mon[90085]: from='client.44687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-mon[90085]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:10 smithi134 ceph-mon[90085]: osdmap e545: 8 total, 7 up, 8 in 2024-09-21T09:46:11.200 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:10 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[125118]: 2024-09-21T09:46:10.970+0000 7f3b879c2640 -1 osd.2 534 log_to_monitors true 2024-09-21T09:46:11.242 INFO:tasks.rados.rados.0.smithi017.stdout:1913: done (1 left) 2024-09-21T09:46:11.242 INFO:tasks.rados.rados.0.smithi017.stdout:1914: delete oid 10 current snap is 245 2024-09-21T09:46:11.243 INFO:tasks.rados.rados.0.smithi017.stdout:1912: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:46:11.243 INFO:tasks.rados.rados.0.smithi017.stdout:1912: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:46:11.243 INFO:tasks.rados.rados.0.smithi017.stdout:1912: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:46:11.243 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 530 (ObjNum 583 snap 181 seq_num 583) dirty exists 2024-09-21T09:46:11.248 INFO:tasks.rados.rados.0.smithi017.stdout:1912: done (1 left) 2024-09-21T09:46:11.248 INFO:tasks.rados.rados.0.smithi017.stdout:1914: done (0 left) 2024-09-21T09:46:11.248 INFO:tasks.rados.rados.0.smithi017.stdout:1915: snap_remove snap 228 2024-09-21T09:46:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:11 smithi134 ceph-mon[90085]: pgmap v396: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 158 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 257 KiB/s wr, 1 op/s; 31/294 objects degraded (10.544%) 2024-09-21T09:46:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:11 smithi134 ceph-mon[90085]: from='osd.2 [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:46:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:11 smithi134 ceph-mon[90085]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:46:11.612 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[125118]: 2024-09-21T09:46:11.257+0000 7f3b7f76c640 -1 osd.2 534 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:46:11.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[102913]: pgmap v396: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 158 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 257 KiB/s wr, 1 op/s; 31/294 objects degraded (10.544%) 2024-09-21T09:46:11.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:11.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:11.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[102913]: from='osd.2 [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:46:11.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[102913]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:46:11.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[105151]: pgmap v396: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 158 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 257 KiB/s wr, 1 op/s; 31/294 objects degraded (10.544%) 2024-09-21T09:46:11.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:11.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:11.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[105151]: from='osd.2 [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:46:11.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:11 smithi017 ceph-mon[105151]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:46:11.932 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:12.250 INFO:tasks.rados.rados.0.smithi017.stdout:1915: done (0 left) 2024-09-21T09:46:12.250 INFO:tasks.rados.rados.0.smithi017.stdout:1916: rollback oid 38 current snap is 245 2024-09-21T09:46:12.251 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 238 2024-09-21T09:46:12.251 INFO:tasks.rados.rados.0.smithi017.stdout:1917: read oid 36 snap -1 2024-09-21T09:46:12.251 INFO:tasks.rados.rados.0.smithi017.stdout:1917: expect (ObjNum 703 snap 228 seq_num 703) 2024-09-21T09:46:12.251 INFO:tasks.rados.rados.0.smithi017.stdout:1918: delete oid 20 current snap is 245 2024-09-21T09:46:12.259 INFO:tasks.rados.rados.0.smithi017.stdout:1918: done (2 left) 2024-09-21T09:46:12.259 INFO:tasks.rados.rados.0.smithi017.stdout:1919: read oid 28 snap -1 2024-09-21T09:46:12.260 INFO:tasks.rados.rados.0.smithi017.stdout:1919: expect deleted 2024-09-21T09:46:12.260 INFO:tasks.rados.rados.0.smithi017.stdout:1920: snap_remove snap 229 2024-09-21T09:46:12.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[102913]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:46:12.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[102913]: osdmap e546: 8 total, 7 up, 8 in 2024-09-21T09:46:12.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[102913]: from='osd.2 [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:46:12.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[102913]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:46:12.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1570560228' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:12.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[105151]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:46:12.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[105151]: osdmap e546: 8 total, 7 up, 8 in 2024-09-21T09:46:12.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[105151]: from='osd.2 [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:46:12.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[105151]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:46:12.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:12 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1570560228' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:12 smithi134 ceph-mon[90085]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:46:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:12 smithi134 ceph-mon[90085]: osdmap e546: 8 total, 7 up, 8 in 2024-09-21T09:46:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:12 smithi134 ceph-mon[90085]: from='osd.2 [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:46:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:12 smithi134 ceph-mon[90085]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:46:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:12 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1570560228' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:13.250 INFO:tasks.rados.rados.0.smithi017.stdout:1920: done (3 left) 2024-09-21T09:46:13.250 INFO:tasks.rados.rados.0.smithi017.stdout:1921: delete oid 17 current snap is 245 2024-09-21T09:46:13.250 INFO:tasks.rados.rados.0.smithi017.stdout:1917: expect (ObjNum 703 snap 228 seq_num 703) 2024-09-21T09:46:13.445 INFO:tasks.rados.rados.0.smithi017.stdout:1916: finishing rollback tid 0 to smithi01785587-38 2024-09-21T09:46:13.445 INFO:tasks.rados.rados.0.smithi017.stdout:1916: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:46:13.445 INFO:tasks.rados.rados.0.smithi017.stdout:1916: finishing rollback tid 2 to smithi01785587-38 2024-09-21T09:46:13.445 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 568 (ObjNum 701 snap 227 seq_num 701) dirty exists 2024-09-21T09:46:13.445 INFO:tasks.rados.rados.0.smithi017.stdout:1916: done (3 left) 2024-09-21T09:46:13.445 INFO:tasks.rados.rados.0.smithi017.stdout:1917: done (2 left) 2024-09-21T09:46:13.446 INFO:tasks.rados.rados.0.smithi017.stdout:1919: done (1 left) 2024-09-21T09:46:13.446 INFO:tasks.rados.rados.0.smithi017.stdout:1921: done (0 left) 2024-09-21T09:46:13.446 INFO:tasks.rados.rados.0.smithi017.stdout:1922: rollback oid 47 current snap is 245 2024-09-21T09:46:13.446 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 47 to 232 2024-09-21T09:46:13.446 INFO:tasks.rados.rados.0.smithi017.stdout:1923: write oid 48 current snap is 245 2024-09-21T09:46:13.446 INFO:tasks.rados.rados.0.smithi017.stdout:1923: seq_num 732 ranges {702724=652156,1951525=351199} 2024-09-21T09:46:13.450 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:46:13.207+0000 7f3131349640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T09:46:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: pgmap v399: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 167 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 912 KiB/s wr, 13 op/s; 33/318 objects degraded (10.377%) 2024-09-21T09:46:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='client.44714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 33/318 objects degraded (10.377%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: osd.2 [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954] boot 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: pgmap v399: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 167 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 912 KiB/s wr, 13 op/s; 33/318 objects degraded (10.377%) 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='client.44714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 33/318 objects degraded (10.377%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: osd.2 [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954] boot 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:13.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:13.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:13.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:13.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:13.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:13.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:13.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:46:13.453 INFO:tasks.rados.rados.0.smithi017.stdout:1923: writing smithi01785587-48 from 702724 to 1354880 tid 1 2024-09-21T09:46:13.458 INFO:tasks.rados.rados.0.smithi017.stdout:1923: writing smithi01785587-48 from 1951525 to 2302724 tid 2 2024-09-21T09:46:13.458 INFO:tasks.rados.rados.0.smithi017.stdout:1924: delete oid 35 current snap is 245 2024-09-21T09:46:13.458 INFO:tasks.rados.rados.0.smithi017.stdout:1922: finishing rollback tid 0 to smithi01785587-47 2024-09-21T09:46:13.458 INFO:tasks.rados.rados.0.smithi017.stdout:1922: finishing rollback tid 1 to smithi01785587-47 2024-09-21T09:46:13.458 INFO:tasks.rados.rados.0.smithi017.stdout:1922: finishing rollback tid 2 to smithi01785587-47 2024-09-21T09:46:13.458 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 674 (ObjNum 664 snap 209 seq_num 664) dirty exists 2024-09-21T09:46:13.458 INFO:tasks.rados.rados.0.smithi017.stdout:1923: finishing write tid 1 to smithi01785587-48 2024-09-21T09:46:13.460 INFO:tasks.rados.rados.0.smithi017.stdout:1922: done (2 left) 2024-09-21T09:46:13.460 INFO:tasks.rados.rados.0.smithi017.stdout:1924: done (1 left) 2024-09-21T09:46:13.460 INFO:tasks.rados.rados.0.smithi017.stdout:1925: delete oid 1 current snap is 245 2024-09-21T09:46:13.460 INFO:tasks.rados.rados.0.smithi017.stdout:1923: finishing write tid 2 to smithi01785587-48 2024-09-21T09:46:13.461 INFO:tasks.rados.rados.0.smithi017.stdout:1923: finishing write tid 3 to smithi01785587-48 2024-09-21T09:46:13.461 INFO:tasks.rados.rados.0.smithi017.stdout:1923: finishing write tid 4 to smithi01785587-48 2024-09-21T09:46:13.461 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 764 (ObjNum 732 snap 245 seq_num 732) dirty exists 2024-09-21T09:46:13.462 INFO:tasks.rados.rados.0.smithi017.stdout:1923: left oid 48 (ObjNum 732 snap 245 seq_num 732) 2024-09-21T09:46:13.462 INFO:tasks.rados.rados.0.smithi017.stdout:1923: done (1 left) 2024-09-21T09:46:13.462 INFO:tasks.rados.rados.0.smithi017.stdout:1925: done (0 left) 2024-09-21T09:46:13.462 INFO:tasks.rados.rados.0.smithi017.stdout:1926: snap_create 2024-09-21T09:46:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: pgmap v399: 89 pgs: 18 active+undersized, 8 active+undersized+degraded, 63 active+clean; 167 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 912 KiB/s wr, 13 op/s; 33/318 objects degraded (10.377%) 2024-09-21T09:46:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='client.34845 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='client.44714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 33/318 objects degraded (10.377%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:46:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: osd.2 [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954] boot 2024-09-21T09:46:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:46:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:13.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:46:14.269 INFO:tasks.rados.rados.0.smithi017.stdout:1926: done (0 left) 2024-09-21T09:46:14.270 INFO:tasks.rados.rados.0.smithi017.stdout:1927: read oid 34 snap -1 2024-09-21T09:46:14.270 INFO:tasks.rados.rados.0.smithi017.stdout:1927: expect (ObjNum 699 snap 225 seq_num 699) 2024-09-21T09:46:14.270 INFO:tasks.rados.rados.0.smithi017.stdout:1928: read oid 33 snap -1 2024-09-21T09:46:14.270 INFO:tasks.rados.rados.0.smithi017.stdout:1928: expect (ObjNum 638 snap 202 seq_num 638) 2024-09-21T09:46:14.270 INFO:tasks.rados.rados.0.smithi017.stdout:1929: rollback oid 12 current snap is 246 2024-09-21T09:46:14.270 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 12 to 220 2024-09-21T09:46:14.270 INFO:tasks.rados.rados.0.smithi017.stdout:1930: write oid 26 current snap is 246 2024-09-21T09:46:14.270 INFO:tasks.rados.rados.0.smithi017.stdout:1930: seq_num 735 ranges {753145=477903,2010618=342527} 2024-09-21T09:46:14.277 INFO:tasks.rados.rados.0.smithi017.stdout:1930: writing smithi01785587-26 from 753145 to 1231048 tid 1 2024-09-21T09:46:14.284 INFO:tasks.rados.rados.0.smithi017.stdout:1930: writing smithi01785587-26 from 2010618 to 2353145 tid 2 2024-09-21T09:46:14.284 INFO:tasks.rados.rados.0.smithi017.stdout:1931: read oid 48 snap -1 2024-09-21T09:46:14.284 INFO:tasks.rados.rados.0.smithi017.stdout:1931: expect (ObjNum 732 snap 245 seq_num 732) 2024-09-21T09:46:14.284 INFO:tasks.rados.rados.0.smithi017.stdout:1932: read oid 9 snap 237 2024-09-21T09:46:14.284 INFO:tasks.rados.rados.0.smithi017.stdout:1932: expect (ObjNum 704 snap 228 seq_num 704) 2024-09-21T09:46:14.285 INFO:tasks.rados.rados.0.smithi017.stdout:1933: read oid 20 snap 236 2024-09-21T09:46:14.285 INFO:tasks.rados.rados.0.smithi017.stdout:1933: expect deleted 2024-09-21T09:46:14.285 INFO:tasks.rados.rados.0.smithi017.stdout:1934: snap_remove snap 226 2024-09-21T09:46:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:14 smithi017 ceph-mon[105151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:46:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:14 smithi017 ceph-mon[105151]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T09:46:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:14 smithi017 ceph-mon[105151]: osdmap e548: 8 total, 8 up, 8 in 2024-09-21T09:46:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:14 smithi017 ceph-mon[102913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:46:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:14 smithi017 ceph-mon[102913]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T09:46:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:14 smithi017 ceph-mon[102913]: osdmap e548: 8 total, 8 up, 8 in 2024-09-21T09:46:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:14 smithi134 ceph-mon[90085]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:46:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:14 smithi134 ceph-mon[90085]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T09:46:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:14 smithi134 ceph-mon[90085]: osdmap e548: 8 total, 8 up, 8 in 2024-09-21T09:46:14.844 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:15.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:46:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:46:14] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.51.0" 2024-09-21T09:46:15.274 INFO:tasks.rados.rados.0.smithi017.stdout:1934: done (7 left) 2024-09-21T09:46:15.274 INFO:tasks.rados.rados.0.smithi017.stdout:1935: delete oid 44 current snap is 246 2024-09-21T09:46:15.274 INFO:tasks.rados.rados.0.smithi017.stdout:1928: expect (ObjNum 638 snap 202 seq_num 638) 2024-09-21T09:46:15.371 INFO:tasks.rados.rados.0.smithi017.stdout:1929: finishing rollback tid 0 to smithi01785587-12 2024-09-21T09:46:15.371 INFO:tasks.rados.rados.0.smithi017.stdout:1929: finishing rollback tid 1 to smithi01785587-12 2024-09-21T09:46:15.371 INFO:tasks.rados.rados.0.smithi017.stdout:1929: finishing rollback tid 2 to smithi01785587-12 2024-09-21T09:46:15.371 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 678 (ObjNum 659 snap 208 seq_num 659) dirty exists 2024-09-21T09:46:15.371 INFO:tasks.rados.rados.0.smithi017.stdout:1928: done (7 left) 2024-09-21T09:46:15.371 INFO:tasks.rados.rados.0.smithi017.stdout:1929: done (6 left) 2024-09-21T09:46:15.371 INFO:tasks.rados.rados.0.smithi017.stdout:1935: done (5 left) 2024-09-21T09:46:15.371 INFO:tasks.rados.rados.0.smithi017.stdout:1936: write oid 5 current snap is 246 2024-09-21T09:46:15.371 INFO:tasks.rados.rados.0.smithi017.stdout:1927: expect (ObjNum 699 snap 225 seq_num 699) 2024-09-21T09:46:15.674 INFO:tasks.rados.rados.0.smithi017.stdout:1930: finishing write tid 1 to smithi01785587-26 2024-09-21T09:46:15.674 INFO:tasks.rados.rados.0.smithi017.stdout:1930: finishing write tid 2 to smithi01785587-26 2024-09-21T09:46:15.674 INFO:tasks.rados.rados.0.smithi017.stdout:1930: finishing write tid 3 to smithi01785587-26 2024-09-21T09:46:15.674 INFO:tasks.rados.rados.0.smithi017.stdout:1930: finishing write tid 4 to smithi01785587-26 2024-09-21T09:46:15.674 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 875 (ObjNum 735 snap 246 seq_num 735) dirty exists 2024-09-21T09:46:15.674 INFO:tasks.rados.rados.0.smithi017.stdout:1930: left oid 26 (ObjNum 735 snap 246 seq_num 735) 2024-09-21T09:46:15.674 INFO:tasks.rados.rados.0.smithi017.stdout:1931: expect (ObjNum 732 snap 245 seq_num 732) 2024-09-21T09:46:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:15 smithi017 ceph-mon[102913]: pgmap v402: 89 pgs: 2 peering, 17 active+undersized, 7 active+undersized+degraded, 63 active+clean; 168 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.4 MiB/s wr, 17 op/s; 31/324 objects degraded (9.568%) 2024-09-21T09:46:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:15 smithi017 ceph-mon[102913]: osdmap e549: 8 total, 8 up, 8 in 2024-09-21T09:46:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:15 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2773859936' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:15 smithi017 ceph-mon[105151]: pgmap v402: 89 pgs: 2 peering, 17 active+undersized, 7 active+undersized+degraded, 63 active+clean; 168 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.4 MiB/s wr, 17 op/s; 31/324 objects degraded (9.568%) 2024-09-21T09:46:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:15 smithi017 ceph-mon[105151]: osdmap e549: 8 total, 8 up, 8 in 2024-09-21T09:46:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:15 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2773859936' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:15.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:15 smithi134 ceph-mon[90085]: pgmap v402: 89 pgs: 2 peering, 17 active+undersized, 7 active+undersized+degraded, 63 active+clean; 168 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.4 MiB/s wr, 17 op/s; 31/324 objects degraded (9.568%) 2024-09-21T09:46:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:15 smithi134 ceph-mon[90085]: osdmap e549: 8 total, 8 up, 8 in 2024-09-21T09:46:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:15 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2773859936' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:15.776 INFO:tasks.rados.rados.0.smithi017.stdout:1932: expect (ObjNum 704 snap 228 seq_num 704) 2024-09-21T09:46:15.904 INFO:tasks.rados.rados.0.smithi017.stdout:1936: seq_num 737 ranges {786759=628401,2011229=375530} 2024-09-21T09:46:15.911 INFO:tasks.rados.rados.0.smithi017.stdout:1936: writing smithi01785587-5 from 786759 to 1415160 tid 1 2024-09-21T09:46:15.917 INFO:tasks.rados.rados.0.smithi017.stdout:1936: writing smithi01785587-5 from 2011229 to 2386759 tid 2 2024-09-21T09:46:15.917 INFO:tasks.rados.rados.0.smithi017.stdout:1927: done (5 left) 2024-09-21T09:46:15.917 INFO:tasks.rados.rados.0.smithi017.stdout:1930: done (4 left) 2024-09-21T09:46:15.917 INFO:tasks.rados.rados.0.smithi017.stdout:1931: done (3 left) 2024-09-21T09:46:15.917 INFO:tasks.rados.rados.0.smithi017.stdout:1932: done (2 left) 2024-09-21T09:46:15.917 INFO:tasks.rados.rados.0.smithi017.stdout:1933: done (1 left) 2024-09-21T09:46:15.918 INFO:tasks.rados.rados.0.smithi017.stdout:1937: read oid 39 snap -1 2024-09-21T09:46:15.918 INFO:tasks.rados.rados.0.smithi017.stdout:1937: expect (ObjNum 715 snap 240 seq_num 715) 2024-09-21T09:46:15.918 INFO:tasks.rados.rados.0.smithi017.stdout:1938: snap_create 2024-09-21T09:46:15.918 INFO:tasks.rados.rados.0.smithi017.stdout:1936: finishing write tid 1 to smithi01785587-5 2024-09-21T09:46:15.919 INFO:tasks.rados.rados.0.smithi017.stdout:1936: finishing write tid 2 to smithi01785587-5 2024-09-21T09:46:15.919 INFO:tasks.rados.rados.0.smithi017.stdout:1936: finishing write tid 3 to smithi01785587-5 2024-09-21T09:46:15.920 INFO:tasks.rados.rados.0.smithi017.stdout:1936: finishing write tid 4 to smithi01785587-5 2024-09-21T09:46:15.920 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 915 (ObjNum 737 snap 246 seq_num 737) dirty exists 2024-09-21T09:46:15.920 INFO:tasks.rados.rados.0.smithi017.stdout:1936: left oid 5 (ObjNum 737 snap 246 seq_num 737) 2024-09-21T09:46:15.921 INFO:tasks.rados.rados.0.smithi017.stdout:1937: expect (ObjNum 715 snap 240 seq_num 715) 2024-09-21T09:46:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:1936: done (2 left) 2024-09-21T09:46:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:1937: done (1 left) 2024-09-21T09:46:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:1938: done (0 left) 2024-09-21T09:46:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:1939: read oid 49 snap -1 2024-09-21T09:46:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:1939: expect (ObjNum 640 snap 202 seq_num 640) 2024-09-21T09:46:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:1940: snap_create 2024-09-21T09:46:16.284 INFO:tasks.rados.rados.0.smithi017.stdout:1939: expect (ObjNum 640 snap 202 seq_num 640) 2024-09-21T09:46:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:16 smithi017 ceph-mon[105151]: from='client.44726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:16 smithi017 ceph-mon[105151]: from='client.44732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:16 smithi017 ceph-mon[105151]: osdmap e550: 8 total, 8 up, 8 in 2024-09-21T09:46:16.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:16 smithi017 ceph-mon[102913]: from='client.44726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:16.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:16 smithi017 ceph-mon[102913]: from='client.44732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:16.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:16 smithi017 ceph-mon[102913]: osdmap e550: 8 total, 8 up, 8 in 2024-09-21T09:46:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:16 smithi134 ceph-mon[90085]: from='client.44726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:16 smithi134 ceph-mon[90085]: from='client.44732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:16 smithi134 ceph-mon[90085]: osdmap e550: 8 total, 8 up, 8 in 2024-09-21T09:46:17.281 INFO:tasks.rados.rados.0.smithi017.stdout:1939: done (1 left) 2024-09-21T09:46:17.281 INFO:tasks.rados.rados.0.smithi017.stdout:1940: done (0 left) 2024-09-21T09:46:17.281 INFO:tasks.rados.rados.0.smithi017.stdout:1941: rollback oid 34 current snap is 248 2024-09-21T09:46:17.281 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 220 2024-09-21T09:46:17.281 INFO:tasks.rados.rados.0.smithi017.stdout:1942: snap_create 2024-09-21T09:46:17.285 INFO:tasks.rados.rados.0.smithi017.stdout:1941: finishing rollback tid 0 to smithi01785587-34 2024-09-21T09:46:17.285 INFO:tasks.rados.rados.0.smithi017.stdout:1941: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:46:17.286 INFO:tasks.rados.rados.0.smithi017.stdout:1941: finishing rollback tid 2 to smithi01785587-34 2024-09-21T09:46:17.286 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 969 (ObjNum 677 snap 212 seq_num 677) dirty exists 2024-09-21T09:46:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:17 smithi017 ceph-mon[102913]: pgmap v405: 89 pgs: 2 peering, 8 active+undersized, 4 active+undersized+degraded, 75 active+clean; 168 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 939 KiB/s wr, 11 op/s; 26/333 objects degraded (7.808%) 2024-09-21T09:46:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:17 smithi017 ceph-mon[102913]: osdmap e551: 8 total, 8 up, 8 in 2024-09-21T09:46:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:17 smithi017 ceph-mon[105151]: pgmap v405: 89 pgs: 2 peering, 8 active+undersized, 4 active+undersized+degraded, 75 active+clean; 168 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 939 KiB/s wr, 11 op/s; 26/333 objects degraded (7.808%) 2024-09-21T09:46:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:17 smithi017 ceph-mon[105151]: osdmap e551: 8 total, 8 up, 8 in 2024-09-21T09:46:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:17 smithi134 ceph-mon[90085]: pgmap v405: 89 pgs: 2 peering, 8 active+undersized, 4 active+undersized+degraded, 75 active+clean; 168 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 939 KiB/s wr, 11 op/s; 26/333 objects degraded (7.808%) 2024-09-21T09:46:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:17 smithi134 ceph-mon[90085]: osdmap e551: 8 total, 8 up, 8 in 2024-09-21T09:46:17.744 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:18.290 INFO:tasks.rados.rados.0.smithi017.stdout:1941: done (1 left) 2024-09-21T09:46:18.290 INFO:tasks.rados.rados.0.smithi017.stdout:1942: done (0 left) 2024-09-21T09:46:18.290 INFO:tasks.rados.rados.0.smithi017.stdout:1943: read oid 37 snap -1 2024-09-21T09:46:18.290 INFO:tasks.rados.rados.0.smithi017.stdout:1943: expect (ObjNum 698 snap 221 seq_num 698) 2024-09-21T09:46:18.290 INFO:tasks.rados.rados.0.smithi017.stdout:1944: write oid 18 current snap is 249 2024-09-21T09:46:18.290 INFO:tasks.rados.rados.0.smithi017.stdout:1944: seq_num 738 ranges {403566=420003,1386064=546918} 2024-09-21T09:46:18.294 INFO:tasks.rados.rados.0.smithi017.stdout:1944: writing smithi01785587-18 from 403566 to 823569 tid 1 2024-09-21T09:46:18.301 INFO:tasks.rados.rados.0.smithi017.stdout:1944: writing smithi01785587-18 from 1386064 to 1932982 tid 2 2024-09-21T09:46:18.301 INFO:tasks.rados.rados.0.smithi017.stdout:1945: delete oid 29 current snap is 249 2024-09-21T09:46:18.301 INFO:tasks.rados.rados.0.smithi017.stdout:1943: expect (ObjNum 698 snap 221 seq_num 698) 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1944: finishing write tid 1 to smithi01785587-18 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1944: finishing write tid 2 to smithi01785587-18 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1944: finishing write tid 3 to smithi01785587-18 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1944: finishing write tid 4 to smithi01785587-18 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 151 (ObjNum 738 snap 249 seq_num 738) dirty exists 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1944: left oid 18 (ObjNum 738 snap 249 seq_num 738) 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1943: done (2 left) 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1944: done (1 left) 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1945: done (0 left) 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1946: read oid 12 snap -1 2024-09-21T09:46:18.493 INFO:tasks.rados.rados.0.smithi017.stdout:1946: expect (ObjNum 659 snap 208 seq_num 659) 2024-09-21T09:46:18.494 INFO:tasks.rados.rados.0.smithi017.stdout:1947: read oid 17 snap 237 2024-09-21T09:46:18.494 INFO:tasks.rados.rados.0.smithi017.stdout:1947: expect (ObjNum 662 snap 209 seq_num 662) 2024-09-21T09:46:18.494 INFO:tasks.rados.rados.0.smithi017.stdout:1948: read oid 23 snap -1 2024-09-21T09:46:18.494 INFO:tasks.rados.rados.0.smithi017.stdout:1948: expect (ObjNum 663 snap 209 seq_num 663) 2024-09-21T09:46:18.494 INFO:tasks.rados.rados.0.smithi017.stdout:1949: snap_create 2024-09-21T09:46:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:1947: expect (ObjNum 662 snap 209 seq_num 662) 2024-09-21T09:46:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:18 smithi017 ceph-mon[102913]: osdmap e552: 8 total, 8 up, 8 in 2024-09-21T09:46:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:18 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2649336719' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:18 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 26/333 objects degraded (7.808%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:18 smithi017 ceph-mon[105151]: osdmap e552: 8 total, 8 up, 8 in 2024-09-21T09:46:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:18 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2649336719' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:18 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 26/333 objects degraded (7.808%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:18.735 INFO:tasks.rados.rados.0.smithi017.stdout:1948: expect (ObjNum 663 snap 209 seq_num 663) 2024-09-21T09:46:18.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:18 smithi134 ceph-mon[90085]: osdmap e552: 8 total, 8 up, 8 in 2024-09-21T09:46:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:18 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2649336719' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:18 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 26/333 objects degraded (7.808%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:18.877 INFO:tasks.rados.rados.0.smithi017.stdout:1946: expect (ObjNum 659 snap 208 seq_num 659) 2024-09-21T09:46:19.291 INFO:tasks.rados.rados.0.smithi017.stdout:1946: done (3 left) 2024-09-21T09:46:19.291 INFO:tasks.rados.rados.0.smithi017.stdout:1947: done (2 left) 2024-09-21T09:46:19.291 INFO:tasks.rados.rados.0.smithi017.stdout:1948: done (1 left) 2024-09-21T09:46:19.291 INFO:tasks.rados.rados.0.smithi017.stdout:1949: done (0 left) 2024-09-21T09:46:19.291 INFO:tasks.rados.rados.0.smithi017.stdout:1950: snap_remove snap 230 2024-09-21T09:46:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[102913]: from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[102913]: pgmap v408: 89 pgs: 2 peering, 87 active+clean; 161 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 445 KiB/s wr, 10 op/s; 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:46:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[102913]: from='client.44750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/333 objects degraded (7.808%), 4 pgs degraded) 2024-09-21T09:46:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T09:46:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[102913]: osdmap e553: 8 total, 8 up, 8 in 2024-09-21T09:46:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[105151]: from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[105151]: pgmap v408: 89 pgs: 2 peering, 87 active+clean; 161 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 445 KiB/s wr, 10 op/s; 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:46:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[105151]: from='client.44750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/333 objects degraded (7.808%), 4 pgs degraded) 2024-09-21T09:46:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T09:46:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:19 smithi017 ceph-mon[105151]: osdmap e553: 8 total, 8 up, 8 in 2024-09-21T09:46:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:19 smithi134 ceph-mon[90085]: from='client.44744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:19 smithi134 ceph-mon[90085]: pgmap v408: 89 pgs: 2 peering, 87 active+clean; 161 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 445 KiB/s wr, 10 op/s; 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:46:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:19 smithi134 ceph-mon[90085]: from='client.44750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:19 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/333 objects degraded (7.808%), 4 pgs degraded) 2024-09-21T09:46:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:19 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T09:46:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:19 smithi134 ceph-mon[90085]: osdmap e553: 8 total, 8 up, 8 in 2024-09-21T09:46:20.302 INFO:tasks.rados.rados.0.smithi017.stdout:1950: done (0 left) 2024-09-21T09:46:20.302 INFO:tasks.rados.rados.0.smithi017.stdout:1951: read oid 43 snap 245 2024-09-21T09:46:20.302 INFO:tasks.rados.rados.0.smithi017.stdout:1951: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:46:20.302 INFO:tasks.rados.rados.0.smithi017.stdout:1952: delete oid 11 current snap is 250 2024-09-21T09:46:20.314 INFO:tasks.rados.rados.0.smithi017.stdout:1952: done (1 left) 2024-09-21T09:46:20.314 INFO:tasks.rados.rados.0.smithi017.stdout:1953: read oid 21 snap 235 2024-09-21T09:46:20.314 INFO:tasks.rados.rados.0.smithi017.stdout:1953: expect (ObjNum 711 snap 233 seq_num 711) 2024-09-21T09:46:20.314 INFO:tasks.rados.rados.0.smithi017.stdout:1954: snap_remove snap 247 2024-09-21T09:46:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:20 smithi134 ceph-mon[90085]: osdmap e554: 8 total, 8 up, 8 in 2024-09-21T09:46:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:46:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:46:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:20.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:20 smithi017 ceph-mon[102913]: osdmap e554: 8 total, 8 up, 8 in 2024-09-21T09:46:20.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:20 smithi017 ceph-mon[105151]: osdmap e554: 8 total, 8 up, 8 in 2024-09-21T09:46:20.655 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:21.305 INFO:tasks.rados.rados.0.smithi017.stdout:1954: done (2 left) 2024-09-21T09:46:21.305 INFO:tasks.rados.rados.0.smithi017.stdout:1955: snap_remove snap 238 2024-09-21T09:46:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:21 smithi017 ceph-mon[102913]: pgmap v411: 89 pgs: 89 active+clean; 161 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 0 B/s wr, 5 op/s; 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:46:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:21 smithi017 ceph-mon[102913]: osdmap e555: 8 total, 8 up, 8 in 2024-09-21T09:46:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:21 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3253964328' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:21 smithi017 ceph-mon[105151]: pgmap v411: 89 pgs: 89 active+clean; 161 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 0 B/s wr, 5 op/s; 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:46:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:21 smithi017 ceph-mon[105151]: osdmap e555: 8 total, 8 up, 8 in 2024-09-21T09:46:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:21 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3253964328' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:21 smithi134 ceph-mon[90085]: pgmap v411: 89 pgs: 89 active+clean; 161 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 0 B/s wr, 5 op/s; 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:46:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:21 smithi134 ceph-mon[90085]: osdmap e555: 8 total, 8 up, 8 in 2024-09-21T09:46:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:21 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3253964328' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:22.313 INFO:tasks.rados.rados.0.smithi017.stdout:1955: done (2 left) 2024-09-21T09:46:22.313 INFO:tasks.rados.rados.0.smithi017.stdout:1956: write oid 24 current snap is 250 2024-09-21T09:46:22.314 INFO:tasks.rados.rados.0.smithi017.stdout:1956: seq_num 741 ranges {453987=645750,1845157=608830} 2024-09-21T09:46:22.320 INFO:tasks.rados.rados.0.smithi017.stdout:1956: writing smithi01785587-24 from 453987 to 1099737 tid 1 2024-09-21T09:46:22.328 INFO:tasks.rados.rados.0.smithi017.stdout:1956: writing smithi01785587-24 from 1845157 to 2453987 tid 2 2024-09-21T09:46:22.328 INFO:tasks.rados.rados.0.smithi017.stdout:1957: write oid 29 current snap is 250 2024-09-21T09:46:22.328 INFO:tasks.rados.rados.0.smithi017.stdout:1957: seq_num 742 ranges {470794=720999,1903639=554256} 2024-09-21T09:46:22.335 INFO:tasks.rados.rados.0.smithi017.stdout:1957: writing smithi01785587-29 from 470794 to 1191793 tid 1 2024-09-21T09:46:22.342 INFO:tasks.rados.rados.0.smithi017.stdout:1957: writing smithi01785587-29 from 1903639 to 2457895 tid 2 2024-09-21T09:46:22.342 INFO:tasks.rados.rados.0.smithi017.stdout:1958: snap_remove snap 231 2024-09-21T09:46:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:22 smithi017 ceph-mon[105151]: from='client.34899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:22 smithi017 ceph-mon[105151]: from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:22 smithi017 ceph-mon[105151]: osdmap e556: 8 total, 8 up, 8 in 2024-09-21T09:46:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:22 smithi017 ceph-mon[102913]: from='client.34899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:22 smithi017 ceph-mon[102913]: from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:22 smithi017 ceph-mon[102913]: osdmap e556: 8 total, 8 up, 8 in 2024-09-21T09:46:22.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:22 smithi134 ceph-mon[90085]: from='client.34899 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:22 smithi134 ceph-mon[90085]: from='client.34905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:22 smithi134 ceph-mon[90085]: osdmap e556: 8 total, 8 up, 8 in 2024-09-21T09:46:23.085 INFO:tasks.rados.rados.0.smithi017.stdout:1958: done (4 left) 2024-09-21T09:46:23.085 INFO:tasks.rados.rados.0.smithi017.stdout:1959: read oid 40 snap 219 2024-09-21T09:46:23.085 INFO:tasks.rados.rados.0.smithi017.stdout:1959: expect deleted 2024-09-21T09:46:23.085 INFO:tasks.rados.rados.0.smithi017.stdout:1960: write oid 2 current snap is 250 2024-09-21T09:46:23.085 INFO:tasks.rados.rados.0.smithi017.stdout:1951: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:46:23.201 INFO:tasks.rados.rados.0.smithi017.stdout:1953: expect (ObjNum 711 snap 233 seq_num 711) 2024-09-21T09:46:23.333 INFO:tasks.rados.rados.0.smithi017.stdout:1956: finishing write tid 1 to smithi01785587-24 2024-09-21T09:46:23.333 INFO:tasks.rados.rados.0.smithi017.stdout:1956: finishing write tid 2 to smithi01785587-24 2024-09-21T09:46:23.333 INFO:tasks.rados.rados.0.smithi017.stdout:1956: finishing write tid 3 to smithi01785587-24 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:1956: finishing write tid 4 to smithi01785587-24 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 584 (ObjNum 741 snap 250 seq_num 741) dirty exists 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:1956: left oid 24 (ObjNum 741 snap 250 seq_num 741) 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:1957: finishing write tid 1 to smithi01785587-29 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:1957: finishing write tid 2 to smithi01785587-29 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:1957: finishing write tid 3 to smithi01785587-29 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:1957: finishing write tid 4 to smithi01785587-29 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 424 (ObjNum 742 snap 250 seq_num 742) dirty exists 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:1957: left oid 29 (ObjNum 742 snap 250 seq_num 742) 2024-09-21T09:46:23.334 INFO:tasks.rados.rados.0.smithi017.stdout:1960: seq_num 743 ranges {487601=796248,1962121=525480} 2024-09-21T09:46:23.340 INFO:tasks.rados.rados.0.smithi017.stdout:1960: writing smithi01785587-2 from 487601 to 1283849 tid 1 2024-09-21T09:46:23.347 INFO:tasks.rados.rados.0.smithi017.stdout:1960: writing smithi01785587-2 from 1962121 to 2487601 tid 2 2024-09-21T09:46:23.348 INFO:tasks.rados.rados.0.smithi017.stdout:1951: done (5 left) 2024-09-21T09:46:23.348 INFO:tasks.rados.rados.0.smithi017.stdout:1953: done (4 left) 2024-09-21T09:46:23.348 INFO:tasks.rados.rados.0.smithi017.stdout:1956: done (3 left) 2024-09-21T09:46:23.348 INFO:tasks.rados.rados.0.smithi017.stdout:1957: done (2 left) 2024-09-21T09:46:23.348 INFO:tasks.rados.rados.0.smithi017.stdout:1961: write oid 33 current snap is 250 2024-09-21T09:46:23.348 INFO:tasks.rados.rados.0.smithi017.stdout:1961: seq_num 744 ranges {504408=471497,1620603=557925} 2024-09-21T09:46:23.353 INFO:tasks.rados.rados.0.smithi017.stdout:1961: writing smithi01785587-33 from 504408 to 975905 tid 1 2024-09-21T09:46:23.360 INFO:tasks.rados.rados.0.smithi017.stdout:1961: writing smithi01785587-33 from 1620603 to 2178528 tid 2 2024-09-21T09:46:23.360 INFO:tasks.rados.rados.0.smithi017.stdout:1962: snap_create 2024-09-21T09:46:23.360 INFO:tasks.rados.rados.0.smithi017.stdout:1960: finishing write tid 1 to smithi01785587-2 2024-09-21T09:46:23.360 INFO:tasks.rados.rados.0.smithi017.stdout:1960: finishing write tid 2 to smithi01785587-2 2024-09-21T09:46:23.360 INFO:tasks.rados.rados.0.smithi017.stdout:1960: finishing write tid 3 to smithi01785587-2 2024-09-21T09:46:23.360 INFO:tasks.rados.rados.0.smithi017.stdout:1960: finishing write tid 4 to smithi01785587-2 2024-09-21T09:46:23.360 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 358 (ObjNum 743 snap 250 seq_num 743) dirty exists 2024-09-21T09:46:23.360 INFO:tasks.rados.rados.0.smithi017.stdout:1960: left oid 2 (ObjNum 743 snap 250 seq_num 743) 2024-09-21T09:46:23.360 INFO:tasks.rados.rados.0.smithi017.stdout:1961: finishing write tid 1 to smithi01785587-33 2024-09-21T09:46:23.363 INFO:tasks.rados.rados.0.smithi017.stdout:1961: finishing write tid 2 to smithi01785587-33 2024-09-21T09:46:23.363 INFO:tasks.rados.rados.0.smithi017.stdout:1961: finishing write tid 3 to smithi01785587-33 2024-09-21T09:46:23.363 INFO:tasks.rados.rados.0.smithi017.stdout:1961: finishing write tid 4 to smithi01785587-33 2024-09-21T09:46:23.363 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 899 (ObjNum 744 snap 250 seq_num 744) dirty exists 2024-09-21T09:46:23.364 INFO:tasks.rados.rados.0.smithi017.stdout:1961: left oid 33 (ObjNum 744 snap 250 seq_num 744) 2024-09-21T09:46:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:23 smithi017 ceph-mon[105151]: pgmap v414: 89 pgs: 89 active+clean; 166 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 482 KiB/s wr, 16 op/s 2024-09-21T09:46:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:23 smithi017 ceph-mon[105151]: osdmap e557: 8 total, 8 up, 8 in 2024-09-21T09:46:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:23 smithi017 ceph-mon[105151]: osdmap e558: 8 total, 8 up, 8 in 2024-09-21T09:46:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:23 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/939495727' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:23 smithi017 ceph-mon[102913]: pgmap v414: 89 pgs: 89 active+clean; 166 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 482 KiB/s wr, 16 op/s 2024-09-21T09:46:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:23 smithi017 ceph-mon[102913]: osdmap e557: 8 total, 8 up, 8 in 2024-09-21T09:46:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:23 smithi017 ceph-mon[102913]: osdmap e558: 8 total, 8 up, 8 in 2024-09-21T09:46:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:23 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/939495727' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:23.568 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:23 smithi134 ceph-mon[90085]: pgmap v414: 89 pgs: 89 active+clean; 166 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 482 KiB/s wr, 16 op/s 2024-09-21T09:46:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:23 smithi134 ceph-mon[90085]: osdmap e557: 8 total, 8 up, 8 in 2024-09-21T09:46:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:23 smithi134 ceph-mon[90085]: osdmap e558: 8 total, 8 up, 8 in 2024-09-21T09:46:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:23 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/939495727' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:24.318 INFO:tasks.rados.rados.0.smithi017.stdout:1959: done (3 left) 2024-09-21T09:46:24.318 INFO:tasks.rados.rados.0.smithi017.stdout:1960: done (2 left) 2024-09-21T09:46:24.318 INFO:tasks.rados.rados.0.smithi017.stdout:1961: done (1 left) 2024-09-21T09:46:24.318 INFO:tasks.rados.rados.0.smithi017.stdout:1962: done (0 left) 2024-09-21T09:46:24.318 INFO:tasks.rados.rados.0.smithi017.stdout:1963: snap_remove snap 220 2024-09-21T09:46:25.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:46:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:46:24] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.51.0" 2024-09-21T09:46:25.322 INFO:tasks.rados.rados.0.smithi017.stdout:1963: done (0 left) 2024-09-21T09:46:25.322 INFO:tasks.rados.rados.0.smithi017.stdout:1964: snap_create 2024-09-21T09:46:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[102913]: from='client.34917 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[102913]: pgmap v417: 89 pgs: 89 active+clean; 166 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 482 KiB/s wr, 18 op/s 2024-09-21T09:46:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[102913]: from='client.34923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[102913]: osdmap e559: 8 total, 8 up, 8 in 2024-09-21T09:46:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[105151]: from='client.34917 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[105151]: pgmap v417: 89 pgs: 89 active+clean; 166 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 482 KiB/s wr, 18 op/s 2024-09-21T09:46:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[105151]: from='client.34923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[105151]: osdmap e559: 8 total, 8 up, 8 in 2024-09-21T09:46:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:25.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:25 smithi134 ceph-mon[90085]: from='client.34917 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:25 smithi134 ceph-mon[90085]: pgmap v417: 89 pgs: 89 active+clean; 166 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 482 KiB/s wr, 18 op/s 2024-09-21T09:46:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:25 smithi134 ceph-mon[90085]: from='client.34923 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:25 smithi134 ceph-mon[90085]: osdmap e559: 8 total, 8 up, 8 in 2024-09-21T09:46:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:26.330 INFO:tasks.rados.rados.0.smithi017.stdout:1964: done (0 left) 2024-09-21T09:46:26.331 INFO:tasks.rados.rados.0.smithi017.stdout:1965: read oid 49 snap -1 2024-09-21T09:46:26.331 INFO:tasks.rados.rados.0.smithi017.stdout:1965: expect (ObjNum 640 snap 202 seq_num 640) 2024-09-21T09:46:26.331 INFO:tasks.rados.rados.0.smithi017.stdout:1966: read oid 35 snap 234 2024-09-21T09:46:26.331 INFO:tasks.rados.rados.0.smithi017.stdout:1966: expect (ObjNum 685 snap 217 seq_num 685) 2024-09-21T09:46:26.331 INFO:tasks.rados.rados.0.smithi017.stdout:1967: snap_remove snap 250 2024-09-21T09:46:26.473 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:26 smithi017 ceph-mon[102913]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:46:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:26 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2342103946' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:26 smithi017 ceph-mon[105151]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:46:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:26 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2342103946' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:26.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:26 smithi134 ceph-mon[90085]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:46:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:26 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2342103946' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:27.335 INFO:tasks.rados.rados.0.smithi017.stdout:1967: done (2 left) 2024-09-21T09:46:27.335 INFO:tasks.rados.rados.0.smithi017.stdout:1968: read oid 28 snap -1 2024-09-21T09:46:27.335 INFO:tasks.rados.rados.0.smithi017.stdout:1968: expect deleted 2024-09-21T09:46:27.335 INFO:tasks.rados.rados.0.smithi017.stdout:1969: write oid 45 current snap is 252 2024-09-21T09:46:27.335 INFO:tasks.rados.rados.0.smithi017.stdout:1969: seq_num 745 ranges {521215=546746,1562732=701583} 2024-09-21T09:46:27.340 INFO:tasks.rados.rados.0.smithi017.stdout:1969: writing smithi01785587-45 from 521215 to 1067961 tid 1 2024-09-21T09:46:27.347 INFO:tasks.rados.rados.0.smithi017.stdout:1969: writing smithi01785587-45 from 1562732 to 2264315 tid 2 2024-09-21T09:46:27.347 INFO:tasks.rados.rados.0.smithi017.stdout:1970: snap_remove snap 236 2024-09-21T09:46:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:27 smithi017 ceph-mon[102913]: pgmap v420: 89 pgs: 89 active+clean; 167 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 563 KiB/s wr, 6 op/s 2024-09-21T09:46:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:27 smithi017 ceph-mon[102913]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:46:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:27 smithi017 ceph-mon[102913]: from='client.34935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:27 smithi017 ceph-mon[105151]: pgmap v420: 89 pgs: 89 active+clean; 167 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 563 KiB/s wr, 6 op/s 2024-09-21T09:46:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:27 smithi017 ceph-mon[105151]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:46:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:27 smithi017 ceph-mon[105151]: from='client.34935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:27 smithi134 ceph-mon[90085]: pgmap v420: 89 pgs: 89 active+clean; 167 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 563 KiB/s wr, 6 op/s 2024-09-21T09:46:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:27 smithi134 ceph-mon[90085]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:46:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:27 smithi134 ceph-mon[90085]: from='client.34935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:28.088 INFO:tasks.rados.rados.0.smithi017.stdout:1970: done (4 left) 2024-09-21T09:46:28.088 INFO:tasks.rados.rados.0.smithi017.stdout:1971: write oid 40 current snap is 252 2024-09-21T09:46:28.088 INFO:tasks.rados.rados.0.smithi017.stdout:1971: seq_num 746 ranges {538022=738348,1737567=561594} 2024-09-21T09:46:28.096 INFO:tasks.rados.rados.0.smithi017.stdout:1971: writing smithi01785587-40 from 538022 to 1276370 tid 1 2024-09-21T09:46:28.103 INFO:tasks.rados.rados.0.smithi017.stdout:1971: writing smithi01785587-40 from 1737567 to 2299161 tid 2 2024-09-21T09:46:28.103 INFO:tasks.rados.rados.0.smithi017.stdout:1972: rollback oid 1 current snap is 252 2024-09-21T09:46:28.103 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 1 to 235 2024-09-21T09:46:28.103 INFO:tasks.rados.rados.0.smithi017.stdout:1965: expect (ObjNum 640 snap 202 seq_num 640) 2024-09-21T09:46:28.348 INFO:tasks.rados.rados.0.smithi017.stdout:1966: expect (ObjNum 685 snap 217 seq_num 685) 2024-09-21T09:46:28.518 INFO:tasks.rados.rados.0.smithi017.stdout:1969: finishing write tid 1 to smithi01785587-45 2024-09-21T09:46:28.518 INFO:tasks.rados.rados.0.smithi017.stdout:1969: finishing write tid 2 to smithi01785587-45 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1969: finishing write tid 3 to smithi01785587-45 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1969: finishing write tid 4 to smithi01785587-45 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 604 (ObjNum 745 snap 252 seq_num 745) dirty exists 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1969: left oid 45 (ObjNum 745 snap 252 seq_num 745) 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1972: finishing rollback tid 1 to smithi01785587-1 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1972: finishing rollback tid 2 to smithi01785587-1 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 569 (ObjNum 652 snap 205 seq_num 652) dirty exists 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1971: finishing write tid 1 to smithi01785587-40 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1965: done (5 left) 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1966: done (4 left) 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1968: done (3 left) 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1969: done (2 left) 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1972: done (1 left) 2024-09-21T09:46:28.519 INFO:tasks.rados.rados.0.smithi017.stdout:1973: snap_create 2024-09-21T09:46:28.520 INFO:tasks.rados.rados.0.smithi017.stdout:1971: finishing write tid 2 to smithi01785587-40 2024-09-21T09:46:28.520 INFO:tasks.rados.rados.0.smithi017.stdout:1971: finishing write tid 3 to smithi01785587-40 2024-09-21T09:46:28.520 INFO:tasks.rados.rados.0.smithi017.stdout:1971: finishing write tid 4 to smithi01785587-40 2024-09-21T09:46:28.520 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1024 (ObjNum 746 snap 252 seq_num 746) dirty exists 2024-09-21T09:46:28.520 INFO:tasks.rados.rados.0.smithi017.stdout:1971: left oid 40 (ObjNum 746 snap 252 seq_num 746) 2024-09-21T09:46:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:28 smithi017 ceph-mon[102913]: from='client.34941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:28 smithi017 ceph-mon[102913]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:46:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:28 smithi017 ceph-mon[102913]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:46:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:28 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:46:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:28 smithi017 ceph-mon[105151]: from='client.34941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:28 smithi017 ceph-mon[105151]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:46:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:28 smithi017 ceph-mon[105151]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:46:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:28 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:46:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:28 smithi134 ceph-mon[90085]: from='client.34941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:28 smithi134 ceph-mon[90085]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:46:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:28 smithi134 ceph-mon[90085]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:46:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:28 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:46:29.343 INFO:tasks.rados.rados.0.smithi017.stdout:1971: done (1 left) 2024-09-21T09:46:29.343 INFO:tasks.rados.rados.0.smithi017.stdout:1973: done (0 left) 2024-09-21T09:46:29.343 INFO:tasks.rados.rados.0.smithi017.stdout:1974: read oid 22 snap -1 2024-09-21T09:46:29.344 INFO:tasks.rados.rados.0.smithi017.stdout:1974: expect (ObjNum 726 snap 245 seq_num 726) 2024-09-21T09:46:29.344 INFO:tasks.rados.rados.0.smithi017.stdout:1975: delete oid 21 current snap is 253 2024-09-21T09:46:29.347 INFO:tasks.rados.rados.0.smithi017.stdout:1975: done (1 left) 2024-09-21T09:46:29.347 INFO:tasks.rados.rados.0.smithi017.stdout:1976: delete oid 28 current snap is 253 2024-09-21T09:46:29.352 INFO:tasks.rados.rados.0.smithi017.stdout:1974: expect (ObjNum 726 snap 245 seq_num 726) 2024-09-21T09:46:29.367 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:29.458 INFO:tasks.rados.rados.0.smithi017.stdout:1974: done (1 left) 2024-09-21T09:46:29.458 INFO:tasks.rados.rados.0.smithi017.stdout:1976: done (0 left) 2024-09-21T09:46:29.458 INFO:tasks.rados.rados.0.smithi017.stdout:1977: snap_remove snap 215 2024-09-21T09:46:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:29 smithi017 ceph-mon[105151]: pgmap v423: 89 pgs: 89 active+clean; 170 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 4 op/s 2024-09-21T09:46:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:29 smithi017 ceph-mon[105151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:46:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:29 smithi017 ceph-mon[105151]: Upgrade: osd.3 is safe to restart 2024-09-21T09:46:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:29 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1347986883' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:29 smithi017 ceph-mon[102913]: pgmap v423: 89 pgs: 89 active+clean; 170 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 4 op/s 2024-09-21T09:46:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:29 smithi017 ceph-mon[102913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:46:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:29 smithi017 ceph-mon[102913]: Upgrade: osd.3 is safe to restart 2024-09-21T09:46:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:29 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1347986883' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:29 smithi134 ceph-mon[90085]: pgmap v423: 89 pgs: 89 active+clean; 170 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 4 op/s 2024-09-21T09:46:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:29 smithi134 ceph-mon[90085]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:46:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:29 smithi134 ceph-mon[90085]: Upgrade: osd.3 is safe to restart 2024-09-21T09:46:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:29 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1347986883' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:30.350 INFO:tasks.rados.rados.0.smithi017.stdout:1977: done (0 left) 2024-09-21T09:46:30.350 INFO:tasks.rados.rados.0.smithi017.stdout:1978: write oid 6 current snap is 253 2024-09-21T09:46:30.350 INFO:tasks.rados.rados.0.smithi017.stdout:1978: seq_num 749 ranges {588443=564095,1796660=708921} 2024-09-21T09:46:30.357 INFO:tasks.rados.rados.0.smithi017.stdout:1978: writing smithi01785587-6 from 588443 to 1152538 tid 1 2024-09-21T09:46:30.366 INFO:tasks.rados.rados.0.smithi017.stdout:1978: writing smithi01785587-6 from 1796660 to 2505581 tid 2 2024-09-21T09:46:30.367 INFO:tasks.rados.rados.0.smithi017.stdout:1979: read oid 41 snap -1 2024-09-21T09:46:30.367 INFO:tasks.rados.rados.0.smithi017.stdout:1979: expect (ObjNum 721 snap 244 seq_num 721) 2024-09-21T09:46:30.367 INFO:tasks.rados.rados.0.smithi017.stdout:1980: read oid 27 snap 245 2024-09-21T09:46:30.367 INFO:tasks.rados.rados.0.smithi017.stdout:1980: expect deleted 2024-09-21T09:46:30.367 INFO:tasks.rados.rados.0.smithi017.stdout:1981: read oid 8 snap -1 2024-09-21T09:46:30.367 INFO:tasks.rados.rados.0.smithi017.stdout:1981: expect deleted 2024-09-21T09:46:30.367 INFO:tasks.rados.rados.0.smithi017.stdout:1982: delete oid 15 current snap is 253 2024-09-21T09:46:30.367 INFO:tasks.rados.rados.0.smithi017.stdout:1978: finishing write tid 1 to smithi01785587-6 2024-09-21T09:46:30.371 INFO:tasks.rados.rados.0.smithi017.stdout:1978: finishing write tid 2 to smithi01785587-6 2024-09-21T09:46:30.371 INFO:tasks.rados.rados.0.smithi017.stdout:1978: finishing write tid 3 to smithi01785587-6 2024-09-21T09:46:30.371 INFO:tasks.rados.rados.0.smithi017.stdout:1978: finishing write tid 4 to smithi01785587-6 2024-09-21T09:46:30.371 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 577 (ObjNum 749 snap 253 seq_num 749) dirty exists 2024-09-21T09:46:30.371 INFO:tasks.rados.rados.0.smithi017.stdout:1978: left oid 6 (ObjNum 749 snap 253 seq_num 749) 2024-09-21T09:46:30.372 INFO:tasks.rados.rados.0.smithi017.stdout:1979: expect (ObjNum 721 snap 244 seq_num 721) 2024-09-21T09:46:30.445 INFO:tasks.rados.rados.0.smithi017.stdout:1978: done (4 left) 2024-09-21T09:46:30.445 INFO:tasks.rados.rados.0.smithi017.stdout:1979: done (3 left) 2024-09-21T09:46:30.445 INFO:tasks.rados.rados.0.smithi017.stdout:1980: done (2 left) 2024-09-21T09:46:30.446 INFO:tasks.rados.rados.0.smithi017.stdout:1981: done (1 left) 2024-09-21T09:46:30.446 INFO:tasks.rados.rados.0.smithi017.stdout:1982: done (0 left) 2024-09-21T09:46:30.446 INFO:tasks.rados.rados.0.smithi017.stdout:1983: read oid 12 snap -1 2024-09-21T09:46:30.446 INFO:tasks.rados.rados.0.smithi017.stdout:1983: expect (ObjNum 659 snap 208 seq_num 659) 2024-09-21T09:46:30.446 INFO:tasks.rados.rados.0.smithi017.stdout:1984: read oid 27 snap 245 2024-09-21T09:46:30.446 INFO:tasks.rados.rados.0.smithi017.stdout:1984: expect deleted 2024-09-21T09:46:30.446 INFO:tasks.rados.rados.0.smithi017.stdout:1985: snap_remove snap 249 2024-09-21T09:46:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:30 smithi134 ceph-mon[90085]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:46:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:30 smithi134 ceph-mon[90085]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:46:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:46:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:46:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[102913]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:46:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[102913]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:46:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[105151]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:46:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[105151]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:46:30.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:31.362 INFO:tasks.rados.rados.0.smithi017.stdout:1985: done (2 left) 2024-09-21T09:46:31.362 INFO:tasks.rados.rados.0.smithi017.stdout:1986: rollback oid 25 current snap is 253 2024-09-21T09:46:31.362 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 245 2024-09-21T09:46:31.362 INFO:tasks.rados.rados.0.smithi017.stdout:1983: expect (ObjNum 659 snap 208 seq_num 659) 2024-09-21T09:46:31.587 INFO:tasks.rados.rados.0.smithi017.stdout:1986: finishing rollback tid 0 to smithi01785587-25 2024-09-21T09:46:31.587 INFO:tasks.rados.rados.0.smithi017.stdout:1986: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:46:31.588 INFO:tasks.rados.rados.0.smithi017.stdout:1986: finishing rollback tid 2 to smithi01785587-25 2024-09-21T09:46:31.588 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 444 (ObjNum 716 snap 242 seq_num 716) dirty exists 2024-09-21T09:46:31.588 INFO:tasks.rados.rados.0.smithi017.stdout:1983: done (2 left) 2024-09-21T09:46:31.588 INFO:tasks.rados.rados.0.smithi017.stdout:1984: done (1 left) 2024-09-21T09:46:31.588 INFO:tasks.rados.rados.0.smithi017.stdout:1986: done (0 left) 2024-09-21T09:46:31.588 INFO:tasks.rados.rados.0.smithi017.stdout:1987: read oid 14 snap -1 2024-09-21T09:46:31.588 INFO:tasks.rados.rados.0.smithi017.stdout:1987: expect (ObjNum 667 snap 209 seq_num 667) 2024-09-21T09:46:31.588 INFO:tasks.rados.rados.0.smithi017.stdout:1988: write oid 33 current snap is 253 2024-09-21T09:46:31.588 INFO:tasks.rados.rados.0.smithi017.stdout:1988: seq_num 751 ranges {622057=714593,1913624=708433} 2024-09-21T09:46:31.594 INFO:tasks.rados.rados.0.smithi017.stdout:1988: writing smithi01785587-33 from 622057 to 1336650 tid 1 2024-09-21T09:46:31.601 INFO:tasks.rados.rados.0.smithi017.stdout:1988: writing smithi01785587-33 from 1913624 to 2622057 tid 2 2024-09-21T09:46:31.602 INFO:tasks.rados.rados.0.smithi017.stdout:1989: snap_create 2024-09-21T09:46:31.602 INFO:tasks.rados.rados.0.smithi017.stdout:1987: expect (ObjNum 667 snap 209 seq_num 667) 2024-09-21T09:46:31.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[105151]: from='client.44822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:31.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[105151]: Upgrade: Updating osd.3 2024-09-21T09:46:31.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[105151]: Deploying daemon osd.3 on smithi017 2024-09-21T09:46:31.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[105151]: pgmap v426: 89 pgs: 89 active+clean; 175 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 4 op/s 2024-09-21T09:46:31.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[105151]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:46:31.629 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[102913]: from='client.44822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:31.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[102913]: Upgrade: Updating osd.3 2024-09-21T09:46:31.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[102913]: Deploying daemon osd.3 on smithi017 2024-09-21T09:46:31.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[102913]: pgmap v426: 89 pgs: 89 active+clean; 175 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 4 op/s 2024-09-21T09:46:31.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:31 smithi017 ceph-mon[102913]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:46:31.710 INFO:tasks.rados.rados.0.smithi017.stdout:1988: finishing write tid 1 to smithi01785587-33 2024-09-21T09:46:31.710 INFO:tasks.rados.rados.0.smithi017.stdout:1988: finishing write tid 2 to smithi01785587-33 2024-09-21T09:46:31.710 INFO:tasks.rados.rados.0.smithi017.stdout:1988: finishing write tid 3 to smithi01785587-33 2024-09-21T09:46:31.710 INFO:tasks.rados.rados.0.smithi017.stdout:1988: finishing write tid 4 to smithi01785587-33 2024-09-21T09:46:31.710 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 924 (ObjNum 751 snap 253 seq_num 751) dirty exists 2024-09-21T09:46:31.710 INFO:tasks.rados.rados.0.smithi017.stdout:1988: left oid 33 (ObjNum 751 snap 253 seq_num 751) 2024-09-21T09:46:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:31 smithi134 ceph-mon[90085]: from='client.44822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:31 smithi134 ceph-mon[90085]: Upgrade: Updating osd.3 2024-09-21T09:46:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:31 smithi134 ceph-mon[90085]: Deploying daemon osd.3 on smithi017 2024-09-21T09:46:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:31 smithi134 ceph-mon[90085]: pgmap v426: 89 pgs: 89 active+clean; 175 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 4 op/s 2024-09-21T09:46:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:31 smithi134 ceph-mon[90085]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:46:32.262 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:32.366 INFO:tasks.rados.rados.0.smithi017.stdout:1987: done (2 left) 2024-09-21T09:46:32.367 INFO:tasks.rados.rados.0.smithi017.stdout:1988: done (1 left) 2024-09-21T09:46:32.367 INFO:tasks.rados.rados.0.smithi017.stdout:1989: done (0 left) 2024-09-21T09:46:32.367 INFO:tasks.rados.rados.0.smithi017.stdout:1990: snap_remove snap 217 2024-09-21T09:46:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:32 smithi017 ceph-mon[102913]: osdmap e566: 8 total, 8 up, 8 in 2024-09-21T09:46:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:32 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2779812353' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:32 smithi017 ceph-mon[105151]: osdmap e566: 8 total, 8 up, 8 in 2024-09-21T09:46:32.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:32 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2779812353' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:32 smithi134 ceph-mon[90085]: osdmap e566: 8 total, 8 up, 8 in 2024-09-21T09:46:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:32 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2779812353' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:33.093 INFO:tasks.rados.rados.0.smithi017.stdout:1990: done (0 left) 2024-09-21T09:46:33.093 INFO:tasks.rados.rados.0.smithi017.stdout:1991: read oid 40 snap 219 2024-09-21T09:46:33.093 INFO:tasks.rados.rados.0.smithi017.stdout:1991: expect deleted 2024-09-21T09:46:33.093 INFO:tasks.rados.rados.0.smithi017.stdout:1992: delete oid 42 current snap is 254 2024-09-21T09:46:33.096 INFO:tasks.rados.rados.0.smithi017.stdout:1992: done (1 left) 2024-09-21T09:46:33.096 INFO:tasks.rados.rados.0.smithi017.stdout:1993: snap_create 2024-09-21T09:46:33.364 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:33 smithi017 systemd[1]: Stopping Ceph osd.3 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:46:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-mon[105151]: pgmap v429: 89 pgs: 89 active+clean; 175 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 934 KiB/s wr, 14 op/s 2024-09-21T09:46:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-mon[105151]: from='client.44834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-mon[105151]: osdmap e567: 8 total, 8 up, 8 in 2024-09-21T09:46:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-mon[105151]: osdmap e568: 8 total, 8 up, 8 in 2024-09-21T09:46:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-mon[102913]: pgmap v429: 89 pgs: 89 active+clean; 175 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 934 KiB/s wr, 14 op/s 2024-09-21T09:46:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-mon[102913]: from='client.44834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-mon[102913]: osdmap e567: 8 total, 8 up, 8 in 2024-09-21T09:46:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-mon[102913]: osdmap e568: 8 total, 8 up, 8 in 2024-09-21T09:46:33.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[64024]: 2024-09-21T09:46:33.371+0000 7fb0bbd11640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:46:33.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[64024]: 2024-09-21T09:46:33.371+0000 7fb0bbd11640 -1 osd.3 568 *** Got signal Terminated *** 2024-09-21T09:46:33.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[64024]: 2024-09-21T09:46:33.371+0000 7fb0bbd11640 -1 osd.3 568 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:46:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:33 smithi134 ceph-mon[90085]: pgmap v429: 89 pgs: 89 active+clean; 175 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 934 KiB/s wr, 14 op/s 2024-09-21T09:46:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:33 smithi134 ceph-mon[90085]: from='client.44834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:33 smithi134 ceph-mon[90085]: osdmap e567: 8 total, 8 up, 8 in 2024-09-21T09:46:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:33 smithi134 ceph-mon[90085]: osdmap e568: 8 total, 8 up, 8 in 2024-09-21T09:46:34.096 INFO:tasks.rados.rados.0.smithi017.stdout:1991: done (1 left) 2024-09-21T09:46:34.096 INFO:tasks.rados.rados.0.smithi017.stdout:1993: done (0 left) 2024-09-21T09:46:34.096 INFO:tasks.rados.rados.0.smithi017.stdout:1994: rollback oid 31 current snap is 255 2024-09-21T09:46:34.096 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 31 to 237 2024-09-21T09:46:34.096 INFO:tasks.rados.rados.0.smithi017.stdout:1995: delete oid 14 current snap is 255 2024-09-21T09:46:34.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:34 smithi017 ceph-mon[105151]: from='client.44840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:34.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:34 smithi017 ceph-mon[105151]: osd.3 marked itself down and dead 2024-09-21T09:46:34.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:34 smithi017 ceph-mon[105151]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:46:34.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:34 smithi017 ceph-mon[105151]: osdmap e569: 8 total, 7 up, 8 in 2024-09-21T09:46:34.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:34 smithi017 ceph-mon[102913]: from='client.44840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:34.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:34 smithi017 ceph-mon[102913]: osd.3 marked itself down and dead 2024-09-21T09:46:34.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:34 smithi017 ceph-mon[102913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:46:34.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:34 smithi017 ceph-mon[102913]: osdmap e569: 8 total, 7 up, 8 in 2024-09-21T09:46:34.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:34 smithi134 ceph-mon[90085]: from='client.44840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:34.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:34 smithi134 ceph-mon[90085]: osd.3 marked itself down and dead 2024-09-21T09:46:34.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:34 smithi134 ceph-mon[90085]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:46:34.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:34 smithi134 ceph-mon[90085]: osdmap e569: 8 total, 7 up, 8 in 2024-09-21T09:46:34.930 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:34 smithi017 podman[129839]: 2024-09-21 09:46:34.639052537 +0000 UTC m=+1.391626662 container died 44d3b78acaa4292819209c70b938a1f56f5cf2a22a949cd9b85562a920040450 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-21T09:46:34.931 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:46:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:46:34] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.51.0" 2024-09-21T09:46:35.106 INFO:tasks.rados.rados.0.smithi017.stdout:1994: finishing rollback tid 1 to smithi01785587-31 2024-09-21T09:46:35.106 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 0 (ObjNum 129 snap 0 seq_num 748891912) dirty dne 2024-09-21T09:46:35.106 INFO:tasks.rados.rados.0.smithi017.stdout:1994: done (1 left) 2024-09-21T09:46:35.106 INFO:tasks.rados.rados.0.smithi017.stdout:1995: done (0 left) 2024-09-21T09:46:35.106 INFO:tasks.rados.rados.0.smithi017.stdout:1996: write oid 26 current snap is 255 2024-09-21T09:46:35.106 INFO:tasks.rados.rados.0.smithi017.stdout:1996: seq_num 754 ranges {672478=656693,2089070=459917} 2024-09-21T09:46:35.114 INFO:tasks.rados.rados.0.smithi017.stdout:1996: writing smithi01785587-26 from 672478 to 1329171 tid 1 2024-09-21T09:46:35.122 INFO:tasks.rados.rados.0.smithi017.stdout:1996: writing smithi01785587-26 from 2089070 to 2548987 tid 2 2024-09-21T09:46:35.122 INFO:tasks.rados.rados.0.smithi017.stdout:1997: write oid 48 current snap is 255 2024-09-21T09:46:35.123 INFO:tasks.rados.rados.0.smithi017.stdout:1996: finishing write tid 1 to smithi01785587-26 2024-09-21T09:46:35.123 INFO:tasks.rados.rados.0.smithi017.stdout:1997: seq_num 755 ranges {689285=731942,2147552=541733} 2024-09-21T09:46:35.132 INFO:tasks.rados.rados.0.smithi017.stdout:1997: writing smithi01785587-48 from 689285 to 1421227 tid 1 2024-09-21T09:46:35.139 INFO:tasks.rados.rados.0.smithi017.stdout:1997: writing smithi01785587-48 from 2147552 to 2689285 tid 2 2024-09-21T09:46:35.139 INFO:tasks.rados.rados.0.smithi017.stdout:1998: snap_remove snap 253 2024-09-21T09:46:35.188 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:35 smithi017 podman[129839]: 2024-09-21 09:46:35.019346842 +0000 UTC m=+1.771920966 container cleanup 44d3b78acaa4292819209c70b938a1f56f5cf2a22a949cd9b85562a920040450 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2) 2024-09-21T09:46:35.188 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:35 smithi017 bash[129839]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3 2024-09-21T09:46:35.189 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:35.450 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:35 smithi017 podman[129865]: 2024-09-21 09:46:35.329424894 +0000 UTC m=+0.682334051 container remove 44d3b78acaa4292819209c70b938a1f56f5cf2a22a949cd9b85562a920040450 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True) 2024-09-21T09:46:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:35 smithi017 ceph-mon[102913]: pgmap v432: 89 pgs: 89 active+clean; 173 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 311 KiB/s wr, 12 op/s 2024-09-21T09:46:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/49918291' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:35 smithi017 ceph-mon[102913]: osdmap e570: 8 total, 7 up, 8 in 2024-09-21T09:46:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:35 smithi017 ceph-mon[105151]: pgmap v432: 89 pgs: 89 active+clean; 173 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 311 KiB/s wr, 12 op/s 2024-09-21T09:46:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/49918291' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:35 smithi017 ceph-mon[105151]: osdmap e570: 8 total, 7 up, 8 in 2024-09-21T09:46:35.659 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:35 smithi134 ceph-mon[90085]: pgmap v432: 89 pgs: 89 active+clean; 173 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 311 KiB/s wr, 12 op/s 2024-09-21T09:46:35.659 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/49918291' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:35.659 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:35 smithi134 ceph-mon[90085]: osdmap e570: 8 total, 7 up, 8 in 2024-09-21T09:46:35.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:35 smithi017 podman[130026]: 2024-09-21 09:46:35.565014242 +0000 UTC m=+0.016564879 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:35.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:35 smithi017 podman[130026]: 2024-09-21 09:46:35.663800063 +0000 UTC m=+0.115350691 container create a09adc18fc15ed641fc966242ba0e8d85a430c07a5c981c500d41ba34a1e6e0f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-21T09:46:35.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:35 smithi017 podman[130026]: 2024-09-21 09:46:35.874826684 +0000 UTC m=+0.326377313 container init a09adc18fc15ed641fc966242ba0e8d85a430c07a5c981c500d41ba34a1e6e0f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-62da54b) 2024-09-21T09:46:35.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:35 smithi017 podman[130026]: 2024-09-21 09:46:35.879875192 +0000 UTC m=+0.331425820 container start a09adc18fc15ed641fc966242ba0e8d85a430c07a5c981c500d41ba34a1e6e0f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T09:46:36.094 INFO:tasks.rados.rados.0.smithi017.stdout:1998: done (2 left) 2024-09-21T09:46:36.094 INFO:tasks.rados.rados.0.smithi017.stdout:1999: delete oid 49 current snap is 255 2024-09-21T09:46:36.094 INFO:tasks.rados.rados.0.smithi017.stdout:1996: finishing write tid 2 to smithi01785587-26 2024-09-21T09:46:36.094 INFO:tasks.rados.rados.0.smithi017.stdout:1996: finishing write tid 3 to smithi01785587-26 2024-09-21T09:46:36.094 INFO:tasks.rados.rados.0.smithi017.stdout:1996: finishing write tid 4 to smithi01785587-26 2024-09-21T09:46:36.094 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 932 (ObjNum 754 snap 255 seq_num 754) dirty exists 2024-09-21T09:46:36.095 INFO:tasks.rados.rados.0.smithi017.stdout:1996: left oid 26 (ObjNum 754 snap 255 seq_num 754) 2024-09-21T09:46:36.095 INFO:tasks.rados.rados.0.smithi017.stdout:1997: finishing write tid 1 to smithi01785587-48 2024-09-21T09:46:36.095 INFO:tasks.rados.rados.0.smithi017.stdout:1997: finishing write tid 2 to smithi01785587-48 2024-09-21T09:46:36.095 INFO:tasks.rados.rados.0.smithi017.stdout:1997: finishing write tid 3 to smithi01785587-48 2024-09-21T09:46:36.095 INFO:tasks.rados.rados.0.smithi017.stdout:1997: finishing write tid 4 to smithi01785587-48 2024-09-21T09:46:36.095 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 794 (ObjNum 755 snap 255 seq_num 755) dirty exists 2024-09-21T09:46:36.095 INFO:tasks.rados.rados.0.smithi017.stdout:1997: left oid 48 (ObjNum 755 snap 255 seq_num 755) 2024-09-21T09:46:36.097 INFO:tasks.rados.rados.0.smithi017.stdout:1996: done (2 left) 2024-09-21T09:46:36.097 INFO:tasks.rados.rados.0.smithi017.stdout:1997: done (1 left) 2024-09-21T09:46:36.097 INFO:tasks.rados.rados.0.smithi017.stdout:1999: done (0 left) 2024-09-21T09:46:36.097 INFO:tasks.rados.rados.0.smithi017.stdout:2000: rollback oid 25 current snap is 255 2024-09-21T09:46:36.097 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 245 2024-09-21T09:46:36.097 INFO:tasks.rados.rados.0.smithi017.stdout:2001: snap_create 2024-09-21T09:46:36.101 INFO:tasks.rados.rados.0.smithi017.stdout:2000: finishing rollback tid 0 to smithi01785587-25 2024-09-21T09:46:36.101 INFO:tasks.rados.rados.0.smithi017.stdout:2000: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:46:36.101 INFO:tasks.rados.rados.0.smithi017.stdout:2000: finishing rollback tid 2 to smithi01785587-25 2024-09-21T09:46:36.101 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 452 (ObjNum 716 snap 242 seq_num 716) dirty exists 2024-09-21T09:46:36.378 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:35 smithi017 podman[130026]: 2024-09-21 09:46:35.957806176 +0000 UTC m=+0.409356820 container attach a09adc18fc15ed641fc966242ba0e8d85a430c07a5c981c500d41ba34a1e6e0f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.vendor=CentOS) 2024-09-21T09:46:36.378 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:36 smithi017 podman[130026]: 2024-09-21 09:46:36.075305592 +0000 UTC m=+0.526856229 container died a09adc18fc15ed641fc966242ba0e8d85a430c07a5c981c500d41ba34a1e6e0f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T09:46:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:36 smithi017 ceph-mon[105151]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:36 smithi017 ceph-mon[105151]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:36 smithi017 ceph-mon[105151]: osdmap e571: 8 total, 7 up, 8 in 2024-09-21T09:46:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:36 smithi017 ceph-mon[102913]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:36 smithi017 ceph-mon[102913]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:36 smithi017 ceph-mon[102913]: osdmap e571: 8 total, 7 up, 8 in 2024-09-21T09:46:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:36 smithi134 ceph-mon[90085]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:36 smithi134 ceph-mon[90085]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:36 smithi134 ceph-mon[90085]: osdmap e571: 8 total, 7 up, 8 in 2024-09-21T09:46:37.094 INFO:tasks.rados.rados.0.smithi017.stdout:2000: done (1 left) 2024-09-21T09:46:37.094 INFO:tasks.rados.rados.0.smithi017.stdout:2001: done (0 left) 2024-09-21T09:46:37.094 INFO:tasks.rados.rados.0.smithi017.stdout:2002: write oid 42 current snap is 256 2024-09-21T09:46:37.094 INFO:tasks.rados.rados.0.smithi017.stdout:2002: seq_num 757 ranges {722899=482440,1748163=723597} 2024-09-21T09:46:37.100 INFO:tasks.rados.rados.0.smithi017.stdout:2002: writing smithi01785587-42 from 722899 to 1205339 tid 1 2024-09-21T09:46:37.107 INFO:tasks.rados.rados.0.smithi017.stdout:2002: writing smithi01785587-42 from 1748163 to 2471760 tid 2 2024-09-21T09:46:37.107 INFO:tasks.rados.rados.0.smithi017.stdout:2003: delete oid 14 current snap is 256 2024-09-21T09:46:37.107 INFO:tasks.rados.rados.0.smithi017.stdout:2002: finishing write tid 1 to smithi01785587-42 2024-09-21T09:46:37.111 INFO:tasks.rados.rados.0.smithi017.stdout:2002: finishing write tid 2 to smithi01785587-42 2024-09-21T09:46:37.111 INFO:tasks.rados.rados.0.smithi017.stdout:2003: done (1 left) 2024-09-21T09:46:37.111 INFO:tasks.rados.rados.0.smithi017.stdout:2004: read oid 3 snap -1 2024-09-21T09:46:37.111 INFO:tasks.rados.rados.0.smithi017.stdout:2004: expect (ObjNum 725 snap 245 seq_num 725) 2024-09-21T09:46:37.111 INFO:tasks.rados.rados.0.smithi017.stdout:2005: write oid 23 current snap is 256 2024-09-21T09:46:37.112 INFO:tasks.rados.rados.0.smithi017.stdout:2002: finishing write tid 3 to smithi01785587-42 2024-09-21T09:46:37.112 INFO:tasks.rados.rados.0.smithi017.stdout:2005: seq_num 759 ranges {756513=632938,1865127=727266} 2024-09-21T09:46:37.119 INFO:tasks.rados.rados.0.smithi017.stdout:2005: writing smithi01785587-23 from 756513 to 1389451 tid 1 2024-09-21T09:46:37.126 INFO:tasks.rados.rados.0.smithi017.stdout:2005: writing smithi01785587-23 from 1865127 to 2592393 tid 2 2024-09-21T09:46:37.126 INFO:tasks.rados.rados.0.smithi017.stdout:2006: write oid 27 current snap is 256 2024-09-21T09:46:37.126 INFO:tasks.rados.rados.0.smithi017.stdout:2006: seq_num 760 ranges {773320=708187,1923609=470924} 2024-09-21T09:46:37.133 INFO:tasks.rados.rados.0.smithi017.stdout:2006: writing smithi01785587-27 from 773320 to 1481507 tid 1 2024-09-21T09:46:37.139 INFO:tasks.rados.rados.0.smithi017.stdout:2006: writing smithi01785587-27 from 1923609 to 2394533 tid 2 2024-09-21T09:46:37.140 INFO:tasks.rados.rados.0.smithi017.stdout:2007: write oid 50 current snap is 256 2024-09-21T09:46:37.140 INFO:tasks.rados.rados.0.smithi017.stdout:2002: finishing write tid 4 to smithi01785587-42 2024-09-21T09:46:37.140 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 714 (ObjNum 757 snap 256 seq_num 757) dirty exists 2024-09-21T09:46:37.140 INFO:tasks.rados.rados.0.smithi017.stdout:2002: left oid 42 (ObjNum 757 snap 256 seq_num 757) 2024-09-21T09:46:37.140 INFO:tasks.rados.rados.0.smithi017.stdout:2007: seq_num 761 ranges {790127=499789,1698444=730935} 2024-09-21T09:46:37.146 INFO:tasks.rados.rados.0.smithi017.stdout:2007: writing smithi01785587-50 from 790127 to 1289916 tid 1 2024-09-21T09:46:37.153 INFO:tasks.rados.rados.0.smithi017.stdout:2007: writing smithi01785587-50 from 1698444 to 2429379 tid 2 2024-09-21T09:46:37.153 INFO:tasks.rados.rados.0.smithi017.stdout:2002: done (4 left) 2024-09-21T09:46:37.153 INFO:tasks.rados.rados.0.smithi017.stdout:2008: read oid 25 snap -1 2024-09-21T09:46:37.153 INFO:tasks.rados.rados.0.smithi017.stdout:2008: expect (ObjNum 716 snap 242 seq_num 716) 2024-09-21T09:46:37.153 INFO:tasks.rados.rados.0.smithi017.stdout:2009: snap_create 2024-09-21T09:46:37.153 INFO:tasks.rados.rados.0.smithi017.stdout:2004: expect (ObjNum 725 snap 245 seq_num 725) 2024-09-21T09:46:37.169 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:36 smithi017 podman[130070]: 2024-09-21 09:46:36.721295769 +0000 UTC m=+0.640166967 container remove a09adc18fc15ed641fc966242ba0e8d85a430c07a5c981c500d41ba34a1e6e0f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T09:46:37.169 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:36 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T09:46:37.169 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:36 smithi017 systemd[1]: Stopped Ceph osd.3 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:46:37.169 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:36 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.3.service: Consumed 10.790s CPU time. 2024-09-21T09:46:37.169 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:36 smithi017 systemd[1]: Starting Ceph osd.3 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:46:37.227 INFO:tasks.rados.rados.0.smithi017.stdout:2005: finishing write tid 1 to smithi01785587-23 2024-09-21T09:46:37.227 INFO:tasks.rados.rados.0.smithi017.stdout:2005: finishing write tid 2 to smithi01785587-23 2024-09-21T09:46:37.227 INFO:tasks.rados.rados.0.smithi017.stdout:2005: finishing write tid 3 to smithi01785587-23 2024-09-21T09:46:37.227 INFO:tasks.rados.rados.0.smithi017.stdout:2005: finishing write tid 4 to smithi01785587-23 2024-09-21T09:46:37.227 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 818 (ObjNum 759 snap 256 seq_num 759) dirty exists 2024-09-21T09:46:37.227 INFO:tasks.rados.rados.0.smithi017.stdout:2005: left oid 23 (ObjNum 759 snap 256 seq_num 759) 2024-09-21T09:46:37.227 INFO:tasks.rados.rados.0.smithi017.stdout:2006: finishing write tid 1 to smithi01785587-27 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2006: finishing write tid 2 to smithi01785587-27 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2006: finishing write tid 3 to smithi01785587-27 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2006: finishing write tid 4 to smithi01785587-27 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 613 (ObjNum 760 snap 256 seq_num 760) dirty exists 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2006: left oid 27 (ObjNum 760 snap 256 seq_num 760) 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2007: finishing write tid 1 to smithi01785587-50 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2007: finishing write tid 2 to smithi01785587-50 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2007: finishing write tid 3 to smithi01785587-50 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2007: finishing write tid 4 to smithi01785587-50 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 610 (ObjNum 761 snap 256 seq_num 761) dirty exists 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2007: left oid 50 (ObjNum 761 snap 256 seq_num 761) 2024-09-21T09:46:37.228 INFO:tasks.rados.rados.0.smithi017.stdout:2008: expect (ObjNum 716 snap 242 seq_num 716) 2024-09-21T09:46:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:37 smithi017 ceph-mon[102913]: pgmap v435: 89 pgs: 12 active+undersized, 8 stale+active+clean, 3 active+undersized+degraded, 66 active+clean; 174 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 932 KiB/s wr, 4 op/s; 10/351 objects degraded (2.849%) 2024-09-21T09:46:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:37 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 10/351 objects degraded (2.849%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:37 smithi017 ceph-mon[102913]: osdmap e572: 8 total, 7 up, 8 in 2024-09-21T09:46:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:37 smithi017 ceph-mon[105151]: pgmap v435: 89 pgs: 12 active+undersized, 8 stale+active+clean, 3 active+undersized+degraded, 66 active+clean; 174 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 932 KiB/s wr, 4 op/s; 10/351 objects degraded (2.849%) 2024-09-21T09:46:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:37 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 10/351 objects degraded (2.849%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:37 smithi017 ceph-mon[105151]: osdmap e572: 8 total, 7 up, 8 in 2024-09-21T09:46:37.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:37 smithi017 podman[130177]: 2024-09-21 09:46:37.070402962 +0000 UTC m=+0.015981693 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:37.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:37 smithi017 podman[130177]: 2024-09-21 09:46:37.204937805 +0000 UTC m=+0.150516525 container create be6ccc8faf44f179ecdc01339e0e478d1692f255d403438eaa634cb514f0bcb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:46:37.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:37 smithi017 podman[130177]: 2024-09-21 09:46:37.407431771 +0000 UTC m=+0.353010491 container init be6ccc8faf44f179ecdc01339e0e478d1692f255d403438eaa634cb514f0bcb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T09:46:37.451 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:37 smithi017 podman[130177]: 2024-09-21 09:46:37.412355196 +0000 UTC m=+0.357933916 container start be6ccc8faf44f179ecdc01339e0e478d1692f255d403438eaa634cb514f0bcb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate, ceph=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T09:46:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:37 smithi134 ceph-mon[90085]: pgmap v435: 89 pgs: 12 active+undersized, 8 stale+active+clean, 3 active+undersized+degraded, 66 active+clean; 174 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 932 KiB/s wr, 4 op/s; 10/351 objects degraded (2.849%) 2024-09-21T09:46:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:37 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 10/351 objects degraded (2.849%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:37 smithi134 ceph-mon[90085]: osdmap e572: 8 total, 7 up, 8 in 2024-09-21T09:46:37.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:37 smithi017 podman[130177]: 2024-09-21 09:46:37.499235513 +0000 UTC m=+0.444814232 container attach be6ccc8faf44f179ecdc01339e0e478d1692f255d403438eaa634cb514f0bcb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T09:46:37.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate[130192]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:46:37.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:37 smithi017 bash[130177]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:46:37.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate[130192]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:46:37.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:37 smithi017 bash[130177]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:46:38.091 INFO:tasks.rados.rados.0.smithi017.stdout:2004: done (5 left) 2024-09-21T09:46:38.091 INFO:tasks.rados.rados.0.smithi017.stdout:2005: done (4 left) 2024-09-21T09:46:38.091 INFO:tasks.rados.rados.0.smithi017.stdout:2006: done (3 left) 2024-09-21T09:46:38.091 INFO:tasks.rados.rados.0.smithi017.stdout:2007: done (2 left) 2024-09-21T09:46:38.091 INFO:tasks.rados.rados.0.smithi017.stdout:2008: done (1 left) 2024-09-21T09:46:38.091 INFO:tasks.rados.rados.0.smithi017.stdout:2009: done (0 left) 2024-09-21T09:46:38.091 INFO:tasks.rados.rados.0.smithi017.stdout:2010: snap_remove snap 244 2024-09-21T09:46:38.109 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:38.322 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate[130192]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:46:38.322 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 bash[130177]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:46:38.323 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate[130192]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:46:38.323 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 bash[130177]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:46:38.323 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate[130192]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:46:38.323 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 bash[130177]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:46:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3392237993' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-mon[102913]: osdmap e573: 8 total, 7 up, 8 in 2024-09-21T09:46:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3392237993' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-mon[105151]: osdmap e573: 8 total, 7 up, 8 in 2024-09-21T09:46:38.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate[130192]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:46:38.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 bash[130177]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:46:38.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate[130192]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T09:46:38.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 bash[130177]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T09:46:38.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate[130192]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:46:38.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 bash[130177]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:46:38.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate[130192]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T09:46:38.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 bash[130177]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T09:46:38.701 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 podman[130177]: 2024-09-21 09:46:38.350243661 +0000 UTC m=+1.295822391 container died be6ccc8faf44f179ecdc01339e0e478d1692f255d403438eaa634cb514f0bcb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T09:46:38.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:38 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3392237993' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:38 smithi134 ceph-mon[90085]: osdmap e573: 8 total, 7 up, 8 in 2024-09-21T09:46:39.091 INFO:tasks.rados.rados.0.smithi017.stdout:2010: done (0 left) 2024-09-21T09:46:39.091 INFO:tasks.rados.rados.0.smithi017.stdout:2011: delete oid 10 current snap is 257 2024-09-21T09:46:39.098 INFO:tasks.rados.rados.0.smithi017.stdout:2011: done (0 left) 2024-09-21T09:46:39.099 INFO:tasks.rados.rados.0.smithi017.stdout:2012: read oid 18 snap -1 2024-09-21T09:46:39.099 INFO:tasks.rados.rados.0.smithi017.stdout:2012: expect (ObjNum 738 snap 249 seq_num 738) 2024-09-21T09:46:39.099 INFO:tasks.rados.rados.0.smithi017.stdout:2013: read oid 28 snap -1 2024-09-21T09:46:39.099 INFO:tasks.rados.rados.0.smithi017.stdout:2013: expect deleted 2024-09-21T09:46:39.099 INFO:tasks.rados.rados.0.smithi017.stdout:2014: snap_remove snap 232 2024-09-21T09:46:39.246 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:38 smithi017 podman[130413]: 2024-09-21 09:46:38.994642702 +0000 UTC m=+0.637824485 container remove be6ccc8faf44f179ecdc01339e0e478d1692f255d403438eaa634cb514f0bcb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-activate, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T09:46:39.247 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:39 smithi017 podman[130483]: 2024-09-21 09:46:39.114337494 +0000 UTC m=+0.020466231 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:39.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:39 smithi017 ceph-mon[102913]: pgmap v438: 89 pgs: 22 active+undersized, 2 stale+active+clean, 5 active+undersized+degraded, 60 active+clean; 172 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 932 KiB/s wr, 10 op/s; 22/351 objects degraded (6.268%) 2024-09-21T09:46:39.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:39 smithi017 ceph-mon[102913]: from='client.44870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:39.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:39 smithi017 ceph-mon[102913]: from='client.44876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:39.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:39 smithi017 ceph-mon[102913]: osdmap e574: 8 total, 7 up, 8 in 2024-09-21T09:46:39.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:39 smithi017 ceph-mon[105151]: pgmap v438: 89 pgs: 22 active+undersized, 2 stale+active+clean, 5 active+undersized+degraded, 60 active+clean; 172 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 932 KiB/s wr, 10 op/s; 22/351 objects degraded (6.268%) 2024-09-21T09:46:39.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:39 smithi017 ceph-mon[105151]: from='client.44870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:39.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:39 smithi017 ceph-mon[105151]: from='client.44876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:39.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:39 smithi017 ceph-mon[105151]: osdmap e574: 8 total, 7 up, 8 in 2024-09-21T09:46:39.512 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:39 smithi017 podman[130483]: 2024-09-21 09:46:39.245649599 +0000 UTC m=+0.151778335 container create 673e3e8b7c0aed588d30d7e1ff542e1466f68f77c086e9c3afd1db81c8934664 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T09:46:39.512 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:39 smithi017 podman[130483]: 2024-09-21 09:46:39.481808924 +0000 UTC m=+0.387937663 container init 673e3e8b7c0aed588d30d7e1ff542e1466f68f77c086e9c3afd1db81c8934664 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, ceph=True) 2024-09-21T09:46:39.512 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:39 smithi017 podman[130483]: 2024-09-21 09:46:39.487253871 +0000 UTC m=+0.393382608 container start 673e3e8b7c0aed588d30d7e1ff542e1466f68f77c086e9c3afd1db81c8934664 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T09:46:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:39 smithi134 ceph-mon[90085]: pgmap v438: 89 pgs: 22 active+undersized, 2 stale+active+clean, 5 active+undersized+degraded, 60 active+clean; 172 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 932 KiB/s wr, 10 op/s; 22/351 objects degraded (6.268%) 2024-09-21T09:46:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:39 smithi134 ceph-mon[90085]: from='client.44870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:39 smithi134 ceph-mon[90085]: from='client.44876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:39 smithi134 ceph-mon[90085]: osdmap e574: 8 total, 7 up, 8 in 2024-09-21T09:46:39.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:39 smithi017 ceph-osd[130521]: -- 172.21.15.17:0/395058468 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555e65521a40 con 0x555e654ffc00 2024-09-21T09:46:39.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:39 smithi017 bash[130483]: 673e3e8b7c0aed588d30d7e1ff542e1466f68f77c086e9c3afd1db81c8934664 2024-09-21T09:46:39.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:39 smithi017 systemd[1]: Started Ceph osd.3 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:46:40.096 INFO:tasks.rados.rados.0.smithi017.stdout:2014: done (2 left) 2024-09-21T09:46:40.096 INFO:tasks.rados.rados.0.smithi017.stdout:2015: read oid 10 snap -1 2024-09-21T09:46:40.096 INFO:tasks.rados.rados.0.smithi017.stdout:2015: expect deleted 2024-09-21T09:46:40.096 INFO:tasks.rados.rados.0.smithi017.stdout:2016: write oid 6 current snap is 257 2024-09-21T09:46:40.096 INFO:tasks.rados.rados.0.smithi017.stdout:2016: seq_num 763 ranges {423741=650287,1699055=618251} 2024-09-21T09:46:40.102 INFO:tasks.rados.rados.0.smithi017.stdout:2016: writing smithi01785587-6 from 423741 to 1074028 tid 1 2024-09-21T09:46:40.109 INFO:tasks.rados.rados.0.smithi017.stdout:2016: writing smithi01785587-6 from 1699055 to 2317306 tid 2 2024-09-21T09:46:40.109 INFO:tasks.rados.rados.0.smithi017.stdout:2017: read oid 45 snap -1 2024-09-21T09:46:40.109 INFO:tasks.rados.rados.0.smithi017.stdout:2017: expect (ObjNum 745 snap 252 seq_num 745) 2024-09-21T09:46:40.110 INFO:tasks.rados.rados.0.smithi017.stdout:2018: read oid 49 snap -1 2024-09-21T09:46:40.110 INFO:tasks.rados.rados.0.smithi017.stdout:2018: expect deleted 2024-09-21T09:46:40.110 INFO:tasks.rados.rados.0.smithi017.stdout:2019: snap_create 2024-09-21T09:46:40.110 INFO:tasks.rados.rados.0.smithi017.stdout:2012: expect (ObjNum 738 snap 249 seq_num 738) 2024-09-21T09:46:40.192 INFO:tasks.rados.rados.0.smithi017.stdout:2016: finishing write tid 1 to smithi01785587-6 2024-09-21T09:46:40.192 INFO:tasks.rados.rados.0.smithi017.stdout:2016: finishing write tid 2 to smithi01785587-6 2024-09-21T09:46:40.192 INFO:tasks.rados.rados.0.smithi017.stdout:2016: finishing write tid 3 to smithi01785587-6 2024-09-21T09:46:40.192 INFO:tasks.rados.rados.0.smithi017.stdout:2016: finishing write tid 4 to smithi01785587-6 2024-09-21T09:46:40.192 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 587 (ObjNum 763 snap 257 seq_num 763) dirty exists 2024-09-21T09:46:40.192 INFO:tasks.rados.rados.0.smithi017.stdout:2016: left oid 6 (ObjNum 763 snap 257 seq_num 763) 2024-09-21T09:46:40.192 INFO:tasks.rados.rados.0.smithi017.stdout:2017: expect (ObjNum 745 snap 252 seq_num 745) 2024-09-21T09:46:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:46:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:46:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[102913]: pgmap v441: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 175 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 288 KiB/s wr, 7 op/s; 23/354 objects degraded (6.497%) 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[102913]: osdmap e575: 8 total, 7 up, 8 in 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[105151]: pgmap v441: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 175 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 288 KiB/s wr, 7 op/s; 23/354 objects degraded (6.497%) 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[105151]: osdmap e575: 8 total, 7 up, 8 in 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:40 smithi134 ceph-mon[90085]: pgmap v441: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 175 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 288 KiB/s wr, 7 op/s; 23/354 objects degraded (6.497%) 2024-09-21T09:46:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:40 smithi134 ceph-mon[90085]: osdmap e575: 8 total, 7 up, 8 in 2024-09-21T09:46:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:41.075 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:41.095 INFO:tasks.rados.rados.0.smithi017.stdout:2012: done (6 left) 2024-09-21T09:46:41.095 INFO:tasks.rados.rados.0.smithi017.stdout:2013: done (5 left) 2024-09-21T09:46:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:2015: done (4 left) 2024-09-21T09:46:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:2016: done (3 left) 2024-09-21T09:46:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:2017: done (2 left) 2024-09-21T09:46:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:2018: done (1 left) 2024-09-21T09:46:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:2019: done (0 left) 2024-09-21T09:46:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:2020: rollback oid 21 current snap is 258 2024-09-21T09:46:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 237 2024-09-21T09:46:41.096 INFO:tasks.rados.rados.0.smithi017.stdout:2021: snap_create 2024-09-21T09:46:41.099 INFO:tasks.rados.rados.0.smithi017.stdout:2020: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:46:41.099 INFO:tasks.rados.rados.0.smithi017.stdout:2020: finishing rollback tid 2 to smithi01785587-21 2024-09-21T09:46:41.099 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1031 (ObjNum 711 snap 233 seq_num 711) dirty exists 2024-09-21T09:46:41.332 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:41 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[130497]: 2024-09-21T09:46:41.142+0000 7faa19d1d640 -1 Falling back to public interface 2024-09-21T09:46:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:41 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/730513025' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:41 smithi017 ceph-mon[102913]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:41 smithi017 ceph-mon[102913]: osdmap e576: 8 total, 7 up, 8 in 2024-09-21T09:46:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:41 smithi017 ceph-mon[102913]: from='client.44897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:41 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/730513025' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:41 smithi017 ceph-mon[105151]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:41.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:41 smithi017 ceph-mon[105151]: osdmap e576: 8 total, 7 up, 8 in 2024-09-21T09:46:41.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:41 smithi017 ceph-mon[105151]: from='client.44897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:41 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/730513025' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:41 smithi134 ceph-mon[90085]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:41 smithi134 ceph-mon[90085]: osdmap e576: 8 total, 7 up, 8 in 2024-09-21T09:46:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:41 smithi134 ceph-mon[90085]: from='client.44897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:42.096 INFO:tasks.rados.rados.0.smithi017.stdout:2020: done (1 left) 2024-09-21T09:46:42.096 INFO:tasks.rados.rados.0.smithi017.stdout:2021: done (0 left) 2024-09-21T09:46:42.096 INFO:tasks.rados.rados.0.smithi017.stdout:2022: read oid 43 snap -1 2024-09-21T09:46:42.096 INFO:tasks.rados.rados.0.smithi017.stdout:2022: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:46:42.096 INFO:tasks.rados.rados.0.smithi017.stdout:2023: write oid 8 current snap is 259 2024-09-21T09:46:42.096 INFO:tasks.rados.rados.0.smithi017.stdout:2023: seq_num 764 ranges {440548=725536,1757537=478262,2813990=26558} 2024-09-21T09:46:42.102 INFO:tasks.rados.rados.0.smithi017.stdout:2023: writing smithi01785587-8 from 440548 to 1166084 tid 1 2024-09-21T09:46:42.108 INFO:tasks.rados.rados.0.smithi017.stdout:2023: writing smithi01785587-8 from 1757537 to 2235799 tid 2 2024-09-21T09:46:42.111 INFO:tasks.rados.rados.0.smithi017.stdout:2023: writing smithi01785587-8 from 2813990 to 2840548 tid 3 2024-09-21T09:46:42.111 INFO:tasks.rados.rados.0.smithi017.stdout:2024: snap_remove snap 237 2024-09-21T09:46:43.091 INFO:tasks.rados.rados.0.smithi017.stdout:2024: done (2 left) 2024-09-21T09:46:43.091 INFO:tasks.rados.rados.0.smithi017.stdout:2025: write oid 25 current snap is 259 2024-09-21T09:46:43.091 INFO:tasks.rados.rados.0.smithi017.stdout:2025: seq_num 765 ranges {457355=400785,1416019=621920,2725060=132295} 2024-09-21T09:46:43.097 INFO:tasks.rados.rados.0.smithi017.stdout:2025: writing smithi01785587-25 from 457355 to 858140 tid 1 2024-09-21T09:46:43.104 INFO:tasks.rados.rados.0.smithi017.stdout:2025: writing smithi01785587-25 from 1416019 to 2037939 tid 2 2024-09-21T09:46:43.110 INFO:tasks.rados.rados.0.smithi017.stdout:2025: writing smithi01785587-25 from 2725060 to 2857355 tid 3 2024-09-21T09:46:43.110 INFO:tasks.rados.rados.0.smithi017.stdout:2026: rollback oid 14 current snap is 259 2024-09-21T09:46:43.110 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 252 2024-09-21T09:46:43.110 INFO:tasks.rados.rados.0.smithi017.stdout:2022: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:46:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:43 smithi017 ceph-mon[102913]: pgmap v444: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 185 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.5 MiB/s wr, 11 op/s; 25/366 objects degraded (6.831%) 2024-09-21T09:46:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:43 smithi017 ceph-mon[102913]: osdmap e577: 8 total, 7 up, 8 in 2024-09-21T09:46:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:43 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 25/366 objects degraded (6.831%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:43 smithi017 ceph-mon[102913]: osdmap e578: 8 total, 7 up, 8 in 2024-09-21T09:46:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:43 smithi017 ceph-mon[105151]: pgmap v444: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 185 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.5 MiB/s wr, 11 op/s; 25/366 objects degraded (6.831%) 2024-09-21T09:46:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:43 smithi017 ceph-mon[105151]: osdmap e577: 8 total, 7 up, 8 in 2024-09-21T09:46:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:43 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 25/366 objects degraded (6.831%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:43.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:43 smithi017 ceph-mon[105151]: osdmap e578: 8 total, 7 up, 8 in 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2023: finishing write tid 1 to smithi01785587-8 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2023: finishing write tid 2 to smithi01785587-8 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2023: finishing write tid 3 to smithi01785587-8 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2023: finishing write tid 4 to smithi01785587-8 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2023: finishing write tid 5 to smithi01785587-8 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 822 (ObjNum 764 snap 259 seq_num 764) dirty exists 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2023: left oid 8 (ObjNum 764 snap 259 seq_num 764) 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2022: done (3 left) 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2023: done (2 left) 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2027: delete oid 29 current snap is 259 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2025: finishing write tid 1 to smithi01785587-25 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2025: finishing write tid 2 to smithi01785587-25 2024-09-21T09:46:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:2025: finishing write tid 3 to smithi01785587-25 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:2025: finishing write tid 4 to smithi01785587-25 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:2026: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:2026: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 945 (ObjNum 667 snap 209 seq_num 667) dirty exists 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:2025: finishing write tid 5 to smithi01785587-25 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 469 (ObjNum 765 snap 259 seq_num 765) dirty exists 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:2025: left oid 25 (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:2025: done (2 left) 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:2026: done (1 left) 2024-09-21T09:46:43.223 INFO:tasks.rados.rados.0.smithi017.stdout:2027: done (0 left) 2024-09-21T09:46:43.224 INFO:tasks.rados.rados.0.smithi017.stdout:2028: snap_remove snap 257 2024-09-21T09:46:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:43 smithi134 ceph-mon[90085]: pgmap v444: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 185 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.5 MiB/s wr, 11 op/s; 25/366 objects degraded (6.831%) 2024-09-21T09:46:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:43 smithi134 ceph-mon[90085]: osdmap e577: 8 total, 7 up, 8 in 2024-09-21T09:46:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:43 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 25/366 objects degraded (6.831%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:43 smithi134 ceph-mon[90085]: osdmap e578: 8 total, 7 up, 8 in 2024-09-21T09:46:43.995 INFO:teuthology.orchestra.run.smithi017.stdout:true 2024-09-21T09:46:44.099 INFO:tasks.rados.rados.0.smithi017.stdout:2028: done (0 left) 2024-09-21T09:46:44.099 INFO:tasks.rados.rados.0.smithi017.stdout:2029: write oid 7 current snap is 259 2024-09-21T09:46:44.099 INFO:tasks.rados.rados.0.smithi017.stdout:2029: seq_num 767 ranges {490969=551283,1816630=625589} 2024-09-21T09:46:44.105 INFO:tasks.rados.rados.0.smithi017.stdout:2029: writing smithi01785587-7 from 490969 to 1042252 tid 1 2024-09-21T09:46:44.114 INFO:tasks.rados.rados.0.smithi017.stdout:2029: writing smithi01785587-7 from 1816630 to 2442219 tid 2 2024-09-21T09:46:44.114 INFO:tasks.rados.rados.0.smithi017.stdout:2030: rollback oid 45 current snap is 259 2024-09-21T09:46:44.114 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 45 to 239 2024-09-21T09:46:44.115 INFO:tasks.rados.rados.0.smithi017.stdout:2029: finishing write tid 1 to smithi01785587-7 2024-09-21T09:46:44.115 INFO:tasks.rados.rados.0.smithi017.stdout:2031: read oid 17 snap -1 2024-09-21T09:46:44.115 INFO:tasks.rados.rados.0.smithi017.stdout:2031: expect deleted 2024-09-21T09:46:44.115 INFO:tasks.rados.rados.0.smithi017.stdout:2032: read oid 32 snap -1 2024-09-21T09:46:44.115 INFO:tasks.rados.rados.0.smithi017.stdout:2032: expect (ObjNum 607 snap 190 seq_num 607) 2024-09-21T09:46:44.115 INFO:tasks.rados.rados.0.smithi017.stdout:2033: snap_create 2024-09-21T09:46:44.120 INFO:tasks.rados.rados.0.smithi017.stdout:2032: expect (ObjNum 607 snap 190 seq_num 607) 2024-09-21T09:46:44.346 INFO:tasks.rados.rados.0.smithi017.stdout:2029: finishing write tid 2 to smithi01785587-7 2024-09-21T09:46:44.347 INFO:tasks.rados.rados.0.smithi017.stdout:2029: finishing write tid 3 to smithi01785587-7 2024-09-21T09:46:44.347 INFO:tasks.rados.rados.0.smithi017.stdout:2029: finishing write tid 4 to smithi01785587-7 2024-09-21T09:46:44.347 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1043 (ObjNum 767 snap 259 seq_num 767) dirty exists 2024-09-21T09:46:44.347 INFO:tasks.rados.rados.0.smithi017.stdout:2029: left oid 7 (ObjNum 767 snap 259 seq_num 767) 2024-09-21T09:46:44.347 INFO:tasks.rados.rados.0.smithi017.stdout:2030: finishing rollback tid 0 to smithi01785587-45 2024-09-21T09:46:44.347 INFO:tasks.rados.rados.0.smithi017.stdout:2030: finishing rollback tid 1 to smithi01785587-45 2024-09-21T09:46:44.347 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 622 (ObjNum 129 snap 0 seq_num 2399473400) dirty dne 2024-09-21T09:46:44.637 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[130497]: 2024-09-21T09:46:44.482+0000 7faa19d1d640 -1 osd.3 568 log_to_monitors true 2024-09-21T09:46:44.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3570963702' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[102913]: osdmap e579: 8 total, 7 up, 8 in 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3570963702' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-mon[105151]: osdmap e579: 8 total, 7 up, 8 in 2024-09-21T09:46:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:44 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:44 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3570963702' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:44 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:44 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:44 smithi134 ceph-mon[90085]: osdmap e579: 8 total, 7 up, 8 in 2024-09-21T09:46:44.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:46:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:46:44] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-09-21T09:46:45.101 INFO:tasks.rados.rados.0.smithi017.stdout:2029: done (4 left) 2024-09-21T09:46:45.102 INFO:tasks.rados.rados.0.smithi017.stdout:2030: done (3 left) 2024-09-21T09:46:45.102 INFO:tasks.rados.rados.0.smithi017.stdout:2031: done (2 left) 2024-09-21T09:46:45.102 INFO:tasks.rados.rados.0.smithi017.stdout:2032: done (1 left) 2024-09-21T09:46:45.102 INFO:tasks.rados.rados.0.smithi017.stdout:2033: done (0 left) 2024-09-21T09:46:45.102 INFO:tasks.rados.rados.0.smithi017.stdout:2034: write oid 20 current snap is 260 2024-09-21T09:46:45.102 INFO:tasks.rados.rados.0.smithi017.stdout:2034: seq_num 768 ranges {507776=742885,1991465=485600} 2024-09-21T09:46:45.109 INFO:tasks.rados.rados.0.smithi017.stdout:2034: writing smithi01785587-20 from 507776 to 1250661 tid 1 2024-09-21T09:46:45.116 INFO:tasks.rados.rados.0.smithi017.stdout:2034: writing smithi01785587-20 from 1991465 to 2477065 tid 2 2024-09-21T09:46:45.116 INFO:tasks.rados.rados.0.smithi017.stdout:2035: write oid 36 current snap is 260 2024-09-21T09:46:45.117 INFO:tasks.rados.rados.0.smithi017.stdout:2035: seq_num 769 ranges {524583=418134,1649947=629258,2834752=89831} 2024-09-21T09:46:45.121 INFO:tasks.rados.rados.0.smithi017.stdout:2035: writing smithi01785587-36 from 524583 to 942717 tid 1 2024-09-21T09:46:45.129 INFO:tasks.rados.rados.0.smithi017.stdout:2035: writing smithi01785587-36 from 1649947 to 2279205 tid 2 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:2035: writing smithi01785587-36 from 2834752 to 2924583 tid 3 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:2036: delete oid 2 current snap is 260 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:2034: finishing write tid 1 to smithi01785587-20 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:2034: finishing write tid 2 to smithi01785587-20 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:2034: finishing write tid 3 to smithi01785587-20 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:2034: finishing write tid 4 to smithi01785587-20 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 478 (ObjNum 768 snap 260 seq_num 768) dirty exists 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:2034: left oid 20 (ObjNum 768 snap 260 seq_num 768) 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:2035: finishing write tid 1 to smithi01785587-36 2024-09-21T09:46:45.133 INFO:tasks.rados.rados.0.smithi017.stdout:2035: finishing write tid 2 to smithi01785587-36 2024-09-21T09:46:45.135 INFO:tasks.rados.rados.0.smithi017.stdout:2034: done (2 left) 2024-09-21T09:46:45.135 INFO:tasks.rados.rados.0.smithi017.stdout:2036: done (1 left) 2024-09-21T09:46:45.135 INFO:tasks.rados.rados.0.smithi017.stdout:2037: read oid 20 snap -1 2024-09-21T09:46:45.135 INFO:tasks.rados.rados.0.smithi017.stdout:2037: expect (ObjNum 768 snap 260 seq_num 768) 2024-09-21T09:46:45.135 INFO:tasks.rados.rados.0.smithi017.stdout:2038: snap_create 2024-09-21T09:46:45.135 INFO:tasks.rados.rados.0.smithi017.stdout:2035: finishing write tid 3 to smithi01785587-36 2024-09-21T09:46:45.136 INFO:tasks.rados.rados.0.smithi017.stdout:2035: finishing write tid 4 to smithi01785587-36 2024-09-21T09:46:45.136 INFO:tasks.rados.rados.0.smithi017.stdout:2035: finishing write tid 5 to smithi01785587-36 2024-09-21T09:46:45.136 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1061 (ObjNum 769 snap 260 seq_num 769) dirty exists 2024-09-21T09:46:45.136 INFO:tasks.rados.rados.0.smithi017.stdout:2035: left oid 36 (ObjNum 769 snap 260 seq_num 769) 2024-09-21T09:46:45.144 INFO:tasks.rados.rados.0.smithi017.stdout:2037: expect (ObjNum 768 snap 260 seq_num 768) 2024-09-21T09:46:45.299 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[130497]: 2024-09-21T09:46:45.118+0000 7faa11ac7640 -1 osd.3 568 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:46:45.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[102913]: pgmap v447: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.2 MiB/s wr, 12 op/s; 25/363 objects degraded (6.887%) 2024-09-21T09:46:45.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[102913]: from='client.44909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:45.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[102913]: from='client.35052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:45.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[102913]: from='osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:46:45.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[102913]: from='osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:46:45.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[102913]: osdmap e580: 8 total, 7 up, 8 in 2024-09-21T09:46:45.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[102913]: from='osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:46:45.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[105151]: pgmap v447: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.2 MiB/s wr, 12 op/s; 25/363 objects degraded (6.887%) 2024-09-21T09:46:45.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[105151]: from='client.44909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:45.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[105151]: from='client.35052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:45.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[105151]: from='osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:46:45.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[105151]: from='osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:46:45.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[105151]: osdmap e580: 8 total, 7 up, 8 in 2024-09-21T09:46:45.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:45 smithi017 ceph-mon[105151]: from='osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:46:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:45 smithi134 ceph-mon[90085]: pgmap v447: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.2 MiB/s wr, 12 op/s; 25/363 objects degraded (6.887%) 2024-09-21T09:46:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:45 smithi134 ceph-mon[90085]: from='client.44909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:45 smithi134 ceph-mon[90085]: from='client.35052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:45 smithi134 ceph-mon[90085]: from='osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:46:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:45 smithi134 ceph-mon[90085]: from='osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:46:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:45 smithi134 ceph-mon[90085]: osdmap e580: 8 total, 7 up, 8 in 2024-09-21T09:46:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:45 smithi134 ceph-mon[90085]: from='osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T09:46:46.108 INFO:tasks.rados.rados.0.smithi017.stdout:2035: done (2 left) 2024-09-21T09:46:46.108 INFO:tasks.rados.rados.0.smithi017.stdout:2037: done (1 left) 2024-09-21T09:46:46.108 INFO:tasks.rados.rados.0.smithi017.stdout:2038: done (0 left) 2024-09-21T09:46:46.108 INFO:tasks.rados.rados.0.smithi017.stdout:2039: read oid 22 snap -1 2024-09-21T09:46:46.108 INFO:tasks.rados.rados.0.smithi017.stdout:2039: expect (ObjNum 726 snap 245 seq_num 726) 2024-09-21T09:46:46.109 INFO:tasks.rados.rados.0.smithi017.stdout:2040: delete oid 41 current snap is 261 2024-09-21T09:46:46.113 INFO:tasks.rados.rados.0.smithi017.stdout:2040: done (1 left) 2024-09-21T09:46:46.113 INFO:tasks.rados.rados.0.smithi017.stdout:2041: snap_create 2024-09-21T09:46:46.124 INFO:tasks.rados.rados.0.smithi017.stdout:2039: expect (ObjNum 726 snap 245 seq_num 726) 2024-09-21T09:46:46.623 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:46:46.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:46 smithi017 ceph-mon[102913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:46:46.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:46 smithi017 ceph-mon[102913]: osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816] boot 2024-09-21T09:46:46.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:46 smithi017 ceph-mon[102913]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T09:46:46.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:46:46.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:46 smithi017 ceph-mon[105151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:46:46.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:46 smithi017 ceph-mon[105151]: osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816] boot 2024-09-21T09:46:46.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:46 smithi017 ceph-mon[105151]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T09:46:46.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:46:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:46 smithi134 ceph-mon[90085]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:46:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:46 smithi134 ceph-mon[90085]: osd.3 [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816] boot 2024-09-21T09:46:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:46 smithi134 ceph-mon[90085]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T09:46:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:46:47.106 INFO:tasks.rados.rados.0.smithi017.stdout:2039: done (1 left) 2024-09-21T09:46:47.106 INFO:tasks.rados.rados.0.smithi017.stdout:2041: done (0 left) 2024-09-21T09:46:47.106 INFO:tasks.rados.rados.0.smithi017.stdout:2042: read oid 25 snap -1 2024-09-21T09:46:47.106 INFO:tasks.rados.rados.0.smithi017.stdout:2042: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:46:47.106 INFO:tasks.rados.rados.0.smithi017.stdout:2043: read oid 11 snap -1 2024-09-21T09:46:47.106 INFO:tasks.rados.rados.0.smithi017.stdout:2043: expect deleted 2024-09-21T09:46:47.107 INFO:tasks.rados.rados.0.smithi017.stdout:2044: snap_remove snap 255 2024-09-21T09:46:47.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T09:46:47.439+0000 7f3131349640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T09:46:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[102913]: pgmap v450: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 188 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 583 KiB/s wr, 9 op/s; 25/372 objects degraded (6.720%) 2024-09-21T09:46:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/692308869' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[102913]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[105151]: pgmap v450: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 188 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 583 KiB/s wr, 9 op/s; 25/372 objects degraded (6.720%) 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/692308869' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[105151]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T09:46:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:47.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:47.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:47.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:47.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:47 smithi134 ceph-mon[90085]: pgmap v450: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 188 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 583 KiB/s wr, 9 op/s; 25/372 objects degraded (6.720%) 2024-09-21T09:46:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:47 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/692308869' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-09-21T09:46:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:47 smithi134 ceph-mon[90085]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T09:46:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:47.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade pause' 2024-09-21T09:46:48.086 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:48.106 INFO:tasks.rados.rados.0.smithi017.stdout:2044: done (2 left) 2024-09-21T09:46:48.106 INFO:tasks.rados.rados.0.smithi017.stdout:2045: read oid 31 snap -1 2024-09-21T09:46:48.106 INFO:tasks.rados.rados.0.smithi017.stdout:2045: expect deleted 2024-09-21T09:46:48.106 INFO:tasks.rados.rados.0.smithi017.stdout:2046: read oid 21 snap -1 2024-09-21T09:46:48.107 INFO:tasks.rados.rados.0.smithi017.stdout:2046: expect (ObjNum 711 snap 233 seq_num 711) 2024-09-21T09:46:48.107 INFO:tasks.rados.rados.0.smithi017.stdout:2047: read oid 47 snap -1 2024-09-21T09:46:48.107 INFO:tasks.rados.rados.0.smithi017.stdout:2047: expect (ObjNum 664 snap 209 seq_num 664) 2024-09-21T09:46:48.107 INFO:tasks.rados.rados.0.smithi017.stdout:2048: delete oid 42 current snap is 262 2024-09-21T09:46:48.107 INFO:tasks.rados.rados.0.smithi017.stdout:2042: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:46:48.264 INFO:tasks.rados.rados.0.smithi017.stdout:2042: done (5 left) 2024-09-21T09:46:48.265 INFO:tasks.rados.rados.0.smithi017.stdout:2043: done (4 left) 2024-09-21T09:46:48.265 INFO:tasks.rados.rados.0.smithi017.stdout:2048: done (3 left) 2024-09-21T09:46:48.265 INFO:tasks.rados.rados.0.smithi017.stdout:2049: write oid 19 current snap is 262 2024-09-21T09:46:48.265 INFO:tasks.rados.rados.0.smithi017.stdout:2047: expect (ObjNum 664 snap 209 seq_num 664) 2024-09-21T09:46:48.376 INFO:tasks.rados.rados.0.smithi017.stdout:2046: expect (ObjNum 711 snap 233 seq_num 711) 2024-09-21T09:46:48.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:48.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:48.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:48.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:48.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:46:48.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[102913]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:46:48.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[102913]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T09:46:48.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 25/372 objects degraded (6.720%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:48.481 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[102913]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T09:46:48.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:48.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:48.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:48.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:48.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:46:48.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[105151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:46:48.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[105151]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T09:46:48.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 25/372 objects degraded (6.720%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:48.482 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:48 smithi017 ceph-mon[105151]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T09:46:48.516 INFO:tasks.rados.rados.0.smithi017.stdout:2049: seq_num 773 ranges {591811=719130,1767522=636596,2828091=163720} 2024-09-21T09:46:48.523 INFO:tasks.rados.rados.0.smithi017.stdout:2049: writing smithi01785587-19 from 591811 to 1310941 tid 1 2024-09-21T09:46:48.529 INFO:tasks.rados.rados.0.smithi017.stdout:2049: writing smithi01785587-19 from 1767522 to 2404118 tid 2 2024-09-21T09:46:48.532 INFO:tasks.rados.rados.0.smithi017.stdout:2049: writing smithi01785587-19 from 2828091 to 2991811 tid 3 2024-09-21T09:46:48.533 INFO:tasks.rados.rados.0.smithi017.stdout:2045: done (3 left) 2024-09-21T09:46:48.533 INFO:tasks.rados.rados.0.smithi017.stdout:2046: done (2 left) 2024-09-21T09:46:48.533 INFO:tasks.rados.rados.0.smithi017.stdout:2047: done (1 left) 2024-09-21T09:46:48.533 INFO:tasks.rados.rados.0.smithi017.stdout:2050: read oid 20 snap 246 2024-09-21T09:46:48.533 INFO:tasks.rados.rados.0.smithi017.stdout:2050: expect deleted 2024-09-21T09:46:48.533 INFO:tasks.rados.rados.0.smithi017.stdout:2051: snap_create 2024-09-21T09:46:48.533 INFO:tasks.rados.rados.0.smithi017.stdout:2049: finishing write tid 1 to smithi01785587-19 2024-09-21T09:46:48.533 INFO:tasks.rados.rados.0.smithi017.stdout:2049: finishing write tid 2 to smithi01785587-19 2024-09-21T09:46:48.535 INFO:tasks.rados.rados.0.smithi017.stdout:2049: finishing write tid 3 to smithi01785587-19 2024-09-21T09:46:48.535 INFO:tasks.rados.rados.0.smithi017.stdout:2049: finishing write tid 4 to smithi01785587-19 2024-09-21T09:46:48.535 INFO:tasks.rados.rados.0.smithi017.stdout:2049: finishing write tid 5 to smithi01785587-19 2024-09-21T09:46:48.535 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 806 (ObjNum 773 snap 262 seq_num 773) dirty exists 2024-09-21T09:46:48.535 INFO:tasks.rados.rados.0.smithi017.stdout:2049: left oid 19 (ObjNum 773 snap 262 seq_num 773) 2024-09-21T09:46:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:46:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:48 smithi134 ceph-mon[90085]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:46:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:48 smithi134 ceph-mon[90085]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T09:46:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:48 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 25/372 objects degraded (6.720%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:46:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:48 smithi134 ceph-mon[90085]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T09:46:49.372 INFO:tasks.rados.rados.0.smithi017.stdout:2049: done (2 left) 2024-09-21T09:46:49.372 INFO:tasks.rados.rados.0.smithi017.stdout:2050: done (1 left) 2024-09-21T09:46:49.373 INFO:tasks.rados.rados.0.smithi017.stdout:2051: done (0 left) 2024-09-21T09:46:49.373 INFO:tasks.rados.rados.0.smithi017.stdout:2052: snap_remove snap 259 2024-09-21T09:46:49.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:49 smithi017 ceph-mon[102913]: pgmap v453: 89 pgs: 3 peering, 23 active+undersized, 5 active+undersized+degraded, 58 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.1 MiB/s wr, 12 op/s; 20/381 objects degraded (5.249%) 2024-09-21T09:46:49.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:49 smithi017 ceph-mon[105151]: pgmap v453: 89 pgs: 3 peering, 23 active+undersized, 5 active+undersized+degraded, 58 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.1 MiB/s wr, 12 op/s; 20/381 objects degraded (5.249%) 2024-09-21T09:46:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:49 smithi134 ceph-mon[90085]: pgmap v453: 89 pgs: 3 peering, 23 active+undersized, 5 active+undersized+degraded, 58 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.1 MiB/s wr, 12 op/s; 20/381 objects degraded (5.249%) 2024-09-21T09:46:49.973 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:50.379 INFO:tasks.rados.rados.0.smithi017.stdout:2052: done (0 left) 2024-09-21T09:46:50.379 INFO:tasks.rados.rados.0.smithi017.stdout:2053: rollback oid 10 current snap is 263 2024-09-21T09:46:50.379 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 256 2024-09-21T09:46:50.379 INFO:tasks.rados.rados.0.smithi017.stdout:2054: read oid 44 snap 219 2024-09-21T09:46:50.379 INFO:tasks.rados.rados.0.smithi017.stdout:2054: expect deleted 2024-09-21T09:46:50.379 INFO:tasks.rados.rados.0.smithi017.stdout:2055: rollback oid 43 current snap is 263 2024-09-21T09:46:50.379 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 251 2024-09-21T09:46:50.380 INFO:tasks.rados.rados.0.smithi017.stdout:2056: snap_create 2024-09-21T09:46:50.380 INFO:tasks.rados.rados.0.smithi017.stdout:2053: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:46:50.380 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 0 (ObjNum 97 snap 0 seq_num 2399539824) dirty dne 2024-09-21T09:46:50.385 INFO:tasks.rados.rados.0.smithi017.stdout:2055: finishing rollback tid 0 to smithi01785587-43 2024-09-21T09:46:50.386 INFO:tasks.rados.rados.0.smithi017.stdout:2055: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:46:50.386 INFO:tasks.rados.rados.0.smithi017.stdout:2055: finishing rollback tid 2 to smithi01785587-43 2024-09-21T09:46:50.386 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 599 (ObjNum 512 snap 160 seq_num 512) dirty exists 2024-09-21T09:46:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:50 smithi134 ceph-mon[90085]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T09:46:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:50 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:46:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:46:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:46:50.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:50 smithi017 ceph-mon[102913]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T09:46:50.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:50.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:50 smithi017 ceph-mon[105151]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T09:46:50.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:50 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:50.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch ps' 2024-09-21T09:46:50.980 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:51.378 INFO:tasks.rados.rados.0.smithi017.stdout:2053: done (3 left) 2024-09-21T09:46:51.378 INFO:tasks.rados.rados.0.smithi017.stdout:2054: done (2 left) 2024-09-21T09:46:51.378 INFO:tasks.rados.rados.0.smithi017.stdout:2055: done (1 left) 2024-09-21T09:46:51.378 INFO:tasks.rados.rados.0.smithi017.stdout:2056: done (0 left) 2024-09-21T09:46:51.378 INFO:tasks.rados.rados.0.smithi017.stdout:2057: snap_remove snap 262 2024-09-21T09:46:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:51 smithi017 ceph-mon[102913]: pgmap v456: 89 pgs: 3 peering, 18 active+undersized, 5 active+undersized+degraded, 63 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 866 KiB/s wr, 6 op/s; 20/387 objects degraded (5.168%) 2024-09-21T09:46:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:51 smithi017 ceph-mon[102913]: from='client.44930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:51 smithi017 ceph-mon[102913]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:51.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:51 smithi017 ceph-mon[102913]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T09:46:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:51 smithi017 ceph-mon[105151]: pgmap v456: 89 pgs: 3 peering, 18 active+undersized, 5 active+undersized+degraded, 63 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 866 KiB/s wr, 6 op/s; 20/387 objects degraded (5.168%) 2024-09-21T09:46:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:51 smithi017 ceph-mon[105151]: from='client.44930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:51 smithi017 ceph-mon[105151]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:51 smithi017 ceph-mon[105151]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T09:46:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:51 smithi134 ceph-mon[90085]: pgmap v456: 89 pgs: 3 peering, 18 active+undersized, 5 active+undersized+degraded, 63 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 866 KiB/s wr, 6 op/s; 20/387 objects degraded (5.168%) 2024-09-21T09:46:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:51 smithi134 ceph-mon[90085]: from='client.44930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:51 smithi134 ceph-mon[90085]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:46:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:51 smithi134 ceph-mon[90085]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T09:46:52.392 INFO:tasks.rados.rados.0.smithi017.stdout:2057: done (0 left) 2024-09-21T09:46:52.392 INFO:tasks.rados.rados.0.smithi017.stdout:2058: write oid 3 current snap is 264 2024-09-21T09:46:52.392 INFO:tasks.rados.rados.0.smithi017.stdout:2058: seq_num 774 ranges {608618=794379,2109651=780254} 2024-09-21T09:46:52.399 INFO:tasks.rados.rados.0.smithi017.stdout:2058: writing smithi01785587-3 from 608618 to 1402997 tid 1 2024-09-21T09:46:52.408 INFO:tasks.rados.rados.0.smithi017.stdout:2058: writing smithi01785587-3 from 2109651 to 2889905 tid 2 2024-09-21T09:46:52.408 INFO:tasks.rados.rados.0.smithi017.stdout:2059: snap_create 2024-09-21T09:46:52.408 INFO:tasks.rados.rados.0.smithi017.stdout:2058: finishing write tid 1 to smithi01785587-3 2024-09-21T09:46:52.411 INFO:tasks.rados.rados.0.smithi017.stdout:2058: finishing write tid 2 to smithi01785587-3 2024-09-21T09:46:52.411 INFO:tasks.rados.rados.0.smithi017.stdout:2058: finishing write tid 3 to smithi01785587-3 2024-09-21T09:46:52.411 INFO:tasks.rados.rados.0.smithi017.stdout:2058: finishing write tid 4 to smithi01785587-3 2024-09-21T09:46:52.411 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 957 (ObjNum 774 snap 264 seq_num 774) dirty exists 2024-09-21T09:46:52.411 INFO:tasks.rados.rados.0.smithi017.stdout:2058: left oid 3 (ObjNum 774 snap 264 seq_num 774) 2024-09-21T09:46:52.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:52 smithi017 ceph-mon[102913]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T09:46:52.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:52 smithi017 ceph-mon[105151]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T09:46:52.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:52 smithi134 ceph-mon[90085]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (7m) 9s ago 11m 19.1M - 0.27.0 11f11916f8cd 0a4d8256f8d7 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi134 *:3000 running (6m) 42s ago 11m 85.6M - 9.4.7 954c08fa6188 8d2c807b0308 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.coodep smithi017 running (9m) 9s ago 9m 19.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 81acf97499eb 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi134.bwovvb smithi134 running (9m) 42s ago 9m 17.3M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be bf739a2a98de 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi134 *:8443,9283,8765 running (5m) 42s ago 15m 455M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 4feb5ade269b 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (6m) 9s ago 16m 519M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 14c61243c535 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (5m) 9s ago 17m 65.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0590143b06cf 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi134 running (3m) 42s ago 15m 44.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 10638b01de94 2024-09-21T09:46:52.909 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (4m) 9s ago 15m 54.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 013cf9450c57 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (7m) 9s ago 10m 10.2M - 1.7.0 72c9c2088986 057b63034a9e 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi134 *:9100 running (6m) 42s ago 10m 11.3M - 1.7.0 72c9c2088986 c6b046d44374 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (115s) 9s ago 14m 73.9M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 68ddc8c6bab9 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (81s) 9s ago 14m 57.0M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 228e704d8681 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (47s) 9s ago 14m 57.6M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 f98642c0dccf 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (13s) 9s ago 13m 11.1M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 673e3e8b7c0a 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi134 running (13m) 42s ago 13m 108M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 7ece30084c41 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi134 running (12m) 42s ago 12m 90.0M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5dec2255fcd7 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi134 running (12m) 42s ago 12m 73.2M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 80d4436a0b8a 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi134 running (12m) 42s ago 12m 94.5M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed2e26ef5698 2024-09-21T09:46:52.910 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi134 *:9095 running (5m) 42s ago 11m 54.5M - 2.51.0 1d3b7f56885b 061a3294723d 2024-09-21T09:46:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:2058: done (1 left) 2024-09-21T09:46:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:2059: done (0 left) 2024-09-21T09:46:53.094 INFO:tasks.rados.rados.0.smithi017.stdout:2060: delete oid 20 current snap is 265 2024-09-21T09:46:53.099 INFO:tasks.rados.rados.0.smithi017.stdout:2060: done (0 left) 2024-09-21T09:46:53.099 INFO:tasks.rados.rados.0.smithi017.stdout:2061: rollback oid 21 current snap is 265 2024-09-21T09:46:53.099 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 219 2024-09-21T09:46:53.099 INFO:tasks.rados.rados.0.smithi017.stdout:2062: snap_remove snap 245 2024-09-21T09:46:53.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[102913]: pgmap v459: 89 pgs: 89 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 649 KiB/s wr, 12 op/s; 2.5 MiB/s, 1 objects/s recovering 2024-09-21T09:46:53.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/387 objects degraded (5.168%), 5 pgs degraded) 2024-09-21T09:46:53.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T09:46:53.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[102913]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T09:46:53.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[102913]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T09:46:53.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[105151]: pgmap v459: 89 pgs: 89 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 649 KiB/s wr, 12 op/s; 2.5 MiB/s, 1 objects/s recovering 2024-09-21T09:46:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/387 objects degraded (5.168%), 5 pgs degraded) 2024-09-21T09:46:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T09:46:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[105151]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T09:46:53.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:53 smithi017 ceph-mon[105151]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T09:46:53.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch ps' 2024-09-21T09:46:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:53 smithi134 ceph-mon[90085]: pgmap v459: 89 pgs: 89 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 649 KiB/s wr, 12 op/s; 2.5 MiB/s, 1 objects/s recovering 2024-09-21T09:46:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:53 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/387 objects degraded (5.168%), 5 pgs degraded) 2024-09-21T09:46:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:53 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T09:46:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:53 smithi134 ceph-mon[90085]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T09:46:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:53 smithi134 ceph-mon[90085]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T09:46:53.933 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:54.096 INFO:tasks.rados.rados.0.smithi017.stdout:2062: done (1 left) 2024-09-21T09:46:54.096 INFO:tasks.rados.rados.0.smithi017.stdout:2063: snap_create 2024-09-21T09:46:54.097 INFO:tasks.rados.rados.0.smithi017.stdout:2061: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:46:54.097 INFO:tasks.rados.rados.0.smithi017.stdout:2061: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:46:54.097 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1048 (ObjNum 129 snap 0 seq_num 2399506136) dirty dne 2024-09-21T09:46:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:54 smithi017 ceph-mon[102913]: from='client.44936 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:54 smithi017 ceph-mon[102913]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T09:46:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:54 smithi017 ceph-mon[105151]: from='client.44936 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:54 smithi017 ceph-mon[105151]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T09:46:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:54 smithi134 ceph-mon[90085]: from='client.44936 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:54 smithi134 ceph-mon[90085]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T09:46:55.099 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:46:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:46:54] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-09-21T09:46:55.099 INFO:tasks.rados.rados.0.smithi017.stdout:2061: done (1 left) 2024-09-21T09:46:55.099 INFO:tasks.rados.rados.0.smithi017.stdout:2063: done (0 left) 2024-09-21T09:46:55.099 INFO:tasks.rados.rados.0.smithi017.stdout:2064: write oid 8 current snap is 266 2024-09-21T09:46:55.099 INFO:tasks.rados.rados.0.smithi017.stdout:2064: seq_num 776 ranges {642232=661230,1942968=783923} 2024-09-21T09:46:55.105 INFO:tasks.rados.rados.0.smithi017.stdout:2064: writing smithi01785587-8 from 642232 to 1303462 tid 1 2024-09-21T09:46:55.114 INFO:tasks.rados.rados.0.smithi017.stdout:2064: writing smithi01785587-8 from 1942968 to 2726891 tid 2 2024-09-21T09:46:55.115 INFO:tasks.rados.rados.0.smithi017.stdout:2065: snap_create 2024-09-21T09:46:55.115 INFO:tasks.rados.rados.0.smithi017.stdout:2064: finishing write tid 1 to smithi01785587-8 2024-09-21T09:46:55.119 INFO:tasks.rados.rados.0.smithi017.stdout:2064: finishing write tid 2 to smithi01785587-8 2024-09-21T09:46:55.119 INFO:tasks.rados.rados.0.smithi017.stdout:2064: finishing write tid 3 to smithi01785587-8 2024-09-21T09:46:55.119 INFO:tasks.rados.rados.0.smithi017.stdout:2064: finishing write tid 4 to smithi01785587-8 2024-09-21T09:46:55.120 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 834 (ObjNum 776 snap 266 seq_num 776) dirty exists 2024-09-21T09:46:55.120 INFO:tasks.rados.rados.0.smithi017.stdout:2064: left oid 8 (ObjNum 776 snap 266 seq_num 776) 2024-09-21T09:46:55.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:55 smithi017 ceph-mon[102913]: pgmap v462: 89 pgs: 89 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 371 KiB/s wr, 10 op/s; 2.5 MiB/s, 1 objects/s recovering 2024-09-21T09:46:55.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:55 smithi017 ceph-mon[102913]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T09:46:55.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:55.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:55.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:55 smithi017 ceph-mon[105151]: pgmap v462: 89 pgs: 89 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 371 KiB/s wr, 10 op/s; 2.5 MiB/s, 1 objects/s recovering 2024-09-21T09:46:55.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:55 smithi017 ceph-mon[105151]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T09:46:55.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:55.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:55 smithi134 ceph-mon[90085]: pgmap v462: 89 pgs: 89 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 371 KiB/s wr, 10 op/s; 2.5 MiB/s, 1 objects/s recovering 2024-09-21T09:46:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:55 smithi134 ceph-mon[90085]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T09:46:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:46:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:55.850 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:46:55.850 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (7m) 12s ago 11m 19.1M - 0.27.0 11f11916f8cd 0a4d8256f8d7 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi134 *:3000 running (6m) 45s ago 11m 85.6M - 9.4.7 954c08fa6188 8d2c807b0308 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.coodep smithi017 running (10m) 12s ago 10m 19.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 81acf97499eb 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi134.bwovvb smithi134 running (9m) 45s ago 9m 17.3M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be bf739a2a98de 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi134 *:8443,9283,8765 running (5m) 45s ago 15m 455M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 4feb5ade269b 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (6m) 12s ago 17m 519M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 14c61243c535 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (5m) 12s ago 17m 65.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 0590143b06cf 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi134 running (3m) 45s ago 15m 44.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 10638b01de94 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (5m) 12s ago 15m 54.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 013cf9450c57 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (7m) 12s ago 10m 10.2M - 1.7.0 72c9c2088986 057b63034a9e 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi134 *:9100 running (6m) 45s ago 10m 11.3M - 1.7.0 72c9c2088986 c6b046d44374 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (118s) 12s ago 14m 73.9M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 68ddc8c6bab9 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (84s) 12s ago 14m 57.0M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 228e704d8681 2024-09-21T09:46:55.851 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (50s) 12s ago 14m 57.6M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 f98642c0dccf 2024-09-21T09:46:55.852 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (16s) 12s ago 13m 11.1M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 673e3e8b7c0a 2024-09-21T09:46:55.852 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi134 running (13m) 45s ago 13m 108M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 7ece30084c41 2024-09-21T09:46:55.852 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi134 running (12m) 45s ago 12m 90.0M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 5dec2255fcd7 2024-09-21T09:46:55.852 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi134 running (12m) 45s ago 12m 73.2M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 80d4436a0b8a 2024-09-21T09:46:55.852 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi134 running (12m) 45s ago 12m 94.5M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ed2e26ef5698 2024-09-21T09:46:55.852 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi134 *:9095 running (5m) 45s ago 11m 54.5M - 2.51.0 1d3b7f56885b 061a3294723d 2024-09-21T09:46:56.100 INFO:tasks.rados.rados.0.smithi017.stdout:2064: done (1 left) 2024-09-21T09:46:56.100 INFO:tasks.rados.rados.0.smithi017.stdout:2065: done (0 left) 2024-09-21T09:46:56.100 INFO:tasks.rados.rados.0.smithi017.stdout:2066: write oid 10 current snap is 267 2024-09-21T09:46:56.100 INFO:tasks.rados.rados.0.smithi017.stdout:2066: seq_num 777 ranges {659039=736479,2001450=643934} 2024-09-21T09:46:56.107 INFO:tasks.rados.rados.0.smithi017.stdout:2066: writing smithi01785587-10 from 659039 to 1395518 tid 1 2024-09-21T09:46:56.115 INFO:tasks.rados.rados.0.smithi017.stdout:2066: writing smithi01785587-10 from 2001450 to 2645384 tid 2 2024-09-21T09:46:56.115 INFO:tasks.rados.rados.0.smithi017.stdout:2067: delete oid 47 current snap is 267 2024-09-21T09:46:56.115 INFO:tasks.rados.rados.0.smithi017.stdout:2066: finishing write tid 1 to smithi01785587-10 2024-09-21T09:46:56.117 INFO:tasks.rados.rados.0.smithi017.stdout:2067: done (1 left) 2024-09-21T09:46:56.117 INFO:tasks.rados.rados.0.smithi017.stdout:2068: snap_create 2024-09-21T09:46:56.117 INFO:tasks.rados.rados.0.smithi017.stdout:2066: finishing write tid 2 to smithi01785587-10 2024-09-21T09:46:56.117 INFO:tasks.rados.rados.0.smithi017.stdout:2066: finishing write tid 3 to smithi01785587-10 2024-09-21T09:46:56.118 INFO:tasks.rados.rados.0.smithi017.stdout:2066: finishing write tid 4 to smithi01785587-10 2024-09-21T09:46:56.118 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 146 (ObjNum 777 snap 267 seq_num 777) dirty exists 2024-09-21T09:46:56.118 INFO:tasks.rados.rados.0.smithi017.stdout:2066: left oid 10 (ObjNum 777 snap 267 seq_num 777) 2024-09-21T09:46:56.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph versions' 2024-09-21T09:46:57.049 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:57.105 INFO:tasks.rados.rados.0.smithi017.stdout:2066: done (1 left) 2024-09-21T09:46:57.106 INFO:tasks.rados.rados.0.smithi017.stdout:2068: done (0 left) 2024-09-21T09:46:57.106 INFO:tasks.rados.rados.0.smithi017.stdout:2069: rollback oid 33 current snap is 268 2024-09-21T09:46:57.106 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 267 2024-09-21T09:46:57.106 INFO:tasks.rados.rados.0.smithi017.stdout:2070: read oid 44 snap -1 2024-09-21T09:46:57.106 INFO:tasks.rados.rados.0.smithi017.stdout:2070: expect deleted 2024-09-21T09:46:57.106 INFO:tasks.rados.rados.0.smithi017.stdout:2071: write oid 23 current snap is 268 2024-09-21T09:46:57.106 INFO:tasks.rados.rados.0.smithi017.stdout:2071: seq_num 779 ranges {692653=486977,1602061=647603,2992629=100024} 2024-09-21T09:46:57.112 INFO:tasks.rados.rados.0.smithi017.stdout:2071: writing smithi01785587-23 from 692653 to 1179630 tid 1 2024-09-21T09:46:57.118 INFO:tasks.rados.rados.0.smithi017.stdout:2071: writing smithi01785587-23 from 1602061 to 2249664 tid 2 2024-09-21T09:46:57.122 INFO:tasks.rados.rados.0.smithi017.stdout:2071: writing smithi01785587-23 from 2992629 to 3092653 tid 3 2024-09-21T09:46:57.123 INFO:tasks.rados.rados.0.smithi017.stdout:2072: snap_remove snap 234 2024-09-21T09:46:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:57 smithi017 ceph-mon[102913]: from='client.35079 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:57 smithi017 ceph-mon[102913]: pgmap v465: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 384 KiB/s wr, 1 op/s 2024-09-21T09:46:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:57 smithi017 ceph-mon[102913]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T09:46:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:57 smithi017 ceph-mon[105151]: from='client.35079 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:57 smithi017 ceph-mon[105151]: pgmap v465: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 384 KiB/s wr, 1 op/s 2024-09-21T09:46:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:57 smithi017 ceph-mon[105151]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T09:46:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:57 smithi134 ceph-mon[90085]: from='client.35079 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:57 smithi134 ceph-mon[90085]: pgmap v465: 89 pgs: 89 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 384 KiB/s wr, 1 op/s 2024-09-21T09:46:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:57 smithi134 ceph-mon[90085]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T09:46:58.100 INFO:tasks.rados.rados.0.smithi017.stdout:2069: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:46:58.100 INFO:tasks.rados.rados.0.smithi017.stdout:2069: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:46:58.100 INFO:tasks.rados.rados.0.smithi017.stdout:2069: finishing rollback tid 2 to smithi01785587-33 2024-09-21T09:46:58.100 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 960 (ObjNum 751 snap 253 seq_num 751) dirty exists 2024-09-21T09:46:58.100 INFO:tasks.rados.rados.0.smithi017.stdout:2071: finishing write tid 1 to smithi01785587-23 2024-09-21T09:46:58.100 INFO:tasks.rados.rados.0.smithi017.stdout:2071: finishing write tid 2 to smithi01785587-23 2024-09-21T09:46:58.100 INFO:tasks.rados.rados.0.smithi017.stdout:2071: finishing write tid 3 to smithi01785587-23 2024-09-21T09:46:58.100 INFO:tasks.rados.rados.0.smithi017.stdout:2071: finishing write tid 4 to smithi01785587-23 2024-09-21T09:46:58.100 INFO:tasks.rados.rados.0.smithi017.stdout:2071: finishing write tid 5 to smithi01785587-23 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 839 (ObjNum 779 snap 268 seq_num 779) dirty exists 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2071: left oid 23 (ObjNum 779 snap 268 seq_num 779) 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2069: done (3 left) 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2070: done (2 left) 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2071: done (1 left) 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2072: done (0 left) 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2073: read oid 9 snap 251 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2073: expect (ObjNum 724 snap 245 seq_num 724) 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2074: read oid 2 snap -1 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2074: expect deleted 2024-09-21T09:46:58.101 INFO:tasks.rados.rados.0.smithi017.stdout:2075: delete oid 34 current snap is 268 2024-09-21T09:46:58.111 INFO:tasks.rados.rados.0.smithi017.stdout:2073: expect (ObjNum 724 snap 245 seq_num 724) 2024-09-21T09:46:58.215 INFO:tasks.rados.rados.0.smithi017.stdout:2073: done (2 left) 2024-09-21T09:46:58.215 INFO:tasks.rados.rados.0.smithi017.stdout:2074: done (1 left) 2024-09-21T09:46:58.215 INFO:tasks.rados.rados.0.smithi017.stdout:2075: done (0 left) 2024-09-21T09:46:58.215 INFO:tasks.rados.rados.0.smithi017.stdout:2076: snap_remove snap 263 2024-09-21T09:46:58.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:58 smithi017 ceph-mon[102913]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T09:46:58.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:58 smithi017 ceph-mon[102913]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T09:46:58.355 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:58 smithi017 ceph-mon[105151]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T09:46:58.355 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:58 smithi017 ceph-mon[105151]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T09:46:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:58 smithi134 ceph-mon[90085]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T09:46:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:58 smithi134 ceph-mon[90085]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: "mon": { 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr": { 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: "osd": { 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 4, 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:46:59.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:46:59.104 INFO:teuthology.orchestra.run.smithi017.stdout: "mds": { 2024-09-21T09:46:59.104 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:46:59.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:46:59.104 INFO:teuthology.orchestra.run.smithi017.stdout: "overall": { 2024-09-21T09:46:59.104 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 6, 2024-09-21T09:46:59.104 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 9 2024-09-21T09:46:59.104 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:46:59.104 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:46:59.119 INFO:tasks.rados.rados.0.smithi017.stdout:2076: done (0 left) 2024-09-21T09:46:59.119 INFO:tasks.rados.rados.0.smithi017.stdout:2077: snap_remove snap 260 2024-09-21T09:46:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:59 smithi017 ceph-mon[102913]: pgmap v468: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-21T09:46:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:46:59 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1659997233' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:59 smithi017 ceph-mon[105151]: pgmap v468: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-21T09:46:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:46:59 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1659997233' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:59 smithi134 ceph-mon[90085]: pgmap v468: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-21T09:46:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:46:59 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1659997233' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:46:59.775 DEBUG:teuthology.parallel:result is None 2024-09-21T09:47:00.129 INFO:tasks.rados.rados.0.smithi017.stdout:2077: done (0 left) 2024-09-21T09:47:00.129 INFO:tasks.rados.rados.0.smithi017.stdout:2078: write oid 20 current snap is 268 2024-09-21T09:47:00.129 INFO:tasks.rados.rados.0.smithi017.stdout:2078: seq_num 781 ranges {726267=637475,2119025=651272} 2024-09-21T09:47:00.137 INFO:tasks.rados.rados.0.smithi017.stdout:2078: writing smithi01785587-20 from 726267 to 1363742 tid 1 2024-09-21T09:47:00.145 INFO:tasks.rados.rados.0.smithi017.stdout:2078: writing smithi01785587-20 from 2119025 to 2770297 tid 2 2024-09-21T09:47:00.145 INFO:tasks.rados.rados.0.smithi017.stdout:2079: snap_create 2024-09-21T09:47:00.145 INFO:tasks.rados.rados.0.smithi017.stdout:2078: finishing write tid 1 to smithi01785587-20 2024-09-21T09:47:00.149 INFO:tasks.rados.rados.0.smithi017.stdout:2078: finishing write tid 2 to smithi01785587-20 2024-09-21T09:47:00.149 INFO:tasks.rados.rados.0.smithi017.stdout:2078: finishing write tid 3 to smithi01785587-20 2024-09-21T09:47:00.149 INFO:tasks.rados.rados.0.smithi017.stdout:2078: finishing write tid 4 to smithi01785587-20 2024-09-21T09:47:00.149 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 499 (ObjNum 781 snap 268 seq_num 781) dirty exists 2024-09-21T09:47:00.149 INFO:tasks.rados.rados.0.smithi017.stdout:2078: left oid 20 (ObjNum 781 snap 268 seq_num 781) 2024-09-21T09:47:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:00 smithi017 ceph-mon[102913]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T09:47:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:00 smithi017 ceph-mon[105151]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T09:47:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:00 smithi134 ceph-mon[90085]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T09:47:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:47:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:47:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:01.135 INFO:tasks.rados.rados.0.smithi017.stdout:2078: done (1 left) 2024-09-21T09:47:01.135 INFO:tasks.rados.rados.0.smithi017.stdout:2079: done (0 left) 2024-09-21T09:47:01.135 INFO:tasks.rados.rados.0.smithi017.stdout:2080: snap_remove snap 219 2024-09-21T09:47:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:01 smithi017 ceph-mon[102913]: pgmap v471: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 690 KiB/s wr, 3 op/s 2024-09-21T09:47:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:01 smithi017 ceph-mon[102913]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T09:47:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:01 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:47:01.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:01 smithi017 ceph-mon[105151]: pgmap v471: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 690 KiB/s wr, 3 op/s 2024-09-21T09:47:01.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:01 smithi017 ceph-mon[105151]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T09:47:01.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:01 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:47:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:01 smithi134 ceph-mon[90085]: pgmap v471: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 690 KiB/s wr, 3 op/s 2024-09-21T09:47:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:01 smithi134 ceph-mon[90085]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T09:47:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:01 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:47:02.144 INFO:tasks.rados.rados.0.smithi017.stdout:2080: done (0 left) 2024-09-21T09:47:02.145 INFO:tasks.rados.rados.0.smithi017.stdout:2081: write oid 8 current snap is 269 2024-09-21T09:47:02.145 INFO:tasks.rados.rados.0.smithi017.stdout:2081: seq_num 782 ranges {743074=712724,2061154=794930} 2024-09-21T09:47:02.153 INFO:tasks.rados.rados.0.smithi017.stdout:2081: writing smithi01785587-8 from 743074 to 1455798 tid 1 2024-09-21T09:47:02.161 INFO:tasks.rados.rados.0.smithi017.stdout:2081: writing smithi01785587-8 from 2061154 to 2856084 tid 2 2024-09-21T09:47:02.162 INFO:tasks.rados.rados.0.smithi017.stdout:2082: delete oid 15 current snap is 269 2024-09-21T09:47:02.162 INFO:tasks.rados.rados.0.smithi017.stdout:2081: finishing write tid 1 to smithi01785587-8 2024-09-21T09:47:02.163 INFO:tasks.rados.rados.0.smithi017.stdout:2082: done (1 left) 2024-09-21T09:47:02.163 INFO:tasks.rados.rados.0.smithi017.stdout:2083: delete oid 34 current snap is 269 2024-09-21T09:47:02.165 INFO:tasks.rados.rados.0.smithi017.stdout:2081: finishing write tid 2 to smithi01785587-8 2024-09-21T09:47:02.166 INFO:tasks.rados.rados.0.smithi017.stdout:2081: finishing write tid 3 to smithi01785587-8 2024-09-21T09:47:02.166 INFO:tasks.rados.rados.0.smithi017.stdout:2083: done (1 left) 2024-09-21T09:47:02.166 INFO:tasks.rados.rados.0.smithi017.stdout:2084: write oid 9 current snap is 269 2024-09-21T09:47:02.166 INFO:tasks.rados.rados.0.smithi017.stdout:2084: seq_num 785 ranges {793495=654824,1952953=658610,3157167=36328} 2024-09-21T09:47:02.173 INFO:tasks.rados.rados.0.smithi017.stdout:2084: writing smithi01785587-9 from 793495 to 1448319 tid 1 2024-09-21T09:47:02.180 INFO:tasks.rados.rados.0.smithi017.stdout:2084: writing smithi01785587-9 from 1952953 to 2611563 tid 2 2024-09-21T09:47:02.184 INFO:tasks.rados.rados.0.smithi017.stdout:2084: writing smithi01785587-9 from 3157167 to 3193495 tid 3 2024-09-21T09:47:02.184 INFO:tasks.rados.rados.0.smithi017.stdout:2085: snap_create 2024-09-21T09:47:02.184 INFO:tasks.rados.rados.0.smithi017.stdout:2081: finishing write tid 4 to smithi01785587-8 2024-09-21T09:47:02.184 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 859 (ObjNum 782 snap 269 seq_num 782) dirty exists 2024-09-21T09:47:02.184 INFO:tasks.rados.rados.0.smithi017.stdout:2081: left oid 8 (ObjNum 782 snap 269 seq_num 782) 2024-09-21T09:47:02.184 INFO:tasks.rados.rados.0.smithi017.stdout:2084: finishing write tid 1 to smithi01785587-9 2024-09-21T09:47:02.185 INFO:tasks.rados.rados.0.smithi017.stdout:2084: finishing write tid 2 to smithi01785587-9 2024-09-21T09:47:02.185 INFO:tasks.rados.rados.0.smithi017.stdout:2084: finishing write tid 3 to smithi01785587-9 2024-09-21T09:47:02.185 INFO:tasks.rados.rados.0.smithi017.stdout:2084: finishing write tid 4 to smithi01785587-9 2024-09-21T09:47:02.185 INFO:tasks.rados.rados.0.smithi017.stdout:2084: finishing write tid 5 to smithi01785587-9 2024-09-21T09:47:02.185 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 385 (ObjNum 785 snap 269 seq_num 785) dirty exists 2024-09-21T09:47:02.186 INFO:tasks.rados.rados.0.smithi017.stdout:2084: left oid 9 (ObjNum 785 snap 269 seq_num 785) 2024-09-21T09:47:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:02 smithi017 ceph-mon[105151]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T09:47:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:02 smithi017 ceph-mon[102913]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T09:47:02.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:02 smithi134 ceph-mon[90085]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T09:47:03.099 INFO:tasks.rados.rados.0.smithi017.stdout:2081: done (2 left) 2024-09-21T09:47:03.099 INFO:tasks.rados.rados.0.smithi017.stdout:2084: done (1 left) 2024-09-21T09:47:03.099 INFO:tasks.rados.rados.0.smithi017.stdout:2085: done (0 left) 2024-09-21T09:47:03.100 INFO:tasks.rados.rados.0.smithi017.stdout:2086: write oid 18 current snap is 270 2024-09-21T09:47:03.100 INFO:tasks.rados.rados.0.smithi017.stdout:2086: seq_num 786 ranges {410302=730073,1895082=402268,2951884=258418} 2024-09-21T09:47:03.105 INFO:tasks.rados.rados.0.smithi017.stdout:2086: writing smithi01785587-18 from 410302 to 1140375 tid 1 2024-09-21T09:47:03.112 INFO:tasks.rados.rados.0.smithi017.stdout:2086: writing smithi01785587-18 from 1895082 to 2297350 tid 2 2024-09-21T09:47:03.118 INFO:tasks.rados.rados.0.smithi017.stdout:2086: writing smithi01785587-18 from 2951884 to 3210302 tid 3 2024-09-21T09:47:03.118 INFO:tasks.rados.rados.0.smithi017.stdout:2087: snap_remove snap 264 2024-09-21T09:47:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: pgmap v474: 89 pgs: 89 active+clean; 209 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 617 KiB/s wr, 3 op/s 2024-09-21T09:47:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T09:47:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[102913]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: pgmap v474: 89 pgs: 89 active+clean; 209 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 617 KiB/s wr, 3 op/s 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T09:47:03.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:03.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:03.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:03.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:47:03.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:03.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:03.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:03.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:47:03.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:03 smithi017 ceph-mon[105151]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: pgmap v474: 89 pgs: 89 active+clean; 209 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 617 KiB/s wr, 3 op/s 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:47:03.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:03 smithi134 ceph-mon[90085]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T09:47:04.100 INFO:tasks.rados.rados.0.smithi017.stdout:2087: done (1 left) 2024-09-21T09:47:04.100 INFO:tasks.rados.rados.0.smithi017.stdout:2088: delete oid 38 current snap is 270 2024-09-21T09:47:04.101 INFO:tasks.rados.rados.0.smithi017.stdout:2086: finishing write tid 1 to smithi01785587-18 2024-09-21T09:47:04.101 INFO:tasks.rados.rados.0.smithi017.stdout:2086: finishing write tid 2 to smithi01785587-18 2024-09-21T09:47:04.101 INFO:tasks.rados.rados.0.smithi017.stdout:2086: finishing write tid 3 to smithi01785587-18 2024-09-21T09:47:04.101 INFO:tasks.rados.rados.0.smithi017.stdout:2086: finishing write tid 4 to smithi01785587-18 2024-09-21T09:47:04.101 INFO:tasks.rados.rados.0.smithi017.stdout:2086: finishing write tid 5 to smithi01785587-18 2024-09-21T09:47:04.101 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 164 (ObjNum 786 snap 270 seq_num 786) dirty exists 2024-09-21T09:47:04.101 INFO:tasks.rados.rados.0.smithi017.stdout:2086: left oid 18 (ObjNum 786 snap 270 seq_num 786) 2024-09-21T09:47:04.110 INFO:tasks.rados.rados.0.smithi017.stdout:2086: done (1 left) 2024-09-21T09:47:04.110 INFO:tasks.rados.rados.0.smithi017.stdout:2088: done (0 left) 2024-09-21T09:47:04.110 INFO:tasks.rados.rados.0.smithi017.stdout:2089: read oid 43 snap -1 2024-09-21T09:47:04.110 INFO:tasks.rados.rados.0.smithi017.stdout:2089: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:47:04.110 INFO:tasks.rados.rados.0.smithi017.stdout:2090: snap_remove snap 248 2024-09-21T09:47:05.103 INFO:tasks.rados.rados.0.smithi017.stdout:2090: done (1 left) 2024-09-21T09:47:05.103 INFO:tasks.rados.rados.0.smithi017.stdout:2091: rollback oid 46 current snap is 270 2024-09-21T09:47:05.104 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 258 2024-09-21T09:47:05.104 INFO:tasks.rados.rados.0.smithi017.stdout:2089: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:47:05.106 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:47:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:47:04] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.51.0" 2024-09-21T09:47:05.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:05 smithi017 ceph-mon[102913]: pgmap v477: 89 pgs: 89 active+clean; 206 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 848 KiB/s rd, 617 KiB/s wr, 5 op/s 2024-09-21T09:47:05.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:05 smithi017 ceph-mon[102913]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T09:47:05.217 INFO:tasks.rados.rados.0.smithi017.stdout:2091: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:47:05.217 INFO:tasks.rados.rados.0.smithi017.stdout:2091: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:47:05.217 INFO:tasks.rados.rados.0.smithi017.stdout:2089: done (1 left) 2024-09-21T09:47:05.217 INFO:tasks.rados.rados.0.smithi017.stdout:2092: snap_create 2024-09-21T09:47:05.217 INFO:tasks.rados.rados.0.smithi017.stdout:2091: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:47:05.217 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 867 (ObjNum 537 snap 169 seq_num 537) dirty exists 2024-09-21T09:47:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:05 smithi017 ceph-mon[105151]: pgmap v477: 89 pgs: 89 active+clean; 206 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 848 KiB/s rd, 617 KiB/s wr, 5 op/s 2024-09-21T09:47:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:05 smithi017 ceph-mon[105151]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T09:47:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:05 smithi134 ceph-mon[90085]: pgmap v477: 89 pgs: 89 active+clean; 206 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 848 KiB/s rd, 617 KiB/s wr, 5 op/s 2024-09-21T09:47:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:05 smithi134 ceph-mon[90085]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T09:47:06.115 INFO:tasks.rados.rados.0.smithi017.stdout:2091: done (1 left) 2024-09-21T09:47:06.115 INFO:tasks.rados.rados.0.smithi017.stdout:2092: done (0 left) 2024-09-21T09:47:06.115 INFO:tasks.rados.rados.0.smithi017.stdout:2093: read oid 24 snap -1 2024-09-21T09:47:06.115 INFO:tasks.rados.rados.0.smithi017.stdout:2093: expect (ObjNum 741 snap 250 seq_num 741) 2024-09-21T09:47:06.115 INFO:tasks.rados.rados.0.smithi017.stdout:2094: read oid 23 snap -1 2024-09-21T09:47:06.115 INFO:tasks.rados.rados.0.smithi017.stdout:2094: expect (ObjNum 779 snap 268 seq_num 779) 2024-09-21T09:47:06.115 INFO:tasks.rados.rados.0.smithi017.stdout:2095: delete oid 46 current snap is 271 2024-09-21T09:47:06.123 INFO:tasks.rados.rados.0.smithi017.stdout:2093: expect (ObjNum 741 snap 250 seq_num 741) 2024-09-21T09:47:06.248 INFO:tasks.rados.rados.0.smithi017.stdout:2094: expect (ObjNum 779 snap 268 seq_num 779) 2024-09-21T09:47:06.429 INFO:tasks.rados.rados.0.smithi017.stdout:2093: done (2 left) 2024-09-21T09:47:06.429 INFO:tasks.rados.rados.0.smithi017.stdout:2094: done (1 left) 2024-09-21T09:47:06.429 INFO:tasks.rados.rados.0.smithi017.stdout:2095: done (0 left) 2024-09-21T09:47:06.430 INFO:tasks.rados.rados.0.smithi017.stdout:2096: write oid 24 current snap is 271 2024-09-21T09:47:06.430 INFO:tasks.rados.rados.0.smithi017.stdout:2096: seq_num 789 ranges {460723=555820,1670528=549595,2634153=626570} 2024-09-21T09:47:06.435 INFO:tasks.rados.rados.0.smithi017.stdout:2096: writing smithi01785587-24 from 460723 to 1016543 tid 1 2024-09-21T09:47:06.441 INFO:tasks.rados.rados.0.smithi017.stdout:2096: writing smithi01785587-24 from 1670528 to 2220123 tid 2 2024-09-21T09:47:06.447 INFO:tasks.rados.rados.0.smithi017.stdout:2096: writing smithi01785587-24 from 2634153 to 3260723 tid 3 2024-09-21T09:47:06.447 INFO:tasks.rados.rados.0.smithi017.stdout:2097: snap_remove snap 265 2024-09-21T09:47:06.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:06 smithi017 ceph-mon[102913]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T09:47:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:06 smithi017 ceph-mon[105151]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T09:47:06.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:06 smithi134 ceph-mon[90085]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T09:47:07.121 INFO:tasks.rados.rados.0.smithi017.stdout:2097: done (1 left) 2024-09-21T09:47:07.121 INFO:tasks.rados.rados.0.smithi017.stdout:2098: snap_remove snap 261 2024-09-21T09:47:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:07 smithi017 ceph-mon[105151]: pgmap v480: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 340 KiB/s wr, 5 op/s 2024-09-21T09:47:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:07 smithi017 ceph-mon[105151]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T09:47:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:07 smithi017 ceph-mon[102913]: pgmap v480: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 340 KiB/s wr, 5 op/s 2024-09-21T09:47:07.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:07 smithi017 ceph-mon[102913]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T09:47:07.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:07 smithi134 ceph-mon[90085]: pgmap v480: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 340 KiB/s wr, 5 op/s 2024-09-21T09:47:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:07 smithi134 ceph-mon[90085]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:2098: done (1 left) 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:2099: rollback oid 33 current snap is 271 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 235 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:2096: finishing write tid 1 to smithi01785587-24 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:2096: finishing write tid 2 to smithi01785587-24 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:2096: finishing write tid 3 to smithi01785587-24 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:2096: finishing write tid 4 to smithi01785587-24 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:2096: finishing write tid 5 to smithi01785587-24 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 659 (ObjNum 789 snap 271 seq_num 789) dirty exists 2024-09-21T09:47:08.102 INFO:tasks.rados.rados.0.smithi017.stdout:2096: left oid 24 (ObjNum 789 snap 271 seq_num 789) 2024-09-21T09:47:08.103 INFO:tasks.rados.rados.0.smithi017.stdout:2096: done (1 left) 2024-09-21T09:47:08.103 INFO:tasks.rados.rados.0.smithi017.stdout:2100: write oid 45 current snap is 271 2024-09-21T09:47:08.103 INFO:tasks.rados.rados.0.smithi017.stdout:2100: seq_num 790 ranges {477530=631069,1612657=409606,2545223=517596} 2024-09-21T09:47:08.108 INFO:tasks.rados.rados.0.smithi017.stdout:2100: writing smithi01785587-45 from 477530 to 1108599 tid 1 2024-09-21T09:47:08.113 INFO:tasks.rados.rados.0.smithi017.stdout:2100: writing smithi01785587-45 from 1612657 to 2022263 tid 2 2024-09-21T09:47:08.120 INFO:tasks.rados.rados.0.smithi017.stdout:2100: writing smithi01785587-45 from 2545223 to 3062819 tid 3 2024-09-21T09:47:08.120 INFO:tasks.rados.rados.0.smithi017.stdout:2101: snap_remove snap 267 2024-09-21T09:47:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:08 smithi017 ceph-mon[105151]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T09:47:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:08 smithi017 ceph-mon[105151]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T09:47:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:08 smithi017 ceph-mon[102913]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T09:47:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:08 smithi017 ceph-mon[102913]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T09:47:08.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:08 smithi134 ceph-mon[90085]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T09:47:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:08 smithi134 ceph-mon[90085]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T09:47:09.127 INFO:tasks.rados.rados.0.smithi017.stdout:2101: done (2 left) 2024-09-21T09:47:09.127 INFO:tasks.rados.rados.0.smithi017.stdout:2102: read oid 5 snap -1 2024-09-21T09:47:09.128 INFO:tasks.rados.rados.0.smithi017.stdout:2102: expect (ObjNum 737 snap 246 seq_num 737) 2024-09-21T09:47:09.128 INFO:tasks.rados.rados.0.smithi017.stdout:2103: snap_remove snap 269 2024-09-21T09:47:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:09 smithi017 ceph-mon[105151]: pgmap v483: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:47:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:09 smithi017 ceph-mon[102913]: pgmap v483: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:47:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:09 smithi134 ceph-mon[90085]: pgmap v483: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 197 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:47:10.141 INFO:tasks.rados.rados.0.smithi017.stdout:2103: done (3 left) 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2104: read oid 4 snap -1 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2104: expect deleted 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2105: write oid 46 current snap is 271 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2099: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2099: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2099: finishing rollback tid 2 to smithi01785587-33 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 990 (ObjNum 638 snap 202 seq_num 638) dirty exists 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2100: finishing write tid 1 to smithi01785587-45 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2100: finishing write tid 2 to smithi01785587-45 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2100: finishing write tid 3 to smithi01785587-45 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2100: finishing write tid 4 to smithi01785587-45 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:2100: finishing write tid 5 to smithi01785587-45 2024-09-21T09:47:10.142 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 632 (ObjNum 790 snap 271 seq_num 790) dirty exists 2024-09-21T09:47:10.143 INFO:tasks.rados.rados.0.smithi017.stdout:2100: left oid 45 (ObjNum 790 snap 271 seq_num 790) 2024-09-21T09:47:10.143 INFO:tasks.rados.rados.0.smithi017.stdout:2102: expect (ObjNum 737 snap 246 seq_num 737) 2024-09-21T09:47:10.266 INFO:tasks.rados.rados.0.smithi017.stdout:2105: seq_num 791 ranges {494337=422671,1387492=553264,2688999=445221} 2024-09-21T09:47:10.271 INFO:tasks.rados.rados.0.smithi017.stdout:2105: writing smithi01785587-46 from 494337 to 917008 tid 1 2024-09-21T09:47:10.277 INFO:tasks.rados.rados.0.smithi017.stdout:2105: writing smithi01785587-46 from 1387492 to 1940756 tid 2 2024-09-21T09:47:10.284 INFO:tasks.rados.rados.0.smithi017.stdout:2105: writing smithi01785587-46 from 2688999 to 3134220 tid 3 2024-09-21T09:47:10.285 INFO:tasks.rados.rados.0.smithi017.stdout:2099: done (4 left) 2024-09-21T09:47:10.285 INFO:tasks.rados.rados.0.smithi017.stdout:2100: done (3 left) 2024-09-21T09:47:10.285 INFO:tasks.rados.rados.0.smithi017.stdout:2102: done (2 left) 2024-09-21T09:47:10.285 INFO:tasks.rados.rados.0.smithi017.stdout:2104: done (1 left) 2024-09-21T09:47:10.285 INFO:tasks.rados.rados.0.smithi017.stdout:2106: read oid 25 snap -1 2024-09-21T09:47:10.285 INFO:tasks.rados.rados.0.smithi017.stdout:2106: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:47:10.285 INFO:tasks.rados.rados.0.smithi017.stdout:2107: delete oid 32 current snap is 271 2024-09-21T09:47:10.285 INFO:tasks.rados.rados.0.smithi017.stdout:2105: finishing write tid 1 to smithi01785587-46 2024-09-21T09:47:10.285 INFO:tasks.rados.rados.0.smithi017.stdout:2105: finishing write tid 2 to smithi01785587-46 2024-09-21T09:47:10.288 INFO:tasks.rados.rados.0.smithi017.stdout:2105: finishing write tid 3 to smithi01785587-46 2024-09-21T09:47:10.288 INFO:tasks.rados.rados.0.smithi017.stdout:2105: finishing write tid 4 to smithi01785587-46 2024-09-21T09:47:10.288 INFO:tasks.rados.rados.0.smithi017.stdout:2105: finishing write tid 5 to smithi01785587-46 2024-09-21T09:47:10.289 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 895 (ObjNum 791 snap 271 seq_num 791) dirty exists 2024-09-21T09:47:10.289 INFO:tasks.rados.rados.0.smithi017.stdout:2105: left oid 46 (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:47:10.289 INFO:tasks.rados.rados.0.smithi017.stdout:2105: done (2 left) 2024-09-21T09:47:10.289 INFO:tasks.rados.rados.0.smithi017.stdout:2107: done (1 left) 2024-09-21T09:47:10.289 INFO:tasks.rados.rados.0.smithi017.stdout:2108: write oid 30 current snap is 271 2024-09-21T09:47:10.290 INFO:tasks.rados.rados.0.smithi017.stdout:2108: seq_num 793 ranges {527951=573169,1504456=556933,2743845=467765} 2024-09-21T09:47:10.295 INFO:tasks.rados.rados.0.smithi017.stdout:2108: writing smithi01785587-30 from 527951 to 1101120 tid 1 2024-09-21T09:47:10.300 INFO:tasks.rados.rados.0.smithi017.stdout:2108: writing smithi01785587-30 from 1504456 to 2061389 tid 2 2024-09-21T09:47:10.306 INFO:tasks.rados.rados.0.smithi017.stdout:2108: writing smithi01785587-30 from 2743845 to 3211610 tid 3 2024-09-21T09:47:10.307 INFO:tasks.rados.rados.0.smithi017.stdout:2109: snap_remove snap 268 2024-09-21T09:47:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:10 smithi017 ceph-mon[102913]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T09:47:10.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:10 smithi017 ceph-mon[105151]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T09:47:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:10 smithi134 ceph-mon[90085]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T09:47:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:47:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:47:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:11.142 INFO:tasks.rados.rados.0.smithi017.stdout:2109: done (2 left) 2024-09-21T09:47:11.142 INFO:tasks.rados.rados.0.smithi017.stdout:2110: delete oid 38 current snap is 271 2024-09-21T09:47:11.142 INFO:tasks.rados.rados.0.smithi017.stdout:2106: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:47:11.296 INFO:tasks.rados.rados.0.smithi017.stdout:2108: finishing write tid 1 to smithi01785587-30 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2108: finishing write tid 2 to smithi01785587-30 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2108: finishing write tid 3 to smithi01785587-30 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2108: finishing write tid 4 to smithi01785587-30 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2108: finishing write tid 5 to smithi01785587-30 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 819 (ObjNum 793 snap 271 seq_num 793) dirty exists 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2108: left oid 30 (ObjNum 793 snap 271 seq_num 793) 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2106: done (2 left) 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2108: done (1 left) 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2110: done (0 left) 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2111: read oid 20 snap -1 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2111: expect (ObjNum 781 snap 268 seq_num 781) 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2112: read oid 33 snap -1 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2112: expect (ObjNum 638 snap 202 seq_num 638) 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2113: read oid 42 snap -1 2024-09-21T09:47:11.297 INFO:tasks.rados.rados.0.smithi017.stdout:2113: expect deleted 2024-09-21T09:47:11.298 INFO:tasks.rados.rados.0.smithi017.stdout:2114: snap_remove snap 258 2024-09-21T09:47:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:11 smithi017 ceph-mon[102913]: pgmap v486: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:47:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:11 smithi017 ceph-mon[102913]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T09:47:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:11 smithi017 ceph-mon[105151]: pgmap v486: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:47:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:11 smithi017 ceph-mon[105151]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T09:47:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:11 smithi134 ceph-mon[90085]: pgmap v486: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 200 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:47:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:11 smithi134 ceph-mon[90085]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T09:47:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:12.157 INFO:tasks.rados.rados.0.smithi017.stdout:2112: expect (ObjNum 638 snap 202 seq_num 638) 2024-09-21T09:47:12.253 INFO:tasks.rados.rados.0.smithi017.stdout:2111: expect (ObjNum 781 snap 268 seq_num 781) 2024-09-21T09:47:12.359 INFO:tasks.rados.rados.0.smithi017.stdout:2111: done (3 left) 2024-09-21T09:47:12.359 INFO:tasks.rados.rados.0.smithi017.stdout:2112: done (2 left) 2024-09-21T09:47:12.359 INFO:tasks.rados.rados.0.smithi017.stdout:2113: done (1 left) 2024-09-21T09:47:12.359 INFO:tasks.rados.rados.0.smithi017.stdout:2114: done (0 left) 2024-09-21T09:47:12.359 INFO:tasks.rados.rados.0.smithi017.stdout:2115: delete oid 36 current snap is 271 2024-09-21T09:47:12.361 INFO:tasks.rados.rados.0.smithi017.stdout:2115: done (0 left) 2024-09-21T09:47:12.361 INFO:tasks.rados.rados.0.smithi017.stdout:2116: snap_remove snap 254 2024-09-21T09:47:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:12 smithi017 ceph-mon[102913]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T09:47:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:12 smithi017 ceph-mon[105151]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T09:47:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:12 smithi134 ceph-mon[90085]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T09:47:13.103 INFO:tasks.rados.rados.0.smithi017.stdout:2116: done (0 left) 2024-09-21T09:47:13.103 INFO:tasks.rados.rados.0.smithi017.stdout:2117: snap_create 2024-09-21T09:47:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:13 smithi017 ceph-mon[105151]: pgmap v489: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-09-21T09:47:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:13 smithi017 ceph-mon[105151]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T09:47:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:13 smithi017 ceph-mon[105151]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T09:47:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:13 smithi017 ceph-mon[102913]: pgmap v489: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-09-21T09:47:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:13 smithi017 ceph-mon[102913]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T09:47:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:13 smithi017 ceph-mon[102913]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T09:47:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:13 smithi134 ceph-mon[90085]: pgmap v489: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-09-21T09:47:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:13 smithi134 ceph-mon[90085]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T09:47:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:13 smithi134 ceph-mon[90085]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T09:47:14.103 INFO:tasks.rados.rados.0.smithi017.stdout:2117: done (0 left) 2024-09-21T09:47:14.103 INFO:tasks.rados.rados.0.smithi017.stdout:2118: write oid 17 current snap is 272 2024-09-21T09:47:14.104 INFO:tasks.rados.rados.0.smithi017.stdout:2118: seq_num 796 ranges {578372=798916,1963549=420613,2826114=417934} 2024-09-21T09:47:14.111 INFO:tasks.rados.rados.0.smithi017.stdout:2118: writing smithi01785587-17 from 578372 to 1377288 tid 1 2024-09-21T09:47:14.116 INFO:tasks.rados.rados.0.smithi017.stdout:2118: writing smithi01785587-17 from 1963549 to 2384162 tid 2 2024-09-21T09:47:14.121 INFO:tasks.rados.rados.0.smithi017.stdout:2118: writing smithi01785587-17 from 2826114 to 3244048 tid 3 2024-09-21T09:47:14.121 INFO:tasks.rados.rados.0.smithi017.stdout:2119: write oid 49 current snap is 272 2024-09-21T09:47:14.121 INFO:tasks.rados.rados.0.smithi017.stdout:2118: finishing write tid 1 to smithi01785587-17 2024-09-21T09:47:14.122 INFO:tasks.rados.rados.0.smithi017.stdout:2119: seq_num 797 ranges {595179=474165,1622031=564271,2737184=629206} 2024-09-21T09:47:14.127 INFO:tasks.rados.rados.0.smithi017.stdout:2119: writing smithi01785587-49 from 595179 to 1069344 tid 1 2024-09-21T09:47:14.133 INFO:tasks.rados.rados.0.smithi017.stdout:2119: writing smithi01785587-49 from 1622031 to 2186302 tid 2 2024-09-21T09:47:14.140 INFO:tasks.rados.rados.0.smithi017.stdout:2119: writing smithi01785587-49 from 2737184 to 3366390 tid 3 2024-09-21T09:47:14.140 INFO:tasks.rados.rados.0.smithi017.stdout:2120: delete oid 4 current snap is 272 2024-09-21T09:47:14.140 INFO:tasks.rados.rados.0.smithi017.stdout:2118: finishing write tid 2 to smithi01785587-17 2024-09-21T09:47:14.141 INFO:tasks.rados.rados.0.smithi017.stdout:2118: finishing write tid 3 to smithi01785587-17 2024-09-21T09:47:14.141 INFO:tasks.rados.rados.0.smithi017.stdout:2118: finishing write tid 4 to smithi01785587-17 2024-09-21T09:47:14.141 INFO:tasks.rados.rados.0.smithi017.stdout:2118: finishing write tid 5 to smithi01785587-17 2024-09-21T09:47:14.141 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 640 (ObjNum 796 snap 272 seq_num 796) dirty exists 2024-09-21T09:47:14.141 INFO:tasks.rados.rados.0.smithi017.stdout:2118: left oid 17 (ObjNum 796 snap 272 seq_num 796) 2024-09-21T09:47:14.141 INFO:tasks.rados.rados.0.smithi017.stdout:2119: finishing write tid 1 to smithi01785587-49 2024-09-21T09:47:14.141 INFO:tasks.rados.rados.0.smithi017.stdout:2119: finishing write tid 2 to smithi01785587-49 2024-09-21T09:47:14.144 INFO:tasks.rados.rados.0.smithi017.stdout:2119: finishing write tid 3 to smithi01785587-49 2024-09-21T09:47:14.144 INFO:tasks.rados.rados.0.smithi017.stdout:2119: finishing write tid 4 to smithi01785587-49 2024-09-21T09:47:14.145 INFO:tasks.rados.rados.0.smithi017.stdout:2119: finishing write tid 5 to smithi01785587-49 2024-09-21T09:47:14.145 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1117 (ObjNum 797 snap 272 seq_num 797) dirty exists 2024-09-21T09:47:14.145 INFO:tasks.rados.rados.0.smithi017.stdout:2119: left oid 49 (ObjNum 797 snap 272 seq_num 797) 2024-09-21T09:47:14.145 INFO:tasks.rados.rados.0.smithi017.stdout:2118: done (2 left) 2024-09-21T09:47:14.145 INFO:tasks.rados.rados.0.smithi017.stdout:2119: done (1 left) 2024-09-21T09:47:14.145 INFO:tasks.rados.rados.0.smithi017.stdout:2120: done (0 left) 2024-09-21T09:47:14.145 INFO:tasks.rados.rados.0.smithi017.stdout:2121: snap_create 2024-09-21T09:47:15.101 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:47:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:47:14] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.51.0" 2024-09-21T09:47:15.109 INFO:tasks.rados.rados.0.smithi017.stdout:2121: done (0 left) 2024-09-21T09:47:15.109 INFO:tasks.rados.rados.0.smithi017.stdout:2122: read oid 4 snap 270 2024-09-21T09:47:15.109 INFO:tasks.rados.rados.0.smithi017.stdout:2122: expect deleted 2024-09-21T09:47:15.109 INFO:tasks.rados.rados.0.smithi017.stdout:2123: snap_create 2024-09-21T09:47:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:15 smithi017 ceph-mon[102913]: pgmap v492: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:47:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:15 smithi017 ceph-mon[102913]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T09:47:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:15 smithi017 ceph-mon[105151]: pgmap v492: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:47:15.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:15 smithi017 ceph-mon[105151]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T09:47:15.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:15 smithi134 ceph-mon[90085]: pgmap v492: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:47:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:15 smithi134 ceph-mon[90085]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T09:47:16.114 INFO:tasks.rados.rados.0.smithi017.stdout:2122: done (1 left) 2024-09-21T09:47:16.115 INFO:tasks.rados.rados.0.smithi017.stdout:2123: done (0 left) 2024-09-21T09:47:16.115 INFO:tasks.rados.rados.0.smithi017.stdout:2124: snap_remove snap 235 2024-09-21T09:47:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:16 smithi017 ceph-mon[102913]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T09:47:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:16 smithi017 ceph-mon[105151]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T09:47:16.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:16 smithi134 ceph-mon[90085]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T09:47:17.116 INFO:tasks.rados.rados.0.smithi017.stdout:2124: done (0 left) 2024-09-21T09:47:17.116 INFO:tasks.rados.rados.0.smithi017.stdout:2125: snap_remove snap 270 2024-09-21T09:47:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:17 smithi017 ceph-mon[105151]: pgmap v495: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-09-21T09:47:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:17 smithi017 ceph-mon[105151]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T09:47:17.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:17 smithi017 ceph-mon[102913]: pgmap v495: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-09-21T09:47:17.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:17 smithi017 ceph-mon[102913]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T09:47:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:17 smithi134 ceph-mon[90085]: pgmap v495: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-09-21T09:47:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:17 smithi134 ceph-mon[90085]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T09:47:18.105 INFO:tasks.rados.rados.0.smithi017.stdout:2125: done (0 left) 2024-09-21T09:47:18.105 INFO:tasks.rados.rados.0.smithi017.stdout:2126: delete oid 34 current snap is 274 2024-09-21T09:47:18.112 INFO:tasks.rados.rados.0.smithi017.stdout:2126: done (0 left) 2024-09-21T09:47:18.112 INFO:tasks.rados.rados.0.smithi017.stdout:2127: write oid 31 current snap is 274 2024-09-21T09:47:18.112 INFO:tasks.rados.rados.0.smithi017.stdout:2127: seq_num 800 ranges {645600=416265,1797477=711598,3219453=226147} 2024-09-21T09:47:18.118 INFO:tasks.rados.rados.0.smithi017.stdout:2127: writing smithi01785587-31 from 645600 to 1061865 tid 1 2024-09-21T09:47:18.126 INFO:tasks.rados.rados.0.smithi017.stdout:2127: writing smithi01785587-31 from 1797477 to 2509075 tid 2 2024-09-21T09:47:18.132 INFO:tasks.rados.rados.0.smithi017.stdout:2127: writing smithi01785587-31 from 3219453 to 3445600 tid 3 2024-09-21T09:47:18.132 INFO:tasks.rados.rados.0.smithi017.stdout:2128: read oid 23 snap 256 2024-09-21T09:47:18.132 INFO:tasks.rados.rados.0.smithi017.stdout:2128: expect (ObjNum 759 snap 256 seq_num 759) 2024-09-21T09:47:18.132 INFO:tasks.rados.rados.0.smithi017.stdout:2129: snap_create 2024-09-21T09:47:18.132 INFO:tasks.rados.rados.0.smithi017.stdout:2127: finishing write tid 1 to smithi01785587-31 2024-09-21T09:47:18.132 INFO:tasks.rados.rados.0.smithi017.stdout:2127: finishing write tid 2 to smithi01785587-31 2024-09-21T09:47:18.134 INFO:tasks.rados.rados.0.smithi017.stdout:2127: finishing write tid 3 to smithi01785587-31 2024-09-21T09:47:18.135 INFO:tasks.rados.rados.0.smithi017.stdout:2127: finishing write tid 4 to smithi01785587-31 2024-09-21T09:47:18.135 INFO:tasks.rados.rados.0.smithi017.stdout:2127: finishing write tid 5 to smithi01785587-31 2024-09-21T09:47:18.135 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1001 (ObjNum 800 snap 274 seq_num 800) dirty exists 2024-09-21T09:47:18.135 INFO:tasks.rados.rados.0.smithi017.stdout:2127: left oid 31 (ObjNum 800 snap 274 seq_num 800) 2024-09-21T09:47:18.137 INFO:tasks.rados.rados.0.smithi017.stdout:2128: expect (ObjNum 759 snap 256 seq_num 759) 2024-09-21T09:47:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:18 smithi017 ceph-mon[105151]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T09:47:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:18 smithi017 ceph-mon[105151]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T09:47:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:18 smithi017 ceph-mon[102913]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T09:47:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:18 smithi017 ceph-mon[102913]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T09:47:18.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:18 smithi134 ceph-mon[90085]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T09:47:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:18 smithi134 ceph-mon[90085]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T09:47:19.128 INFO:tasks.rados.rados.0.smithi017.stdout:2127: done (2 left) 2024-09-21T09:47:19.129 INFO:tasks.rados.rados.0.smithi017.stdout:2128: done (1 left) 2024-09-21T09:47:19.129 INFO:tasks.rados.rados.0.smithi017.stdout:2129: done (0 left) 2024-09-21T09:47:19.129 INFO:tasks.rados.rados.0.smithi017.stdout:2130: write oid 9 current snap is 275 2024-09-21T09:47:19.129 INFO:tasks.rados.rados.0.smithi017.stdout:2130: seq_num 801 ranges {662407=491514,1855959=571609,2846876=615531} 2024-09-21T09:47:19.135 INFO:tasks.rados.rados.0.smithi017.stdout:2130: writing smithi01785587-9 from 662407 to 1153921 tid 1 2024-09-21T09:47:19.142 INFO:tasks.rados.rados.0.smithi017.stdout:2130: writing smithi01785587-9 from 1855959 to 2427568 tid 2 2024-09-21T09:47:19.148 INFO:tasks.rados.rados.0.smithi017.stdout:2130: writing smithi01785587-9 from 2846876 to 3462407 tid 3 2024-09-21T09:47:19.148 INFO:tasks.rados.rados.0.smithi017.stdout:2131: read oid 10 snap 266 2024-09-21T09:47:19.148 INFO:tasks.rados.rados.0.smithi017.stdout:2131: expect deleted 2024-09-21T09:47:19.148 INFO:tasks.rados.rados.0.smithi017.stdout:2132: snap_create 2024-09-21T09:47:19.148 INFO:tasks.rados.rados.0.smithi017.stdout:2130: finishing write tid 1 to smithi01785587-9 2024-09-21T09:47:19.148 INFO:tasks.rados.rados.0.smithi017.stdout:2130: finishing write tid 2 to smithi01785587-9 2024-09-21T09:47:19.151 INFO:tasks.rados.rados.0.smithi017.stdout:2130: finishing write tid 3 to smithi01785587-9 2024-09-21T09:47:19.151 INFO:tasks.rados.rados.0.smithi017.stdout:2130: finishing write tid 4 to smithi01785587-9 2024-09-21T09:47:19.151 INFO:tasks.rados.rados.0.smithi017.stdout:2130: finishing write tid 5 to smithi01785587-9 2024-09-21T09:47:19.151 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 414 (ObjNum 801 snap 275 seq_num 801) dirty exists 2024-09-21T09:47:19.151 INFO:tasks.rados.rados.0.smithi017.stdout:2130: left oid 9 (ObjNum 801 snap 275 seq_num 801) 2024-09-21T09:47:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:19 smithi017 ceph-mon[105151]: pgmap v498: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 408 KiB/s wr, 5 op/s 2024-09-21T09:47:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:19 smithi017 ceph-mon[102913]: pgmap v498: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 408 KiB/s wr, 5 op/s 2024-09-21T09:47:19.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:19 smithi134 ceph-mon[90085]: pgmap v498: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 408 KiB/s wr, 5 op/s 2024-09-21T09:47:20.140 INFO:tasks.rados.rados.0.smithi017.stdout:2130: done (2 left) 2024-09-21T09:47:20.140 INFO:tasks.rados.rados.0.smithi017.stdout:2131: done (1 left) 2024-09-21T09:47:20.140 INFO:tasks.rados.rados.0.smithi017.stdout:2132: done (0 left) 2024-09-21T09:47:20.140 INFO:tasks.rados.rados.0.smithi017.stdout:2133: read oid 3 snap -1 2024-09-21T09:47:20.140 INFO:tasks.rados.rados.0.smithi017.stdout:2133: expect (ObjNum 774 snap 264 seq_num 774) 2024-09-21T09:47:20.140 INFO:tasks.rados.rados.0.smithi017.stdout:2134: write oid 39 current snap is 276 2024-09-21T09:47:20.140 INFO:tasks.rados.rados.0.smithi017.stdout:2134: seq_num 802 ranges {679214=566763,1798088=715267,3157946=321268} 2024-09-21T09:47:20.146 INFO:tasks.rados.rados.0.smithi017.stdout:2134: writing smithi01785587-39 from 679214 to 1245977 tid 1 2024-09-21T09:47:20.153 INFO:tasks.rados.rados.0.smithi017.stdout:2134: writing smithi01785587-39 from 1798088 to 2513355 tid 2 2024-09-21T09:47:20.159 INFO:tasks.rados.rados.0.smithi017.stdout:2134: writing smithi01785587-39 from 3157946 to 3479214 tid 3 2024-09-21T09:47:20.159 INFO:tasks.rados.rados.0.smithi017.stdout:2135: read oid 5 snap 272 2024-09-21T09:47:20.159 INFO:tasks.rados.rados.0.smithi017.stdout:2135: expect (ObjNum 737 snap 246 seq_num 737) 2024-09-21T09:47:20.160 INFO:tasks.rados.rados.0.smithi017.stdout:2136: snap_create 2024-09-21T09:47:20.160 INFO:tasks.rados.rados.0.smithi017.stdout:2133: expect (ObjNum 774 snap 264 seq_num 774) 2024-09-21T09:47:20.297 INFO:tasks.rados.rados.0.smithi017.stdout:2134: finishing write tid 1 to smithi01785587-39 2024-09-21T09:47:20.297 INFO:tasks.rados.rados.0.smithi017.stdout:2134: finishing write tid 2 to smithi01785587-39 2024-09-21T09:47:20.297 INFO:tasks.rados.rados.0.smithi017.stdout:2134: finishing write tid 3 to smithi01785587-39 2024-09-21T09:47:20.298 INFO:tasks.rados.rados.0.smithi017.stdout:2134: finishing write tid 4 to smithi01785587-39 2024-09-21T09:47:20.298 INFO:tasks.rados.rados.0.smithi017.stdout:2135: expect (ObjNum 737 snap 246 seq_num 737) 2024-09-21T09:47:20.421 INFO:tasks.rados.rados.0.smithi017.stdout:2134: finishing write tid 5 to smithi01785587-39 2024-09-21T09:47:20.421 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1104 (ObjNum 802 snap 276 seq_num 802) dirty exists 2024-09-21T09:47:20.421 INFO:tasks.rados.rados.0.smithi017.stdout:2134: left oid 39 (ObjNum 802 snap 276 seq_num 802) 2024-09-21T09:47:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:20 smithi017 ceph-mon[102913]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T09:47:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:20 smithi017 ceph-mon[105151]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T09:47:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:20 smithi134 ceph-mon[90085]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T09:47:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:47:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:47:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:21.145 INFO:tasks.rados.rados.0.smithi017.stdout:2133: done (3 left) 2024-09-21T09:47:21.145 INFO:tasks.rados.rados.0.smithi017.stdout:2134: done (2 left) 2024-09-21T09:47:21.145 INFO:tasks.rados.rados.0.smithi017.stdout:2135: done (1 left) 2024-09-21T09:47:21.146 INFO:tasks.rados.rados.0.smithi017.stdout:2136: done (0 left) 2024-09-21T09:47:21.146 INFO:tasks.rados.rados.0.smithi017.stdout:2137: delete oid 11 current snap is 277 2024-09-21T09:47:21.149 INFO:tasks.rados.rados.0.smithi017.stdout:2137: done (0 left) 2024-09-21T09:47:21.149 INFO:tasks.rados.rados.0.smithi017.stdout:2138: snap_create 2024-09-21T09:47:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:21 smithi017 ceph-mon[102913]: pgmap v501: 89 pgs: 89 active+clean; 207 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 808 KiB/s wr, 2 op/s 2024-09-21T09:47:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:21 smithi017 ceph-mon[102913]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T09:47:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:21 smithi017 ceph-mon[105151]: pgmap v501: 89 pgs: 89 active+clean; 207 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 808 KiB/s wr, 2 op/s 2024-09-21T09:47:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:21 smithi017 ceph-mon[105151]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T09:47:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:21 smithi134 ceph-mon[90085]: pgmap v501: 89 pgs: 89 active+clean; 207 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 808 KiB/s wr, 2 op/s 2024-09-21T09:47:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:21 smithi134 ceph-mon[90085]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T09:47:22.152 INFO:tasks.rados.rados.0.smithi017.stdout:2138: done (0 left) 2024-09-21T09:47:22.152 INFO:tasks.rados.rados.0.smithi017.stdout:2139: read oid 43 snap -1 2024-09-21T09:47:22.152 INFO:tasks.rados.rados.0.smithi017.stdout:2139: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:47:22.152 INFO:tasks.rados.rados.0.smithi017.stdout:2140: read oid 1 snap -1 2024-09-21T09:47:22.152 INFO:tasks.rados.rados.0.smithi017.stdout:2140: expect (ObjNum 652 snap 205 seq_num 652) 2024-09-21T09:47:22.152 INFO:tasks.rados.rados.0.smithi017.stdout:2141: write oid 42 current snap is 278 2024-09-21T09:47:22.152 INFO:tasks.rados.rados.0.smithi017.stdout:2141: seq_num 804 ranges {712828=717261,1915052=718936,3212792=300036} 2024-09-21T09:47:22.159 INFO:tasks.rados.rados.0.smithi017.stdout:2141: writing smithi01785587-42 from 712828 to 1430089 tid 1 2024-09-21T09:47:22.166 INFO:tasks.rados.rados.0.smithi017.stdout:2141: writing smithi01785587-42 from 1915052 to 2633988 tid 2 2024-09-21T09:47:22.172 INFO:tasks.rados.rados.0.smithi017.stdout:2141: writing smithi01785587-42 from 3212792 to 3512828 tid 3 2024-09-21T09:47:22.172 INFO:tasks.rados.rados.0.smithi017.stdout:2142: delete oid 34 current snap is 278 2024-09-21T09:47:22.172 INFO:tasks.rados.rados.0.smithi017.stdout:2139: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:47:22.283 INFO:tasks.rados.rados.0.smithi017.stdout:2141: finishing write tid 1 to smithi01785587-42 2024-09-21T09:47:22.283 INFO:tasks.rados.rados.0.smithi017.stdout:2141: finishing write tid 2 to smithi01785587-42 2024-09-21T09:47:22.283 INFO:tasks.rados.rados.0.smithi017.stdout:2140: expect (ObjNum 652 snap 205 seq_num 652) 2024-09-21T09:47:22.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:22 smithi017 ceph-mon[102913]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T09:47:22.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:22 smithi017 ceph-mon[105151]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T09:47:22.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:22 smithi134 ceph-mon[90085]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2141: finishing write tid 3 to smithi01785587-42 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2141: finishing write tid 4 to smithi01785587-42 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2141: finishing write tid 5 to smithi01785587-42 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 772 (ObjNum 804 snap 278 seq_num 804) dirty exists 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2141: left oid 42 (ObjNum 804 snap 278 seq_num 804) 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2139: done (3 left) 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2140: done (2 left) 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2141: done (1 left) 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2142: done (0 left) 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2143: rollback oid 37 current snap is 278 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 239 2024-09-21T09:47:22.533 INFO:tasks.rados.rados.0.smithi017.stdout:2144: snap_remove snap 252 2024-09-21T09:47:23.107 INFO:tasks.rados.rados.0.smithi017.stdout:2143: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:47:23.107 INFO:tasks.rados.rados.0.smithi017.stdout:2143: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:47:23.107 INFO:tasks.rados.rados.0.smithi017.stdout:2143: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:47:23.107 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1108 (ObjNum 698 snap 221 seq_num 698) dirty exists 2024-09-21T09:47:23.107 INFO:tasks.rados.rados.0.smithi017.stdout:2143: done (1 left) 2024-09-21T09:47:23.107 INFO:tasks.rados.rados.0.smithi017.stdout:2144: done (0 left) 2024-09-21T09:47:23.107 INFO:tasks.rados.rados.0.smithi017.stdout:2145: read oid 28 snap -1 2024-09-21T09:47:23.107 INFO:tasks.rados.rados.0.smithi017.stdout:2145: expect deleted 2024-09-21T09:47:23.107 INFO:tasks.rados.rados.0.smithi017.stdout:2146: snap_create 2024-09-21T09:47:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:23 smithi017 ceph-mon[102913]: pgmap v504: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 731 KiB/s wr, 4 op/s 2024-09-21T09:47:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:23 smithi017 ceph-mon[102913]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T09:47:23.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:23 smithi017 ceph-mon[102913]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T09:47:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:23 smithi017 ceph-mon[105151]: pgmap v504: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 731 KiB/s wr, 4 op/s 2024-09-21T09:47:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:23 smithi017 ceph-mon[105151]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T09:47:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:23 smithi017 ceph-mon[105151]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T09:47:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:23 smithi134 ceph-mon[90085]: pgmap v504: 89 pgs: 89 active+clean; 208 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 731 KiB/s wr, 4 op/s 2024-09-21T09:47:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:23 smithi134 ceph-mon[90085]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T09:47:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:23 smithi134 ceph-mon[90085]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T09:47:24.106 INFO:tasks.rados.rados.0.smithi017.stdout:2145: done (1 left) 2024-09-21T09:47:24.106 INFO:tasks.rados.rados.0.smithi017.stdout:2146: done (0 left) 2024-09-21T09:47:24.106 INFO:tasks.rados.rados.0.smithi017.stdout:2147: delete oid 13 current snap is 279 2024-09-21T09:47:24.110 INFO:tasks.rados.rados.0.smithi017.stdout:2147: done (0 left) 2024-09-21T09:47:24.110 INFO:tasks.rados.rados.0.smithi017.stdout:2148: write oid 20 current snap is 279 2024-09-21T09:47:24.110 INFO:tasks.rados.rados.0.smithi017.stdout:2148: seq_num 807 ranges {763249=659361,2090498=582616,3295061=268188} 2024-09-21T09:47:24.118 INFO:tasks.rados.rados.0.smithi017.stdout:2148: writing smithi01785587-20 from 763249 to 1422610 tid 1 2024-09-21T09:47:24.125 INFO:tasks.rados.rados.0.smithi017.stdout:2148: writing smithi01785587-20 from 2090498 to 2673114 tid 2 2024-09-21T09:47:24.130 INFO:tasks.rados.rados.0.smithi017.stdout:2148: writing smithi01785587-20 from 3295061 to 3563249 tid 3 2024-09-21T09:47:24.130 INFO:tasks.rados.rados.0.smithi017.stdout:2149: rollback oid 43 current snap is 279 2024-09-21T09:47:24.130 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 277 2024-09-21T09:47:24.130 INFO:tasks.rados.rados.0.smithi017.stdout:2148: finishing write tid 1 to smithi01785587-20 2024-09-21T09:47:24.130 INFO:tasks.rados.rados.0.smithi017.stdout:2150: rollback oid 5 current snap is 279 2024-09-21T09:47:24.130 INFO:tasks.rados.rados.0.smithi017.stdout:2148: finishing write tid 2 to smithi01785587-20 2024-09-21T09:47:24.131 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 266 2024-09-21T09:47:24.131 INFO:tasks.rados.rados.0.smithi017.stdout:2151: write oid 16 current snap is 279 2024-09-21T09:47:24.131 INFO:tasks.rados.rados.0.smithi017.stdout:2151: seq_num 808 ranges {780056=734610,2148980=726274,3322484=257572} 2024-09-21T09:47:24.139 INFO:tasks.rados.rados.0.smithi017.stdout:2151: writing smithi01785587-16 from 780056 to 1514666 tid 1 2024-09-21T09:47:24.146 INFO:tasks.rados.rados.0.smithi017.stdout:2151: writing smithi01785587-16 from 2148980 to 2875254 tid 2 2024-09-21T09:47:24.151 INFO:tasks.rados.rados.0.smithi017.stdout:2151: writing smithi01785587-16 from 3322484 to 3580056 tid 3 2024-09-21T09:47:24.151 INFO:tasks.rados.rados.0.smithi017.stdout:2152: read oid 42 snap 266 2024-09-21T09:47:24.151 INFO:tasks.rados.rados.0.smithi017.stdout:2152: expect deleted 2024-09-21T09:47:24.151 INFO:tasks.rados.rados.0.smithi017.stdout:2153: write oid 17 current snap is 279 2024-09-21T09:47:24.151 INFO:tasks.rados.rados.0.smithi017.stdout:2148: finishing write tid 3 to smithi01785587-20 2024-09-21T09:47:24.152 INFO:tasks.rados.rados.0.smithi017.stdout:2148: finishing write tid 4 to smithi01785587-20 2024-09-21T09:47:24.152 INFO:tasks.rados.rados.0.smithi017.stdout:2148: finishing write tid 5 to smithi01785587-20 2024-09-21T09:47:24.152 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 532 (ObjNum 807 snap 279 seq_num 807) dirty exists 2024-09-21T09:47:24.152 INFO:tasks.rados.rados.0.smithi017.stdout:2148: left oid 20 (ObjNum 807 snap 279 seq_num 807) 2024-09-21T09:47:24.152 INFO:tasks.rados.rados.0.smithi017.stdout:2149: finishing rollback tid 0 to smithi01785587-43 2024-09-21T09:47:24.152 INFO:tasks.rados.rados.0.smithi017.stdout:2149: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:47:24.152 INFO:tasks.rados.rados.0.smithi017.stdout:2153: seq_num 809 ranges {796863=409859,1807462=586285,3066260=530603} 2024-09-21T09:47:24.157 INFO:tasks.rados.rados.0.smithi017.stdout:2153: writing smithi01785587-17 from 796863 to 1206722 tid 1 2024-09-21T09:47:24.164 INFO:tasks.rados.rados.0.smithi017.stdout:2153: writing smithi01785587-17 from 1807462 to 2393747 tid 2 2024-09-21T09:47:24.171 INFO:tasks.rados.rados.0.smithi017.stdout:2153: writing smithi01785587-17 from 3066260 to 3596863 tid 3 2024-09-21T09:47:24.171 INFO:tasks.rados.rados.0.smithi017.stdout:2148: done (5 left) 2024-09-21T09:47:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2154: write oid 31 current snap is 279 2024-09-21T09:47:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2149: finishing rollback tid 2 to smithi01785587-43 2024-09-21T09:47:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 645 (ObjNum 512 snap 160 seq_num 512) dirty exists 2024-09-21T09:47:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2150: finishing rollback tid 0 to smithi01785587-5 2024-09-21T09:47:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2150: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:47:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2154: seq_num 810 ranges {413670=485108,1465944=729943,2977330=524801} 2024-09-21T09:47:24.176 INFO:tasks.rados.rados.0.smithi017.stdout:2154: writing smithi01785587-31 from 413670 to 898778 tid 1 2024-09-21T09:47:24.183 INFO:tasks.rados.rados.0.smithi017.stdout:2154: writing smithi01785587-31 from 1465944 to 2195887 tid 2 2024-09-21T09:47:24.191 INFO:tasks.rados.rados.0.smithi017.stdout:2154: writing smithi01785587-31 from 2977330 to 3502131 tid 3 2024-09-21T09:47:24.191 INFO:tasks.rados.rados.0.smithi017.stdout:2149: done (5 left) 2024-09-21T09:47:24.191 INFO:tasks.rados.rados.0.smithi017.stdout:2155: write oid 11 current snap is 279 2024-09-21T09:47:24.191 INFO:tasks.rados.rados.0.smithi017.stdout:2155: seq_num 811 ranges {430477=560357,1408073=473601,2488400=736073} 2024-09-21T09:47:24.196 INFO:tasks.rados.rados.0.smithi017.stdout:2155: writing smithi01785587-11 from 430477 to 990834 tid 1 2024-09-21T09:47:24.201 INFO:tasks.rados.rados.0.smithi017.stdout:2155: writing smithi01785587-11 from 1408073 to 1881674 tid 2 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2155: writing smithi01785587-11 from 2488400 to 3224473 tid 3 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2156: delete oid 27 current snap is 279 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2150: finishing rollback tid 2 to smithi01785587-5 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1009 (ObjNum 737 snap 246 seq_num 737) dirty exists 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2151: finishing write tid 1 to smithi01785587-16 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2151: finishing write tid 2 to smithi01785587-16 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2151: finishing write tid 3 to smithi01785587-16 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2151: finishing write tid 4 to smithi01785587-16 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2151: finishing write tid 5 to smithi01785587-16 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1022 (ObjNum 808 snap 279 seq_num 808) dirty exists 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2151: left oid 16 (ObjNum 808 snap 279 seq_num 808) 2024-09-21T09:47:24.210 INFO:tasks.rados.rados.0.smithi017.stdout:2153: finishing write tid 1 to smithi01785587-17 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2153: finishing write tid 2 to smithi01785587-17 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2153: finishing write tid 3 to smithi01785587-17 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2153: finishing write tid 4 to smithi01785587-17 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2153: finishing write tid 5 to smithi01785587-17 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 649 (ObjNum 809 snap 279 seq_num 809) dirty exists 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2153: left oid 17 (ObjNum 809 snap 279 seq_num 809) 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2154: finishing write tid 1 to smithi01785587-31 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2154: finishing write tid 2 to smithi01785587-31 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2154: finishing write tid 3 to smithi01785587-31 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2154: finishing write tid 4 to smithi01785587-31 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:2154: finishing write tid 5 to smithi01785587-31 2024-09-21T09:47:24.211 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1014 (ObjNum 810 snap 279 seq_num 810) dirty exists 2024-09-21T09:47:24.212 INFO:tasks.rados.rados.0.smithi017.stdout:2154: left oid 31 (ObjNum 810 snap 279 seq_num 810) 2024-09-21T09:47:24.212 INFO:tasks.rados.rados.0.smithi017.stdout:2155: finishing write tid 1 to smithi01785587-11 2024-09-21T09:47:24.212 INFO:tasks.rados.rados.0.smithi017.stdout:2155: finishing write tid 2 to smithi01785587-11 2024-09-21T09:47:24.214 INFO:tasks.rados.rados.0.smithi017.stdout:2150: done (6 left) 2024-09-21T09:47:24.214 INFO:tasks.rados.rados.0.smithi017.stdout:2151: done (5 left) 2024-09-21T09:47:24.214 INFO:tasks.rados.rados.0.smithi017.stdout:2152: done (4 left) 2024-09-21T09:47:24.214 INFO:tasks.rados.rados.0.smithi017.stdout:2153: done (3 left) 2024-09-21T09:47:24.214 INFO:tasks.rados.rados.0.smithi017.stdout:2154: done (2 left) 2024-09-21T09:47:24.214 INFO:tasks.rados.rados.0.smithi017.stdout:2156: done (1 left) 2024-09-21T09:47:24.214 INFO:tasks.rados.rados.0.smithi017.stdout:2157: delete oid 9 current snap is 279 2024-09-21T09:47:24.214 INFO:tasks.rados.rados.0.smithi017.stdout:2155: finishing write tid 3 to smithi01785587-11 2024-09-21T09:47:24.215 INFO:tasks.rados.rados.0.smithi017.stdout:2155: finishing write tid 4 to smithi01785587-11 2024-09-21T09:47:24.215 INFO:tasks.rados.rados.0.smithi017.stdout:2155: finishing write tid 5 to smithi01785587-11 2024-09-21T09:47:24.215 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 936 (ObjNum 811 snap 279 seq_num 811) dirty exists 2024-09-21T09:47:24.215 INFO:tasks.rados.rados.0.smithi017.stdout:2155: left oid 11 (ObjNum 811 snap 279 seq_num 811) 2024-09-21T09:47:24.215 INFO:tasks.rados.rados.0.smithi017.stdout:2155: done (1 left) 2024-09-21T09:47:24.215 INFO:tasks.rados.rados.0.smithi017.stdout:2157: done (0 left) 2024-09-21T09:47:24.215 INFO:tasks.rados.rados.0.smithi017.stdout:2158: read oid 43 snap -1 2024-09-21T09:47:24.215 INFO:tasks.rados.rados.0.smithi017.stdout:2158: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:47:24.215 INFO:tasks.rados.rados.0.smithi017.stdout:2159: read oid 19 snap -1 2024-09-21T09:47:24.216 INFO:tasks.rados.rados.0.smithi017.stdout:2159: expect (ObjNum 773 snap 262 seq_num 773) 2024-09-21T09:47:24.216 INFO:tasks.rados.rados.0.smithi017.stdout:2160: read oid 35 snap -1 2024-09-21T09:47:24.216 INFO:tasks.rados.rados.0.smithi017.stdout:2160: expect deleted 2024-09-21T09:47:24.216 INFO:tasks.rados.rados.0.smithi017.stdout:2161: rollback oid 22 current snap is 279 2024-09-21T09:47:24.216 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 22 to 266 2024-09-21T09:47:24.216 INFO:tasks.rados.rados.0.smithi017.stdout:2162: read oid 31 snap -1 2024-09-21T09:47:24.216 INFO:tasks.rados.rados.0.smithi017.stdout:2162: expect (ObjNum 810 snap 279 seq_num 810) 2024-09-21T09:47:24.216 INFO:tasks.rados.rados.0.smithi017.stdout:2163: snap_remove snap 275 2024-09-21T09:47:25.104 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:47:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:47:24] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.51.0" 2024-09-21T09:47:25.155 INFO:tasks.rados.rados.0.smithi017.stdout:2163: done (5 left) 2024-09-21T09:47:25.155 INFO:tasks.rados.rados.0.smithi017.stdout:2164: read oid 17 snap -1 2024-09-21T09:47:25.155 INFO:tasks.rados.rados.0.smithi017.stdout:2164: expect (ObjNum 809 snap 279 seq_num 809) 2024-09-21T09:47:25.155 INFO:tasks.rados.rados.0.smithi017.stdout:2165: write oid 2 current snap is 279 2024-09-21T09:47:25.155 INFO:tasks.rados.rados.0.smithi017.stdout:2165: seq_num 814 ranges {480898=502457,1699872=737281,3087022=593876} 2024-09-21T09:47:25.161 INFO:tasks.rados.rados.0.smithi017.stdout:2165: writing smithi01785587-2 from 480898 to 983355 tid 1 2024-09-21T09:47:25.170 INFO:tasks.rados.rados.0.smithi017.stdout:2165: writing smithi01785587-2 from 1699872 to 2437153 tid 2 2024-09-21T09:47:25.177 INFO:tasks.rados.rados.0.smithi017.stdout:2165: writing smithi01785587-2 from 3087022 to 3680898 tid 3 2024-09-21T09:47:25.177 INFO:tasks.rados.rados.0.smithi017.stdout:2166: snap_remove snap 271 2024-09-21T09:47:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:25 smithi017 ceph-mon[102913]: pgmap v507: 89 pgs: 89 active+clean; 211 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-21T09:47:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:25 smithi017 ceph-mon[102913]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T09:47:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:25 smithi017 ceph-mon[105151]: pgmap v507: 89 pgs: 89 active+clean; 211 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-21T09:47:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:25 smithi017 ceph-mon[105151]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T09:47:25.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:25 smithi134 ceph-mon[90085]: pgmap v507: 89 pgs: 89 active+clean; 211 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-21T09:47:25.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:25 smithi134 ceph-mon[90085]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T09:47:26.162 INFO:tasks.rados.rados.0.smithi017.stdout:2166: done (7 left) 2024-09-21T09:47:26.162 INFO:tasks.rados.rados.0.smithi017.stdout:2167: delete oid 24 current snap is 279 2024-09-21T09:47:26.162 INFO:tasks.rados.rados.0.smithi017.stdout:2161: finishing rollback tid 0 to smithi01785587-22 2024-09-21T09:47:26.162 INFO:tasks.rados.rados.0.smithi017.stdout:2161: finishing rollback tid 1 to smithi01785587-22 2024-09-21T09:47:26.162 INFO:tasks.rados.rados.0.smithi017.stdout:2162: expect (ObjNum 810 snap 279 seq_num 810) 2024-09-21T09:47:26.334 INFO:tasks.rados.rados.0.smithi017.stdout:2158: expect (ObjNum 512 snap 160 seq_num 512) 2024-09-21T09:47:26.444 INFO:tasks.rados.rados.0.smithi017.stdout:2161: finishing rollback tid 2 to smithi01785587-22 2024-09-21T09:47:26.444 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1026 (ObjNum 726 snap 245 seq_num 726) dirty exists 2024-09-21T09:47:26.444 INFO:tasks.rados.rados.0.smithi017.stdout:2159: expect (ObjNum 773 snap 262 seq_num 773) 2024-09-21T09:47:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:26 smithi017 ceph-mon[102913]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T09:47:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:26 smithi017 ceph-mon[105151]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T09:47:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:26 smithi134 ceph-mon[90085]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T09:47:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:26.541 INFO:tasks.rados.rados.0.smithi017.stdout:2164: expect (ObjNum 809 snap 279 seq_num 809) 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2165: finishing write tid 1 to smithi01785587-2 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2165: finishing write tid 2 to smithi01785587-2 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2165: finishing write tid 3 to smithi01785587-2 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2165: finishing write tid 4 to smithi01785587-2 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2165: finishing write tid 5 to smithi01785587-2 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 426 (ObjNum 814 snap 279 seq_num 814) dirty exists 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2165: left oid 2 (ObjNum 814 snap 279 seq_num 814) 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2158: done (7 left) 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2159: done (6 left) 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2160: done (5 left) 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2161: done (4 left) 2024-09-21T09:47:26.704 INFO:tasks.rados.rados.0.smithi017.stdout:2162: done (3 left) 2024-09-21T09:47:26.705 INFO:tasks.rados.rados.0.smithi017.stdout:2164: done (2 left) 2024-09-21T09:47:26.705 INFO:tasks.rados.rados.0.smithi017.stdout:2165: done (1 left) 2024-09-21T09:47:26.705 INFO:tasks.rados.rados.0.smithi017.stdout:2167: done (0 left) 2024-09-21T09:47:26.705 INFO:tasks.rados.rados.0.smithi017.stdout:2168: rollback oid 13 current snap is 279 2024-09-21T09:47:26.705 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 13 to 239 2024-09-21T09:47:26.705 INFO:tasks.rados.rados.0.smithi017.stdout:2169: delete oid 33 current snap is 279 2024-09-21T09:47:26.706 INFO:tasks.rados.rados.0.smithi017.stdout:2169: done (1 left) 2024-09-21T09:47:26.706 INFO:tasks.rados.rados.0.smithi017.stdout:2170: snap_create 2024-09-21T09:47:26.706 INFO:tasks.rados.rados.0.smithi017.stdout:2168: finishing rollback tid 1 to smithi01785587-13 2024-09-21T09:47:26.706 INFO:tasks.rados.rados.0.smithi017.stdout:2168: finishing rollback tid 2 to smithi01785587-13 2024-09-21T09:47:26.706 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 845 (ObjNum 632 snap 200 seq_num 632) dirty exists 2024-09-21T09:47:27.166 INFO:tasks.rados.rados.0.smithi017.stdout:2168: done (1 left) 2024-09-21T09:47:27.166 INFO:tasks.rados.rados.0.smithi017.stdout:2170: done (0 left) 2024-09-21T09:47:27.166 INFO:tasks.rados.rados.0.smithi017.stdout:2171: write oid 4 current snap is 280 2024-09-21T09:47:27.166 INFO:tasks.rados.rados.0.smithi017.stdout:2171: seq_num 817 ranges {531319=728204,1758965=484608,2652938=520058} 2024-09-21T09:47:27.172 INFO:tasks.rados.rados.0.smithi017.stdout:2171: writing smithi01785587-4 from 531319 to 1259523 tid 1 2024-09-21T09:47:27.178 INFO:tasks.rados.rados.0.smithi017.stdout:2171: writing smithi01785587-4 from 1758965 to 2243573 tid 2 2024-09-21T09:47:27.184 INFO:tasks.rados.rados.0.smithi017.stdout:2171: writing smithi01785587-4 from 2652938 to 3172996 tid 3 2024-09-21T09:47:27.184 INFO:tasks.rados.rados.0.smithi017.stdout:2172: read oid 37 snap -1 2024-09-21T09:47:27.184 INFO:tasks.rados.rados.0.smithi017.stdout:2172: expect (ObjNum 698 snap 221 seq_num 698) 2024-09-21T09:47:27.184 INFO:tasks.rados.rados.0.smithi017.stdout:2173: snap_remove snap 266 2024-09-21T09:47:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:27 smithi017 ceph-mon[102913]: pgmap v510: 89 pgs: 89 active+clean; 216 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.0 MiB/s wr, 24 op/s 2024-09-21T09:47:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:27 smithi017 ceph-mon[102913]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T09:47:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:27 smithi017 ceph-mon[105151]: pgmap v510: 89 pgs: 89 active+clean; 216 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.0 MiB/s wr, 24 op/s 2024-09-21T09:47:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:27 smithi017 ceph-mon[105151]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T09:47:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:27 smithi134 ceph-mon[90085]: pgmap v510: 89 pgs: 89 active+clean; 216 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.0 MiB/s wr, 24 op/s 2024-09-21T09:47:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:27 smithi134 ceph-mon[90085]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T09:47:28.108 INFO:tasks.rados.rados.0.smithi017.stdout:2173: done (2 left) 2024-09-21T09:47:28.108 INFO:tasks.rados.rados.0.smithi017.stdout:2174: write oid 33 current snap is 280 2024-09-21T09:47:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:2171: finishing write tid 1 to smithi01785587-4 2024-09-21T09:47:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:2171: finishing write tid 2 to smithi01785587-4 2024-09-21T09:47:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:2171: finishing write tid 3 to smithi01785587-4 2024-09-21T09:47:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:2171: finishing write tid 4 to smithi01785587-4 2024-09-21T09:47:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:2171: finishing write tid 5 to smithi01785587-4 2024-09-21T09:47:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1040 (ObjNum 817 snap 280 seq_num 817) dirty exists 2024-09-21T09:47:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:2171: left oid 4 (ObjNum 817 snap 280 seq_num 817) 2024-09-21T09:47:28.110 INFO:tasks.rados.rados.0.smithi017.stdout:2172: expect (ObjNum 698 snap 221 seq_num 698) 2024-09-21T09:47:28.305 INFO:tasks.rados.rados.0.smithi017.stdout:2174: seq_num 818 ranges {548126=403453,1417447=744619,2680361=447683,3680450=67676} 2024-09-21T09:47:28.310 INFO:tasks.rados.rados.0.smithi017.stdout:2174: writing smithi01785587-33 from 548126 to 951579 tid 1 2024-09-21T09:47:28.316 INFO:tasks.rados.rados.0.smithi017.stdout:2174: writing smithi01785587-33 from 1417447 to 2162066 tid 2 2024-09-21T09:47:28.321 INFO:tasks.rados.rados.0.smithi017.stdout:2174: writing smithi01785587-33 from 2680361 to 3128044 tid 3 2024-09-21T09:47:28.325 INFO:tasks.rados.rados.0.smithi017.stdout:2174: writing smithi01785587-33 from 3680450 to 3748126 tid 4 2024-09-21T09:47:28.325 INFO:tasks.rados.rados.0.smithi017.stdout:2171: done (2 left) 2024-09-21T09:47:28.325 INFO:tasks.rados.rados.0.smithi017.stdout:2172: done (1 left) 2024-09-21T09:47:28.325 INFO:tasks.rados.rados.0.smithi017.stdout:2175: rollback oid 49 current snap is 280 2024-09-21T09:47:28.326 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 279 2024-09-21T09:47:28.326 INFO:tasks.rados.rados.0.smithi017.stdout:2174: finishing write tid 1 to smithi01785587-33 2024-09-21T09:47:28.326 INFO:tasks.rados.rados.0.smithi017.stdout:2174: finishing write tid 2 to smithi01785587-33 2024-09-21T09:47:28.326 INFO:tasks.rados.rados.0.smithi017.stdout:2174: finishing write tid 3 to smithi01785587-33 2024-09-21T09:47:28.326 INFO:tasks.rados.rados.0.smithi017.stdout:2176: snap_create 2024-09-21T09:47:28.327 INFO:tasks.rados.rados.0.smithi017.stdout:2174: finishing write tid 4 to smithi01785587-33 2024-09-21T09:47:28.327 INFO:tasks.rados.rados.0.smithi017.stdout:2174: finishing write tid 5 to smithi01785587-33 2024-09-21T09:47:28.328 INFO:tasks.rados.rados.0.smithi017.stdout:2174: finishing write tid 6 to smithi01785587-33 2024-09-21T09:47:28.328 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1051 (ObjNum 818 snap 280 seq_num 818) dirty exists 2024-09-21T09:47:28.328 INFO:tasks.rados.rados.0.smithi017.stdout:2174: left oid 33 (ObjNum 818 snap 280 seq_num 818) 2024-09-21T09:47:28.328 INFO:tasks.rados.rados.0.smithi017.stdout:2175: finishing rollback tid 0 to smithi01785587-49 2024-09-21T09:47:28.328 INFO:tasks.rados.rados.0.smithi017.stdout:2175: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:47:28.329 INFO:tasks.rados.rados.0.smithi017.stdout:2175: finishing rollback tid 2 to smithi01785587-49 2024-09-21T09:47:28.329 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1141 (ObjNum 797 snap 272 seq_num 797) dirty exists 2024-09-21T09:47:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:28 smithi017 ceph-mon[102913]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T09:47:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:28 smithi017 ceph-mon[102913]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T09:47:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:28 smithi017 ceph-mon[105151]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T09:47:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:28 smithi017 ceph-mon[105151]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T09:47:28.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:28 smithi134 ceph-mon[90085]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T09:47:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:28 smithi134 ceph-mon[90085]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T09:47:29.176 INFO:tasks.rados.rados.0.smithi017.stdout:2174: done (2 left) 2024-09-21T09:47:29.177 INFO:tasks.rados.rados.0.smithi017.stdout:2175: done (1 left) 2024-09-21T09:47:29.177 INFO:tasks.rados.rados.0.smithi017.stdout:2176: done (0 left) 2024-09-21T09:47:29.177 INFO:tasks.rados.rados.0.smithi017.stdout:2177: write oid 29 current snap is 281 2024-09-21T09:47:29.177 INFO:tasks.rados.rados.0.smithi017.stdout:2177: seq_num 819 ranges {564933=478702,1759576=488277,2991431=658955} 2024-09-21T09:47:29.182 INFO:tasks.rados.rados.0.smithi017.stdout:2177: writing smithi01785587-29 from 564933 to 1043635 tid 1 2024-09-21T09:47:29.189 INFO:tasks.rados.rados.0.smithi017.stdout:2177: writing smithi01785587-29 from 1759576 to 2247853 tid 2 2024-09-21T09:47:29.197 INFO:tasks.rados.rados.0.smithi017.stdout:2177: writing smithi01785587-29 from 2991431 to 3650386 tid 3 2024-09-21T09:47:29.197 INFO:tasks.rados.rados.0.smithi017.stdout:2178: read oid 21 snap 246 2024-09-21T09:47:29.197 INFO:tasks.rados.rados.0.smithi017.stdout:2178: expect (ObjNum 719 snap 243 seq_num 719) 2024-09-21T09:47:29.197 INFO:tasks.rados.rados.0.smithi017.stdout:2179: rollback oid 46 current snap is 281 2024-09-21T09:47:29.197 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 272 2024-09-21T09:47:29.197 INFO:tasks.rados.rados.0.smithi017.stdout:2177: finishing write tid 1 to smithi01785587-29 2024-09-21T09:47:29.198 INFO:tasks.rados.rados.0.smithi017.stdout:2177: finishing write tid 2 to smithi01785587-29 2024-09-21T09:47:29.198 INFO:tasks.rados.rados.0.smithi017.stdout:2180: write oid 34 current snap is 281 2024-09-21T09:47:29.198 INFO:tasks.rados.rados.0.smithi017.stdout:2180: seq_num 820 ranges {581740=553951,1818058=748288,3018854=470227} 2024-09-21T09:47:29.204 INFO:tasks.rados.rados.0.smithi017.stdout:2180: writing smithi01785587-34 from 581740 to 1135691 tid 1 2024-09-21T09:47:29.212 INFO:tasks.rados.rados.0.smithi017.stdout:2180: writing smithi01785587-34 from 1818058 to 2566346 tid 2 2024-09-21T09:47:29.218 INFO:tasks.rados.rados.0.smithi017.stdout:2180: writing smithi01785587-34 from 3018854 to 3489081 tid 3 2024-09-21T09:47:29.218 INFO:tasks.rados.rados.0.smithi017.stdout:2181: delete oid 28 current snap is 281 2024-09-21T09:47:29.218 INFO:tasks.rados.rados.0.smithi017.stdout:2177: finishing write tid 3 to smithi01785587-29 2024-09-21T09:47:29.218 INFO:tasks.rados.rados.0.smithi017.stdout:2177: finishing write tid 4 to smithi01785587-29 2024-09-21T09:47:29.218 INFO:tasks.rados.rados.0.smithi017.stdout:2177: finishing write tid 5 to smithi01785587-29 2024-09-21T09:47:29.218 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 540 (ObjNum 819 snap 281 seq_num 819) dirty exists 2024-09-21T09:47:29.219 INFO:tasks.rados.rados.0.smithi017.stdout:2177: left oid 29 (ObjNum 819 snap 281 seq_num 819) 2024-09-21T09:47:29.219 INFO:tasks.rados.rados.0.smithi017.stdout:2179: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:47:29.219 INFO:tasks.rados.rados.0.smithi017.stdout:2179: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:47:29.219 INFO:tasks.rados.rados.0.smithi017.stdout:2178: expect (ObjNum 719 snap 243 seq_num 719) 2024-09-21T09:47:29.307 INFO:tasks.rados.rados.0.smithi017.stdout:2179: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:47:29.307 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 948 (ObjNum 791 snap 271 seq_num 791) dirty exists 2024-09-21T09:47:29.309 INFO:tasks.rados.rados.0.smithi017.stdout:2180: finishing write tid 1 to smithi01785587-34 2024-09-21T09:47:29.309 INFO:tasks.rados.rados.0.smithi017.stdout:2180: finishing write tid 2 to smithi01785587-34 2024-09-21T09:47:29.309 INFO:tasks.rados.rados.0.smithi017.stdout:2180: finishing write tid 3 to smithi01785587-34 2024-09-21T09:47:29.309 INFO:tasks.rados.rados.0.smithi017.stdout:2180: finishing write tid 4 to smithi01785587-34 2024-09-21T09:47:29.310 INFO:tasks.rados.rados.0.smithi017.stdout:2180: finishing write tid 5 to smithi01785587-34 2024-09-21T09:47:29.310 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1145 (ObjNum 820 snap 281 seq_num 820) dirty exists 2024-09-21T09:47:29.310 INFO:tasks.rados.rados.0.smithi017.stdout:2180: left oid 34 (ObjNum 820 snap 281 seq_num 820) 2024-09-21T09:47:29.310 INFO:tasks.rados.rados.0.smithi017.stdout:2177: done (4 left) 2024-09-21T09:47:29.310 INFO:tasks.rados.rados.0.smithi017.stdout:2178: done (3 left) 2024-09-21T09:47:29.310 INFO:tasks.rados.rados.0.smithi017.stdout:2179: done (2 left) 2024-09-21T09:47:29.310 INFO:tasks.rados.rados.0.smithi017.stdout:2180: done (1 left) 2024-09-21T09:47:29.310 INFO:tasks.rados.rados.0.smithi017.stdout:2181: done (0 left) 2024-09-21T09:47:29.310 INFO:tasks.rados.rados.0.smithi017.stdout:2182: snap_create 2024-09-21T09:47:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:29 smithi017 ceph-mon[102913]: pgmap v513: 89 pgs: 89 active+clean; 227 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.5 MiB/s wr, 26 op/s 2024-09-21T09:47:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:29 smithi017 ceph-mon[105151]: pgmap v513: 89 pgs: 89 active+clean; 227 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.5 MiB/s wr, 26 op/s 2024-09-21T09:47:29.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:29 smithi134 ceph-mon[90085]: pgmap v513: 89 pgs: 89 active+clean; 227 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.5 MiB/s wr, 26 op/s 2024-09-21T09:47:30.182 INFO:tasks.rados.rados.0.smithi017.stdout:2182: done (0 left) 2024-09-21T09:47:30.182 INFO:tasks.rados.rados.0.smithi017.stdout:2183: read oid 40 snap -1 2024-09-21T09:47:30.182 INFO:tasks.rados.rados.0.smithi017.stdout:2183: expect (ObjNum 746 snap 252 seq_num 746) 2024-09-21T09:47:30.182 INFO:tasks.rados.rados.0.smithi017.stdout:2184: delete oid 33 current snap is 282 2024-09-21T09:47:30.186 INFO:tasks.rados.rados.0.smithi017.stdout:2184: done (1 left) 2024-09-21T09:47:30.186 INFO:tasks.rados.rados.0.smithi017.stdout:2185: write oid 32 current snap is 282 2024-09-21T09:47:30.186 INFO:tasks.rados.rados.0.smithi017.stdout:2185: seq_num 823 ranges {632161=496051,1593504=495615,2701123=420396,3611645=220516} 2024-09-21T09:47:30.193 INFO:tasks.rados.rados.0.smithi017.stdout:2185: writing smithi01785587-32 from 632161 to 1128212 tid 1 2024-09-21T09:47:30.198 INFO:tasks.rados.rados.0.smithi017.stdout:2185: writing smithi01785587-32 from 1593504 to 2089119 tid 2 2024-09-21T09:47:30.204 INFO:tasks.rados.rados.0.smithi017.stdout:2185: writing smithi01785587-32 from 2701123 to 3121519 tid 3 2024-09-21T09:47:30.208 INFO:tasks.rados.rados.0.smithi017.stdout:2185: writing smithi01785587-32 from 3611645 to 3832161 tid 4 2024-09-21T09:47:30.208 INFO:tasks.rados.rados.0.smithi017.stdout:2186: snap_remove snap 273 2024-09-21T09:47:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:30 smithi017 ceph-mon[102913]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T09:47:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:30 smithi017 ceph-mon[105151]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T09:47:30.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:30 smithi134 ceph-mon[90085]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T09:47:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:47:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:47:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:31.188 INFO:tasks.rados.rados.0.smithi017.stdout:2183: expect (ObjNum 746 snap 252 seq_num 746) 2024-09-21T09:47:31.274 INFO:tasks.rados.rados.0.smithi017.stdout:2185: finishing write tid 1 to smithi01785587-32 2024-09-21T09:47:31.274 INFO:tasks.rados.rados.0.smithi017.stdout:2185: finishing write tid 2 to smithi01785587-32 2024-09-21T09:47:31.274 INFO:tasks.rados.rados.0.smithi017.stdout:2185: finishing write tid 3 to smithi01785587-32 2024-09-21T09:47:31.274 INFO:tasks.rados.rados.0.smithi017.stdout:2185: finishing write tid 4 to smithi01785587-32 2024-09-21T09:47:31.274 INFO:tasks.rados.rados.0.smithi017.stdout:2183: done (2 left) 2024-09-21T09:47:31.274 INFO:tasks.rados.rados.0.smithi017.stdout:2186: done (1 left) 2024-09-21T09:47:31.274 INFO:tasks.rados.rados.0.smithi017.stdout:2187: read oid 31 snap 279 2024-09-21T09:47:31.275 INFO:tasks.rados.rados.0.smithi017.stdout:2187: expect (ObjNum 810 snap 279 seq_num 810) 2024-09-21T09:47:31.275 INFO:tasks.rados.rados.0.smithi017.stdout:2188: write oid 22 current snap is 282 2024-09-21T09:47:31.275 INFO:tasks.rados.rados.0.smithi017.stdout:2185: finishing write tid 5 to smithi01785587-32 2024-09-21T09:47:31.275 INFO:tasks.rados.rados.0.smithi017.stdout:2188: seq_num 824 ranges {648968=571300,1651986=639273,2728546=631668} 2024-09-21T09:47:31.280 INFO:tasks.rados.rados.0.smithi017.stdout:2188: writing smithi01785587-22 from 648968 to 1220268 tid 1 2024-09-21T09:47:31.286 INFO:tasks.rados.rados.0.smithi017.stdout:2188: writing smithi01785587-22 from 1651986 to 2291259 tid 2 2024-09-21T09:47:31.292 INFO:tasks.rados.rados.0.smithi017.stdout:2188: writing smithi01785587-22 from 2728546 to 3360214 tid 3 2024-09-21T09:47:31.293 INFO:tasks.rados.rados.0.smithi017.stdout:2189: read oid 35 snap -1 2024-09-21T09:47:31.293 INFO:tasks.rados.rados.0.smithi017.stdout:2189: expect deleted 2024-09-21T09:47:31.293 INFO:tasks.rados.rados.0.smithi017.stdout:2190: write oid 8 current snap is 282 2024-09-21T09:47:31.293 INFO:tasks.rados.rados.0.smithi017.stdout:2185: finishing write tid 6 to smithi01785587-32 2024-09-21T09:47:31.293 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 953 (ObjNum 823 snap 282 seq_num 823) dirty exists 2024-09-21T09:47:31.293 INFO:tasks.rados.rados.0.smithi017.stdout:2185: left oid 32 (ObjNum 823 snap 282 seq_num 823) 2024-09-21T09:47:31.293 INFO:tasks.rados.rados.0.smithi017.stdout:2187: expect (ObjNum 810 snap 279 seq_num 810) 2024-09-21T09:47:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:31 smithi017 ceph-mon[102913]: pgmap v516: 89 pgs: 89 active+clean; 227 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:47:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:31 smithi017 ceph-mon[102913]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T09:47:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:31 smithi017 ceph-mon[105151]: pgmap v516: 89 pgs: 89 active+clean; 227 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:47:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:31 smithi017 ceph-mon[105151]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T09:47:31.463 INFO:tasks.rados.rados.0.smithi017.stdout:2188: finishing write tid 1 to smithi01785587-22 2024-09-21T09:47:31.463 INFO:tasks.rados.rados.0.smithi017.stdout:2188: finishing write tid 2 to smithi01785587-22 2024-09-21T09:47:31.463 INFO:tasks.rados.rados.0.smithi017.stdout:2188: finishing write tid 3 to smithi01785587-22 2024-09-21T09:47:31.463 INFO:tasks.rados.rados.0.smithi017.stdout:2188: finishing write tid 4 to smithi01785587-22 2024-09-21T09:47:31.463 INFO:tasks.rados.rados.0.smithi017.stdout:2188: finishing write tid 5 to smithi01785587-22 2024-09-21T09:47:31.463 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1065 (ObjNum 824 snap 282 seq_num 824) dirty exists 2024-09-21T09:47:31.463 INFO:tasks.rados.rados.0.smithi017.stdout:2188: left oid 22 (ObjNum 824 snap 282 seq_num 824) 2024-09-21T09:47:31.463 INFO:tasks.rados.rados.0.smithi017.stdout:2190: seq_num 825 ranges {665775=646549,2110468=499284,3155969=442940} 2024-09-21T09:47:31.470 INFO:tasks.rados.rados.0.smithi017.stdout:2190: writing smithi01785587-8 from 665775 to 1312324 tid 1 2024-09-21T09:47:31.477 INFO:tasks.rados.rados.0.smithi017.stdout:2190: writing smithi01785587-8 from 2110468 to 2609752 tid 2 2024-09-21T09:47:31.483 INFO:tasks.rados.rados.0.smithi017.stdout:2190: writing smithi01785587-8 from 3155969 to 3598909 tid 3 2024-09-21T09:47:31.483 INFO:tasks.rados.rados.0.smithi017.stdout:2185: done (4 left) 2024-09-21T09:47:31.483 INFO:tasks.rados.rados.0.smithi017.stdout:2187: done (3 left) 2024-09-21T09:47:31.483 INFO:tasks.rados.rados.0.smithi017.stdout:2188: done (2 left) 2024-09-21T09:47:31.483 INFO:tasks.rados.rados.0.smithi017.stdout:2189: done (1 left) 2024-09-21T09:47:31.483 INFO:tasks.rados.rados.0.smithi017.stdout:2191: delete oid 49 current snap is 282 2024-09-21T09:47:31.483 INFO:tasks.rados.rados.0.smithi017.stdout:2190: finishing write tid 1 to smithi01785587-8 2024-09-21T09:47:31.484 INFO:tasks.rados.rados.0.smithi017.stdout:2190: finishing write tid 2 to smithi01785587-8 2024-09-21T09:47:31.485 INFO:tasks.rados.rados.0.smithi017.stdout:2191: done (1 left) 2024-09-21T09:47:31.485 INFO:tasks.rados.rados.0.smithi017.stdout:2192: snap_remove snap 239 2024-09-21T09:47:31.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:31 smithi134 ceph-mon[90085]: pgmap v516: 89 pgs: 89 active+clean; 227 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:47:31.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:31 smithi134 ceph-mon[90085]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T09:47:32.194 INFO:tasks.rados.rados.0.smithi017.stdout:2192: done (1 left) 2024-09-21T09:47:32.195 INFO:tasks.rados.rados.0.smithi017.stdout:2193: write oid 33 current snap is 282 2024-09-21T09:47:32.195 INFO:tasks.rados.rados.0.smithi017.stdout:2193: seq_num 827 ranges {699389=797047,2111079=502953,3094462=581837} 2024-09-21T09:47:32.204 INFO:tasks.rados.rados.0.smithi017.stdout:2193: writing smithi01785587-33 from 699389 to 1496436 tid 1 2024-09-21T09:47:32.211 INFO:tasks.rados.rados.0.smithi017.stdout:2193: writing smithi01785587-33 from 2111079 to 2614032 tid 2 2024-09-21T09:47:32.217 INFO:tasks.rados.rados.0.smithi017.stdout:2193: writing smithi01785587-33 from 3094462 to 3676299 tid 3 2024-09-21T09:47:32.217 INFO:tasks.rados.rados.0.smithi017.stdout:2194: read oid 4 snap -1 2024-09-21T09:47:32.217 INFO:tasks.rados.rados.0.smithi017.stdout:2194: expect (ObjNum 817 snap 280 seq_num 817) 2024-09-21T09:47:32.217 INFO:tasks.rados.rados.0.smithi017.stdout:2195: write oid 16 current snap is 282 2024-09-21T09:47:32.217 INFO:tasks.rados.rados.0.smithi017.stdout:2190: finishing write tid 3 to smithi01785587-8 2024-09-21T09:47:32.217 INFO:tasks.rados.rados.0.smithi017.stdout:2190: finishing write tid 4 to smithi01785587-8 2024-09-21T09:47:32.218 INFO:tasks.rados.rados.0.smithi017.stdout:2190: finishing write tid 5 to smithi01785587-8 2024-09-21T09:47:32.218 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 960 (ObjNum 825 snap 282 seq_num 825) dirty exists 2024-09-21T09:47:32.218 INFO:tasks.rados.rados.0.smithi017.stdout:2190: left oid 8 (ObjNum 825 snap 282 seq_num 825) 2024-09-21T09:47:32.218 INFO:tasks.rados.rados.0.smithi017.stdout:2193: finishing write tid 1 to smithi01785587-33 2024-09-21T09:47:32.218 INFO:tasks.rados.rados.0.smithi017.stdout:2193: finishing write tid 2 to smithi01785587-33 2024-09-21T09:47:32.218 INFO:tasks.rados.rados.0.smithi017.stdout:2195: seq_num 828 ranges {716196=472296,1769561=646611,3121885=793109} 2024-09-21T09:47:32.223 INFO:tasks.rados.rados.0.smithi017.stdout:2195: writing smithi01785587-16 from 716196 to 1188492 tid 1 2024-09-21T09:47:32.231 INFO:tasks.rados.rados.0.smithi017.stdout:2195: writing smithi01785587-16 from 1769561 to 2416172 tid 2 2024-09-21T09:47:32.239 INFO:tasks.rados.rados.0.smithi017.stdout:2195: writing smithi01785587-16 from 3121885 to 3914994 tid 3 2024-09-21T09:47:32.239 INFO:tasks.rados.rados.0.smithi017.stdout:2190: done (3 left) 2024-09-21T09:47:32.239 INFO:tasks.rados.rados.0.smithi017.stdout:2196: delete oid 43 current snap is 282 2024-09-21T09:47:32.239 INFO:tasks.rados.rados.0.smithi017.stdout:2193: finishing write tid 3 to smithi01785587-33 2024-09-21T09:47:32.239 INFO:tasks.rados.rados.0.smithi017.stdout:2193: finishing write tid 4 to smithi01785587-33 2024-09-21T09:47:32.240 INFO:tasks.rados.rados.0.smithi017.stdout:2193: finishing write tid 5 to smithi01785587-33 2024-09-21T09:47:32.240 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1077 (ObjNum 827 snap 282 seq_num 827) dirty exists 2024-09-21T09:47:32.240 INFO:tasks.rados.rados.0.smithi017.stdout:2193: left oid 33 (ObjNum 827 snap 282 seq_num 827) 2024-09-21T09:47:32.240 INFO:tasks.rados.rados.0.smithi017.stdout:2194: expect (ObjNum 817 snap 280 seq_num 817) 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2195: finishing write tid 1 to smithi01785587-16 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2195: finishing write tid 2 to smithi01785587-16 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2195: finishing write tid 3 to smithi01785587-16 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2195: finishing write tid 4 to smithi01785587-16 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2195: finishing write tid 5 to smithi01785587-16 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1082 (ObjNum 828 snap 282 seq_num 828) dirty exists 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2195: left oid 16 (ObjNum 828 snap 282 seq_num 828) 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2193: done (3 left) 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2194: done (2 left) 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2195: done (1 left) 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2196: done (0 left) 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2197: write oid 12 current snap is 282 2024-09-21T09:47:32.363 INFO:tasks.rados.rados.0.smithi017.stdout:2197: seq_num 830 ranges {749810=739147,2002878=650280,3293084=415653} 2024-09-21T09:47:32.371 INFO:tasks.rados.rados.0.smithi017.stdout:2197: writing smithi01785587-12 from 749810 to 1488957 tid 1 2024-09-21T09:47:32.377 INFO:tasks.rados.rados.0.smithi017.stdout:2197: writing smithi01785587-12 from 2002878 to 2653158 tid 2 2024-09-21T09:47:32.383 INFO:tasks.rados.rados.0.smithi017.stdout:2197: writing smithi01785587-12 from 3293084 to 3708737 tid 3 2024-09-21T09:47:32.383 INFO:tasks.rados.rados.0.smithi017.stdout:2198: rollback oid 38 current snap is 282 2024-09-21T09:47:32.383 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 281 2024-09-21T09:47:32.383 INFO:tasks.rados.rados.0.smithi017.stdout:2197: finishing write tid 1 to smithi01785587-12 2024-09-21T09:47:32.383 INFO:tasks.rados.rados.0.smithi017.stdout:2197: finishing write tid 2 to smithi01785587-12 2024-09-21T09:47:32.384 INFO:tasks.rados.rados.0.smithi017.stdout:2199: read oid 34 snap 277 2024-09-21T09:47:32.384 INFO:tasks.rados.rados.0.smithi017.stdout:2199: expect deleted 2024-09-21T09:47:32.384 INFO:tasks.rados.rados.0.smithi017.stdout:2200: snap_create 2024-09-21T09:47:32.384 INFO:tasks.rados.rados.0.smithi017.stdout:2198: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:47:32.384 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 648 (ObjNum 129 snap 0 seq_num 2399478144) dirty dne 2024-09-21T09:47:32.385 INFO:tasks.rados.rados.0.smithi017.stdout:2197: finishing write tid 3 to smithi01785587-12 2024-09-21T09:47:32.385 INFO:tasks.rados.rados.0.smithi017.stdout:2197: finishing write tid 4 to smithi01785587-12 2024-09-21T09:47:32.386 INFO:tasks.rados.rados.0.smithi017.stdout:2197: finishing write tid 5 to smithi01785587-12 2024-09-21T09:47:32.386 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 785 (ObjNum 830 snap 282 seq_num 830) dirty exists 2024-09-21T09:47:32.386 INFO:tasks.rados.rados.0.smithi017.stdout:2197: left oid 12 (ObjNum 830 snap 282 seq_num 830) 2024-09-21T09:47:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:32 smithi017 ceph-mon[102913]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T09:47:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:32 smithi017 ceph-mon[105151]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T09:47:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:32 smithi134 ceph-mon[90085]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T09:47:33.108 INFO:tasks.rados.rados.0.smithi017.stdout:2197: done (3 left) 2024-09-21T09:47:33.109 INFO:tasks.rados.rados.0.smithi017.stdout:2198: done (2 left) 2024-09-21T09:47:33.109 INFO:tasks.rados.rados.0.smithi017.stdout:2199: done (1 left) 2024-09-21T09:47:33.109 INFO:tasks.rados.rados.0.smithi017.stdout:2200: done (0 left) 2024-09-21T09:47:33.109 INFO:tasks.rados.rados.0.smithi017.stdout:2201: rollback oid 48 current snap is 283 2024-09-21T09:47:33.109 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 280 2024-09-21T09:47:33.109 INFO:tasks.rados.rados.0.smithi017.stdout:2202: snap_remove snap 274 2024-09-21T09:47:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:33 smithi017 ceph-mon[102913]: pgmap v519: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-21T09:47:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:33 smithi017 ceph-mon[102913]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T09:47:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:33 smithi017 ceph-mon[102913]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T09:47:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:33 smithi017 ceph-mon[105151]: pgmap v519: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-21T09:47:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:33 smithi017 ceph-mon[105151]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T09:47:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:33 smithi017 ceph-mon[105151]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T09:47:33.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:33 smithi134 ceph-mon[90085]: pgmap v519: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-21T09:47:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:33 smithi134 ceph-mon[90085]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T09:47:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:33 smithi134 ceph-mon[90085]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T09:47:34.109 INFO:tasks.rados.rados.0.smithi017.stdout:2202: done (1 left) 2024-09-21T09:47:34.109 INFO:tasks.rados.rados.0.smithi017.stdout:2203: delete oid 33 current snap is 283 2024-09-21T09:47:34.110 INFO:tasks.rados.rados.0.smithi017.stdout:2201: finishing rollback tid 0 to smithi01785587-48 2024-09-21T09:47:34.110 INFO:tasks.rados.rados.0.smithi017.stdout:2201: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:47:34.110 INFO:tasks.rados.rados.0.smithi017.stdout:2201: finishing rollback tid 2 to smithi01785587-48 2024-09-21T09:47:34.110 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 861 (ObjNum 755 snap 255 seq_num 755) dirty exists 2024-09-21T09:47:34.117 INFO:tasks.rados.rados.0.smithi017.stdout:2201: done (1 left) 2024-09-21T09:47:34.117 INFO:tasks.rados.rados.0.smithi017.stdout:2203: done (0 left) 2024-09-21T09:47:34.117 INFO:tasks.rados.rados.0.smithi017.stdout:2204: read oid 35 snap -1 2024-09-21T09:47:34.117 INFO:tasks.rados.rados.0.smithi017.stdout:2204: expect deleted 2024-09-21T09:47:34.117 INFO:tasks.rados.rados.0.smithi017.stdout:2205: read oid 30 snap -1 2024-09-21T09:47:34.117 INFO:tasks.rados.rados.0.smithi017.stdout:2205: expect (ObjNum 793 snap 271 seq_num 793) 2024-09-21T09:47:34.117 INFO:tasks.rados.rados.0.smithi017.stdout:2206: snap_create 2024-09-21T09:47:34.130 INFO:tasks.rados.rados.0.smithi017.stdout:2205: expect (ObjNum 793 snap 271 seq_num 793) 2024-09-21T09:47:35.110 INFO:tasks.rados.rados.0.smithi017.stdout:2204: done (2 left) 2024-09-21T09:47:35.111 INFO:tasks.rados.rados.0.smithi017.stdout:2205: done (1 left) 2024-09-21T09:47:35.111 INFO:tasks.rados.rados.0.smithi017.stdout:2206: done (0 left) 2024-09-21T09:47:35.111 INFO:tasks.rados.rados.0.smithi017.stdout:2207: snap_create 2024-09-21T09:47:35.115 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:47:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:47:34] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.51.0" 2024-09-21T09:47:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:35 smithi017 ceph-mon[102913]: pgmap v522: 89 pgs: 89 active+clean; 238 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-21T09:47:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:35 smithi017 ceph-mon[102913]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T09:47:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:35 smithi017 ceph-mon[105151]: pgmap v522: 89 pgs: 89 active+clean; 238 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-21T09:47:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:35 smithi017 ceph-mon[105151]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T09:47:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:35 smithi134 ceph-mon[90085]: pgmap v522: 89 pgs: 89 active+clean; 238 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-21T09:47:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:35 smithi134 ceph-mon[90085]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T09:47:36.124 INFO:tasks.rados.rados.0.smithi017.stdout:2207: done (0 left) 2024-09-21T09:47:36.124 INFO:tasks.rados.rados.0.smithi017.stdout:2208: rollback oid 14 current snap is 285 2024-09-21T09:47:36.124 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 282 2024-09-21T09:47:36.124 INFO:tasks.rados.rados.0.smithi017.stdout:2209: snap_create 2024-09-21T09:47:36.127 INFO:tasks.rados.rados.0.smithi017.stdout:2208: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:47:36.128 INFO:tasks.rados.rados.0.smithi017.stdout:2208: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:47:36.128 INFO:tasks.rados.rados.0.smithi017.stdout:2208: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:47:36.128 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1038 (ObjNum 667 snap 209 seq_num 667) dirty exists 2024-09-21T09:47:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:36 smithi017 ceph-mon[102913]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T09:47:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:36 smithi017 ceph-mon[105151]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T09:47:36.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:36 smithi134 ceph-mon[90085]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T09:47:37.125 INFO:tasks.rados.rados.0.smithi017.stdout:2208: done (1 left) 2024-09-21T09:47:37.125 INFO:tasks.rados.rados.0.smithi017.stdout:2209: done (0 left) 2024-09-21T09:47:37.125 INFO:tasks.rados.rados.0.smithi017.stdout:2210: rollback oid 41 current snap is 286 2024-09-21T09:47:37.126 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 281 2024-09-21T09:47:37.126 INFO:tasks.rados.rados.0.smithi017.stdout:2211: snap_remove snap 276 2024-09-21T09:47:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:37 smithi017 ceph-mon[102913]: pgmap v525: 89 pgs: 89 active+clean; 225 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:47:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:37 smithi017 ceph-mon[102913]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T09:47:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:37 smithi017 ceph-mon[105151]: pgmap v525: 89 pgs: 89 active+clean; 225 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:47:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:37 smithi017 ceph-mon[105151]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T09:47:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:37 smithi134 ceph-mon[90085]: pgmap v525: 89 pgs: 89 active+clean; 225 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:47:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:37 smithi134 ceph-mon[90085]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T09:47:38.111 INFO:tasks.rados.rados.0.smithi017.stdout:2211: done (1 left) 2024-09-21T09:47:38.111 INFO:tasks.rados.rados.0.smithi017.stdout:2212: read oid 21 snap -1 2024-09-21T09:47:38.111 INFO:tasks.rados.rados.0.smithi017.stdout:2212: expect deleted 2024-09-21T09:47:38.111 INFO:tasks.rados.rados.0.smithi017.stdout:2213: write oid 44 current snap is 286 2024-09-21T09:47:38.111 INFO:tasks.rados.rados.0.smithi017.stdout:2210: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:47:38.111 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 312 (ObjNum 129 snap 0 seq_num 2399472976) dirty dne 2024-09-21T09:47:38.111 INFO:tasks.rados.rados.0.smithi017.stdout:2213: seq_num 832 ranges {783424=489645,2003489=653949,3231577=554550} 2024-09-21T09:47:38.119 INFO:tasks.rados.rados.0.smithi017.stdout:2213: writing smithi01785587-44 from 783424 to 1273069 tid 1 2024-09-21T09:47:38.127 INFO:tasks.rados.rados.0.smithi017.stdout:2213: writing smithi01785587-44 from 2003489 to 2657438 tid 2 2024-09-21T09:47:38.134 INFO:tasks.rados.rados.0.smithi017.stdout:2213: writing smithi01785587-44 from 3231577 to 3786127 tid 3 2024-09-21T09:47:38.134 INFO:tasks.rados.rados.0.smithi017.stdout:2210: done (2 left) 2024-09-21T09:47:38.134 INFO:tasks.rados.rados.0.smithi017.stdout:2214: rollback oid 49 current snap is 286 2024-09-21T09:47:38.134 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 283 2024-09-21T09:47:38.134 INFO:tasks.rados.rados.0.smithi017.stdout:2213: finishing write tid 1 to smithi01785587-44 2024-09-21T09:47:38.134 INFO:tasks.rados.rados.0.smithi017.stdout:2213: finishing write tid 2 to smithi01785587-44 2024-09-21T09:47:38.134 INFO:tasks.rados.rados.0.smithi017.stdout:2212: done (2 left) 2024-09-21T09:47:38.134 INFO:tasks.rados.rados.0.smithi017.stdout:2215: write oid 42 current snap is 286 2024-09-21T09:47:38.134 INFO:tasks.rados.rados.0.smithi017.stdout:2215: seq_num 833 ranges {400231=564894,1661971=338260} 2024-09-21T09:47:38.139 INFO:tasks.rados.rados.0.smithi017.stdout:2215: writing smithi01785587-42 from 400231 to 965125 tid 1 2024-09-21T09:47:38.145 INFO:tasks.rados.rados.0.smithi017.stdout:2215: writing smithi01785587-42 from 1661971 to 2000231 tid 2 2024-09-21T09:47:38.145 INFO:tasks.rados.rados.0.smithi017.stdout:2216: read oid 45 snap -1 2024-09-21T09:47:38.145 INFO:tasks.rados.rados.0.smithi017.stdout:2216: expect (ObjNum 790 snap 271 seq_num 790) 2024-09-21T09:47:38.145 INFO:tasks.rados.rados.0.smithi017.stdout:2217: snap_create 2024-09-21T09:47:38.145 INFO:tasks.rados.rados.0.smithi017.stdout:2214: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:47:38.145 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1149 (ObjNum 449 snap 0 seq_num 2883383314) dirty dne 2024-09-21T09:47:38.146 INFO:tasks.rados.rados.0.smithi017.stdout:2213: finishing write tid 3 to smithi01785587-44 2024-09-21T09:47:38.146 INFO:tasks.rados.rados.0.smithi017.stdout:2213: finishing write tid 4 to smithi01785587-44 2024-09-21T09:47:38.146 INFO:tasks.rados.rados.0.smithi017.stdout:2213: finishing write tid 5 to smithi01785587-44 2024-09-21T09:47:38.146 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1048 (ObjNum 832 snap 286 seq_num 832) dirty exists 2024-09-21T09:47:38.146 INFO:tasks.rados.rados.0.smithi017.stdout:2213: left oid 44 (ObjNum 832 snap 286 seq_num 832) 2024-09-21T09:47:38.146 INFO:tasks.rados.rados.0.smithi017.stdout:2215: finishing write tid 1 to smithi01785587-42 2024-09-21T09:47:38.148 INFO:tasks.rados.rados.0.smithi017.stdout:2215: finishing write tid 2 to smithi01785587-42 2024-09-21T09:47:38.148 INFO:tasks.rados.rados.0.smithi017.stdout:2215: finishing write tid 3 to smithi01785587-42 2024-09-21T09:47:38.148 INFO:tasks.rados.rados.0.smithi017.stdout:2216: expect (ObjNum 790 snap 271 seq_num 790) 2024-09-21T09:47:38.256 INFO:tasks.rados.rados.0.smithi017.stdout:2215: finishing write tid 4 to smithi01785587-42 2024-09-21T09:47:38.256 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 793 (ObjNum 833 snap 286 seq_num 833) dirty exists 2024-09-21T09:47:38.256 INFO:tasks.rados.rados.0.smithi017.stdout:2215: left oid 42 (ObjNum 833 snap 286 seq_num 833) 2024-09-21T09:47:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:38 smithi017 ceph-mon[102913]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T09:47:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:38 smithi017 ceph-mon[102913]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T09:47:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:38 smithi017 ceph-mon[105151]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T09:47:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:38 smithi017 ceph-mon[105151]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T09:47:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:38 smithi134 ceph-mon[90085]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T09:47:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:38 smithi134 ceph-mon[90085]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T09:47:39.144 INFO:tasks.rados.rados.0.smithi017.stdout:2213: done (4 left) 2024-09-21T09:47:39.144 INFO:tasks.rados.rados.0.smithi017.stdout:2214: done (3 left) 2024-09-21T09:47:39.144 INFO:tasks.rados.rados.0.smithi017.stdout:2215: done (2 left) 2024-09-21T09:47:39.144 INFO:tasks.rados.rados.0.smithi017.stdout:2216: done (1 left) 2024-09-21T09:47:39.145 INFO:tasks.rados.rados.0.smithi017.stdout:2217: done (0 left) 2024-09-21T09:47:39.145 INFO:tasks.rados.rados.0.smithi017.stdout:2218: write oid 44 current snap is 287 2024-09-21T09:47:39.145 INFO:tasks.rados.rados.0.smithi017.stdout:2218: seq_num 834 ranges {417038=640143,1720453=296585} 2024-09-21T09:47:39.150 INFO:tasks.rados.rados.0.smithi017.stdout:2218: writing smithi01785587-44 from 417038 to 1057181 tid 1 2024-09-21T09:47:39.156 INFO:tasks.rados.rados.0.smithi017.stdout:2218: writing smithi01785587-44 from 1720453 to 2017038 tid 2 2024-09-21T09:47:39.156 INFO:tasks.rados.rados.0.smithi017.stdout:2219: delete oid 13 current snap is 287 2024-09-21T09:47:39.156 INFO:tasks.rados.rados.0.smithi017.stdout:2218: finishing write tid 1 to smithi01785587-44 2024-09-21T09:47:39.159 INFO:tasks.rados.rados.0.smithi017.stdout:2218: finishing write tid 2 to smithi01785587-44 2024-09-21T09:47:39.159 INFO:tasks.rados.rados.0.smithi017.stdout:2218: finishing write tid 3 to smithi01785587-44 2024-09-21T09:47:39.159 INFO:tasks.rados.rados.0.smithi017.stdout:2218: finishing write tid 4 to smithi01785587-44 2024-09-21T09:47:39.159 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1052 (ObjNum 834 snap 287 seq_num 834) dirty exists 2024-09-21T09:47:39.159 INFO:tasks.rados.rados.0.smithi017.stdout:2218: left oid 44 (ObjNum 834 snap 287 seq_num 834) 2024-09-21T09:47:39.160 INFO:tasks.rados.rados.0.smithi017.stdout:2218: done (1 left) 2024-09-21T09:47:39.160 INFO:tasks.rados.rados.0.smithi017.stdout:2219: done (0 left) 2024-09-21T09:47:39.160 INFO:tasks.rados.rados.0.smithi017.stdout:2220: read oid 47 snap -1 2024-09-21T09:47:39.160 INFO:tasks.rados.rados.0.smithi017.stdout:2220: expect deleted 2024-09-21T09:47:39.160 INFO:tasks.rados.rados.0.smithi017.stdout:2221: read oid 37 snap -1 2024-09-21T09:47:39.160 INFO:tasks.rados.rados.0.smithi017.stdout:2221: expect (ObjNum 698 snap 221 seq_num 698) 2024-09-21T09:47:39.160 INFO:tasks.rados.rados.0.smithi017.stdout:2222: snap_remove snap 279 2024-09-21T09:47:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:39 smithi017 ceph-mon[102913]: pgmap v528: 89 pgs: 89 active+clean; 222 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2024-09-21T09:47:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:39 smithi017 ceph-mon[105151]: pgmap v528: 89 pgs: 89 active+clean; 222 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2024-09-21T09:47:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:39 smithi134 ceph-mon[90085]: pgmap v528: 89 pgs: 89 active+clean; 222 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2024-09-21T09:47:40.152 INFO:tasks.rados.rados.0.smithi017.stdout:2222: done (2 left) 2024-09-21T09:47:40.152 INFO:tasks.rados.rados.0.smithi017.stdout:2223: snap_remove snap 246 2024-09-21T09:47:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:40 smithi017 ceph-mon[102913]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T09:47:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:40 smithi017 ceph-mon[105151]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T09:47:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:40 smithi134 ceph-mon[90085]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T09:47:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:47:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:47:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:41.150 INFO:tasks.rados.rados.0.smithi017.stdout:2223: done (2 left) 2024-09-21T09:47:41.151 INFO:tasks.rados.rados.0.smithi017.stdout:2224: write oid 34 current snap is 287 2024-09-21T09:47:41.151 INFO:tasks.rados.rados.0.smithi017.stdout:2224: seq_num 836 ranges {450652=790641,1721064=329588} 2024-09-21T09:47:41.158 INFO:tasks.rados.rados.0.smithi017.stdout:2224: writing smithi01785587-34 from 450652 to 1241293 tid 1 2024-09-21T09:47:41.164 INFO:tasks.rados.rados.0.smithi017.stdout:2224: writing smithi01785587-34 from 1721064 to 2050652 tid 2 2024-09-21T09:47:41.165 INFO:tasks.rados.rados.0.smithi017.stdout:2225: snap_remove snap 277 2024-09-21T09:47:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:41 smithi017 ceph-mon[102913]: pgmap v531: 89 pgs: 89 active+clean; 222 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 829 KiB/s wr, 3 op/s 2024-09-21T09:47:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:41 smithi017 ceph-mon[102913]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T09:47:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:41 smithi017 ceph-mon[105151]: pgmap v531: 89 pgs: 89 active+clean; 222 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 829 KiB/s wr, 3 op/s 2024-09-21T09:47:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:41 smithi017 ceph-mon[105151]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T09:47:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:41 smithi134 ceph-mon[90085]: pgmap v531: 89 pgs: 89 active+clean; 222 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 829 KiB/s wr, 3 op/s 2024-09-21T09:47:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:41 smithi134 ceph-mon[90085]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T09:47:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:42.166 INFO:tasks.rados.rados.0.smithi017.stdout:2225: done (3 left) 2024-09-21T09:47:42.166 INFO:tasks.rados.rados.0.smithi017.stdout:2226: write oid 27 current snap is 287 2024-09-21T09:47:42.166 INFO:tasks.rados.rados.0.smithi017.stdout:2226: seq_num 837 ranges {467459=582243,1495899=404945} 2024-09-21T09:47:42.172 INFO:tasks.rados.rados.0.smithi017.stdout:2226: writing smithi01785587-27 from 467459 to 1049702 tid 1 2024-09-21T09:47:42.177 INFO:tasks.rados.rados.0.smithi017.stdout:2226: writing smithi01785587-27 from 1495899 to 1900844 tid 2 2024-09-21T09:47:42.178 INFO:tasks.rados.rados.0.smithi017.stdout:2227: delete oid 35 current snap is 287 2024-09-21T09:47:42.178 INFO:tasks.rados.rados.0.smithi017.stdout:2221: expect (ObjNum 698 snap 221 seq_num 698) 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:2224: finishing write tid 1 to smithi01785587-34 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:2224: finishing write tid 2 to smithi01785587-34 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:2224: finishing write tid 3 to smithi01785587-34 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:2224: finishing write tid 4 to smithi01785587-34 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1168 (ObjNum 836 snap 287 seq_num 836) dirty exists 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:2224: left oid 34 (ObjNum 836 snap 287 seq_num 836) 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:2226: finishing write tid 1 to smithi01785587-27 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:2226: finishing write tid 2 to smithi01785587-27 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:2226: finishing write tid 3 to smithi01785587-27 2024-09-21T09:47:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:2226: finishing write tid 4 to smithi01785587-27 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 680 (ObjNum 837 snap 287 seq_num 837) dirty exists 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2226: left oid 27 (ObjNum 837 snap 287 seq_num 837) 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2220: done (4 left) 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2221: done (3 left) 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2224: done (2 left) 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2226: done (1 left) 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2227: done (0 left) 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2228: read oid 16 snap -1 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2228: expect (ObjNum 828 snap 282 seq_num 828) 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2229: write oid 27 current snap is 287 2024-09-21T09:47:42.372 INFO:tasks.rados.rados.0.smithi017.stdout:2229: seq_num 839 ranges {501073=732741,1896510=204563} 2024-09-21T09:47:42.377 INFO:tasks.rados.rados.0.smithi017.stdout:2229: writing smithi01785587-27 from 501073 to 1233814 tid 1 2024-09-21T09:47:42.382 INFO:tasks.rados.rados.0.smithi017.stdout:2229: writing smithi01785587-27 from 1896510 to 2101073 tid 2 2024-09-21T09:47:42.382 INFO:tasks.rados.rados.0.smithi017.stdout:2230: snap_create 2024-09-21T09:47:42.382 INFO:tasks.rados.rados.0.smithi017.stdout:2229: finishing write tid 1 to smithi01785587-27 2024-09-21T09:47:42.384 INFO:tasks.rados.rados.0.smithi017.stdout:2229: finishing write tid 2 to smithi01785587-27 2024-09-21T09:47:42.384 INFO:tasks.rados.rados.0.smithi017.stdout:2229: finishing write tid 3 to smithi01785587-27 2024-09-21T09:47:42.385 INFO:tasks.rados.rados.0.smithi017.stdout:2229: finishing write tid 4 to smithi01785587-27 2024-09-21T09:47:42.385 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 683 (ObjNum 839 snap 287 seq_num 839) dirty exists 2024-09-21T09:47:42.385 INFO:tasks.rados.rados.0.smithi017.stdout:2229: left oid 27 (ObjNum 839 snap 287 seq_num 839) 2024-09-21T09:47:42.385 INFO:tasks.rados.rados.0.smithi017.stdout:2228: expect (ObjNum 828 snap 282 seq_num 828) 2024-09-21T09:47:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:42 smithi017 ceph-mon[102913]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T09:47:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:42 smithi017 ceph-mon[105151]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T09:47:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:42 smithi134 ceph-mon[90085]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T09:47:43.111 INFO:tasks.rados.rados.0.smithi017.stdout:2228: done (2 left) 2024-09-21T09:47:43.112 INFO:tasks.rados.rados.0.smithi017.stdout:2229: done (1 left) 2024-09-21T09:47:43.112 INFO:tasks.rados.rados.0.smithi017.stdout:2230: done (0 left) 2024-09-21T09:47:43.112 INFO:tasks.rados.rados.0.smithi017.stdout:2231: write oid 12 current snap is 288 2024-09-21T09:47:43.112 INFO:tasks.rados.rados.0.smithi017.stdout:2231: seq_num 840 ranges {517880=407990,1554992=562888} 2024-09-21T09:47:43.116 INFO:tasks.rados.rados.0.smithi017.stdout:2231: writing smithi01785587-12 from 517880 to 925870 tid 1 2024-09-21T09:47:43.124 INFO:tasks.rados.rados.0.smithi017.stdout:2231: writing smithi01785587-12 from 1554992 to 2117880 tid 2 2024-09-21T09:47:43.124 INFO:tasks.rados.rados.0.smithi017.stdout:2232: write oid 38 current snap is 288 2024-09-21T09:47:43.124 INFO:tasks.rados.rados.0.smithi017.stdout:2231: finishing write tid 1 to smithi01785587-12 2024-09-21T09:47:43.124 INFO:tasks.rados.rados.0.smithi017.stdout:2232: seq_num 841 ranges {534687=483239,1613474=412283} 2024-09-21T09:47:43.130 INFO:tasks.rados.rados.0.smithi017.stdout:2232: writing smithi01785587-38 from 534687 to 1017926 tid 1 2024-09-21T09:47:43.136 INFO:tasks.rados.rados.0.smithi017.stdout:2232: writing smithi01785587-38 from 1613474 to 2025757 tid 2 2024-09-21T09:47:43.136 INFO:tasks.rados.rados.0.smithi017.stdout:2233: delete oid 39 current snap is 288 2024-09-21T09:47:43.136 INFO:tasks.rados.rados.0.smithi017.stdout:2231: finishing write tid 2 to smithi01785587-12 2024-09-21T09:47:43.136 INFO:tasks.rados.rados.0.smithi017.stdout:2231: finishing write tid 3 to smithi01785587-12 2024-09-21T09:47:43.136 INFO:tasks.rados.rados.0.smithi017.stdout:2231: finishing write tid 4 to smithi01785587-12 2024-09-21T09:47:43.136 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 807 (ObjNum 840 snap 288 seq_num 840) dirty exists 2024-09-21T09:47:43.136 INFO:tasks.rados.rados.0.smithi017.stdout:2231: left oid 12 (ObjNum 840 snap 288 seq_num 840) 2024-09-21T09:47:43.136 INFO:tasks.rados.rados.0.smithi017.stdout:2232: finishing write tid 1 to smithi01785587-38 2024-09-21T09:47:43.139 INFO:tasks.rados.rados.0.smithi017.stdout:2232: finishing write tid 2 to smithi01785587-38 2024-09-21T09:47:43.139 INFO:tasks.rados.rados.0.smithi017.stdout:2231: done (2 left) 2024-09-21T09:47:43.139 INFO:tasks.rados.rados.0.smithi017.stdout:2233: done (1 left) 2024-09-21T09:47:43.139 INFO:tasks.rados.rados.0.smithi017.stdout:2234: write oid 30 current snap is 288 2024-09-21T09:47:43.139 INFO:tasks.rados.rados.0.smithi017.stdout:2234: seq_num 843 ranges {568301=633737,1614085=415952} 2024-09-21T09:47:43.145 INFO:tasks.rados.rados.0.smithi017.stdout:2234: writing smithi01785587-30 from 568301 to 1202038 tid 1 2024-09-21T09:47:43.150 INFO:tasks.rados.rados.0.smithi017.stdout:2234: writing smithi01785587-30 from 1614085 to 2030037 tid 2 2024-09-21T09:47:43.150 INFO:tasks.rados.rados.0.smithi017.stdout:2235: snap_create 2024-09-21T09:47:43.150 INFO:tasks.rados.rados.0.smithi017.stdout:2232: finishing write tid 3 to smithi01785587-38 2024-09-21T09:47:43.150 INFO:tasks.rados.rados.0.smithi017.stdout:2232: finishing write tid 4 to smithi01785587-38 2024-09-21T09:47:43.150 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 734 (ObjNum 841 snap 288 seq_num 841) dirty exists 2024-09-21T09:47:43.151 INFO:tasks.rados.rados.0.smithi017.stdout:2232: left oid 38 (ObjNum 841 snap 288 seq_num 841) 2024-09-21T09:47:43.151 INFO:tasks.rados.rados.0.smithi017.stdout:2234: finishing write tid 1 to smithi01785587-30 2024-09-21T09:47:43.153 INFO:tasks.rados.rados.0.smithi017.stdout:2234: finishing write tid 2 to smithi01785587-30 2024-09-21T09:47:43.153 INFO:tasks.rados.rados.0.smithi017.stdout:2234: finishing write tid 3 to smithi01785587-30 2024-09-21T09:47:43.153 INFO:tasks.rados.rados.0.smithi017.stdout:2234: finishing write tid 4 to smithi01785587-30 2024-09-21T09:47:43.153 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 889 (ObjNum 843 snap 288 seq_num 843) dirty exists 2024-09-21T09:47:43.153 INFO:tasks.rados.rados.0.smithi017.stdout:2234: left oid 30 (ObjNum 843 snap 288 seq_num 843) 2024-09-21T09:47:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:43 smithi017 ceph-mon[102913]: pgmap v534: 89 pgs: 89 active+clean; 224 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:47:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:43 smithi017 ceph-mon[102913]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T09:47:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:43 smithi017 ceph-mon[102913]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T09:47:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:43 smithi017 ceph-mon[105151]: pgmap v534: 89 pgs: 89 active+clean; 224 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:47:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:43 smithi017 ceph-mon[105151]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T09:47:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:43 smithi017 ceph-mon[105151]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T09:47:43.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:43 smithi134 ceph-mon[90085]: pgmap v534: 89 pgs: 89 active+clean; 224 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:47:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:43 smithi134 ceph-mon[90085]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T09:47:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:43 smithi134 ceph-mon[90085]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T09:47:44.113 INFO:tasks.rados.rados.0.smithi017.stdout:2232: done (2 left) 2024-09-21T09:47:44.113 INFO:tasks.rados.rados.0.smithi017.stdout:2234: done (1 left) 2024-09-21T09:47:44.113 INFO:tasks.rados.rados.0.smithi017.stdout:2235: done (0 left) 2024-09-21T09:47:44.114 INFO:tasks.rados.rados.0.smithi017.stdout:2236: read oid 46 snap -1 2024-09-21T09:47:44.114 INFO:tasks.rados.rados.0.smithi017.stdout:2236: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:47:44.114 INFO:tasks.rados.rados.0.smithi017.stdout:2237: write oid 5 current snap is 289 2024-09-21T09:47:44.114 INFO:tasks.rados.rados.0.smithi017.stdout:2237: seq_num 844 ranges {585108=425339,1788920=396188} 2024-09-21T09:47:44.118 INFO:tasks.rados.rados.0.smithi017.stdout:2237: writing smithi01785587-5 from 585108 to 1010447 tid 1 2024-09-21T09:47:44.125 INFO:tasks.rados.rados.0.smithi017.stdout:2237: writing smithi01785587-5 from 1788920 to 2185108 tid 2 2024-09-21T09:47:44.126 INFO:tasks.rados.rados.0.smithi017.stdout:2238: write oid 37 current snap is 289 2024-09-21T09:47:44.126 INFO:tasks.rados.rados.0.smithi017.stdout:2238: seq_num 845 ranges {601915=500588,1847402=354513} 2024-09-21T09:47:44.131 INFO:tasks.rados.rados.0.smithi017.stdout:2238: writing smithi01785587-37 from 601915 to 1102503 tid 1 2024-09-21T09:47:44.138 INFO:tasks.rados.rados.0.smithi017.stdout:2238: writing smithi01785587-37 from 1847402 to 2201915 tid 2 2024-09-21T09:47:44.138 INFO:tasks.rados.rados.0.smithi017.stdout:2239: delete oid 16 current snap is 289 2024-09-21T09:47:44.138 INFO:tasks.rados.rados.0.smithi017.stdout:2236: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:47:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2237: finishing write tid 1 to smithi01785587-5 2024-09-21T09:47:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2236: done (3 left) 2024-09-21T09:47:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2239: done (2 left) 2024-09-21T09:47:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2240: read oid 40 snap 251 2024-09-21T09:47:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2240: expect deleted 2024-09-21T09:47:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2241: snap_remove snap 256 2024-09-21T09:47:45.110 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:47:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:47:44] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.51.0" 2024-09-21T09:47:45.168 INFO:tasks.rados.rados.0.smithi017.stdout:2241: done (3 left) 2024-09-21T09:47:45.168 INFO:tasks.rados.rados.0.smithi017.stdout:2242: delete oid 45 current snap is 289 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:2237: finishing write tid 2 to smithi01785587-5 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:2237: finishing write tid 3 to smithi01785587-5 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:2237: finishing write tid 4 to smithi01785587-5 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1070 (ObjNum 844 snap 289 seq_num 844) dirty exists 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:2237: left oid 5 (ObjNum 844 snap 289 seq_num 844) 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:2238: finishing write tid 1 to smithi01785587-37 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:2238: finishing write tid 2 to smithi01785587-37 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:2238: finishing write tid 3 to smithi01785587-37 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:2238: finishing write tid 4 to smithi01785587-37 2024-09-21T09:47:45.169 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1160 (ObjNum 845 snap 289 seq_num 845) dirty exists 2024-09-21T09:47:45.170 INFO:tasks.rados.rados.0.smithi017.stdout:2238: left oid 37 (ObjNum 845 snap 289 seq_num 845) 2024-09-21T09:47:45.176 INFO:tasks.rados.rados.0.smithi017.stdout:2237: done (3 left) 2024-09-21T09:47:45.176 INFO:tasks.rados.rados.0.smithi017.stdout:2238: done (2 left) 2024-09-21T09:47:45.176 INFO:tasks.rados.rados.0.smithi017.stdout:2240: done (1 left) 2024-09-21T09:47:45.176 INFO:tasks.rados.rados.0.smithi017.stdout:2242: done (0 left) 2024-09-21T09:47:45.176 INFO:tasks.rados.rados.0.smithi017.stdout:2243: rollback oid 10 current snap is 289 2024-09-21T09:47:45.176 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 284 2024-09-21T09:47:45.176 INFO:tasks.rados.rados.0.smithi017.stdout:2244: read oid 16 snap 251 2024-09-21T09:47:45.176 INFO:tasks.rados.rados.0.smithi017.stdout:2244: expect (ObjNum 708 snap 228 seq_num 708) 2024-09-21T09:47:45.176 INFO:tasks.rados.rados.0.smithi017.stdout:2245: snap_remove snap 288 2024-09-21T09:47:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:45 smithi017 ceph-mon[102913]: pgmap v537: 89 pgs: 89 active+clean; 225 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-21T09:47:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:45 smithi017 ceph-mon[102913]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T09:47:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:45 smithi017 ceph-mon[105151]: pgmap v537: 89 pgs: 89 active+clean; 225 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-21T09:47:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:45 smithi017 ceph-mon[105151]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T09:47:45.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:45 smithi134 ceph-mon[90085]: pgmap v537: 89 pgs: 89 active+clean; 225 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-21T09:47:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:45 smithi134 ceph-mon[90085]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T09:47:46.176 INFO:tasks.rados.rados.0.smithi017.stdout:2245: done (2 left) 2024-09-21T09:47:46.176 INFO:tasks.rados.rados.0.smithi017.stdout:2246: delete oid 33 current snap is 289 2024-09-21T09:47:46.176 INFO:tasks.rados.rados.0.smithi017.stdout:2243: finishing rollback tid 0 to smithi01785587-10 2024-09-21T09:47:46.176 INFO:tasks.rados.rados.0.smithi017.stdout:2243: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:47:46.176 INFO:tasks.rados.rados.0.smithi017.stdout:2243: finishing rollback tid 2 to smithi01785587-10 2024-09-21T09:47:46.176 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 150 (ObjNum 777 snap 267 seq_num 777) dirty exists 2024-09-21T09:47:46.176 INFO:tasks.rados.rados.0.smithi017.stdout:2244: expect (ObjNum 708 snap 228 seq_num 708) 2024-09-21T09:47:46.313 INFO:tasks.rados.rados.0.smithi017.stdout:2243: done (2 left) 2024-09-21T09:47:46.313 INFO:tasks.rados.rados.0.smithi017.stdout:2244: done (1 left) 2024-09-21T09:47:46.313 INFO:tasks.rados.rados.0.smithi017.stdout:2246: done (0 left) 2024-09-21T09:47:46.313 INFO:tasks.rados.rados.0.smithi017.stdout:2247: write oid 39 current snap is 289 2024-09-21T09:47:46.313 INFO:tasks.rados.rados.0.smithi017.stdout:2247: seq_num 849 ranges {669143=401584,1564977=426959} 2024-09-21T09:47:46.319 INFO:tasks.rados.rados.0.smithi017.stdout:2247: writing smithi01785587-39 from 669143 to 1070727 tid 1 2024-09-21T09:47:46.324 INFO:tasks.rados.rados.0.smithi017.stdout:2247: writing smithi01785587-39 from 1564977 to 1991936 tid 2 2024-09-21T09:47:46.324 INFO:tasks.rados.rados.0.smithi017.stdout:2248: rollback oid 17 current snap is 289 2024-09-21T09:47:46.324 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 287 2024-09-21T09:47:46.324 INFO:tasks.rados.rados.0.smithi017.stdout:2247: finishing write tid 1 to smithi01785587-39 2024-09-21T09:47:46.324 INFO:tasks.rados.rados.0.smithi017.stdout:2249: snap_remove snap 281 2024-09-21T09:47:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:46 smithi017 ceph-mon[102913]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T09:47:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:46 smithi017 ceph-mon[105151]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T09:47:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:46 smithi134 ceph-mon[90085]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T09:47:47.181 INFO:tasks.rados.rados.0.smithi017.stdout:2249: done (2 left) 2024-09-21T09:47:47.181 INFO:tasks.rados.rados.0.smithi017.stdout:2250: read oid 2 snap -1 2024-09-21T09:47:47.181 INFO:tasks.rados.rados.0.smithi017.stdout:2250: expect (ObjNum 814 snap 279 seq_num 814) 2024-09-21T09:47:47.181 INFO:tasks.rados.rados.0.smithi017.stdout:2251: write oid 9 current snap is 289 2024-09-21T09:47:47.181 INFO:tasks.rados.rados.0.smithi017.stdout:2251: seq_num 850 ranges {685950=476833,1623459=570617} 2024-09-21T09:47:47.188 INFO:tasks.rados.rados.0.smithi017.stdout:2251: writing smithi01785587-9 from 685950 to 1162783 tid 1 2024-09-21T09:47:47.195 INFO:tasks.rados.rados.0.smithi017.stdout:2251: writing smithi01785587-9 from 1623459 to 2194076 tid 2 2024-09-21T09:47:47.195 INFO:tasks.rados.rados.0.smithi017.stdout:2252: write oid 35 current snap is 289 2024-09-21T09:47:47.195 INFO:tasks.rados.rados.0.smithi017.stdout:2247: finishing write tid 2 to smithi01785587-39 2024-09-21T09:47:47.195 INFO:tasks.rados.rados.0.smithi017.stdout:2252: seq_num 851 ranges {702757=552082,1681941=430628} 2024-09-21T09:47:47.202 INFO:tasks.rados.rados.0.smithi017.stdout:2252: writing smithi01785587-35 from 702757 to 1254839 tid 1 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:2252: writing smithi01785587-35 from 1681941 to 2112569 tid 2 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:2253: read oid 4 snap 284 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:2253: expect (ObjNum 817 snap 280 seq_num 817) 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:2254: rollback oid 36 current snap is 289 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 36 to 287 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:2248: finishing rollback tid 0 to smithi01785587-17 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:2247: finishing write tid 3 to smithi01785587-39 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:2248: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:2247: finishing write tid 4 to smithi01785587-39 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1171 (ObjNum 849 snap 289 seq_num 849) dirty exists 2024-09-21T09:47:47.208 INFO:tasks.rados.rados.0.smithi017.stdout:2247: left oid 39 (ObjNum 849 snap 289 seq_num 849) 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2248: finishing rollback tid 2 to smithi01785587-17 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 695 (ObjNum 809 snap 279 seq_num 809) dirty exists 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2247: done (6 left) 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2248: done (5 left) 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2255: read oid 46 snap -1 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2255: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2256: read oid 7 snap -1 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2256: expect (ObjNum 767 snap 259 seq_num 767) 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2257: read oid 17 snap 272 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2257: expect (ObjNum 796 snap 272 seq_num 796) 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2258: rollback oid 32 current snap is 289 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 282 2024-09-21T09:47:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2250: expect (ObjNum 814 snap 279 seq_num 814) 2024-09-21T09:47:47.331 INFO:tasks.rados.rados.0.smithi017.stdout:2251: finishing write tid 1 to smithi01785587-9 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:2251: finishing write tid 2 to smithi01785587-9 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:2251: finishing write tid 3 to smithi01785587-9 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:2251: finishing write tid 4 to smithi01785587-9 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 451 (ObjNum 850 snap 289 seq_num 850) dirty exists 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:2251: left oid 9 (ObjNum 850 snap 289 seq_num 850) 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:2252: finishing write tid 1 to smithi01785587-35 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:2250: done (8 left) 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:2251: done (7 left) 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:2259: write oid 50 current snap is 289 2024-09-21T09:47:47.332 INFO:tasks.rados.rados.0.smithi017.stdout:2255: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:47:47.442 INFO:tasks.rados.rados.0.smithi017.stdout:2254: finishing rollback tid 1 to smithi01785587-36 2024-09-21T09:47:47.442 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1087 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:47:47.442 INFO:tasks.rados.rados.0.smithi017.stdout:2252: finishing write tid 2 to smithi01785587-35 2024-09-21T09:47:47.442 INFO:tasks.rados.rados.0.smithi017.stdout:2252: finishing write tid 3 to smithi01785587-35 2024-09-21T09:47:47.442 INFO:tasks.rados.rados.0.smithi017.stdout:2252: finishing write tid 4 to smithi01785587-35 2024-09-21T09:47:47.442 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 254 (ObjNum 851 snap 289 seq_num 851) dirty exists 2024-09-21T09:47:47.443 INFO:tasks.rados.rados.0.smithi017.stdout:2252: left oid 35 (ObjNum 851 snap 289 seq_num 851) 2024-09-21T09:47:47.443 INFO:tasks.rados.rados.0.smithi017.stdout:2259: seq_num 852 ranges {719564=743684,2140423=179141} 2024-09-21T09:47:47.450 INFO:tasks.rados.rados.0.smithi017.stdout:2259: writing smithi01785587-50 from 719564 to 1463248 tid 1 2024-09-21T09:47:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:47 smithi017 ceph-mon[102913]: pgmap v540: 89 pgs: 89 active+clean; 220 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 927 KiB/s wr, 13 op/s 2024-09-21T09:47:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:47 smithi017 ceph-mon[102913]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T09:47:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:47 smithi017 ceph-mon[105151]: pgmap v540: 89 pgs: 89 active+clean; 220 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 927 KiB/s wr, 13 op/s 2024-09-21T09:47:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:47 smithi017 ceph-mon[105151]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T09:47:47.455 INFO:tasks.rados.rados.0.smithi017.stdout:2259: writing smithi01785587-50 from 2140423 to 2319564 tid 2 2024-09-21T09:47:47.455 INFO:tasks.rados.rados.0.smithi017.stdout:2252: done (7 left) 2024-09-21T09:47:47.455 INFO:tasks.rados.rados.0.smithi017.stdout:2254: done (6 left) 2024-09-21T09:47:47.455 INFO:tasks.rados.rados.0.smithi017.stdout:2255: done (5 left) 2024-09-21T09:47:47.455 INFO:tasks.rados.rados.0.smithi017.stdout:2260: read oid 18 snap 284 2024-09-21T09:47:47.455 INFO:tasks.rados.rados.0.smithi017.stdout:2260: expect (ObjNum 786 snap 270 seq_num 786) 2024-09-21T09:47:47.455 INFO:tasks.rados.rados.0.smithi017.stdout:2261: read oid 24 snap -1 2024-09-21T09:47:47.455 INFO:tasks.rados.rados.0.smithi017.stdout:2261: expect deleted 2024-09-21T09:47:47.456 INFO:tasks.rados.rados.0.smithi017.stdout:2262: delete oid 36 current snap is 289 2024-09-21T09:47:47.456 INFO:tasks.rados.rados.0.smithi017.stdout:2258: finishing rollback tid 0 to smithi01785587-32 2024-09-21T09:47:47.456 INFO:tasks.rados.rados.0.smithi017.stdout:2258: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:47:47.456 INFO:tasks.rados.rados.0.smithi017.stdout:2256: expect (ObjNum 767 snap 259 seq_num 767) 2024-09-21T09:47:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:47 smithi134 ceph-mon[90085]: pgmap v540: 89 pgs: 89 active+clean; 220 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 927 KiB/s wr, 13 op/s 2024-09-21T09:47:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:47 smithi134 ceph-mon[90085]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T09:47:47.656 INFO:tasks.rados.rados.0.smithi017.stdout:2258: finishing rollback tid 2 to smithi01785587-32 2024-09-21T09:47:47.656 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1000 (ObjNum 823 snap 282 seq_num 823) dirty exists 2024-09-21T09:47:47.656 INFO:tasks.rados.rados.0.smithi017.stdout:2257: expect (ObjNum 796 snap 272 seq_num 796) 2024-09-21T09:47:47.767 INFO:tasks.rados.rados.0.smithi017.stdout:2253: expect (ObjNum 817 snap 280 seq_num 817) 2024-09-21T09:47:47.890 INFO:tasks.rados.rados.0.smithi017.stdout:2259: finishing write tid 1 to smithi01785587-50 2024-09-21T09:47:47.891 INFO:tasks.rados.rados.0.smithi017.stdout:2259: finishing write tid 2 to smithi01785587-50 2024-09-21T09:47:47.891 INFO:tasks.rados.rados.0.smithi017.stdout:2259: finishing write tid 3 to smithi01785587-50 2024-09-21T09:47:47.891 INFO:tasks.rados.rados.0.smithi017.stdout:2259: finishing write tid 4 to smithi01785587-50 2024-09-21T09:47:47.891 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 742 (ObjNum 852 snap 289 seq_num 852) dirty exists 2024-09-21T09:47:47.891 INFO:tasks.rados.rados.0.smithi017.stdout:2259: left oid 50 (ObjNum 852 snap 289 seq_num 852) 2024-09-21T09:47:47.891 INFO:tasks.rados.rados.0.smithi017.stdout:2260: expect (ObjNum 786 snap 270 seq_num 786) 2024-09-21T09:47:48.031 INFO:tasks.rados.rados.0.smithi017.stdout:2253: done (7 left) 2024-09-21T09:47:48.031 INFO:tasks.rados.rados.0.smithi017.stdout:2256: done (6 left) 2024-09-21T09:47:48.031 INFO:tasks.rados.rados.0.smithi017.stdout:2257: done (5 left) 2024-09-21T09:47:48.031 INFO:tasks.rados.rados.0.smithi017.stdout:2258: done (4 left) 2024-09-21T09:47:48.031 INFO:tasks.rados.rados.0.smithi017.stdout:2259: done (3 left) 2024-09-21T09:47:48.031 INFO:tasks.rados.rados.0.smithi017.stdout:2260: done (2 left) 2024-09-21T09:47:48.031 INFO:tasks.rados.rados.0.smithi017.stdout:2261: done (1 left) 2024-09-21T09:47:48.031 INFO:tasks.rados.rados.0.smithi017.stdout:2262: done (0 left) 2024-09-21T09:47:48.031 INFO:tasks.rados.rados.0.smithi017.stdout:2263: write oid 14 current snap is 289 2024-09-21T09:47:48.032 INFO:tasks.rados.rados.0.smithi017.stdout:2263: seq_num 854 ranges {753178=494182,1857387=495791} 2024-09-21T09:47:48.037 INFO:tasks.rados.rados.0.smithi017.stdout:2263: writing smithi01785587-14 from 753178 to 1247360 tid 1 2024-09-21T09:47:48.044 INFO:tasks.rados.rados.0.smithi017.stdout:2263: writing smithi01785587-14 from 1857387 to 2353178 tid 2 2024-09-21T09:47:48.044 INFO:tasks.rados.rados.0.smithi017.stdout:2264: delete oid 41 current snap is 289 2024-09-21T09:47:48.044 INFO:tasks.rados.rados.0.smithi017.stdout:2263: finishing write tid 1 to smithi01785587-14 2024-09-21T09:47:48.046 INFO:tasks.rados.rados.0.smithi017.stdout:2264: done (1 left) 2024-09-21T09:47:48.046 INFO:tasks.rados.rados.0.smithi017.stdout:2265: read oid 46 snap -1 2024-09-21T09:47:48.046 INFO:tasks.rados.rados.0.smithi017.stdout:2265: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:47:48.046 INFO:tasks.rados.rados.0.smithi017.stdout:2266: rollback oid 43 current snap is 289 2024-09-21T09:47:48.046 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 287 2024-09-21T09:47:48.046 INFO:tasks.rados.rados.0.smithi017.stdout:2267: read oid 13 snap 285 2024-09-21T09:47:48.046 INFO:tasks.rados.rados.0.smithi017.stdout:2267: expect (ObjNum 632 snap 200 seq_num 632) 2024-09-21T09:47:48.046 INFO:tasks.rados.rados.0.smithi017.stdout:2268: write oid 27 current snap is 289 2024-09-21T09:47:48.046 INFO:tasks.rados.rados.0.smithi017.stdout:2268: seq_num 856 ranges {786792=644680,1857998=528794} 2024-09-21T09:47:48.053 INFO:tasks.rados.rados.0.smithi017.stdout:2268: writing smithi01785587-27 from 786792 to 1431472 tid 1 2024-09-21T09:47:48.059 INFO:tasks.rados.rados.0.smithi017.stdout:2268: writing smithi01785587-27 from 1857998 to 2386792 tid 2 2024-09-21T09:47:48.059 INFO:tasks.rados.rados.0.smithi017.stdout:2269: write oid 29 current snap is 289 2024-09-21T09:47:48.059 INFO:tasks.rados.rados.0.smithi017.stdout:2269: seq_num 857 ranges {403599=719929,1916480=441635} 2024-09-21T09:47:48.065 INFO:tasks.rados.rados.0.smithi017.stdout:2269: writing smithi01785587-29 from 403599 to 1123528 tid 1 2024-09-21T09:47:48.072 INFO:tasks.rados.rados.0.smithi017.stdout:2269: writing smithi01785587-29 from 1916480 to 2358115 tid 2 2024-09-21T09:47:48.073 INFO:tasks.rados.rados.0.smithi017.stdout:2270: snap_create 2024-09-21T09:47:48.073 INFO:tasks.rados.rados.0.smithi017.stdout:2263: finishing write tid 2 to smithi01785587-14 2024-09-21T09:47:48.073 INFO:tasks.rados.rados.0.smithi017.stdout:2263: finishing write tid 3 to smithi01785587-14 2024-09-21T09:47:48.073 INFO:tasks.rados.rados.0.smithi017.stdout:2263: finishing write tid 4 to smithi01785587-14 2024-09-21T09:47:48.073 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1084 (ObjNum 854 snap 289 seq_num 854) dirty exists 2024-09-21T09:47:48.073 INFO:tasks.rados.rados.0.smithi017.stdout:2263: left oid 14 (ObjNum 854 snap 289 seq_num 854) 2024-09-21T09:47:48.073 INFO:tasks.rados.rados.0.smithi017.stdout:2265: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:47:48.184 INFO:tasks.rados.rados.0.smithi017.stdout:2266: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:47:48.185 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 661 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:47:48.185 INFO:tasks.rados.rados.0.smithi017.stdout:2267: expect (ObjNum 632 snap 200 seq_num 632) 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:2268: finishing write tid 1 to smithi01785587-27 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:2268: finishing write tid 2 to smithi01785587-27 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:2268: finishing write tid 3 to smithi01785587-27 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:2268: finishing write tid 4 to smithi01785587-27 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 703 (ObjNum 856 snap 289 seq_num 856) dirty exists 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:2268: left oid 27 (ObjNum 856 snap 289 seq_num 856) 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:2269: finishing write tid 1 to smithi01785587-29 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:2269: finishing write tid 2 to smithi01785587-29 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:2269: finishing write tid 3 to smithi01785587-29 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:2269: finishing write tid 4 to smithi01785587-29 2024-09-21T09:47:48.277 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 564 (ObjNum 857 snap 289 seq_num 857) dirty exists 2024-09-21T09:47:48.278 INFO:tasks.rados.rados.0.smithi017.stdout:2269: left oid 29 (ObjNum 857 snap 289 seq_num 857) 2024-09-21T09:47:48.278 INFO:tasks.rados.rados.0.smithi017.stdout:2263: done (6 left) 2024-09-21T09:47:48.278 INFO:tasks.rados.rados.0.smithi017.stdout:2265: done (5 left) 2024-09-21T09:47:48.278 INFO:tasks.rados.rados.0.smithi017.stdout:2266: done (4 left) 2024-09-21T09:47:48.278 INFO:tasks.rados.rados.0.smithi017.stdout:2267: done (3 left) 2024-09-21T09:47:48.278 INFO:tasks.rados.rados.0.smithi017.stdout:2268: done (2 left) 2024-09-21T09:47:48.278 INFO:tasks.rados.rados.0.smithi017.stdout:2269: done (1 left) 2024-09-21T09:47:48.278 INFO:tasks.rados.rados.0.smithi017.stdout:2270: done (0 left) 2024-09-21T09:47:48.278 INFO:tasks.rados.rados.0.smithi017.stdout:2271: snap_create 2024-09-21T09:47:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:48 smithi017 ceph-mon[102913]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T09:47:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:48 smithi017 ceph-mon[102913]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T09:47:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:48 smithi017 ceph-mon[105151]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T09:47:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:48 smithi017 ceph-mon[105151]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T09:47:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:48 smithi134 ceph-mon[90085]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T09:47:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:48 smithi134 ceph-mon[90085]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T09:47:49.190 INFO:tasks.rados.rados.0.smithi017.stdout:2271: done (0 left) 2024-09-21T09:47:49.191 INFO:tasks.rados.rados.0.smithi017.stdout:2272: delete oid 9 current snap is 291 2024-09-21T09:47:49.195 INFO:tasks.rados.rados.0.smithi017.stdout:2272: done (0 left) 2024-09-21T09:47:49.195 INFO:tasks.rados.rados.0.smithi017.stdout:2273: write oid 48 current snap is 291 2024-09-21T09:47:49.195 INFO:tasks.rados.rados.0.smithi017.stdout:2273: seq_num 859 ranges {437213=470427,1633444=728951} 2024-09-21T09:47:49.200 INFO:tasks.rados.rados.0.smithi017.stdout:2273: writing smithi01785587-48 from 437213 to 907640 tid 1 2024-09-21T09:47:49.207 INFO:tasks.rados.rados.0.smithi017.stdout:2273: writing smithi01785587-48 from 1633444 to 2362395 tid 2 2024-09-21T09:47:49.207 INFO:tasks.rados.rados.0.smithi017.stdout:2274: snap_remove snap 284 2024-09-21T09:47:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:49 smithi017 ceph-mon[102913]: pgmap v543: 89 pgs: 89 active+clean; 220 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 913 KiB/s wr, 11 op/s 2024-09-21T09:47:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:49 smithi017 ceph-mon[105151]: pgmap v543: 89 pgs: 89 active+clean; 220 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 913 KiB/s wr, 11 op/s 2024-09-21T09:47:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:49 smithi134 ceph-mon[90085]: pgmap v543: 89 pgs: 89 active+clean; 220 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 913 KiB/s wr, 11 op/s 2024-09-21T09:47:50.197 INFO:tasks.rados.rados.0.smithi017.stdout:2274: done (1 left) 2024-09-21T09:47:50.197 INFO:tasks.rados.rados.0.smithi017.stdout:2275: read oid 34 snap -1 2024-09-21T09:47:50.197 INFO:tasks.rados.rados.0.smithi017.stdout:2275: expect (ObjNum 836 snap 287 seq_num 836) 2024-09-21T09:47:50.197 INFO:tasks.rados.rados.0.smithi017.stdout:2276: snap_remove snap 278 2024-09-21T09:47:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:50 smithi017 ceph-mon[102913]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T09:47:50.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:50 smithi017 ceph-mon[105151]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T09:47:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:50 smithi134 ceph-mon[90085]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T09:47:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:47:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:47:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:47:51.204 INFO:tasks.rados.rados.0.smithi017.stdout:2276: done (2 left) 2024-09-21T09:47:51.204 INFO:tasks.rados.rados.0.smithi017.stdout:2277: snap_remove snap 283 2024-09-21T09:47:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:51 smithi017 ceph-mon[102913]: pgmap v546: 89 pgs: 89 active+clean; 222 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-09-21T09:47:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:51 smithi017 ceph-mon[102913]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T09:47:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:51 smithi017 ceph-mon[105151]: pgmap v546: 89 pgs: 89 active+clean; 222 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-09-21T09:47:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:51 smithi017 ceph-mon[105151]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T09:47:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:51 smithi134 ceph-mon[90085]: pgmap v546: 89 pgs: 89 active+clean; 222 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-09-21T09:47:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:51 smithi134 ceph-mon[90085]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T09:47:52.209 INFO:tasks.rados.rados.0.smithi017.stdout:2277: done (2 left) 2024-09-21T09:47:52.209 INFO:tasks.rados.rados.0.smithi017.stdout:2278: write oid 7 current snap is 291 2024-09-21T09:47:52.209 INFO:tasks.rados.rados.0.smithi017.stdout:2278: seq_num 860 ranges {454020=662029,1691926=588962} 2024-09-21T09:47:52.216 INFO:tasks.rados.rados.0.smithi017.stdout:2278: writing smithi01785587-7 from 454020 to 1116049 tid 1 2024-09-21T09:47:52.222 INFO:tasks.rados.rados.0.smithi017.stdout:2278: writing smithi01785587-7 from 1691926 to 2280888 tid 2 2024-09-21T09:47:52.223 INFO:tasks.rados.rados.0.smithi017.stdout:2279: snap_remove snap 251 2024-09-21T09:47:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:52 smithi134 ceph-mon[90085]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T09:47:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:52 smithi017 ceph-mon[102913]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T09:47:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:52 smithi017 ceph-mon[105151]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T09:47:53.120 INFO:tasks.rados.rados.0.smithi017.stdout:2279: done (3 left) 2024-09-21T09:47:53.120 INFO:tasks.rados.rados.0.smithi017.stdout:2280: write oid 19 current snap is 291 2024-09-21T09:47:53.120 INFO:tasks.rados.rados.0.smithi017.stdout:2280: seq_num 861 ranges {470827=737278,1750408=720419} 2024-09-21T09:47:53.128 INFO:tasks.rados.rados.0.smithi017.stdout:2280: writing smithi01785587-19 from 470827 to 1208105 tid 1 2024-09-21T09:47:53.135 INFO:tasks.rados.rados.0.smithi017.stdout:2280: writing smithi01785587-19 from 1750408 to 2470827 tid 2 2024-09-21T09:47:53.135 INFO:tasks.rados.rados.0.smithi017.stdout:2281: snap_create 2024-09-21T09:47:53.135 INFO:tasks.rados.rados.0.smithi017.stdout:2273: finishing write tid 1 to smithi01785587-48 2024-09-21T09:47:53.135 INFO:tasks.rados.rados.0.smithi017.stdout:2273: finishing write tid 2 to smithi01785587-48 2024-09-21T09:47:53.136 INFO:tasks.rados.rados.0.smithi017.stdout:2273: finishing write tid 3 to smithi01785587-48 2024-09-21T09:47:53.136 INFO:tasks.rados.rados.0.smithi017.stdout:2273: finishing write tid 4 to smithi01785587-48 2024-09-21T09:47:53.136 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 905 (ObjNum 859 snap 291 seq_num 859) dirty exists 2024-09-21T09:47:53.136 INFO:tasks.rados.rados.0.smithi017.stdout:2273: left oid 48 (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:47:53.136 INFO:tasks.rados.rados.0.smithi017.stdout:2275: expect (ObjNum 836 snap 287 seq_num 836) 2024-09-21T09:47:53.201 INFO:tasks.rados.rados.0.smithi017.stdout:2278: finishing write tid 1 to smithi01785587-7 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:2278: finishing write tid 2 to smithi01785587-7 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:2278: finishing write tid 3 to smithi01785587-7 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:2278: finishing write tid 4 to smithi01785587-7 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1196 (ObjNum 860 snap 291 seq_num 860) dirty exists 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:2278: left oid 7 (ObjNum 860 snap 291 seq_num 860) 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:2280: finishing write tid 1 to smithi01785587-19 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:2280: finishing write tid 2 to smithi01785587-19 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:2280: finishing write tid 3 to smithi01785587-19 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:2280: finishing write tid 4 to smithi01785587-19 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 933 (ObjNum 861 snap 291 seq_num 861) dirty exists 2024-09-21T09:47:53.202 INFO:tasks.rados.rados.0.smithi017.stdout:2280: left oid 19 (ObjNum 861 snap 291 seq_num 861) 2024-09-21T09:47:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:53 smithi134 ceph-mon[90085]: pgmap v549: 89 pgs: 89 active+clean; 228 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.9 MiB/s wr, 28 op/s 2024-09-21T09:47:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:53 smithi134 ceph-mon[90085]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T09:47:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:53 smithi134 ceph-mon[90085]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T09:47:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:53 smithi017 ceph-mon[102913]: pgmap v549: 89 pgs: 89 active+clean; 228 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.9 MiB/s wr, 28 op/s 2024-09-21T09:47:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:53 smithi017 ceph-mon[102913]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T09:47:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:53 smithi017 ceph-mon[102913]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T09:47:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:53 smithi017 ceph-mon[105151]: pgmap v549: 89 pgs: 89 active+clean; 228 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.9 MiB/s wr, 28 op/s 2024-09-21T09:47:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:53 smithi017 ceph-mon[105151]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T09:47:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:53 smithi017 ceph-mon[105151]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T09:47:54.114 INFO:tasks.rados.rados.0.smithi017.stdout:2273: done (4 left) 2024-09-21T09:47:54.115 INFO:tasks.rados.rados.0.smithi017.stdout:2275: done (3 left) 2024-09-21T09:47:54.115 INFO:tasks.rados.rados.0.smithi017.stdout:2278: done (2 left) 2024-09-21T09:47:54.115 INFO:tasks.rados.rados.0.smithi017.stdout:2280: done (1 left) 2024-09-21T09:47:54.115 INFO:tasks.rados.rados.0.smithi017.stdout:2281: done (0 left) 2024-09-21T09:47:54.115 INFO:tasks.rados.rados.0.smithi017.stdout:2282: delete oid 11 current snap is 292 2024-09-21T09:47:54.119 INFO:tasks.rados.rados.0.smithi017.stdout:2282: done (0 left) 2024-09-21T09:47:54.120 INFO:tasks.rados.rados.0.smithi017.stdout:2283: rollback oid 44 current snap is 292 2024-09-21T09:47:54.120 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 272 2024-09-21T09:47:54.120 INFO:tasks.rados.rados.0.smithi017.stdout:2284: write oid 45 current snap is 292 2024-09-21T09:47:54.120 INFO:tasks.rados.rados.0.smithi017.stdout:2284: seq_num 863 ranges {504441=487776,1467372=736289} 2024-09-21T09:47:54.125 INFO:tasks.rados.rados.0.smithi017.stdout:2284: writing smithi01785587-45 from 504441 to 992217 tid 1 2024-09-21T09:47:54.132 INFO:tasks.rados.rados.0.smithi017.stdout:2284: writing smithi01785587-45 from 1467372 to 2203661 tid 2 2024-09-21T09:47:54.132 INFO:tasks.rados.rados.0.smithi017.stdout:2285: rollback oid 41 current snap is 292 2024-09-21T09:47:54.132 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 285 2024-09-21T09:47:54.132 INFO:tasks.rados.rados.0.smithi017.stdout:2283: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:47:54.132 INFO:tasks.rados.rados.0.smithi017.stdout:2283: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:47:54.132 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1107 (ObjNum 129 snap 0 seq_num 2399874736) dirty dne 2024-09-21T09:47:54.133 INFO:tasks.rados.rados.0.smithi017.stdout:2284: finishing write tid 1 to smithi01785587-45 2024-09-21T09:47:54.133 INFO:tasks.rados.rados.0.smithi017.stdout:2283: done (2 left) 2024-09-21T09:47:54.133 INFO:tasks.rados.rados.0.smithi017.stdout:2286: delete oid 17 current snap is 292 2024-09-21T09:47:54.133 INFO:tasks.rados.rados.0.smithi017.stdout:2285: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:47:54.133 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:47:54.136 INFO:tasks.rados.rados.0.smithi017.stdout:2284: finishing write tid 2 to smithi01785587-45 2024-09-21T09:47:54.136 INFO:tasks.rados.rados.0.smithi017.stdout:2284: finishing write tid 3 to smithi01785587-45 2024-09-21T09:47:54.137 INFO:tasks.rados.rados.0.smithi017.stdout:2284: finishing write tid 4 to smithi01785587-45 2024-09-21T09:47:54.137 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 720 (ObjNum 863 snap 292 seq_num 863) dirty exists 2024-09-21T09:47:54.137 INFO:tasks.rados.rados.0.smithi017.stdout:2284: left oid 45 (ObjNum 863 snap 292 seq_num 863) 2024-09-21T09:47:54.137 INFO:tasks.rados.rados.0.smithi017.stdout:2284: done (2 left) 2024-09-21T09:47:54.137 INFO:tasks.rados.rados.0.smithi017.stdout:2285: done (1 left) 2024-09-21T09:47:54.137 INFO:tasks.rados.rados.0.smithi017.stdout:2286: done (0 left) 2024-09-21T09:47:54.137 INFO:tasks.rados.rados.0.smithi017.stdout:2287: snap_create 2024-09-21T09:47:55.112 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:47:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:47:54] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.51.0" 2024-09-21T09:47:55.121 INFO:tasks.rados.rados.0.smithi017.stdout:2287: done (0 left) 2024-09-21T09:47:55.121 INFO:tasks.rados.rados.0.smithi017.stdout:2288: write oid 22 current snap is 293 2024-09-21T09:47:55.121 INFO:tasks.rados.rados.0.smithi017.stdout:2288: seq_num 865 ranges {538055=638274,1867983=670072} 2024-09-21T09:47:55.127 INFO:tasks.rados.rados.0.smithi017.stdout:2288: writing smithi01785587-22 from 538055 to 1176329 tid 1 2024-09-21T09:47:55.135 INFO:tasks.rados.rados.0.smithi017.stdout:2288: writing smithi01785587-22 from 1867983 to 2538055 tid 2 2024-09-21T09:47:55.135 INFO:tasks.rados.rados.0.smithi017.stdout:2289: rollback oid 4 current snap is 293 2024-09-21T09:47:55.135 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 4 to 272 2024-09-21T09:47:55.135 INFO:tasks.rados.rados.0.smithi017.stdout:2288: finishing write tid 1 to smithi01785587-22 2024-09-21T09:47:55.135 INFO:tasks.rados.rados.0.smithi017.stdout:2290: delete oid 42 current snap is 293 2024-09-21T09:47:55.138 INFO:tasks.rados.rados.0.smithi017.stdout:2290: done (2 left) 2024-09-21T09:47:55.138 INFO:tasks.rados.rados.0.smithi017.stdout:2291: write oid 18 current snap is 293 2024-09-21T09:47:55.139 INFO:tasks.rados.rados.0.smithi017.stdout:2291: seq_num 867 ranges {571669=505125,1701300=743627} 2024-09-21T09:47:55.144 INFO:tasks.rados.rados.0.smithi017.stdout:2291: writing smithi01785587-18 from 571669 to 1076794 tid 1 2024-09-21T09:47:55.152 INFO:tasks.rados.rados.0.smithi017.stdout:2291: writing smithi01785587-18 from 1701300 to 2444927 tid 2 2024-09-21T09:47:55.152 INFO:tasks.rados.rados.0.smithi017.stdout:2292: snap_create 2024-09-21T09:47:55.152 INFO:tasks.rados.rados.0.smithi017.stdout:2288: finishing write tid 2 to smithi01785587-22 2024-09-21T09:47:55.152 INFO:tasks.rados.rados.0.smithi017.stdout:2288: finishing write tid 3 to smithi01785587-22 2024-09-21T09:47:55.152 INFO:tasks.rados.rados.0.smithi017.stdout:2288: finishing write tid 4 to smithi01785587-22 2024-09-21T09:47:55.152 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1153 (ObjNum 865 snap 293 seq_num 865) dirty exists 2024-09-21T09:47:55.153 INFO:tasks.rados.rados.0.smithi017.stdout:2288: left oid 22 (ObjNum 865 snap 293 seq_num 865) 2024-09-21T09:47:55.153 INFO:tasks.rados.rados.0.smithi017.stdout:2289: finishing rollback tid 0 to smithi01785587-4 2024-09-21T09:47:55.153 INFO:tasks.rados.rados.0.smithi017.stdout:2289: finishing rollback tid 1 to smithi01785587-4 2024-09-21T09:47:55.153 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1156 (ObjNum 113 snap 0 seq_num 3570625224) dirty dne 2024-09-21T09:47:55.153 INFO:tasks.rados.rados.0.smithi017.stdout:2291: finishing write tid 1 to smithi01785587-18 2024-09-21T09:47:55.156 INFO:tasks.rados.rados.0.smithi017.stdout:2291: finishing write tid 2 to smithi01785587-18 2024-09-21T09:47:55.156 INFO:tasks.rados.rados.0.smithi017.stdout:2291: finishing write tid 3 to smithi01785587-18 2024-09-21T09:47:55.156 INFO:tasks.rados.rados.0.smithi017.stdout:2291: finishing write tid 4 to smithi01785587-18 2024-09-21T09:47:55.156 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 192 (ObjNum 867 snap 293 seq_num 867) dirty exists 2024-09-21T09:47:55.156 INFO:tasks.rados.rados.0.smithi017.stdout:2291: left oid 18 (ObjNum 867 snap 293 seq_num 867) 2024-09-21T09:47:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:55 smithi017 ceph-mon[102913]: pgmap v552: 89 pgs: 89 active+clean; 225 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-09-21T09:47:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:55 smithi017 ceph-mon[102913]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T09:47:55.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:55 smithi017 ceph-mon[105151]: pgmap v552: 89 pgs: 89 active+clean; 225 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-09-21T09:47:55.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:55 smithi017 ceph-mon[105151]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T09:47:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:55 smithi134 ceph-mon[90085]: pgmap v552: 89 pgs: 89 active+clean; 225 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-09-21T09:47:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:55 smithi134 ceph-mon[90085]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T09:47:56.128 INFO:tasks.rados.rados.0.smithi017.stdout:2288: done (3 left) 2024-09-21T09:47:56.128 INFO:tasks.rados.rados.0.smithi017.stdout:2289: done (2 left) 2024-09-21T09:47:56.128 INFO:tasks.rados.rados.0.smithi017.stdout:2291: done (1 left) 2024-09-21T09:47:56.128 INFO:tasks.rados.rados.0.smithi017.stdout:2292: done (0 left) 2024-09-21T09:47:56.128 INFO:tasks.rados.rados.0.smithi017.stdout:2293: read oid 3 snap -1 2024-09-21T09:47:56.128 INFO:tasks.rados.rados.0.smithi017.stdout:2293: expect (ObjNum 774 snap 264 seq_num 774) 2024-09-21T09:47:56.128 INFO:tasks.rados.rados.0.smithi017.stdout:2294: read oid 22 snap -1 2024-09-21T09:47:56.128 INFO:tasks.rados.rados.0.smithi017.stdout:2294: expect (ObjNum 865 snap 293 seq_num 865) 2024-09-21T09:47:56.128 INFO:tasks.rados.rados.0.smithi017.stdout:2295: rollback oid 35 current snap is 294 2024-09-21T09:47:56.129 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 286 2024-09-21T09:47:56.129 INFO:tasks.rados.rados.0.smithi017.stdout:2296: read oid 21 snap -1 2024-09-21T09:47:56.129 INFO:tasks.rados.rados.0.smithi017.stdout:2296: expect deleted 2024-09-21T09:47:56.129 INFO:tasks.rados.rados.0.smithi017.stdout:2297: read oid 8 snap 280 2024-09-21T09:47:56.129 INFO:tasks.rados.rados.0.smithi017.stdout:2297: expect (ObjNum 782 snap 269 seq_num 782) 2024-09-21T09:47:56.129 INFO:tasks.rados.rados.0.smithi017.stdout:2298: snap_create 2024-09-21T09:47:56.132 INFO:tasks.rados.rados.0.smithi017.stdout:2297: expect (ObjNum 782 snap 269 seq_num 782) 2024-09-21T09:47:56.325 INFO:tasks.rados.rados.0.smithi017.stdout:2295: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:47:56.325 INFO:tasks.rados.rados.0.smithi017.stdout:2295: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:47:56.326 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 257 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:47:56.326 INFO:tasks.rados.rados.0.smithi017.stdout:2293: expect (ObjNum 774 snap 264 seq_num 774) 2024-09-21T09:47:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:56 smithi017 ceph-mon[102913]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T09:47:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:56 smithi017 ceph-mon[105151]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T09:47:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:56.462 INFO:tasks.rados.rados.0.smithi017.stdout:2294: expect (ObjNum 865 snap 293 seq_num 865) 2024-09-21T09:47:56.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:56 smithi134 ceph-mon[90085]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T09:47:56.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:57.128 INFO:tasks.rados.rados.0.smithi017.stdout:2293: done (5 left) 2024-09-21T09:47:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:2294: done (4 left) 2024-09-21T09:47:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:2295: done (3 left) 2024-09-21T09:47:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:2296: done (2 left) 2024-09-21T09:47:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:2297: done (1 left) 2024-09-21T09:47:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:2298: done (0 left) 2024-09-21T09:47:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:2299: read oid 48 snap 285 2024-09-21T09:47:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:2299: expect (ObjNum 755 snap 255 seq_num 755) 2024-09-21T09:47:57.129 INFO:tasks.rados.rados.0.smithi017.stdout:2300: delete oid 12 current snap is 295 2024-09-21T09:47:57.132 INFO:tasks.rados.rados.0.smithi017.stdout:2300: done (1 left) 2024-09-21T09:47:57.132 INFO:tasks.rados.rados.0.smithi017.stdout:2301: snap_create 2024-09-21T09:47:57.142 INFO:tasks.rados.rados.0.smithi017.stdout:2299: expect (ObjNum 755 snap 255 seq_num 755) 2024-09-21T09:47:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:57 smithi017 ceph-mon[102913]: pgmap v555: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:47:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:57 smithi017 ceph-mon[102913]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T09:47:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:57 smithi017 ceph-mon[105151]: pgmap v555: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:47:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:57 smithi017 ceph-mon[105151]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T09:47:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:57 smithi134 ceph-mon[90085]: pgmap v555: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:47:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:57 smithi134 ceph-mon[90085]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T09:47:58.115 INFO:tasks.rados.rados.0.smithi017.stdout:2299: done (1 left) 2024-09-21T09:47:58.115 INFO:tasks.rados.rados.0.smithi017.stdout:2301: done (0 left) 2024-09-21T09:47:58.115 INFO:tasks.rados.rados.0.smithi017.stdout:2302: read oid 45 snap 280 2024-09-21T09:47:58.115 INFO:tasks.rados.rados.0.smithi017.stdout:2302: expect (ObjNum 790 snap 271 seq_num 790) 2024-09-21T09:47:58.115 INFO:tasks.rados.rados.0.smithi017.stdout:2303: delete oid 47 current snap is 296 2024-09-21T09:47:58.117 INFO:tasks.rados.rados.0.smithi017.stdout:2303: done (1 left) 2024-09-21T09:47:58.117 INFO:tasks.rados.rados.0.smithi017.stdout:2304: snap_create 2024-09-21T09:47:58.122 INFO:tasks.rados.rados.0.smithi017.stdout:2302: expect (ObjNum 790 snap 271 seq_num 790) 2024-09-21T09:47:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:58 smithi017 ceph-mon[102913]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T09:47:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:58 smithi017 ceph-mon[102913]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T09:47:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:58 smithi017 ceph-mon[105151]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T09:47:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:58 smithi017 ceph-mon[105151]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T09:47:58.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:58 smithi134 ceph-mon[90085]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T09:47:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:58 smithi134 ceph-mon[90085]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T09:47:59.118 INFO:tasks.rados.rados.0.smithi017.stdout:2302: done (1 left) 2024-09-21T09:47:59.118 INFO:tasks.rados.rados.0.smithi017.stdout:2304: done (0 left) 2024-09-21T09:47:59.118 INFO:tasks.rados.rados.0.smithi017.stdout:2305: snap_create 2024-09-21T09:47:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:59 smithi017 ceph-mon[102913]: pgmap v558: 89 pgs: 89 active+clean; 178 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:47:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:47:59 smithi017 ceph-mon[102913]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T09:47:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:59 smithi017 ceph-mon[105151]: pgmap v558: 89 pgs: 89 active+clean; 178 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:47:59.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:47:59 smithi017 ceph-mon[105151]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T09:47:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:59 smithi134 ceph-mon[90085]: pgmap v558: 89 pgs: 89 active+clean; 178 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:47:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:47:59 smithi134 ceph-mon[90085]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T09:48:00.119 INFO:tasks.rados.rados.0.smithi017.stdout:2305: done (0 left) 2024-09-21T09:48:00.119 INFO:tasks.rados.rados.0.smithi017.stdout:2306: read oid 34 snap -1 2024-09-21T09:48:00.119 INFO:tasks.rados.rados.0.smithi017.stdout:2306: expect (ObjNum 836 snap 287 seq_num 836) 2024-09-21T09:48:00.119 INFO:tasks.rados.rados.0.smithi017.stdout:2307: read oid 49 snap -1 2024-09-21T09:48:00.119 INFO:tasks.rados.rados.0.smithi017.stdout:2307: expect deleted 2024-09-21T09:48:00.119 INFO:tasks.rados.rados.0.smithi017.stdout:2308: snap_create 2024-09-21T09:48:00.126 INFO:tasks.rados.rados.0.smithi017.stdout:2306: expect (ObjNum 836 snap 287 seq_num 836) 2024-09-21T09:48:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:48:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:48:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:01.124 INFO:tasks.rados.rados.0.smithi017.stdout:2306: done (2 left) 2024-09-21T09:48:01.124 INFO:tasks.rados.rados.0.smithi017.stdout:2307: done (1 left) 2024-09-21T09:48:01.125 INFO:tasks.rados.rados.0.smithi017.stdout:2308: done (0 left) 2024-09-21T09:48:01.125 INFO:tasks.rados.rados.0.smithi017.stdout:2309: read oid 40 snap 296 2024-09-21T09:48:01.125 INFO:tasks.rados.rados.0.smithi017.stdout:2309: expect (ObjNum 746 snap 252 seq_num 746) 2024-09-21T09:48:01.125 INFO:tasks.rados.rados.0.smithi017.stdout:2310: write oid 38 current snap is 299 2024-09-21T09:48:01.125 INFO:tasks.rados.rados.0.smithi017.stdout:2310: seq_num 870 ranges {622090=730872,1760393=607307} 2024-09-21T09:48:01.132 INFO:tasks.rados.rados.0.smithi017.stdout:2310: writing smithi01785587-38 from 622090 to 1352962 tid 1 2024-09-21T09:48:01.138 INFO:tasks.rados.rados.0.smithi017.stdout:2310: writing smithi01785587-38 from 1760393 to 2367700 tid 2 2024-09-21T09:48:01.138 INFO:tasks.rados.rados.0.smithi017.stdout:2311: read oid 44 snap 295 2024-09-21T09:48:01.139 INFO:tasks.rados.rados.0.smithi017.stdout:2311: expect deleted 2024-09-21T09:48:01.139 INFO:tasks.rados.rados.0.smithi017.stdout:2312: snap_create 2024-09-21T09:48:01.139 INFO:tasks.rados.rados.0.smithi017.stdout:2309: expect (ObjNum 746 snap 252 seq_num 746) 2024-09-21T09:48:01.222 INFO:tasks.rados.rados.0.smithi017.stdout:2310: finishing write tid 1 to smithi01785587-38 2024-09-21T09:48:01.222 INFO:tasks.rados.rados.0.smithi017.stdout:2310: finishing write tid 2 to smithi01785587-38 2024-09-21T09:48:01.222 INFO:tasks.rados.rados.0.smithi017.stdout:2310: finishing write tid 3 to smithi01785587-38 2024-09-21T09:48:01.222 INFO:tasks.rados.rados.0.smithi017.stdout:2310: finishing write tid 4 to smithi01785587-38 2024-09-21T09:48:01.222 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 760 (ObjNum 870 snap 299 seq_num 870) dirty exists 2024-09-21T09:48:01.222 INFO:tasks.rados.rados.0.smithi017.stdout:2310: left oid 38 (ObjNum 870 snap 299 seq_num 870) 2024-09-21T09:48:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:01 smithi017 ceph-mon[102913]: pgmap v561: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 605 KiB/s wr, 7 op/s 2024-09-21T09:48:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:01 smithi017 ceph-mon[102913]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T09:48:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:01 smithi017 ceph-mon[105151]: pgmap v561: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 605 KiB/s wr, 7 op/s 2024-09-21T09:48:01.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:01 smithi017 ceph-mon[105151]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T09:48:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:01 smithi134 ceph-mon[90085]: pgmap v561: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 605 KiB/s wr, 7 op/s 2024-09-21T09:48:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:01 smithi134 ceph-mon[90085]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T09:48:02.129 INFO:tasks.rados.rados.0.smithi017.stdout:2309: done (3 left) 2024-09-21T09:48:02.129 INFO:tasks.rados.rados.0.smithi017.stdout:2310: done (2 left) 2024-09-21T09:48:02.129 INFO:tasks.rados.rados.0.smithi017.stdout:2311: done (1 left) 2024-09-21T09:48:02.130 INFO:tasks.rados.rados.0.smithi017.stdout:2312: done (0 left) 2024-09-21T09:48:02.130 INFO:tasks.rados.rados.0.smithi017.stdout:2313: snap_create 2024-09-21T09:48:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:02 smithi017 ceph-mon[102913]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T09:48:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:02 smithi017 ceph-mon[105151]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T09:48:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:02 smithi134 ceph-mon[90085]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T09:48:03.116 INFO:tasks.rados.rados.0.smithi017.stdout:2313: done (0 left) 2024-09-21T09:48:03.116 INFO:tasks.rados.rados.0.smithi017.stdout:2314: read oid 36 snap 294 2024-09-21T09:48:03.116 INFO:tasks.rados.rados.0.smithi017.stdout:2314: expect deleted 2024-09-21T09:48:03.117 INFO:tasks.rados.rados.0.smithi017.stdout:2315: write oid 21 current snap is 301 2024-09-21T09:48:03.117 INFO:tasks.rados.rados.0.smithi017.stdout:2315: seq_num 871 ranges {638897=406121,1818875=750965} 2024-09-21T09:48:03.122 INFO:tasks.rados.rados.0.smithi017.stdout:2315: writing smithi01785587-21 from 638897 to 1045018 tid 1 2024-09-21T09:48:03.131 INFO:tasks.rados.rados.0.smithi017.stdout:2315: writing smithi01785587-21 from 1818875 to 2569840 tid 2 2024-09-21T09:48:03.131 INFO:tasks.rados.rados.0.smithi017.stdout:2316: snap_create 2024-09-21T09:48:03.131 INFO:tasks.rados.rados.0.smithi017.stdout:2315: finishing write tid 1 to smithi01785587-21 2024-09-21T09:48:03.135 INFO:tasks.rados.rados.0.smithi017.stdout:2315: finishing write tid 2 to smithi01785587-21 2024-09-21T09:48:03.135 INFO:tasks.rados.rados.0.smithi017.stdout:2315: finishing write tid 3 to smithi01785587-21 2024-09-21T09:48:03.135 INFO:tasks.rados.rados.0.smithi017.stdout:2315: finishing write tid 4 to smithi01785587-21 2024-09-21T09:48:03.135 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1207 (ObjNum 871 snap 301 seq_num 871) dirty exists 2024-09-21T09:48:03.136 INFO:tasks.rados.rados.0.smithi017.stdout:2315: left oid 21 (ObjNum 871 snap 301 seq_num 871) 2024-09-21T09:48:03.394 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:03 smithi017 ceph-mon[102913]: pgmap v564: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 626 KiB/s wr, 14 op/s 2024-09-21T09:48:03.394 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:03 smithi017 ceph-mon[102913]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T09:48:03.395 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:03.395 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:03 smithi017 ceph-mon[102913]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T09:48:03.395 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:03 smithi017 ceph-mon[105151]: pgmap v564: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 626 KiB/s wr, 14 op/s 2024-09-21T09:48:03.395 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:03 smithi017 ceph-mon[105151]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T09:48:03.395 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:03.395 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:03 smithi017 ceph-mon[105151]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T09:48:03.464 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:03 smithi134 ceph-mon[90085]: pgmap v564: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 626 KiB/s wr, 14 op/s 2024-09-21T09:48:03.465 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:03 smithi134 ceph-mon[90085]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T09:48:03.465 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:03.465 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:03 smithi134 ceph-mon[90085]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T09:48:04.134 INFO:tasks.rados.rados.0.smithi017.stdout:2314: done (2 left) 2024-09-21T09:48:04.134 INFO:tasks.rados.rados.0.smithi017.stdout:2315: done (1 left) 2024-09-21T09:48:04.134 INFO:tasks.rados.rados.0.smithi017.stdout:2316: done (0 left) 2024-09-21T09:48:04.134 INFO:tasks.rados.rados.0.smithi017.stdout:2317: snap_create 2024-09-21T09:48:05.097 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:48:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:48:04] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.51.0" 2024-09-21T09:48:05.135 INFO:tasks.rados.rados.0.smithi017.stdout:2317: done (0 left) 2024-09-21T09:48:05.135 INFO:tasks.rados.rados.0.smithi017.stdout:2318: snap_remove snap 289 2024-09-21T09:48:05.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:05 smithi134 ceph-mon[90085]: pgmap v567: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 327 KiB/s wr, 13 op/s 2024-09-21T09:48:05.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:05 smithi134 ceph-mon[90085]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T09:48:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:05 smithi017 ceph-mon[102913]: pgmap v567: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 327 KiB/s wr, 13 op/s 2024-09-21T09:48:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:05 smithi017 ceph-mon[102913]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T09:48:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:05 smithi017 ceph-mon[105151]: pgmap v567: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 327 KiB/s wr, 13 op/s 2024-09-21T09:48:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:05 smithi017 ceph-mon[105151]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T09:48:06.145 INFO:tasks.rados.rados.0.smithi017.stdout:2318: done (0 left) 2024-09-21T09:48:06.145 INFO:tasks.rados.rados.0.smithi017.stdout:2319: snap_remove snap 292 2024-09-21T09:48:06.147 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:06 smithi134 ceph-mon[90085]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T09:48:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:06 smithi017 ceph-mon[102913]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T09:48:06.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:06 smithi017 ceph-mon[105151]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T09:48:07.146 INFO:tasks.rados.rados.0.smithi017.stdout:2319: done (0 left) 2024-09-21T09:48:07.146 INFO:tasks.rados.rados.0.smithi017.stdout:2320: snap_create 2024-09-21T09:48:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[105151]: pgmap v570: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2 op/s 2024-09-21T09:48:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[105151]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T09:48:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[102913]: pgmap v570: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2 op/s 2024-09-21T09:48:07.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[102913]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T09:48:07.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:07 smithi134 ceph-mon[90085]: pgmap v570: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2 op/s 2024-09-21T09:48:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:07 smithi134 ceph-mon[90085]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T09:48:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:08.156 INFO:tasks.rados.rados.0.smithi017.stdout:2320: done (0 left) 2024-09-21T09:48:08.157 INFO:tasks.rados.rados.0.smithi017.stdout:2321: snap_remove snap 287 2024-09-21T09:48:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:08 smithi017 ceph-mon[105151]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T09:48:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:08 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:08 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:08 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:08 smithi017 ceph-mon[102913]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T09:48:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:08 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:08 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:08 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:08.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:08 smithi134 ceph-mon[90085]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T09:48:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:08 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:08 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:08 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:48:09.159 INFO:tasks.rados.rados.0.smithi017.stdout:2321: done (0 left) 2024-09-21T09:48:09.159 INFO:tasks.rados.rados.0.smithi017.stdout:2322: write oid 50 current snap is 304 2024-09-21T09:48:09.159 INFO:tasks.rados.rados.0.smithi017.stdout:2322: seq_num 872 ranges {655704=481370,1761004=494623} 2024-09-21T09:48:09.166 INFO:tasks.rados.rados.0.smithi017.stdout:2322: writing smithi01785587-50 from 655704 to 1137074 tid 1 2024-09-21T09:48:09.173 INFO:tasks.rados.rados.0.smithi017.stdout:2322: writing smithi01785587-50 from 1761004 to 2255627 tid 2 2024-09-21T09:48:09.174 INFO:tasks.rados.rados.0.smithi017.stdout:2323: read oid 4 snap -1 2024-09-21T09:48:09.174 INFO:tasks.rados.rados.0.smithi017.stdout:2323: expect deleted 2024-09-21T09:48:09.174 INFO:tasks.rados.rados.0.smithi017.stdout:2324: delete oid 12 current snap is 304 2024-09-21T09:48:09.174 INFO:tasks.rados.rados.0.smithi017.stdout:2322: finishing write tid 1 to smithi01785587-50 2024-09-21T09:48:09.178 INFO:tasks.rados.rados.0.smithi017.stdout:2322: finishing write tid 2 to smithi01785587-50 2024-09-21T09:48:09.178 INFO:tasks.rados.rados.0.smithi017.stdout:2322: finishing write tid 3 to smithi01785587-50 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2322: finishing write tid 4 to smithi01785587-50 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 770 (ObjNum 872 snap 304 seq_num 872) dirty exists 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2322: left oid 50 (ObjNum 872 snap 304 seq_num 872) 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2322: done (2 left) 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2323: done (1 left) 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2324: done (0 left) 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2325: read oid 13 snap -1 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2325: expect deleted 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2326: read oid 27 snap -1 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2326: expect (ObjNum 856 snap 289 seq_num 856) 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2327: read oid 47 snap -1 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2327: expect deleted 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2328: write oid 21 current snap is 304 2024-09-21T09:48:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:2328: seq_num 874 ranges {689318=631868,1877968=498292} 2024-09-21T09:48:09.186 INFO:tasks.rados.rados.0.smithi017.stdout:2328: writing smithi01785587-21 from 689318 to 1321186 tid 1 2024-09-21T09:48:09.192 INFO:tasks.rados.rados.0.smithi017.stdout:2328: writing smithi01785587-21 from 1877968 to 2376260 tid 2 2024-09-21T09:48:09.193 INFO:tasks.rados.rados.0.smithi017.stdout:2329: write oid 30 current snap is 304 2024-09-21T09:48:09.193 INFO:tasks.rados.rados.0.smithi017.stdout:2329: seq_num 875 ranges {706125=423470,1652803=758303} 2024-09-21T09:48:09.199 INFO:tasks.rados.rados.0.smithi017.stdout:2329: writing smithi01785587-30 from 706125 to 1129595 tid 1 2024-09-21T09:48:09.206 INFO:tasks.rados.rados.0.smithi017.stdout:2329: writing smithi01785587-30 from 1652803 to 2411106 tid 2 2024-09-21T09:48:09.206 INFO:tasks.rados.rados.0.smithi017.stdout:2330: snap_remove snap 302 2024-09-21T09:48:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:09 smithi017 ceph-mon[102913]: pgmap v573: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 283 KiB/s wr, 1 op/s 2024-09-21T09:48:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:09 smithi017 ceph-mon[102913]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T09:48:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:09 smithi017 ceph-mon[105151]: pgmap v573: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 283 KiB/s wr, 1 op/s 2024-09-21T09:48:09.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:09 smithi017 ceph-mon[105151]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T09:48:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:09 smithi134 ceph-mon[90085]: pgmap v573: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 283 KiB/s wr, 1 op/s 2024-09-21T09:48:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:09 smithi134 ceph-mon[90085]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T09:48:10.176 INFO:tasks.rados.rados.0.smithi017.stdout:2326: expect (ObjNum 856 snap 289 seq_num 856) 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:2328: finishing write tid 1 to smithi01785587-21 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:2328: finishing write tid 2 to smithi01785587-21 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:2328: finishing write tid 3 to smithi01785587-21 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:2328: finishing write tid 4 to smithi01785587-21 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1215 (ObjNum 874 snap 304 seq_num 874) dirty exists 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:2328: left oid 21 (ObjNum 874 snap 304 seq_num 874) 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:2329: finishing write tid 1 to smithi01785587-30 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:2329: finishing write tid 2 to smithi01785587-30 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:2329: finishing write tid 3 to smithi01785587-30 2024-09-21T09:48:10.291 INFO:tasks.rados.rados.0.smithi017.stdout:2325: done (5 left) 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2326: done (4 left) 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2327: done (3 left) 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2328: done (2 left) 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2330: done (1 left) 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2331: rollback oid 14 current snap is 304 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 290 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2329: finishing write tid 4 to smithi01785587-30 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 947 (ObjNum 875 snap 304 seq_num 875) dirty exists 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2329: left oid 30 (ObjNum 875 snap 304 seq_num 875) 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2329: done (1 left) 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2332: rollback oid 43 current snap is 304 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 301 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:2333: rollback oid 6 current snap is 304 2024-09-21T09:48:10.292 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 298 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:2334: read oid 8 snap 293 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:2334: expect (ObjNum 825 snap 282 seq_num 825) 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:2335: read oid 48 snap 291 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:2335: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:2336: rollback oid 13 current snap is 304 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 13 to 290 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:2337: delete oid 35 current snap is 304 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:2337: done (6 left) 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:2338: write oid 45 current snap is 304 2024-09-21T09:48:10.293 INFO:tasks.rados.rados.0.smithi017.stdout:2338: seq_num 877 ranges {739739=573968,2053414=686325} 2024-09-21T09:48:10.300 INFO:tasks.rados.rados.0.smithi017.stdout:2338: writing smithi01785587-45 from 739739 to 1313707 tid 1 2024-09-21T09:48:10.308 INFO:tasks.rados.rados.0.smithi017.stdout:2338: writing smithi01785587-45 from 2053414 to 2739739 tid 2 2024-09-21T09:48:10.308 INFO:tasks.rados.rados.0.smithi017.stdout:2339: rollback oid 21 current snap is 304 2024-09-21T09:48:10.308 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 290 2024-09-21T09:48:10.308 INFO:tasks.rados.rados.0.smithi017.stdout:2332: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:48:10.308 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 661 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:48:10.309 INFO:tasks.rados.rados.0.smithi017.stdout:2333: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:48:10.309 INFO:tasks.rados.rados.0.smithi017.stdout:2333: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:48:10.309 INFO:tasks.rados.rados.0.smithi017.stdout:2333: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:48:10.309 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 691 (ObjNum 763 snap 257 seq_num 763) dirty exists 2024-09-21T09:48:10.309 INFO:tasks.rados.rados.0.smithi017.stdout:2331: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:48:10.309 INFO:tasks.rados.rados.0.smithi017.stdout:2334: expect (ObjNum 825 snap 282 seq_num 825) 2024-09-21T09:48:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:10 smithi017 ceph-mon[102913]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T09:48:10.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:10 smithi017 ceph-mon[105151]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T09:48:10.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:10 smithi134 ceph-mon[90085]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T09:48:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:48:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:48:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:10.582 INFO:tasks.rados.rados.0.smithi017.stdout:2331: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:48:10.582 INFO:tasks.rados.rados.0.smithi017.stdout:2331: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:48:10.583 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1119 (ObjNum 854 snap 289 seq_num 854) dirty exists 2024-09-21T09:48:10.583 INFO:tasks.rados.rados.0.smithi017.stdout:2335: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:48:10.695 INFO:tasks.rados.rados.0.smithi017.stdout:2336: finishing rollback tid 1 to smithi01785587-13 2024-09-21T09:48:10.695 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 871 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:48:10.695 INFO:tasks.rados.rados.0.smithi017.stdout:2338: finishing write tid 1 to smithi01785587-45 2024-09-21T09:48:10.695 INFO:tasks.rados.rados.0.smithi017.stdout:2339: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:48:10.695 INFO:tasks.rados.rados.0.smithi017.stdout:2339: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:48:10.695 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1219 (ObjNum 241 snap 0 seq_num 2399891056) dirty dne 2024-09-21T09:48:10.695 INFO:tasks.rados.rados.0.smithi017.stdout:2338: finishing write tid 2 to smithi01785587-45 2024-09-21T09:48:10.695 INFO:tasks.rados.rados.0.smithi017.stdout:2338: finishing write tid 3 to smithi01785587-45 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2338: finishing write tid 4 to smithi01785587-45 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 734 (ObjNum 877 snap 304 seq_num 877) dirty exists 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2338: left oid 45 (ObjNum 877 snap 304 seq_num 877) 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2331: done (7 left) 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2332: done (6 left) 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2333: done (5 left) 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2334: done (4 left) 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2335: done (3 left) 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2336: done (2 left) 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2338: done (1 left) 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2339: done (0 left) 2024-09-21T09:48:10.696 INFO:tasks.rados.rados.0.smithi017.stdout:2340: snap_remove snap 297 2024-09-21T09:48:11.174 INFO:tasks.rados.rados.0.smithi017.stdout:2340: done (0 left) 2024-09-21T09:48:11.174 INFO:tasks.rados.rados.0.smithi017.stdout:2341: write oid 1 current snap is 304 2024-09-21T09:48:11.175 INFO:tasks.rados.rados.0.smithi017.stdout:2341: seq_num 878 ranges {756546=649217,2111896=505630} 2024-09-21T09:48:11.182 INFO:tasks.rados.rados.0.smithi017.stdout:2341: writing smithi01785587-1 from 756546 to 1405763 tid 1 2024-09-21T09:48:11.189 INFO:tasks.rados.rados.0.smithi017.stdout:2341: writing smithi01785587-1 from 2111896 to 2617526 tid 2 2024-09-21T09:48:11.189 INFO:tasks.rados.rados.0.smithi017.stdout:2342: snap_remove snap 296 2024-09-21T09:48:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:11 smithi017 ceph-mon[102913]: pgmap v576: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 283 KiB/s wr, 1 op/s 2024-09-21T09:48:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:11 smithi017 ceph-mon[102913]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T09:48:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:11 smithi017 ceph-mon[105151]: pgmap v576: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 283 KiB/s wr, 1 op/s 2024-09-21T09:48:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:11 smithi017 ceph-mon[105151]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T09:48:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:11 smithi134 ceph-mon[90085]: pgmap v576: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 283 KiB/s wr, 1 op/s 2024-09-21T09:48:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:11 smithi134 ceph-mon[90085]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T09:48:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:12.186 INFO:tasks.rados.rados.0.smithi017.stdout:2342: done (1 left) 2024-09-21T09:48:12.186 INFO:tasks.rados.rados.0.smithi017.stdout:2343: snap_create 2024-09-21T09:48:12.186 INFO:tasks.rados.rados.0.smithi017.stdout:2341: finishing write tid 1 to smithi01785587-1 2024-09-21T09:48:12.186 INFO:tasks.rados.rados.0.smithi017.stdout:2341: finishing write tid 2 to smithi01785587-1 2024-09-21T09:48:12.187 INFO:tasks.rados.rados.0.smithi017.stdout:2341: finishing write tid 3 to smithi01785587-1 2024-09-21T09:48:12.187 INFO:tasks.rados.rados.0.smithi017.stdout:2341: finishing write tid 4 to smithi01785587-1 2024-09-21T09:48:12.187 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 697 (ObjNum 878 snap 304 seq_num 878) dirty exists 2024-09-21T09:48:12.187 INFO:tasks.rados.rados.0.smithi017.stdout:2341: left oid 1 (ObjNum 878 snap 304 seq_num 878) 2024-09-21T09:48:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:12 smithi017 ceph-mon[102913]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T09:48:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:12 smithi017 ceph-mon[105151]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T09:48:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:12 smithi134 ceph-mon[90085]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T09:48:13.120 INFO:tasks.rados.rados.0.smithi017.stdout:2341: done (1 left) 2024-09-21T09:48:13.120 INFO:tasks.rados.rados.0.smithi017.stdout:2343: done (0 left) 2024-09-21T09:48:13.120 INFO:tasks.rados.rados.0.smithi017.stdout:2344: delete oid 31 current snap is 305 2024-09-21T09:48:13.125 INFO:tasks.rados.rados.0.smithi017.stdout:2344: done (0 left) 2024-09-21T09:48:13.125 INFO:tasks.rados.rados.0.smithi017.stdout:2345: write oid 40 current snap is 305 2024-09-21T09:48:13.125 INFO:tasks.rados.rados.0.smithi017.stdout:2345: seq_num 880 ranges {790160=799715,2112507=509299} 2024-09-21T09:48:13.133 INFO:tasks.rados.rados.0.smithi017.stdout:2345: writing smithi01785587-40 from 790160 to 1589875 tid 1 2024-09-21T09:48:13.139 INFO:tasks.rados.rados.0.smithi017.stdout:2345: writing smithi01785587-40 from 2112507 to 2621806 tid 2 2024-09-21T09:48:13.139 INFO:tasks.rados.rados.0.smithi017.stdout:2346: read oid 6 snap -1 2024-09-21T09:48:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:2346: expect (ObjNum 763 snap 257 seq_num 763) 2024-09-21T09:48:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:2347: write oid 34 current snap is 305 2024-09-21T09:48:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:2347: seq_num 881 ranges {406967=474964,1370989=769310,2677186=129781} 2024-09-21T09:48:13.144 INFO:tasks.rados.rados.0.smithi017.stdout:2347: writing smithi01785587-34 from 406967 to 881931 tid 1 2024-09-21T09:48:13.152 INFO:tasks.rados.rados.0.smithi017.stdout:2347: writing smithi01785587-34 from 1370989 to 2140299 tid 2 2024-09-21T09:48:13.156 INFO:tasks.rados.rados.0.smithi017.stdout:2347: writing smithi01785587-34 from 2677186 to 2806967 tid 3 2024-09-21T09:48:13.156 INFO:tasks.rados.rados.0.smithi017.stdout:2348: read oid 29 snap 291 2024-09-21T09:48:13.156 INFO:tasks.rados.rados.0.smithi017.stdout:2348: expect (ObjNum 857 snap 289 seq_num 857) 2024-09-21T09:48:13.156 INFO:tasks.rados.rados.0.smithi017.stdout:2349: rollback oid 30 current snap is 305 2024-09-21T09:48:13.156 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 301 2024-09-21T09:48:13.156 INFO:tasks.rados.rados.0.smithi017.stdout:2345: finishing write tid 1 to smithi01785587-40 2024-09-21T09:48:13.156 INFO:tasks.rados.rados.0.smithi017.stdout:2345: finishing write tid 2 to smithi01785587-40 2024-09-21T09:48:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:2345: finishing write tid 3 to smithi01785587-40 2024-09-21T09:48:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:2345: finishing write tid 4 to smithi01785587-40 2024-09-21T09:48:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1203 (ObjNum 880 snap 305 seq_num 880) dirty exists 2024-09-21T09:48:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:2345: left oid 40 (ObjNum 880 snap 305 seq_num 880) 2024-09-21T09:48:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:2347: finishing write tid 1 to smithi01785587-34 2024-09-21T09:48:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:2346: expect (ObjNum 763 snap 257 seq_num 763) 2024-09-21T09:48:13.310 INFO:tasks.rados.rados.0.smithi017.stdout:2347: finishing write tid 2 to smithi01785587-34 2024-09-21T09:48:13.310 INFO:tasks.rados.rados.0.smithi017.stdout:2347: finishing write tid 3 to smithi01785587-34 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2347: finishing write tid 4 to smithi01785587-34 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2347: finishing write tid 5 to smithi01785587-34 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1224 (ObjNum 881 snap 305 seq_num 881) dirty exists 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2347: left oid 34 (ObjNum 881 snap 305 seq_num 881) 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2349: finishing rollback tid 0 to smithi01785587-30 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2349: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2349: finishing rollback tid 2 to smithi01785587-30 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 958 (ObjNum 843 snap 288 seq_num 843) dirty exists 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2345: done (4 left) 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2346: done (3 left) 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2347: done (2 left) 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2349: done (1 left) 2024-09-21T09:48:13.311 INFO:tasks.rados.rados.0.smithi017.stdout:2350: delete oid 47 current snap is 305 2024-09-21T09:48:13.312 INFO:tasks.rados.rados.0.smithi017.stdout:2348: expect (ObjNum 857 snap 289 seq_num 857) 2024-09-21T09:48:13.418 INFO:tasks.rados.rados.0.smithi017.stdout:2348: done (1 left) 2024-09-21T09:48:13.418 INFO:tasks.rados.rados.0.smithi017.stdout:2350: done (0 left) 2024-09-21T09:48:13.418 INFO:tasks.rados.rados.0.smithi017.stdout:2351: snap_create 2024-09-21T09:48:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:13 smithi017 ceph-mon[102913]: pgmap v579: 89 pgs: 89 active+clean; 187 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 804 KiB/s wr, 8 op/s 2024-09-21T09:48:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:13 smithi017 ceph-mon[102913]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T09:48:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:13 smithi017 ceph-mon[102913]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T09:48:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:13 smithi017 ceph-mon[105151]: pgmap v579: 89 pgs: 89 active+clean; 187 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 804 KiB/s wr, 8 op/s 2024-09-21T09:48:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:13 smithi017 ceph-mon[105151]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T09:48:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:13 smithi017 ceph-mon[105151]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T09:48:13.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:13 smithi134 ceph-mon[90085]: pgmap v579: 89 pgs: 89 active+clean; 187 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 804 KiB/s wr, 8 op/s 2024-09-21T09:48:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:13 smithi134 ceph-mon[90085]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T09:48:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:13 smithi134 ceph-mon[90085]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T09:48:14.191 INFO:tasks.rados.rados.0.smithi017.stdout:2351: done (0 left) 2024-09-21T09:48:14.191 INFO:tasks.rados.rados.0.smithi017.stdout:2352: read oid 3 snap -1 2024-09-21T09:48:14.191 INFO:tasks.rados.rados.0.smithi017.stdout:2352: expect (ObjNum 774 snap 264 seq_num 774) 2024-09-21T09:48:14.191 INFO:tasks.rados.rados.0.smithi017.stdout:2353: snap_remove snap 285 2024-09-21T09:48:15.195 INFO:tasks.rados.rados.0.smithi017.stdout:2353: done (1 left) 2024-09-21T09:48:15.195 INFO:tasks.rados.rados.0.smithi017.stdout:2354: read oid 10 snap 305 2024-09-21T09:48:15.195 INFO:tasks.rados.rados.0.smithi017.stdout:2354: expect (ObjNum 777 snap 267 seq_num 777) 2024-09-21T09:48:15.195 INFO:tasks.rados.rados.0.smithi017.stdout:2355: snap_remove snap 291 2024-09-21T09:48:15.196 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:48:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:48:14] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-09-21T09:48:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:15 smithi017 ceph-mon[102913]: pgmap v582: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:48:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:15 smithi017 ceph-mon[102913]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T09:48:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:15 smithi017 ceph-mon[105151]: pgmap v582: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:48:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:15 smithi017 ceph-mon[105151]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T09:48:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:15 smithi134 ceph-mon[90085]: pgmap v582: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:48:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:15 smithi134 ceph-mon[90085]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T09:48:16.205 INFO:tasks.rados.rados.0.smithi017.stdout:2355: done (2 left) 2024-09-21T09:48:16.205 INFO:tasks.rados.rados.0.smithi017.stdout:2356: snap_remove snap 298 2024-09-21T09:48:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:16 smithi134 ceph-mon[90085]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T09:48:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:16 smithi017 ceph-mon[102913]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T09:48:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:16 smithi017 ceph-mon[105151]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T09:48:17.210 INFO:tasks.rados.rados.0.smithi017.stdout:2356: done (2 left) 2024-09-21T09:48:17.211 INFO:tasks.rados.rados.0.smithi017.stdout:2357: snap_create 2024-09-21T09:48:17.211 INFO:tasks.rados.rados.0.smithi017.stdout:2352: expect (ObjNum 774 snap 264 seq_num 774) 2024-09-21T09:48:17.348 INFO:tasks.rados.rados.0.smithi017.stdout:2354: expect (ObjNum 777 snap 267 seq_num 777) 2024-09-21T09:48:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:17 smithi134 ceph-mon[90085]: pgmap v585: 89 pgs: 89 active+clean; 191 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 910 KiB/s wr, 13 op/s 2024-09-21T09:48:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:17 smithi134 ceph-mon[90085]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T09:48:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:17 smithi017 ceph-mon[102913]: pgmap v585: 89 pgs: 89 active+clean; 191 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 910 KiB/s wr, 13 op/s 2024-09-21T09:48:17.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:17 smithi017 ceph-mon[102913]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T09:48:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:17 smithi017 ceph-mon[105151]: pgmap v585: 89 pgs: 89 active+clean; 191 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 910 KiB/s wr, 13 op/s 2024-09-21T09:48:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:17 smithi017 ceph-mon[105151]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T09:48:18.120 INFO:tasks.rados.rados.0.smithi017.stdout:2352: done (2 left) 2024-09-21T09:48:18.120 INFO:tasks.rados.rados.0.smithi017.stdout:2354: done (1 left) 2024-09-21T09:48:18.120 INFO:tasks.rados.rados.0.smithi017.stdout:2357: done (0 left) 2024-09-21T09:48:18.120 INFO:tasks.rados.rados.0.smithi017.stdout:2358: write oid 41 current snap is 307 2024-09-21T09:48:18.120 INFO:tasks.rados.rados.0.smithi017.stdout:2358: seq_num 883 ranges {440581=741815,1604306=656626,2732032=108549} 2024-09-21T09:48:18.126 INFO:tasks.rados.rados.0.smithi017.stdout:2358: writing smithi01785587-41 from 440581 to 1182396 tid 1 2024-09-21T09:48:18.133 INFO:tasks.rados.rados.0.smithi017.stdout:2358: writing smithi01785587-41 from 1604306 to 2260932 tid 2 2024-09-21T09:48:18.137 INFO:tasks.rados.rados.0.smithi017.stdout:2358: writing smithi01785587-41 from 2732032 to 2840581 tid 3 2024-09-21T09:48:18.137 INFO:tasks.rados.rados.0.smithi017.stdout:2359: write oid 32 current snap is 307 2024-09-21T09:48:18.137 INFO:tasks.rados.rados.0.smithi017.stdout:2359: seq_num 884 ranges {457388=417064,1546435=516637,2643102=214286} 2024-09-21T09:48:18.141 INFO:tasks.rados.rados.0.smithi017.stdout:2359: writing smithi01785587-32 from 457388 to 874452 tid 1 2024-09-21T09:48:18.149 INFO:tasks.rados.rados.0.smithi017.stdout:2359: writing smithi01785587-32 from 1546435 to 2063072 tid 2 2024-09-21T09:48:18.154 INFO:tasks.rados.rados.0.smithi017.stdout:2359: writing smithi01785587-32 from 2643102 to 2857388 tid 3 2024-09-21T09:48:18.154 INFO:tasks.rados.rados.0.smithi017.stdout:2360: write oid 15 current snap is 307 2024-09-21T09:48:18.154 INFO:tasks.rados.rados.0.smithi017.stdout:2358: finishing write tid 1 to smithi01785587-41 2024-09-21T09:48:18.154 INFO:tasks.rados.rados.0.smithi017.stdout:2360: seq_num 885 ranges {474195=492313,1604917=660295,2670525=203670} 2024-09-21T09:48:18.159 INFO:tasks.rados.rados.0.smithi017.stdout:2360: writing smithi01785587-15 from 474195 to 966508 tid 1 2024-09-21T09:48:18.166 INFO:tasks.rados.rados.0.smithi017.stdout:2360: writing smithi01785587-15 from 1604917 to 2265212 tid 2 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2360: writing smithi01785587-15 from 2670525 to 2874195 tid 3 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2361: read oid 38 snap -1 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2361: expect (ObjNum 870 snap 299 seq_num 870) 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2362: snap_create 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2358: finishing write tid 2 to smithi01785587-41 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2358: finishing write tid 3 to smithi01785587-41 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2358: finishing write tid 4 to smithi01785587-41 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2358: finishing write tid 5 to smithi01785587-41 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 359 (ObjNum 883 snap 307 seq_num 883) dirty exists 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2358: left oid 41 (ObjNum 883 snap 307 seq_num 883) 2024-09-21T09:48:18.171 INFO:tasks.rados.rados.0.smithi017.stdout:2359: finishing write tid 1 to smithi01785587-32 2024-09-21T09:48:18.172 INFO:tasks.rados.rados.0.smithi017.stdout:2359: finishing write tid 2 to smithi01785587-32 2024-09-21T09:48:18.172 INFO:tasks.rados.rados.0.smithi017.stdout:2359: finishing write tid 3 to smithi01785587-32 2024-09-21T09:48:18.172 INFO:tasks.rados.rados.0.smithi017.stdout:2359: finishing write tid 4 to smithi01785587-32 2024-09-21T09:48:18.172 INFO:tasks.rados.rados.0.smithi017.stdout:2359: finishing write tid 5 to smithi01785587-32 2024-09-21T09:48:18.172 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1033 (ObjNum 884 snap 307 seq_num 884) dirty exists 2024-09-21T09:48:18.172 INFO:tasks.rados.rados.0.smithi017.stdout:2359: left oid 32 (ObjNum 884 snap 307 seq_num 884) 2024-09-21T09:48:18.172 INFO:tasks.rados.rados.0.smithi017.stdout:2360: finishing write tid 1 to smithi01785587-15 2024-09-21T09:48:18.172 INFO:tasks.rados.rados.0.smithi017.stdout:2360: finishing write tid 2 to smithi01785587-15 2024-09-21T09:48:18.173 INFO:tasks.rados.rados.0.smithi017.stdout:2360: finishing write tid 3 to smithi01785587-15 2024-09-21T09:48:18.173 INFO:tasks.rados.rados.0.smithi017.stdout:2360: finishing write tid 4 to smithi01785587-15 2024-09-21T09:48:18.173 INFO:tasks.rados.rados.0.smithi017.stdout:2360: finishing write tid 5 to smithi01785587-15 2024-09-21T09:48:18.173 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 363 (ObjNum 885 snap 307 seq_num 885) dirty exists 2024-09-21T09:48:18.173 INFO:tasks.rados.rados.0.smithi017.stdout:2360: left oid 15 (ObjNum 885 snap 307 seq_num 885) 2024-09-21T09:48:18.174 INFO:tasks.rados.rados.0.smithi017.stdout:2361: expect (ObjNum 870 snap 299 seq_num 870) 2024-09-21T09:48:18.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:18 smithi134 ceph-mon[90085]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T09:48:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:18 smithi134 ceph-mon[90085]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T09:48:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:18 smithi017 ceph-mon[102913]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T09:48:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:18 smithi017 ceph-mon[102913]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T09:48:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:18 smithi017 ceph-mon[105151]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T09:48:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:18 smithi017 ceph-mon[105151]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T09:48:19.123 INFO:tasks.rados.rados.0.smithi017.stdout:2358: done (4 left) 2024-09-21T09:48:19.123 INFO:tasks.rados.rados.0.smithi017.stdout:2359: done (3 left) 2024-09-21T09:48:19.123 INFO:tasks.rados.rados.0.smithi017.stdout:2360: done (2 left) 2024-09-21T09:48:19.123 INFO:tasks.rados.rados.0.smithi017.stdout:2361: done (1 left) 2024-09-21T09:48:19.123 INFO:tasks.rados.rados.0.smithi017.stdout:2362: done (0 left) 2024-09-21T09:48:19.123 INFO:tasks.rados.rados.0.smithi017.stdout:2363: write oid 34 current snap is 308 2024-09-21T09:48:19.123 INFO:tasks.rados.rados.0.smithi017.stdout:2363: seq_num 886 ranges {491002=567562,1663399=520306,2814301=76701} 2024-09-21T09:48:19.128 INFO:tasks.rados.rados.0.smithi017.stdout:2363: writing smithi01785587-34 from 491002 to 1058564 tid 1 2024-09-21T09:48:19.135 INFO:tasks.rados.rados.0.smithi017.stdout:2363: writing smithi01785587-34 from 1663399 to 2183705 tid 2 2024-09-21T09:48:19.139 INFO:tasks.rados.rados.0.smithi017.stdout:2363: writing smithi01785587-34 from 2814301 to 2891002 tid 3 2024-09-21T09:48:19.140 INFO:tasks.rados.rados.0.smithi017.stdout:2364: write oid 13 current snap is 308 2024-09-21T09:48:19.140 INFO:tasks.rados.rados.0.smithi017.stdout:2363: finishing write tid 1 to smithi01785587-34 2024-09-21T09:48:19.140 INFO:tasks.rados.rados.0.smithi017.stdout:2363: finishing write tid 2 to smithi01785587-34 2024-09-21T09:48:19.140 INFO:tasks.rados.rados.0.smithi017.stdout:2364: seq_num 887 ranges {507809=642811,1721881=663964} 2024-09-21T09:48:19.145 INFO:tasks.rados.rados.0.smithi017.stdout:2364: writing smithi01785587-13 from 507809 to 1150620 tid 1 2024-09-21T09:48:19.152 INFO:tasks.rados.rados.0.smithi017.stdout:2364: writing smithi01785587-13 from 1721881 to 2385845 tid 2 2024-09-21T09:48:19.152 INFO:tasks.rados.rados.0.smithi017.stdout:2365: write oid 36 current snap is 308 2024-09-21T09:48:19.153 INFO:tasks.rados.rados.0.smithi017.stdout:2365: seq_num 888 ranges {524616=718060,1664010=523975,2752794=171822} 2024-09-21T09:48:19.159 INFO:tasks.rados.rados.0.smithi017.stdout:2365: writing smithi01785587-36 from 524616 to 1242676 tid 1 2024-09-21T09:48:19.164 INFO:tasks.rados.rados.0.smithi017.stdout:2365: writing smithi01785587-36 from 1664010 to 2187985 tid 2 2024-09-21T09:48:19.169 INFO:tasks.rados.rados.0.smithi017.stdout:2365: writing smithi01785587-36 from 2752794 to 2924616 tid 3 2024-09-21T09:48:19.169 INFO:tasks.rados.rados.0.smithi017.stdout:2366: read oid 15 snap -1 2024-09-21T09:48:19.169 INFO:tasks.rados.rados.0.smithi017.stdout:2366: expect (ObjNum 885 snap 307 seq_num 885) 2024-09-21T09:48:19.169 INFO:tasks.rados.rados.0.smithi017.stdout:2367: write oid 17 current snap is 308 2024-09-21T09:48:19.169 INFO:tasks.rados.rados.0.smithi017.stdout:2363: finishing write tid 3 to smithi01785587-34 2024-09-21T09:48:19.169 INFO:tasks.rados.rados.0.smithi017.stdout:2363: finishing write tid 4 to smithi01785587-34 2024-09-21T09:48:19.169 INFO:tasks.rados.rados.0.smithi017.stdout:2363: finishing write tid 5 to smithi01785587-34 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1237 (ObjNum 886 snap 308 seq_num 886) dirty exists 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:2363: left oid 34 (ObjNum 886 snap 308 seq_num 886) 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:2364: finishing write tid 1 to smithi01785587-13 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:2364: finishing write tid 2 to smithi01785587-13 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:2364: finishing write tid 3 to smithi01785587-13 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:2364: finishing write tid 4 to smithi01785587-13 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 973 (ObjNum 887 snap 308 seq_num 887) dirty exists 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:2364: left oid 13 (ObjNum 887 snap 308 seq_num 887) 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:2365: finishing write tid 1 to smithi01785587-36 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:2365: finishing write tid 2 to smithi01785587-36 2024-09-21T09:48:19.170 INFO:tasks.rados.rados.0.smithi017.stdout:2367: seq_num 889 ranges {541423=793309,2122492=667633} 2024-09-21T09:48:19.177 INFO:tasks.rados.rados.0.smithi017.stdout:2367: writing smithi01785587-17 from 541423 to 1334732 tid 1 2024-09-21T09:48:19.186 INFO:tasks.rados.rados.0.smithi017.stdout:2367: writing smithi01785587-17 from 2122492 to 2790125 tid 2 2024-09-21T09:48:19.186 INFO:tasks.rados.rados.0.smithi017.stdout:2363: done (4 left) 2024-09-21T09:48:19.186 INFO:tasks.rados.rados.0.smithi017.stdout:2364: done (3 left) 2024-09-21T09:48:19.186 INFO:tasks.rados.rados.0.smithi017.stdout:2368: write oid 24 current snap is 308 2024-09-21T09:48:19.186 INFO:tasks.rados.rados.0.smithi017.stdout:2365: finishing write tid 3 to smithi01785587-36 2024-09-21T09:48:19.186 INFO:tasks.rados.rados.0.smithi017.stdout:2368: seq_num 890 ranges {558230=584911,1897327=527644,2923993=34237} 2024-09-21T09:48:19.192 INFO:tasks.rados.rados.0.smithi017.stdout:2368: writing smithi01785587-24 from 558230 to 1143141 tid 1 2024-09-21T09:48:19.199 INFO:tasks.rados.rados.0.smithi017.stdout:2368: writing smithi01785587-24 from 1897327 to 2424971 tid 2 2024-09-21T09:48:19.202 INFO:tasks.rados.rados.0.smithi017.stdout:2368: writing smithi01785587-24 from 2923993 to 2958230 tid 3 2024-09-21T09:48:19.202 INFO:tasks.rados.rados.0.smithi017.stdout:2369: snap_create 2024-09-21T09:48:19.202 INFO:tasks.rados.rados.0.smithi017.stdout:2365: finishing write tid 4 to smithi01785587-36 2024-09-21T09:48:19.202 INFO:tasks.rados.rados.0.smithi017.stdout:2365: finishing write tid 5 to smithi01785587-36 2024-09-21T09:48:19.202 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1217 (ObjNum 888 snap 308 seq_num 888) dirty exists 2024-09-21T09:48:19.203 INFO:tasks.rados.rados.0.smithi017.stdout:2365: left oid 36 (ObjNum 888 snap 308 seq_num 888) 2024-09-21T09:48:19.203 INFO:tasks.rados.rados.0.smithi017.stdout:2366: expect (ObjNum 885 snap 307 seq_num 885) 2024-09-21T09:48:19.299 INFO:tasks.rados.rados.0.smithi017.stdout:2367: finishing write tid 1 to smithi01785587-17 2024-09-21T09:48:19.299 INFO:tasks.rados.rados.0.smithi017.stdout:2367: finishing write tid 2 to smithi01785587-17 2024-09-21T09:48:19.299 INFO:tasks.rados.rados.0.smithi017.stdout:2367: finishing write tid 3 to smithi01785587-17 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:2367: finishing write tid 4 to smithi01785587-17 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 749 (ObjNum 889 snap 308 seq_num 889) dirty exists 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:2367: left oid 17 (ObjNum 889 snap 308 seq_num 889) 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:2368: finishing write tid 1 to smithi01785587-24 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:2368: finishing write tid 2 to smithi01785587-24 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:2368: finishing write tid 3 to smithi01785587-24 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:2368: finishing write tid 4 to smithi01785587-24 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:2368: finishing write tid 5 to smithi01785587-24 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 794 (ObjNum 890 snap 308 seq_num 890) dirty exists 2024-09-21T09:48:19.300 INFO:tasks.rados.rados.0.smithi017.stdout:2368: left oid 24 (ObjNum 890 snap 308 seq_num 890) 2024-09-21T09:48:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:19 smithi134 ceph-mon[90085]: pgmap v588: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 618 KiB/s wr, 12 op/s 2024-09-21T09:48:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:19 smithi134 ceph-mon[90085]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T09:48:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:19 smithi017 ceph-mon[102913]: pgmap v588: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 618 KiB/s wr, 12 op/s 2024-09-21T09:48:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:19 smithi017 ceph-mon[102913]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T09:48:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:19 smithi017 ceph-mon[105151]: pgmap v588: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 618 KiB/s wr, 12 op/s 2024-09-21T09:48:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:19 smithi017 ceph-mon[105151]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T09:48:20.125 INFO:tasks.rados.rados.0.smithi017.stdout:2365: done (4 left) 2024-09-21T09:48:20.125 INFO:tasks.rados.rados.0.smithi017.stdout:2366: done (3 left) 2024-09-21T09:48:20.125 INFO:tasks.rados.rados.0.smithi017.stdout:2367: done (2 left) 2024-09-21T09:48:20.125 INFO:tasks.rados.rados.0.smithi017.stdout:2368: done (1 left) 2024-09-21T09:48:20.125 INFO:tasks.rados.rados.0.smithi017.stdout:2369: done (0 left) 2024-09-21T09:48:20.125 INFO:tasks.rados.rados.0.smithi017.stdout:2370: read oid 25 snap 301 2024-09-21T09:48:20.126 INFO:tasks.rados.rados.0.smithi017.stdout:2370: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:48:20.126 INFO:tasks.rados.rados.0.smithi017.stdout:2371: write oid 39 current snap is 309 2024-09-21T09:48:20.126 INFO:tasks.rados.rados.0.smithi017.stdout:2371: seq_num 891 ranges {575037=660160,1955809=671302} 2024-09-21T09:48:20.132 INFO:tasks.rados.rados.0.smithi017.stdout:2371: writing smithi01785587-39 from 575037 to 1235197 tid 1 2024-09-21T09:48:20.141 INFO:tasks.rados.rados.0.smithi017.stdout:2371: writing smithi01785587-39 from 1955809 to 2627111 tid 2 2024-09-21T09:48:20.141 INFO:tasks.rados.rados.0.smithi017.stdout:2372: read oid 42 snap -1 2024-09-21T09:48:20.141 INFO:tasks.rados.rados.0.smithi017.stdout:2372: expect deleted 2024-09-21T09:48:20.141 INFO:tasks.rados.rados.0.smithi017.stdout:2373: write oid 33 current snap is 309 2024-09-21T09:48:20.141 INFO:tasks.rados.rados.0.smithi017.stdout:2373: seq_num 892 ranges {591844=735409,1897938=531313,2862486=129358} 2024-09-21T09:48:20.148 INFO:tasks.rados.rados.0.smithi017.stdout:2373: writing smithi01785587-33 from 591844 to 1327253 tid 1 2024-09-21T09:48:20.154 INFO:tasks.rados.rados.0.smithi017.stdout:2373: writing smithi01785587-33 from 1897938 to 2429251 tid 2 2024-09-21T09:48:20.158 INFO:tasks.rados.rados.0.smithi017.stdout:2373: writing smithi01785587-33 from 2862486 to 2991844 tid 3 2024-09-21T09:48:20.158 INFO:tasks.rados.rados.0.smithi017.stdout:2374: write oid 23 current snap is 309 2024-09-21T09:48:20.158 INFO:tasks.rados.rados.0.smithi017.stdout:2374: seq_num 893 ranges {608651=410658,1556420=674971,2773556=235095} 2024-09-21T09:48:20.163 INFO:tasks.rados.rados.0.smithi017.stdout:2374: writing smithi01785587-23 from 608651 to 1019309 tid 1 2024-09-21T09:48:20.170 INFO:tasks.rados.rados.0.smithi017.stdout:2374: writing smithi01785587-23 from 1556420 to 2231391 tid 2 2024-09-21T09:48:20.175 INFO:tasks.rados.rados.0.smithi017.stdout:2374: writing smithi01785587-23 from 2773556 to 3008651 tid 3 2024-09-21T09:48:20.175 INFO:tasks.rados.rados.0.smithi017.stdout:2375: read oid 8 snap -1 2024-09-21T09:48:20.175 INFO:tasks.rados.rados.0.smithi017.stdout:2375: expect (ObjNum 825 snap 282 seq_num 825) 2024-09-21T09:48:20.175 INFO:tasks.rados.rados.0.smithi017.stdout:2376: delete oid 21 current snap is 309 2024-09-21T09:48:20.175 INFO:tasks.rados.rados.0.smithi017.stdout:2370: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:48:20.328 INFO:tasks.rados.rados.0.smithi017.stdout:2371: finishing write tid 1 to smithi01785587-39 2024-09-21T09:48:20.328 INFO:tasks.rados.rados.0.smithi017.stdout:2371: finishing write tid 2 to smithi01785587-39 2024-09-21T09:48:20.328 INFO:tasks.rados.rados.0.smithi017.stdout:2371: finishing write tid 3 to smithi01785587-39 2024-09-21T09:48:20.328 INFO:tasks.rados.rados.0.smithi017.stdout:2371: finishing write tid 4 to smithi01785587-39 2024-09-21T09:48:20.328 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1221 (ObjNum 891 snap 309 seq_num 891) dirty exists 2024-09-21T09:48:20.328 INFO:tasks.rados.rados.0.smithi017.stdout:2371: left oid 39 (ObjNum 891 snap 309 seq_num 891) 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2373: finishing write tid 1 to smithi01785587-33 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2370: done (6 left) 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2371: done (5 left) 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2372: done (4 left) 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2376: done (3 left) 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2377: rollback oid 20 current snap is 309 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 290 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2373: finishing write tid 2 to smithi01785587-33 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2373: finishing write tid 3 to smithi01785587-33 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2373: finishing write tid 4 to smithi01785587-33 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2373: finishing write tid 5 to smithi01785587-33 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1184 (ObjNum 892 snap 309 seq_num 892) dirty exists 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2373: left oid 33 (ObjNum 892 snap 309 seq_num 892) 2024-09-21T09:48:20.329 INFO:tasks.rados.rados.0.smithi017.stdout:2374: finishing write tid 1 to smithi01785587-23 2024-09-21T09:48:20.330 INFO:tasks.rados.rados.0.smithi017.stdout:2374: finishing write tid 2 to smithi01785587-23 2024-09-21T09:48:20.330 INFO:tasks.rados.rados.0.smithi017.stdout:2374: finishing write tid 3 to smithi01785587-23 2024-09-21T09:48:20.330 INFO:tasks.rados.rados.0.smithi017.stdout:2374: finishing write tid 4 to smithi01785587-23 2024-09-21T09:48:20.330 INFO:tasks.rados.rados.0.smithi017.stdout:2374: finishing write tid 5 to smithi01785587-23 2024-09-21T09:48:20.330 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1038 (ObjNum 893 snap 309 seq_num 893) dirty exists 2024-09-21T09:48:20.330 INFO:tasks.rados.rados.0.smithi017.stdout:2374: left oid 23 (ObjNum 893 snap 309 seq_num 893) 2024-09-21T09:48:20.330 INFO:tasks.rados.rados.0.smithi017.stdout:2375: expect (ObjNum 825 snap 282 seq_num 825) 2024-09-21T09:48:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:48:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:48:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:20.601 INFO:tasks.rados.rados.0.smithi017.stdout:2377: finishing rollback tid 0 to smithi01785587-20 2024-09-21T09:48:20.601 INFO:tasks.rados.rados.0.smithi017.stdout:2377: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:48:20.601 INFO:tasks.rados.rados.0.smithi017.stdout:2377: finishing rollback tid 2 to smithi01785587-20 2024-09-21T09:48:20.601 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 582 (ObjNum 807 snap 279 seq_num 807) dirty exists 2024-09-21T09:48:20.601 INFO:tasks.rados.rados.0.smithi017.stdout:2373: done (3 left) 2024-09-21T09:48:20.601 INFO:tasks.rados.rados.0.smithi017.stdout:2374: done (2 left) 2024-09-21T09:48:20.601 INFO:tasks.rados.rados.0.smithi017.stdout:2375: done (1 left) 2024-09-21T09:48:20.601 INFO:tasks.rados.rados.0.smithi017.stdout:2377: done (0 left) 2024-09-21T09:48:20.602 INFO:tasks.rados.rados.0.smithi017.stdout:2378: rollback oid 33 current snap is 309 2024-09-21T09:48:20.602 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 305 2024-09-21T09:48:20.602 INFO:tasks.rados.rados.0.smithi017.stdout:2379: write oid 10 current snap is 309 2024-09-21T09:48:20.602 INFO:tasks.rados.rados.0.smithi017.stdout:2379: seq_num 895 ranges {642265=561156,1673384=678640,2828402=213863} 2024-09-21T09:48:20.607 INFO:tasks.rados.rados.0.smithi017.stdout:2379: writing smithi01785587-10 from 642265 to 1203421 tid 1 2024-09-21T09:48:20.613 INFO:tasks.rados.rados.0.smithi017.stdout:2379: writing smithi01785587-10 from 1673384 to 2352024 tid 2 2024-09-21T09:48:20.617 INFO:tasks.rados.rados.0.smithi017.stdout:2379: writing smithi01785587-10 from 2828402 to 3042265 tid 3 2024-09-21T09:48:20.617 INFO:tasks.rados.rados.0.smithi017.stdout:2380: delete oid 16 current snap is 309 2024-09-21T09:48:20.617 INFO:tasks.rados.rados.0.smithi017.stdout:2378: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:48:20.617 INFO:tasks.rados.rados.0.smithi017.stdout:2378: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:48:20.617 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1186 (ObjNum 49 snap 0 seq_num 2399624544) dirty dne 2024-09-21T09:48:20.618 INFO:tasks.rados.rados.0.smithi017.stdout:2379: finishing write tid 1 to smithi01785587-10 2024-09-21T09:48:20.618 INFO:tasks.rados.rados.0.smithi017.stdout:2379: finishing write tid 2 to smithi01785587-10 2024-09-21T09:48:20.619 INFO:tasks.rados.rados.0.smithi017.stdout:2379: finishing write tid 3 to smithi01785587-10 2024-09-21T09:48:20.619 INFO:tasks.rados.rados.0.smithi017.stdout:2379: finishing write tid 4 to smithi01785587-10 2024-09-21T09:48:20.619 INFO:tasks.rados.rados.0.smithi017.stdout:2378: done (2 left) 2024-09-21T09:48:20.619 INFO:tasks.rados.rados.0.smithi017.stdout:2380: done (1 left) 2024-09-21T09:48:20.619 INFO:tasks.rados.rados.0.smithi017.stdout:2381: write oid 19 current snap is 309 2024-09-21T09:48:20.619 INFO:tasks.rados.rados.0.smithi017.stdout:2381: seq_num 897 ranges {675879=711654,2073995=682309} 2024-09-21T09:48:20.626 INFO:tasks.rados.rados.0.smithi017.stdout:2381: writing smithi01785587-19 from 675879 to 1387533 tid 1 2024-09-21T09:48:20.634 INFO:tasks.rados.rados.0.smithi017.stdout:2381: writing smithi01785587-19 from 2073995 to 2756304 tid 2 2024-09-21T09:48:20.634 INFO:tasks.rados.rados.0.smithi017.stdout:2382: read oid 41 snap -1 2024-09-21T09:48:20.634 INFO:tasks.rados.rados.0.smithi017.stdout:2382: expect (ObjNum 883 snap 307 seq_num 883) 2024-09-21T09:48:20.635 INFO:tasks.rados.rados.0.smithi017.stdout:2383: write oid 22 current snap is 309 2024-09-21T09:48:20.635 INFO:tasks.rados.rados.0.smithi017.stdout:2379: finishing write tid 5 to smithi01785587-10 2024-09-21T09:48:20.635 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 169 (ObjNum 895 snap 309 seq_num 895) dirty exists 2024-09-21T09:48:20.635 INFO:tasks.rados.rados.0.smithi017.stdout:2379: left oid 10 (ObjNum 895 snap 309 seq_num 895) 2024-09-21T09:48:20.635 INFO:tasks.rados.rados.0.smithi017.stdout:2381: finishing write tid 1 to smithi01785587-19 2024-09-21T09:48:20.635 INFO:tasks.rados.rados.0.smithi017.stdout:2383: seq_num 898 ranges {692686=503256,1848830=425967,2910671=182015} 2024-09-21T09:48:20.641 INFO:tasks.rados.rados.0.smithi017.stdout:2383: writing smithi01785587-22 from 692686 to 1195942 tid 1 2024-09-21T09:48:20.647 INFO:tasks.rados.rados.0.smithi017.stdout:2383: writing smithi01785587-22 from 1848830 to 2274797 tid 2 2024-09-21T09:48:20.652 INFO:tasks.rados.rados.0.smithi017.stdout:2383: writing smithi01785587-22 from 2910671 to 3092686 tid 3 2024-09-21T09:48:20.652 INFO:tasks.rados.rados.0.smithi017.stdout:2379: done (3 left) 2024-09-21T09:48:20.652 INFO:tasks.rados.rados.0.smithi017.stdout:2384: snap_create 2024-09-21T09:48:20.652 INFO:tasks.rados.rados.0.smithi017.stdout:2381: finishing write tid 2 to smithi01785587-19 2024-09-21T09:48:20.652 INFO:tasks.rados.rados.0.smithi017.stdout:2381: finishing write tid 3 to smithi01785587-19 2024-09-21T09:48:20.653 INFO:tasks.rados.rados.0.smithi017.stdout:2381: finishing write tid 4 to smithi01785587-19 2024-09-21T09:48:20.653 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 977 (ObjNum 897 snap 309 seq_num 897) dirty exists 2024-09-21T09:48:20.653 INFO:tasks.rados.rados.0.smithi017.stdout:2381: left oid 19 (ObjNum 897 snap 309 seq_num 897) 2024-09-21T09:48:20.653 INFO:tasks.rados.rados.0.smithi017.stdout:2382: expect (ObjNum 883 snap 307 seq_num 883) 2024-09-21T09:48:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:2383: finishing write tid 1 to smithi01785587-22 2024-09-21T09:48:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:2383: finishing write tid 2 to smithi01785587-22 2024-09-21T09:48:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:2383: finishing write tid 3 to smithi01785587-22 2024-09-21T09:48:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:2383: finishing write tid 4 to smithi01785587-22 2024-09-21T09:48:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:2383: finishing write tid 5 to smithi01785587-22 2024-09-21T09:48:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1192 (ObjNum 898 snap 309 seq_num 898) dirty exists 2024-09-21T09:48:20.744 INFO:tasks.rados.rados.0.smithi017.stdout:2383: left oid 22 (ObjNum 898 snap 309 seq_num 898) 2024-09-21T09:48:21.219 INFO:tasks.rados.rados.0.smithi017.stdout:2381: done (3 left) 2024-09-21T09:48:21.219 INFO:tasks.rados.rados.0.smithi017.stdout:2382: done (2 left) 2024-09-21T09:48:21.219 INFO:tasks.rados.rados.0.smithi017.stdout:2383: done (1 left) 2024-09-21T09:48:21.219 INFO:tasks.rados.rados.0.smithi017.stdout:2384: done (0 left) 2024-09-21T09:48:21.219 INFO:tasks.rados.rados.0.smithi017.stdout:2385: write oid 9 current snap is 310 2024-09-21T09:48:21.219 INFO:tasks.rados.rados.0.smithi017.stdout:2385: seq_num 899 ranges {709493=578505,1907312=685978} 2024-09-21T09:48:21.225 INFO:tasks.rados.rados.0.smithi017.stdout:2385: writing smithi01785587-9 from 709493 to 1287998 tid 1 2024-09-21T09:48:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:2385: writing smithi01785587-9 from 1907312 to 2593290 tid 2 2024-09-21T09:48:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:2386: delete oid 25 current snap is 310 2024-09-21T09:48:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:2385: finishing write tid 1 to smithi01785587-9 2024-09-21T09:48:21.236 INFO:tasks.rados.rados.0.smithi017.stdout:2385: finishing write tid 2 to smithi01785587-9 2024-09-21T09:48:21.237 INFO:tasks.rados.rados.0.smithi017.stdout:2385: finishing write tid 3 to smithi01785587-9 2024-09-21T09:48:21.237 INFO:tasks.rados.rados.0.smithi017.stdout:2385: finishing write tid 4 to smithi01785587-9 2024-09-21T09:48:21.237 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 464 (ObjNum 899 snap 310 seq_num 899) dirty exists 2024-09-21T09:48:21.237 INFO:tasks.rados.rados.0.smithi017.stdout:2385: left oid 9 (ObjNum 899 snap 310 seq_num 899) 2024-09-21T09:48:21.238 INFO:tasks.rados.rados.0.smithi017.stdout:2385: done (1 left) 2024-09-21T09:48:21.238 INFO:tasks.rados.rados.0.smithi017.stdout:2386: done (0 left) 2024-09-21T09:48:21.238 INFO:tasks.rados.rados.0.smithi017.stdout:2387: read oid 14 snap -1 2024-09-21T09:48:21.238 INFO:tasks.rados.rados.0.smithi017.stdout:2387: expect (ObjNum 854 snap 289 seq_num 854) 2024-09-21T09:48:21.238 INFO:tasks.rados.rados.0.smithi017.stdout:2388: write oid 2 current snap is 310 2024-09-21T09:48:21.238 INFO:tasks.rados.rados.0.smithi017.stdout:2388: seq_num 901 ranges {743107=729003,1907923=689647} 2024-09-21T09:48:21.246 INFO:tasks.rados.rados.0.smithi017.stdout:2388: writing smithi01785587-2 from 743107 to 1472110 tid 1 2024-09-21T09:48:21.253 INFO:tasks.rados.rados.0.smithi017.stdout:2388: writing smithi01785587-2 from 1907923 to 2597570 tid 2 2024-09-21T09:48:21.253 INFO:tasks.rados.rados.0.smithi017.stdout:2389: write oid 19 current snap is 310 2024-09-21T09:48:21.253 INFO:tasks.rados.rados.0.smithi017.stdout:2389: seq_num 902 ranges {759914=404252,1566405=433305,2504010=655904} 2024-09-21T09:48:21.260 INFO:tasks.rados.rados.0.smithi017.stdout:2389: writing smithi01785587-19 from 759914 to 1164166 tid 1 2024-09-21T09:48:21.265 INFO:tasks.rados.rados.0.smithi017.stdout:2389: writing smithi01785587-19 from 1566405 to 1999710 tid 2 2024-09-21T09:48:21.272 INFO:tasks.rados.rados.0.smithi017.stdout:2389: writing smithi01785587-19 from 2504010 to 3159914 tid 3 2024-09-21T09:48:21.272 INFO:tasks.rados.rados.0.smithi017.stdout:2390: read oid 46 snap -1 2024-09-21T09:48:21.272 INFO:tasks.rados.rados.0.smithi017.stdout:2390: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:48:21.272 INFO:tasks.rados.rados.0.smithi017.stdout:2391: read oid 32 snap -1 2024-09-21T09:48:21.272 INFO:tasks.rados.rados.0.smithi017.stdout:2391: expect (ObjNum 884 snap 307 seq_num 884) 2024-09-21T09:48:21.272 INFO:tasks.rados.rados.0.smithi017.stdout:2392: read oid 45 snap -1 2024-09-21T09:48:21.273 INFO:tasks.rados.rados.0.smithi017.stdout:2392: expect (ObjNum 877 snap 304 seq_num 877) 2024-09-21T09:48:21.273 INFO:tasks.rados.rados.0.smithi017.stdout:2393: write oid 8 current snap is 310 2024-09-21T09:48:21.273 INFO:tasks.rados.rados.0.smithi017.stdout:2393: seq_num 903 ranges {776721=479501,2024887=693316} 2024-09-21T09:48:21.279 INFO:tasks.rados.rados.0.smithi017.stdout:2393: writing smithi01785587-8 from 776721 to 1256222 tid 1 2024-09-21T09:48:21.287 INFO:tasks.rados.rados.0.smithi017.stdout:2393: writing smithi01785587-8 from 2024887 to 2718203 tid 2 2024-09-21T09:48:21.287 INFO:tasks.rados.rados.0.smithi017.stdout:2394: delete oid 31 current snap is 310 2024-09-21T09:48:21.287 INFO:tasks.rados.rados.0.smithi017.stdout:2387: expect (ObjNum 854 snap 289 seq_num 854) 2024-09-21T09:48:21.391 INFO:tasks.rados.rados.0.smithi017.stdout:2388: finishing write tid 1 to smithi01785587-2 2024-09-21T09:48:21.391 INFO:tasks.rados.rados.0.smithi017.stdout:2388: finishing write tid 2 to smithi01785587-2 2024-09-21T09:48:21.391 INFO:tasks.rados.rados.0.smithi017.stdout:2388: finishing write tid 3 to smithi01785587-2 2024-09-21T09:48:21.391 INFO:tasks.rados.rados.0.smithi017.stdout:2388: finishing write tid 4 to smithi01785587-2 2024-09-21T09:48:21.391 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 468 (ObjNum 901 snap 310 seq_num 901) dirty exists 2024-09-21T09:48:21.391 INFO:tasks.rados.rados.0.smithi017.stdout:2388: left oid 2 (ObjNum 901 snap 310 seq_num 901) 2024-09-21T09:48:21.391 INFO:tasks.rados.rados.0.smithi017.stdout:2389: finishing write tid 1 to smithi01785587-19 2024-09-21T09:48:21.391 INFO:tasks.rados.rados.0.smithi017.stdout:2389: finishing write tid 2 to smithi01785587-19 2024-09-21T09:48:21.392 INFO:tasks.rados.rados.0.smithi017.stdout:2387: done (7 left) 2024-09-21T09:48:21.392 INFO:tasks.rados.rados.0.smithi017.stdout:2388: done (6 left) 2024-09-21T09:48:21.392 INFO:tasks.rados.rados.0.smithi017.stdout:2394: done (5 left) 2024-09-21T09:48:21.392 INFO:tasks.rados.rados.0.smithi017.stdout:2395: snap_remove snap 280 2024-09-21T09:48:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:21 smithi017 ceph-mon[102913]: pgmap v591: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1013 KiB/s rd, 336 KiB/s wr, 3 op/s 2024-09-21T09:48:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:21 smithi017 ceph-mon[102913]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T09:48:21.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:21 smithi017 ceph-mon[105151]: pgmap v591: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1013 KiB/s rd, 336 KiB/s wr, 3 op/s 2024-09-21T09:48:21.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:21 smithi017 ceph-mon[105151]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T09:48:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:21 smithi134 ceph-mon[90085]: pgmap v591: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1013 KiB/s rd, 336 KiB/s wr, 3 op/s 2024-09-21T09:48:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:21 smithi134 ceph-mon[90085]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T09:48:22.225 INFO:tasks.rados.rados.0.smithi017.stdout:2395: done (5 left) 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2396: read oid 26 snap -1 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2396: expect (ObjNum 754 snap 255 seq_num 754) 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2397: read oid 22 snap -1 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2397: expect (ObjNum 898 snap 309 seq_num 898) 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2398: read oid 20 snap 299 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2398: expect (ObjNum 807 snap 279 seq_num 807) 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2399: delete oid 41 current snap is 310 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2389: finishing write tid 3 to smithi01785587-19 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2389: finishing write tid 4 to smithi01785587-19 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:2389: finishing write tid 5 to smithi01785587-19 2024-09-21T09:48:22.226 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 982 (ObjNum 902 snap 310 seq_num 902) dirty exists 2024-09-21T09:48:22.227 INFO:tasks.rados.rados.0.smithi017.stdout:2389: left oid 19 (ObjNum 902 snap 310 seq_num 902) 2024-09-21T09:48:22.227 INFO:tasks.rados.rados.0.smithi017.stdout:2390: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:48:22.357 INFO:tasks.rados.rados.0.smithi017.stdout:2392: expect (ObjNum 877 snap 304 seq_num 877) 2024-09-21T09:48:22.470 INFO:tasks.rados.rados.0.smithi017.stdout:2391: expect (ObjNum 884 snap 307 seq_num 884) 2024-09-21T09:48:22.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:22 smithi134 ceph-mon[90085]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T09:48:22.597 INFO:tasks.rados.rados.0.smithi017.stdout:2393: finishing write tid 1 to smithi01785587-8 2024-09-21T09:48:22.598 INFO:tasks.rados.rados.0.smithi017.stdout:2393: finishing write tid 2 to smithi01785587-8 2024-09-21T09:48:22.598 INFO:tasks.rados.rados.0.smithi017.stdout:2393: finishing write tid 3 to smithi01785587-8 2024-09-21T09:48:22.598 INFO:tasks.rados.rados.0.smithi017.stdout:2393: finishing write tid 4 to smithi01785587-8 2024-09-21T09:48:22.598 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1042 (ObjNum 903 snap 310 seq_num 903) dirty exists 2024-09-21T09:48:22.598 INFO:tasks.rados.rados.0.smithi017.stdout:2393: left oid 8 (ObjNum 903 snap 310 seq_num 903) 2024-09-21T09:48:22.598 INFO:tasks.rados.rados.0.smithi017.stdout:2398: expect (ObjNum 807 snap 279 seq_num 807) 2024-09-21T09:48:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:22 smithi017 ceph-mon[102913]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T09:48:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:22 smithi017 ceph-mon[105151]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T09:48:22.766 INFO:tasks.rados.rados.0.smithi017.stdout:2396: expect (ObjNum 754 snap 255 seq_num 754) 2024-09-21T09:48:22.921 INFO:tasks.rados.rados.0.smithi017.stdout:2397: expect (ObjNum 898 snap 309 seq_num 898) 2024-09-21T09:48:23.146 INFO:tasks.rados.rados.0.smithi017.stdout:2389: done (8 left) 2024-09-21T09:48:23.146 INFO:tasks.rados.rados.0.smithi017.stdout:2390: done (7 left) 2024-09-21T09:48:23.146 INFO:tasks.rados.rados.0.smithi017.stdout:2391: done (6 left) 2024-09-21T09:48:23.146 INFO:tasks.rados.rados.0.smithi017.stdout:2392: done (5 left) 2024-09-21T09:48:23.146 INFO:tasks.rados.rados.0.smithi017.stdout:2393: done (4 left) 2024-09-21T09:48:23.146 INFO:tasks.rados.rados.0.smithi017.stdout:2396: done (3 left) 2024-09-21T09:48:23.146 INFO:tasks.rados.rados.0.smithi017.stdout:2397: done (2 left) 2024-09-21T09:48:23.146 INFO:tasks.rados.rados.0.smithi017.stdout:2398: done (1 left) 2024-09-21T09:48:23.147 INFO:tasks.rados.rados.0.smithi017.stdout:2399: done (0 left) 2024-09-21T09:48:23.147 INFO:tasks.rados.rados.0.smithi017.stdout:2400: write oid 33 current snap is 310 2024-09-21T09:48:23.147 INFO:tasks.rados.rados.0.smithi017.stdout:2400: seq_num 906 ranges {427142=421601,1400333=440643,2613702=450326} 2024-09-21T09:48:23.150 INFO:tasks.rados.rados.0.smithi017.stdout:2400: writing smithi01785587-33 from 427142 to 848743 tid 1 2024-09-21T09:48:23.156 INFO:tasks.rados.rados.0.smithi017.stdout:2400: writing smithi01785587-33 from 1400333 to 1840976 tid 2 2024-09-21T09:48:23.163 INFO:tasks.rados.rados.0.smithi017.stdout:2400: writing smithi01785587-33 from 2613702 to 3064028 tid 3 2024-09-21T09:48:23.163 INFO:tasks.rados.rados.0.smithi017.stdout:2401: read oid 26 snap 303 2024-09-21T09:48:23.163 INFO:tasks.rados.rados.0.smithi017.stdout:2401: expect (ObjNum 754 snap 255 seq_num 754) 2024-09-21T09:48:23.163 INFO:tasks.rados.rados.0.smithi017.stdout:2402: write oid 12 current snap is 310 2024-09-21T09:48:23.163 INFO:tasks.rados.rados.0.smithi017.stdout:2400: finishing write tid 1 to smithi01785587-33 2024-09-21T09:48:23.163 INFO:tasks.rados.rados.0.smithi017.stdout:2400: finishing write tid 2 to smithi01785587-33 2024-09-21T09:48:23.163 INFO:tasks.rados.rados.0.smithi017.stdout:2402: seq_num 907 ranges {443949=496850,1458815=584301,2641125=602824} 2024-09-21T09:48:23.168 INFO:tasks.rados.rados.0.smithi017.stdout:2402: writing smithi01785587-12 from 443949 to 940799 tid 1 2024-09-21T09:48:23.174 INFO:tasks.rados.rados.0.smithi017.stdout:2402: writing smithi01785587-12 from 1458815 to 2043116 tid 2 2024-09-21T09:48:23.181 INFO:tasks.rados.rados.0.smithi017.stdout:2402: writing smithi01785587-12 from 2641125 to 3243949 tid 3 2024-09-21T09:48:23.181 INFO:tasks.rados.rados.0.smithi017.stdout:2403: rollback oid 50 current snap is 310 2024-09-21T09:48:23.181 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 50 to 295 2024-09-21T09:48:23.181 INFO:tasks.rados.rados.0.smithi017.stdout:2400: finishing write tid 3 to smithi01785587-33 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:2400: finishing write tid 4 to smithi01785587-33 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:2400: finishing write tid 5 to smithi01785587-33 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1204 (ObjNum 906 snap 310 seq_num 906) dirty exists 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:2400: left oid 33 (ObjNum 906 snap 310 seq_num 906) 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:2400: done (3 left) 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:2404: read oid 14 snap -1 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:2404: expect (ObjNum 854 snap 289 seq_num 854) 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:2405: delete oid 24 current snap is 310 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:2402: finishing write tid 1 to smithi01785587-12 2024-09-21T09:48:23.182 INFO:tasks.rados.rados.0.smithi017.stdout:2401: expect (ObjNum 754 snap 255 seq_num 754) 2024-09-21T09:48:23.338 INFO:tasks.rados.rados.0.smithi017.stdout:2402: finishing write tid 2 to smithi01785587-12 2024-09-21T09:48:23.339 INFO:tasks.rados.rados.0.smithi017.stdout:2404: expect (ObjNum 854 snap 289 seq_num 854) 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:2402: finishing write tid 3 to smithi01785587-12 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:2402: finishing write tid 4 to smithi01785587-12 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:2402: finishing write tid 5 to smithi01785587-12 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 868 (ObjNum 907 snap 310 seq_num 907) dirty exists 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:2402: left oid 12 (ObjNum 907 snap 310 seq_num 907) 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:2403: finishing rollback tid 0 to smithi01785587-50 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:2403: finishing rollback tid 1 to smithi01785587-50 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:2401: done (4 left) 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:2402: done (3 left) 2024-09-21T09:48:23.441 INFO:tasks.rados.rados.0.smithi017.stdout:2404: done (2 left) 2024-09-21T09:48:23.442 INFO:tasks.rados.rados.0.smithi017.stdout:2405: done (1 left) 2024-09-21T09:48:23.442 INFO:tasks.rados.rados.0.smithi017.stdout:2406: snap_remove snap 290 2024-09-21T09:48:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:23 smithi134 ceph-mon[90085]: pgmap v594: 89 pgs: 89 active+clean; 212 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.4 MiB/s wr, 27 op/s 2024-09-21T09:48:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:23 smithi134 ceph-mon[90085]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T09:48:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:23 smithi134 ceph-mon[90085]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T09:48:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:23 smithi017 ceph-mon[105151]: pgmap v594: 89 pgs: 89 active+clean; 212 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.4 MiB/s wr, 27 op/s 2024-09-21T09:48:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:23 smithi017 ceph-mon[105151]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T09:48:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:23 smithi017 ceph-mon[105151]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T09:48:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:23 smithi017 ceph-mon[102913]: pgmap v594: 89 pgs: 89 active+clean; 212 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.4 MiB/s wr, 27 op/s 2024-09-21T09:48:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:23 smithi017 ceph-mon[102913]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T09:48:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:23 smithi017 ceph-mon[102913]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T09:48:24.233 INFO:tasks.rados.rados.0.smithi017.stdout:2406: done (1 left) 2024-09-21T09:48:24.233 INFO:tasks.rados.rados.0.smithi017.stdout:2407: read oid 49 snap 308 2024-09-21T09:48:24.233 INFO:tasks.rados.rados.0.smithi017.stdout:2407: expect deleted 2024-09-21T09:48:24.233 INFO:tasks.rados.rados.0.smithi017.stdout:2408: delete oid 35 current snap is 310 2024-09-21T09:48:24.233 INFO:tasks.rados.rados.0.smithi017.stdout:2403: finishing rollback tid 2 to smithi01785587-50 2024-09-21T09:48:24.233 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 800 (ObjNum 852 snap 289 seq_num 852) dirty exists 2024-09-21T09:48:24.243 INFO:tasks.rados.rados.0.smithi017.stdout:2403: done (2 left) 2024-09-21T09:48:24.243 INFO:tasks.rados.rados.0.smithi017.stdout:2407: done (1 left) 2024-09-21T09:48:24.243 INFO:tasks.rados.rados.0.smithi017.stdout:2408: done (0 left) 2024-09-21T09:48:24.243 INFO:tasks.rados.rados.0.smithi017.stdout:2409: rollback oid 37 current snap is 310 2024-09-21T09:48:24.243 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 286 2024-09-21T09:48:24.243 INFO:tasks.rados.rados.0.smithi017.stdout:2410: rollback oid 34 current snap is 310 2024-09-21T09:48:24.244 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 309 2024-09-21T09:48:24.244 INFO:tasks.rados.rados.0.smithi017.stdout:2411: write oid 50 current snap is 310 2024-09-21T09:48:24.244 INFO:tasks.rados.rados.0.smithi017.stdout:2411: seq_num 910 ranges {494370=722597,1917908=447981,3007041=287329} 2024-09-21T09:48:24.250 INFO:tasks.rados.rados.0.smithi017.stdout:2411: writing smithi01785587-50 from 494370 to 1216967 tid 1 2024-09-21T09:48:24.257 INFO:tasks.rados.rados.0.smithi017.stdout:2411: writing smithi01785587-50 from 1917908 to 2365889 tid 2 2024-09-21T09:48:24.262 INFO:tasks.rados.rados.0.smithi017.stdout:2411: writing smithi01785587-50 from 3007041 to 3294370 tid 3 2024-09-21T09:48:24.262 INFO:tasks.rados.rados.0.smithi017.stdout:2412: snap_create 2024-09-21T09:48:24.262 INFO:tasks.rados.rados.0.smithi017.stdout:2409: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:48:24.262 INFO:tasks.rados.rados.0.smithi017.stdout:2410: finishing rollback tid 0 to smithi01785587-34 2024-09-21T09:48:24.262 INFO:tasks.rados.rados.0.smithi017.stdout:2409: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:48:24.262 INFO:tasks.rados.rados.0.smithi017.stdout:2409: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:48:24.262 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1235 (ObjNum 698 snap 221 seq_num 698) dirty exists 2024-09-21T09:48:24.263 INFO:tasks.rados.rados.0.smithi017.stdout:2410: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:48:24.263 INFO:tasks.rados.rados.0.smithi017.stdout:2410: finishing rollback tid 2 to smithi01785587-34 2024-09-21T09:48:24.263 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1250 (ObjNum 886 snap 308 seq_num 886) dirty exists 2024-09-21T09:48:24.263 INFO:tasks.rados.rados.0.smithi017.stdout:2411: finishing write tid 1 to smithi01785587-50 2024-09-21T09:48:24.263 INFO:tasks.rados.rados.0.smithi017.stdout:2411: finishing write tid 2 to smithi01785587-50 2024-09-21T09:48:24.264 INFO:tasks.rados.rados.0.smithi017.stdout:2411: finishing write tid 3 to smithi01785587-50 2024-09-21T09:48:24.264 INFO:tasks.rados.rados.0.smithi017.stdout:2411: finishing write tid 4 to smithi01785587-50 2024-09-21T09:48:24.265 INFO:tasks.rados.rados.0.smithi017.stdout:2411: finishing write tid 5 to smithi01785587-50 2024-09-21T09:48:24.265 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 810 (ObjNum 910 snap 310 seq_num 910) dirty exists 2024-09-21T09:48:24.265 INFO:tasks.rados.rados.0.smithi017.stdout:2411: left oid 50 (ObjNum 910 snap 310 seq_num 910) 2024-09-21T09:48:25.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:48:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:48:24] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-09-21T09:48:25.232 INFO:tasks.rados.rados.0.smithi017.stdout:2409: done (3 left) 2024-09-21T09:48:25.232 INFO:tasks.rados.rados.0.smithi017.stdout:2410: done (2 left) 2024-09-21T09:48:25.232 INFO:tasks.rados.rados.0.smithi017.stdout:2411: done (1 left) 2024-09-21T09:48:25.232 INFO:tasks.rados.rados.0.smithi017.stdout:2412: done (0 left) 2024-09-21T09:48:25.232 INFO:tasks.rados.rados.0.smithi017.stdout:2413: delete oid 24 current snap is 311 2024-09-21T09:48:25.236 INFO:tasks.rados.rados.0.smithi017.stdout:2413: done (0 left) 2024-09-21T09:48:25.236 INFO:tasks.rados.rados.0.smithi017.stdout:2414: rollback oid 18 current snap is 311 2024-09-21T09:48:25.236 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 18 to 307 2024-09-21T09:48:25.236 INFO:tasks.rados.rados.0.smithi017.stdout:2415: snap_remove snap 299 2024-09-21T09:48:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:25 smithi134 ceph-mon[90085]: pgmap v597: 89 pgs: 89 active+clean; 224 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 5.1 MiB/s wr, 36 op/s 2024-09-21T09:48:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:25 smithi134 ceph-mon[90085]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T09:48:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:25 smithi017 ceph-mon[102913]: pgmap v597: 89 pgs: 89 active+clean; 224 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 5.1 MiB/s wr, 36 op/s 2024-09-21T09:48:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:25 smithi017 ceph-mon[102913]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T09:48:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:25 smithi017 ceph-mon[105151]: pgmap v597: 89 pgs: 89 active+clean; 224 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 5.1 MiB/s wr, 36 op/s 2024-09-21T09:48:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:25 smithi017 ceph-mon[105151]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T09:48:26.244 INFO:tasks.rados.rados.0.smithi017.stdout:2415: done (1 left) 2024-09-21T09:48:26.245 INFO:tasks.rados.rados.0.smithi017.stdout:2416: write oid 19 current snap is 311 2024-09-21T09:48:26.245 INFO:tasks.rados.rados.0.smithi017.stdout:2416: seq_num 912 ranges {527984=473095,1634872=451650,2661887=634311} 2024-09-21T09:48:26.250 INFO:tasks.rados.rados.0.smithi017.stdout:2416: writing smithi01785587-19 from 527984 to 1001079 tid 1 2024-09-21T09:48:26.256 INFO:tasks.rados.rados.0.smithi017.stdout:2416: writing smithi01785587-19 from 1634872 to 2086522 tid 2 2024-09-21T09:48:26.263 INFO:tasks.rados.rados.0.smithi017.stdout:2416: writing smithi01785587-19 from 2661887 to 3296198 tid 3 2024-09-21T09:48:26.263 INFO:tasks.rados.rados.0.smithi017.stdout:2417: write oid 43 current snap is 311 2024-09-21T09:48:26.263 INFO:tasks.rados.rados.0.smithi017.stdout:2417: seq_num 913 ranges {544791=664697,1693354=595308,2689310=445583} 2024-09-21T09:48:26.270 INFO:tasks.rados.rados.0.smithi017.stdout:2417: writing smithi01785587-43 from 544791 to 1209488 tid 1 2024-09-21T09:48:26.276 INFO:tasks.rados.rados.0.smithi017.stdout:2417: writing smithi01785587-43 from 1693354 to 2288662 tid 2 2024-09-21T09:48:26.281 INFO:tasks.rados.rados.0.smithi017.stdout:2417: writing smithi01785587-43 from 2689310 to 3134893 tid 3 2024-09-21T09:48:26.281 INFO:tasks.rados.rados.0.smithi017.stdout:2418: read oid 47 snap 295 2024-09-21T09:48:26.282 INFO:tasks.rados.rados.0.smithi017.stdout:2418: expect deleted 2024-09-21T09:48:26.282 INFO:tasks.rados.rados.0.smithi017.stdout:2414: finishing rollback tid 0 to smithi01785587-18 2024-09-21T09:48:26.282 INFO:tasks.rados.rados.0.smithi017.stdout:2414: finishing rollback tid 1 to smithi01785587-18 2024-09-21T09:48:26.282 INFO:tasks.rados.rados.0.smithi017.stdout:2419: rollback oid 35 current snap is 311 2024-09-21T09:48:26.282 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 294 2024-09-21T09:48:26.282 INFO:tasks.rados.rados.0.smithi017.stdout:2414: finishing rollback tid 2 to smithi01785587-18 2024-09-21T09:48:26.283 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 210 (ObjNum 867 snap 293 seq_num 867) dirty exists 2024-09-21T09:48:26.283 INFO:tasks.rados.rados.0.smithi017.stdout:2416: finishing write tid 1 to smithi01785587-19 2024-09-21T09:48:26.283 INFO:tasks.rados.rados.0.smithi017.stdout:2416: finishing write tid 2 to smithi01785587-19 2024-09-21T09:48:26.283 INFO:tasks.rados.rados.0.smithi017.stdout:2416: finishing write tid 3 to smithi01785587-19 2024-09-21T09:48:26.283 INFO:tasks.rados.rados.0.smithi017.stdout:2416: finishing write tid 4 to smithi01785587-19 2024-09-21T09:48:26.283 INFO:tasks.rados.rados.0.smithi017.stdout:2416: finishing write tid 5 to smithi01785587-19 2024-09-21T09:48:26.284 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 1005 (ObjNum 912 snap 311 seq_num 912) dirty exists 2024-09-21T09:48:26.284 INFO:tasks.rados.rados.0.smithi017.stdout:2416: left oid 19 (ObjNum 912 snap 311 seq_num 912) 2024-09-21T09:48:26.284 INFO:tasks.rados.rados.0.smithi017.stdout:2417: finishing write tid 1 to smithi01785587-43 2024-09-21T09:48:26.284 INFO:tasks.rados.rados.0.smithi017.stdout:2417: finishing write tid 2 to smithi01785587-43 2024-09-21T09:48:26.284 INFO:tasks.rados.rados.0.smithi017.stdout:2414: done (4 left) 2024-09-21T09:48:26.284 INFO:tasks.rados.rados.0.smithi017.stdout:2416: done (3 left) 2024-09-21T09:48:26.284 INFO:tasks.rados.rados.0.smithi017.stdout:2420: snap_remove snap 301 2024-09-21T09:48:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:26 smithi134 ceph-mon[90085]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T09:48:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:26 smithi017 ceph-mon[105151]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T09:48:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:26 smithi017 ceph-mon[102913]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T09:48:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:27.248 INFO:tasks.rados.rados.0.smithi017.stdout:2419: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:48:27.248 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 257 (ObjNum 49 snap 0 seq_num 2399504720) dirty dne 2024-09-21T09:48:27.248 INFO:tasks.rados.rados.0.smithi017.stdout:2417: finishing write tid 3 to smithi01785587-43 2024-09-21T09:48:27.248 INFO:tasks.rados.rados.0.smithi017.stdout:2417: finishing write tid 4 to smithi01785587-43 2024-09-21T09:48:27.249 INFO:tasks.rados.rados.0.smithi017.stdout:2417: finishing write tid 5 to smithi01785587-43 2024-09-21T09:48:27.249 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 731 (ObjNum 913 snap 311 seq_num 913) dirty exists 2024-09-21T09:48:27.249 INFO:tasks.rados.rados.0.smithi017.stdout:2417: left oid 43 (ObjNum 913 snap 311 seq_num 913) 2024-09-21T09:48:27.249 INFO:tasks.rados.rados.0.smithi017.stdout:2417: done (3 left) 2024-09-21T09:48:27.249 INFO:tasks.rados.rados.0.smithi017.stdout:2418: done (2 left) 2024-09-21T09:48:27.249 INFO:tasks.rados.rados.0.smithi017.stdout:2419: done (1 left) 2024-09-21T09:48:27.250 INFO:tasks.rados.rados.0.smithi017.stdout:2420: done (0 left) 2024-09-21T09:48:27.250 INFO:tasks.rados.rados.0.smithi017.stdout:2421: snap_remove snap 294 2024-09-21T09:48:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:27 smithi134 ceph-mon[90085]: pgmap v600: 89 pgs: 89 active+clean; 223 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.3 MiB/s wr, 25 op/s 2024-09-21T09:48:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:27 smithi134 ceph-mon[90085]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T09:48:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:27 smithi017 ceph-mon[102913]: pgmap v600: 89 pgs: 89 active+clean; 223 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.3 MiB/s wr, 25 op/s 2024-09-21T09:48:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:27 smithi017 ceph-mon[102913]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T09:48:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:27 smithi017 ceph-mon[105151]: pgmap v600: 89 pgs: 89 active+clean; 223 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.3 MiB/s wr, 25 op/s 2024-09-21T09:48:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:27 smithi017 ceph-mon[105151]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T09:48:28.127 INFO:tasks.rados.rados.0.smithi017.stdout:2421: done (0 left) 2024-09-21T09:48:28.127 INFO:tasks.rados.rados.0.smithi017.stdout:2422: read oid 45 snap 309 2024-09-21T09:48:28.127 INFO:tasks.rados.rados.0.smithi017.stdout:2422: expect (ObjNum 877 snap 304 seq_num 877) 2024-09-21T09:48:28.128 INFO:tasks.rados.rados.0.smithi017.stdout:2423: snap_remove snap 306 2024-09-21T09:48:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:28 smithi017 ceph-mon[102913]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T09:48:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:28 smithi017 ceph-mon[102913]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T09:48:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:28 smithi017 ceph-mon[105151]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T09:48:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:28 smithi017 ceph-mon[105151]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T09:48:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:28 smithi134 ceph-mon[90085]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T09:48:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:28 smithi134 ceph-mon[90085]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T09:48:29.129 INFO:tasks.rados.rados.0.smithi017.stdout:2422: expect (ObjNum 877 snap 304 seq_num 877) 2024-09-21T09:48:29.245 INFO:tasks.rados.rados.0.smithi017.stdout:2422: done (1 left) 2024-09-21T09:48:29.246 INFO:tasks.rados.rados.0.smithi017.stdout:2423: done (0 left) 2024-09-21T09:48:29.246 INFO:tasks.rados.rados.0.smithi017.stdout:2424: write oid 1 current snap is 311 2024-09-21T09:48:29.246 INFO:tasks.rados.rados.0.smithi017.stdout:2424: seq_num 914 ranges {561598=739946,1751836=455319,2716733=644865} 2024-09-21T09:48:29.253 INFO:tasks.rados.rados.0.smithi017.stdout:2424: writing smithi01785587-1 from 561598 to 1301544 tid 1 2024-09-21T09:48:29.258 INFO:tasks.rados.rados.0.smithi017.stdout:2424: writing smithi01785587-1 from 1751836 to 2207155 tid 2 2024-09-21T09:48:29.265 INFO:tasks.rados.rados.0.smithi017.stdout:2424: writing smithi01785587-1 from 2716733 to 3361598 tid 3 2024-09-21T09:48:29.266 INFO:tasks.rados.rados.0.smithi017.stdout:2425: write oid 36 current snap is 311 2024-09-21T09:48:29.266 INFO:tasks.rados.rados.0.smithi017.stdout:2424: finishing write tid 1 to smithi01785587-1 2024-09-21T09:48:29.266 INFO:tasks.rados.rados.0.smithi017.stdout:2424: finishing write tid 2 to smithi01785587-1 2024-09-21T09:48:29.266 INFO:tasks.rados.rados.0.smithi017.stdout:2425: seq_num 915 ranges {578405=415195,1410318=598977,2744156=584480} 2024-09-21T09:48:29.271 INFO:tasks.rados.rados.0.smithi017.stdout:2425: writing smithi01785587-36 from 578405 to 993600 tid 1 2024-09-21T09:48:29.277 INFO:tasks.rados.rados.0.smithi017.stdout:2425: writing smithi01785587-36 from 1410318 to 2009295 tid 2 2024-09-21T09:48:29.284 INFO:tasks.rados.rados.0.smithi017.stdout:2425: writing smithi01785587-36 from 2744156 to 3328636 tid 3 2024-09-21T09:48:29.284 INFO:tasks.rados.rados.0.smithi017.stdout:2426: snap_create 2024-09-21T09:48:29.284 INFO:tasks.rados.rados.0.smithi017.stdout:2424: finishing write tid 3 to smithi01785587-1 2024-09-21T09:48:29.284 INFO:tasks.rados.rados.0.smithi017.stdout:2424: finishing write tid 4 to smithi01785587-1 2024-09-21T09:48:29.285 INFO:tasks.rados.rados.0.smithi017.stdout:2424: finishing write tid 5 to smithi01785587-1 2024-09-21T09:48:29.285 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 744 (ObjNum 914 snap 311 seq_num 914) dirty exists 2024-09-21T09:48:29.285 INFO:tasks.rados.rados.0.smithi017.stdout:2424: left oid 1 (ObjNum 914 snap 311 seq_num 914) 2024-09-21T09:48:29.285 INFO:tasks.rados.rados.0.smithi017.stdout:2425: finishing write tid 1 to smithi01785587-36 2024-09-21T09:48:29.285 INFO:tasks.rados.rados.0.smithi017.stdout:2425: finishing write tid 2 to smithi01785587-36 2024-09-21T09:48:29.287 INFO:tasks.rados.rados.0.smithi017.stdout:2425: finishing write tid 3 to smithi01785587-36 2024-09-21T09:48:29.287 INFO:tasks.rados.rados.0.smithi017.stdout:2425: finishing write tid 4 to smithi01785587-36 2024-09-21T09:48:29.287 INFO:tasks.rados.rados.0.smithi017.stdout:2425: finishing write tid 5 to smithi01785587-36 2024-09-21T09:48:29.287 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1262 (ObjNum 915 snap 311 seq_num 915) dirty exists 2024-09-21T09:48:29.287 INFO:tasks.rados.rados.0.smithi017.stdout:2425: left oid 36 (ObjNum 915 snap 311 seq_num 915) 2024-09-21T09:48:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:29 smithi017 ceph-mon[102913]: pgmap v603: 89 pgs: 89 active+clean; 227 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.7 MiB/s wr, 26 op/s 2024-09-21T09:48:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:29 smithi017 ceph-mon[102913]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T09:48:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:29 smithi017 ceph-mon[105151]: pgmap v603: 89 pgs: 89 active+clean; 227 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.7 MiB/s wr, 26 op/s 2024-09-21T09:48:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:29 smithi017 ceph-mon[105151]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T09:48:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:29 smithi134 ceph-mon[90085]: pgmap v603: 89 pgs: 89 active+clean; 227 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.7 MiB/s wr, 26 op/s 2024-09-21T09:48:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:29 smithi134 ceph-mon[90085]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T09:48:30.258 INFO:tasks.rados.rados.0.smithi017.stdout:2424: done (2 left) 2024-09-21T09:48:30.258 INFO:tasks.rados.rados.0.smithi017.stdout:2425: done (1 left) 2024-09-21T09:48:30.259 INFO:tasks.rados.rados.0.smithi017.stdout:2426: done (0 left) 2024-09-21T09:48:30.259 INFO:tasks.rados.rados.0.smithi017.stdout:2427: write oid 38 current snap is 312 2024-09-21T09:48:30.259 INFO:tasks.rados.rados.0.smithi017.stdout:2427: seq_num 916 ranges {595212=490444,1868800=458988,2887932=507280} 2024-09-21T09:48:30.264 INFO:tasks.rados.rados.0.smithi017.stdout:2427: writing smithi01785587-38 from 595212 to 1085656 tid 1 2024-09-21T09:48:30.271 INFO:tasks.rados.rados.0.smithi017.stdout:2427: writing smithi01785587-38 from 1868800 to 2327788 tid 2 2024-09-21T09:48:30.277 INFO:tasks.rados.rados.0.smithi017.stdout:2427: writing smithi01785587-38 from 2887932 to 3395212 tid 3 2024-09-21T09:48:30.277 INFO:tasks.rados.rados.0.smithi017.stdout:2428: delete oid 17 current snap is 312 2024-09-21T09:48:30.277 INFO:tasks.rados.rados.0.smithi017.stdout:2427: finishing write tid 1 to smithi01785587-38 2024-09-21T09:48:30.277 INFO:tasks.rados.rados.0.smithi017.stdout:2427: finishing write tid 2 to smithi01785587-38 2024-09-21T09:48:30.280 INFO:tasks.rados.rados.0.smithi017.stdout:2428: done (1 left) 2024-09-21T09:48:30.280 INFO:tasks.rados.rados.0.smithi017.stdout:2429: rollback oid 46 current snap is 312 2024-09-21T09:48:30.280 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 308 2024-09-21T09:48:30.280 INFO:tasks.rados.rados.0.smithi017.stdout:2430: snap_create 2024-09-21T09:48:30.281 INFO:tasks.rados.rados.0.smithi017.stdout:2427: finishing write tid 3 to smithi01785587-38 2024-09-21T09:48:30.281 INFO:tasks.rados.rados.0.smithi017.stdout:2427: finishing write tid 4 to smithi01785587-38 2024-09-21T09:48:30.282 INFO:tasks.rados.rados.0.smithi017.stdout:2427: finishing write tid 5 to smithi01785587-38 2024-09-21T09:48:30.282 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 824 (ObjNum 916 snap 312 seq_num 916) dirty exists 2024-09-21T09:48:30.282 INFO:tasks.rados.rados.0.smithi017.stdout:2427: left oid 38 (ObjNum 916 snap 312 seq_num 916) 2024-09-21T09:48:30.283 INFO:tasks.rados.rados.0.smithi017.stdout:2429: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:48:30.284 INFO:tasks.rados.rados.0.smithi017.stdout:2429: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:48:30.284 INFO:tasks.rados.rados.0.smithi017.stdout:2429: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:48:30.285 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1080 (ObjNum 791 snap 271 seq_num 791) dirty exists 2024-09-21T09:48:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:48:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:48:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:31.265 INFO:tasks.rados.rados.0.smithi017.stdout:2427: done (2 left) 2024-09-21T09:48:31.265 INFO:tasks.rados.rados.0.smithi017.stdout:2429: done (1 left) 2024-09-21T09:48:31.265 INFO:tasks.rados.rados.0.smithi017.stdout:2430: done (0 left) 2024-09-21T09:48:31.265 INFO:tasks.rados.rados.0.smithi017.stdout:2431: read oid 24 snap -1 2024-09-21T09:48:31.265 INFO:tasks.rados.rados.0.smithi017.stdout:2431: expect deleted 2024-09-21T09:48:31.265 INFO:tasks.rados.rados.0.smithi017.stdout:2432: delete oid 2 current snap is 313 2024-09-21T09:48:31.269 INFO:tasks.rados.rados.0.smithi017.stdout:2431: done (1 left) 2024-09-21T09:48:31.269 INFO:tasks.rados.rados.0.smithi017.stdout:2432: done (0 left) 2024-09-21T09:48:31.269 INFO:tasks.rados.rados.0.smithi017.stdout:2433: delete oid 40 current snap is 313 2024-09-21T09:48:31.271 INFO:tasks.rados.rados.0.smithi017.stdout:2433: done (0 left) 2024-09-21T09:48:31.272 INFO:tasks.rados.rados.0.smithi017.stdout:2434: read oid 48 snap -1 2024-09-21T09:48:31.272 INFO:tasks.rados.rados.0.smithi017.stdout:2434: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:48:31.272 INFO:tasks.rados.rados.0.smithi017.stdout:2435: write oid 41 current snap is 313 2024-09-21T09:48:31.272 INFO:tasks.rados.rados.0.smithi017.stdout:2435: seq_num 920 ranges {662440=507793,1702728=749973,2881271=557193} 2024-09-21T09:48:31.278 INFO:tasks.rados.rados.0.smithi017.stdout:2435: writing smithi01785587-41 from 662440 to 1170233 tid 1 2024-09-21T09:48:31.285 INFO:tasks.rados.rados.0.smithi017.stdout:2435: writing smithi01785587-41 from 1702728 to 2452701 tid 2 2024-09-21T09:48:31.291 INFO:tasks.rados.rados.0.smithi017.stdout:2435: writing smithi01785587-41 from 2881271 to 3438464 tid 3 2024-09-21T09:48:31.291 INFO:tasks.rados.rados.0.smithi017.stdout:2436: write oid 3 current snap is 313 2024-09-21T09:48:31.291 INFO:tasks.rados.rados.0.smithi017.stdout:2436: seq_num 921 ranges {679247=583042,2044857=609984,3192341=286906} 2024-09-21T09:48:31.297 INFO:tasks.rados.rados.0.smithi017.stdout:2436: writing smithi01785587-3 from 679247 to 1262289 tid 1 2024-09-21T09:48:31.305 INFO:tasks.rados.rados.0.smithi017.stdout:2436: writing smithi01785587-3 from 2044857 to 2654841 tid 2 2024-09-21T09:48:31.310 INFO:tasks.rados.rados.0.smithi017.stdout:2436: writing smithi01785587-3 from 3192341 to 3479247 tid 3 2024-09-21T09:48:31.310 INFO:tasks.rados.rados.0.smithi017.stdout:2437: read oid 33 snap -1 2024-09-21T09:48:31.310 INFO:tasks.rados.rados.0.smithi017.stdout:2437: expect (ObjNum 906 snap 310 seq_num 906) 2024-09-21T09:48:31.311 INFO:tasks.rados.rados.0.smithi017.stdout:2438: read oid 32 snap -1 2024-09-21T09:48:31.311 INFO:tasks.rados.rados.0.smithi017.stdout:2438: expect (ObjNum 884 snap 307 seq_num 884) 2024-09-21T09:48:31.311 INFO:tasks.rados.rados.0.smithi017.stdout:2439: rollback oid 43 current snap is 313 2024-09-21T09:48:31.311 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 300 2024-09-21T09:48:31.311 INFO:tasks.rados.rados.0.smithi017.stdout:2434: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:48:31.424 INFO:tasks.rados.rados.0.smithi017.stdout:2435: finishing write tid 1 to smithi01785587-41 2024-09-21T09:48:31.424 INFO:tasks.rados.rados.0.smithi017.stdout:2435: finishing write tid 2 to smithi01785587-41 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2435: finishing write tid 3 to smithi01785587-41 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2435: finishing write tid 4 to smithi01785587-41 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2435: finishing write tid 5 to smithi01785587-41 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 369 (ObjNum 920 snap 313 seq_num 920) dirty exists 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2435: left oid 41 (ObjNum 920 snap 313 seq_num 920) 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2436: finishing write tid 1 to smithi01785587-3 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2436: finishing write tid 2 to smithi01785587-3 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2434: done (5 left) 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2435: done (4 left) 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2440: delete oid 6 current snap is 313 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2439: finishing rollback tid 0 to smithi01785587-43 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2436: finishing write tid 3 to smithi01785587-3 2024-09-21T09:48:31.425 INFO:tasks.rados.rados.0.smithi017.stdout:2439: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:48:31.426 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 747 (ObjNum 33 snap 0 seq_num 2399960800) dirty dne 2024-09-21T09:48:31.426 INFO:tasks.rados.rados.0.smithi017.stdout:2436: finishing write tid 4 to smithi01785587-3 2024-09-21T09:48:31.426 INFO:tasks.rados.rados.0.smithi017.stdout:2436: finishing write tid 5 to smithi01785587-3 2024-09-21T09:48:31.426 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1171 (ObjNum 921 snap 313 seq_num 921) dirty exists 2024-09-21T09:48:31.426 INFO:tasks.rados.rados.0.smithi017.stdout:2436: left oid 3 (ObjNum 921 snap 313 seq_num 921) 2024-09-21T09:48:31.426 INFO:tasks.rados.rados.0.smithi017.stdout:2438: expect (ObjNum 884 snap 307 seq_num 884) 2024-09-21T09:48:31.551 INFO:tasks.rados.rados.0.smithi017.stdout:2437: expect (ObjNum 906 snap 310 seq_num 906) 2024-09-21T09:48:31.660 INFO:tasks.rados.rados.0.smithi017.stdout:2436: done (4 left) 2024-09-21T09:48:31.661 INFO:tasks.rados.rados.0.smithi017.stdout:2437: done (3 left) 2024-09-21T09:48:31.661 INFO:tasks.rados.rados.0.smithi017.stdout:2438: done (2 left) 2024-09-21T09:48:31.661 INFO:tasks.rados.rados.0.smithi017.stdout:2439: done (1 left) 2024-09-21T09:48:31.661 INFO:tasks.rados.rados.0.smithi017.stdout:2440: done (0 left) 2024-09-21T09:48:31.661 INFO:tasks.rados.rados.0.smithi017.stdout:2441: read oid 42 snap -1 2024-09-21T09:48:31.661 INFO:tasks.rados.rados.0.smithi017.stdout:2441: expect deleted 2024-09-21T09:48:31.661 INFO:tasks.rados.rados.0.smithi017.stdout:2442: read oid 44 snap 311 2024-09-21T09:48:31.661 INFO:tasks.rados.rados.0.smithi017.stdout:2442: expect deleted 2024-09-21T09:48:31.661 INFO:tasks.rados.rados.0.smithi017.stdout:2443: snap_create 2024-09-21T09:48:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:31 smithi017 ceph-mon[102913]: pgmap v606: 89 pgs: 89 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:48:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:31 smithi017 ceph-mon[102913]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T09:48:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:31 smithi017 ceph-mon[105151]: pgmap v606: 89 pgs: 89 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:48:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:31 smithi017 ceph-mon[105151]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T09:48:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:31 smithi134 ceph-mon[90085]: pgmap v606: 89 pgs: 89 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:48:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:31 smithi134 ceph-mon[90085]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T09:48:32.270 INFO:tasks.rados.rados.0.smithi017.stdout:2441: done (2 left) 2024-09-21T09:48:32.270 INFO:tasks.rados.rados.0.smithi017.stdout:2442: done (1 left) 2024-09-21T09:48:32.270 INFO:tasks.rados.rados.0.smithi017.stdout:2443: done (0 left) 2024-09-21T09:48:32.270 INFO:tasks.rados.rados.0.smithi017.stdout:2444: write oid 17 current snap is 314 2024-09-21T09:48:32.270 INFO:tasks.rados.rados.0.smithi017.stdout:2444: seq_num 923 ranges {712861=733540,2161821=613653,3247187=265674} 2024-09-21T09:48:32.277 INFO:tasks.rados.rados.0.smithi017.stdout:2444: writing smithi01785587-17 from 712861 to 1446401 tid 1 2024-09-21T09:48:32.285 INFO:tasks.rados.rados.0.smithi017.stdout:2444: writing smithi01785587-17 from 2161821 to 2775474 tid 2 2024-09-21T09:48:32.289 INFO:tasks.rados.rados.0.smithi017.stdout:2444: writing smithi01785587-17 from 3247187 to 3512861 tid 3 2024-09-21T09:48:32.289 INFO:tasks.rados.rados.0.smithi017.stdout:2445: read oid 39 snap -1 2024-09-21T09:48:32.289 INFO:tasks.rados.rados.0.smithi017.stdout:2445: expect (ObjNum 891 snap 309 seq_num 891) 2024-09-21T09:48:32.289 INFO:tasks.rados.rados.0.smithi017.stdout:2446: read oid 42 snap -1 2024-09-21T09:48:32.289 INFO:tasks.rados.rados.0.smithi017.stdout:2446: expect deleted 2024-09-21T09:48:32.289 INFO:tasks.rados.rados.0.smithi017.stdout:2447: delete oid 1 current snap is 314 2024-09-21T09:48:32.289 INFO:tasks.rados.rados.0.smithi017.stdout:2444: finishing write tid 1 to smithi01785587-17 2024-09-21T09:48:32.290 INFO:tasks.rados.rados.0.smithi017.stdout:2444: finishing write tid 2 to smithi01785587-17 2024-09-21T09:48:32.291 INFO:tasks.rados.rados.0.smithi017.stdout:2444: finishing write tid 3 to smithi01785587-17 2024-09-21T09:48:32.292 INFO:tasks.rados.rados.0.smithi017.stdout:2444: finishing write tid 4 to smithi01785587-17 2024-09-21T09:48:32.293 INFO:tasks.rados.rados.0.smithi017.stdout:2444: finishing write tid 5 to smithi01785587-17 2024-09-21T09:48:32.293 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 767 (ObjNum 923 snap 314 seq_num 923) dirty exists 2024-09-21T09:48:32.293 INFO:tasks.rados.rados.0.smithi017.stdout:2444: left oid 17 (ObjNum 923 snap 314 seq_num 923) 2024-09-21T09:48:32.294 INFO:tasks.rados.rados.0.smithi017.stdout:2445: expect (ObjNum 891 snap 309 seq_num 891) 2024-09-21T09:48:32.432 INFO:tasks.rados.rados.0.smithi017.stdout:2444: done (3 left) 2024-09-21T09:48:32.432 INFO:tasks.rados.rados.0.smithi017.stdout:2445: done (2 left) 2024-09-21T09:48:32.432 INFO:tasks.rados.rados.0.smithi017.stdout:2446: done (1 left) 2024-09-21T09:48:32.432 INFO:tasks.rados.rados.0.smithi017.stdout:2447: done (0 left) 2024-09-21T09:48:32.432 INFO:tasks.rados.rados.0.smithi017.stdout:2448: delete oid 32 current snap is 314 2024-09-21T09:48:32.434 INFO:tasks.rados.rados.0.smithi017.stdout:2448: done (0 left) 2024-09-21T09:48:32.434 INFO:tasks.rados.rados.0.smithi017.stdout:2449: snap_remove snap 309 2024-09-21T09:48:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:32 smithi017 ceph-mon[102913]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T09:48:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:32 smithi017 ceph-mon[105151]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T09:48:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:32 smithi134 ceph-mon[90085]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T09:48:33.127 INFO:tasks.rados.rados.0.smithi017.stdout:2449: done (0 left) 2024-09-21T09:48:33.127 INFO:tasks.rados.rados.0.smithi017.stdout:2450: snap_remove snap 310 2024-09-21T09:48:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:33 smithi017 ceph-mon[105151]: pgmap v609: 89 pgs: 89 active+clean; 237 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-21T09:48:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:33 smithi017 ceph-mon[105151]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T09:48:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:33 smithi017 ceph-mon[105151]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T09:48:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:33 smithi017 ceph-mon[102913]: pgmap v609: 89 pgs: 89 active+clean; 237 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-21T09:48:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:33 smithi017 ceph-mon[102913]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T09:48:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:33 smithi017 ceph-mon[102913]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T09:48:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:33 smithi134 ceph-mon[90085]: pgmap v609: 89 pgs: 89 active+clean; 237 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-21T09:48:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:33 smithi134 ceph-mon[90085]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T09:48:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:33 smithi134 ceph-mon[90085]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T09:48:34.135 INFO:tasks.rados.rados.0.smithi017.stdout:2450: done (0 left) 2024-09-21T09:48:34.135 INFO:tasks.rados.rados.0.smithi017.stdout:2451: read oid 43 snap -1 2024-09-21T09:48:34.135 INFO:tasks.rados.rados.0.smithi017.stdout:2451: expect deleted 2024-09-21T09:48:34.135 INFO:tasks.rados.rados.0.smithi017.stdout:2452: read oid 19 snap -1 2024-09-21T09:48:34.135 INFO:tasks.rados.rados.0.smithi017.stdout:2452: expect (ObjNum 912 snap 311 seq_num 912) 2024-09-21T09:48:34.135 INFO:tasks.rados.rados.0.smithi017.stdout:2453: read oid 29 snap 313 2024-09-21T09:48:34.135 INFO:tasks.rados.rados.0.smithi017.stdout:2453: expect (ObjNum 857 snap 289 seq_num 857) 2024-09-21T09:48:34.135 INFO:tasks.rados.rados.0.smithi017.stdout:2454: write oid 14 current snap is 314 2024-09-21T09:48:34.135 INFO:tasks.rados.rados.0.smithi017.stdout:2454: seq_num 926 ranges {763282=559287,1820914=760980,3213103=350179} 2024-09-21T09:48:34.142 INFO:tasks.rados.rados.0.smithi017.stdout:2454: writing smithi01785587-14 from 763282 to 1322569 tid 1 2024-09-21T09:48:34.149 INFO:tasks.rados.rados.0.smithi017.stdout:2454: writing smithi01785587-14 from 1820914 to 2581894 tid 2 2024-09-21T09:48:34.155 INFO:tasks.rados.rados.0.smithi017.stdout:2454: writing smithi01785587-14 from 3213103 to 3563282 tid 3 2024-09-21T09:48:34.155 INFO:tasks.rados.rados.0.smithi017.stdout:2455: snap_remove snap 308 2024-09-21T09:48:35.127 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:48:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:48:34] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-09-21T09:48:35.138 INFO:tasks.rados.rados.0.smithi017.stdout:2453: expect (ObjNum 857 snap 289 seq_num 857) 2024-09-21T09:48:35.250 INFO:tasks.rados.rados.0.smithi017.stdout:2454: finishing write tid 1 to smithi01785587-14 2024-09-21T09:48:35.250 INFO:tasks.rados.rados.0.smithi017.stdout:2454: finishing write tid 2 to smithi01785587-14 2024-09-21T09:48:35.250 INFO:tasks.rados.rados.0.smithi017.stdout:2452: expect (ObjNum 912 snap 311 seq_num 912) 2024-09-21T09:48:35.449 INFO:tasks.rados.rados.0.smithi017.stdout:2454: finishing write tid 3 to smithi01785587-14 2024-09-21T09:48:35.449 INFO:tasks.rados.rados.0.smithi017.stdout:2454: finishing write tid 4 to smithi01785587-14 2024-09-21T09:48:35.449 INFO:tasks.rados.rados.0.smithi017.stdout:2454: finishing write tid 5 to smithi01785587-14 2024-09-21T09:48:35.449 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1180 (ObjNum 926 snap 314 seq_num 926) dirty exists 2024-09-21T09:48:35.449 INFO:tasks.rados.rados.0.smithi017.stdout:2454: left oid 14 (ObjNum 926 snap 314 seq_num 926) 2024-09-21T09:48:35.450 INFO:tasks.rados.rados.0.smithi017.stdout:2451: done (4 left) 2024-09-21T09:48:35.450 INFO:tasks.rados.rados.0.smithi017.stdout:2452: done (3 left) 2024-09-21T09:48:35.450 INFO:tasks.rados.rados.0.smithi017.stdout:2453: done (2 left) 2024-09-21T09:48:35.450 INFO:tasks.rados.rados.0.smithi017.stdout:2454: done (1 left) 2024-09-21T09:48:35.450 INFO:tasks.rados.rados.0.smithi017.stdout:2455: done (0 left) 2024-09-21T09:48:35.450 INFO:tasks.rados.rados.0.smithi017.stdout:2456: delete oid 32 current snap is 314 2024-09-21T09:48:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:35 smithi017 ceph-mon[102913]: pgmap v612: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 12 op/s 2024-09-21T09:48:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:35 smithi017 ceph-mon[102913]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T09:48:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:35 smithi017 ceph-mon[105151]: pgmap v612: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 12 op/s 2024-09-21T09:48:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:35 smithi017 ceph-mon[105151]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T09:48:35.451 INFO:tasks.rados.rados.0.smithi017.stdout:2456: done (0 left) 2024-09-21T09:48:35.451 INFO:tasks.rados.rados.0.smithi017.stdout:2457: write oid 15 current snap is 314 2024-09-21T09:48:35.452 INFO:tasks.rados.rados.0.smithi017.stdout:2457: seq_num 928 ranges {796896=426138,1654231=764649,2984302=480075} 2024-09-21T09:48:35.458 INFO:tasks.rados.rados.0.smithi017.stdout:2457: writing smithi01785587-15 from 796896 to 1223034 tid 1 2024-09-21T09:48:35.465 INFO:tasks.rados.rados.0.smithi017.stdout:2457: writing smithi01785587-15 from 1654231 to 2418880 tid 2 2024-09-21T09:48:35.471 INFO:tasks.rados.rados.0.smithi017.stdout:2457: writing smithi01785587-15 from 2984302 to 3464377 tid 3 2024-09-21T09:48:35.471 INFO:tasks.rados.rados.0.smithi017.stdout:2458: snap_remove snap 307 2024-09-21T09:48:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:35 smithi134 ceph-mon[90085]: pgmap v612: 89 pgs: 89 active+clean; 242 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.3 MiB/s wr, 12 op/s 2024-09-21T09:48:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:35 smithi134 ceph-mon[90085]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T09:48:36.143 INFO:tasks.rados.rados.0.smithi017.stdout:2457: finishing write tid 1 to smithi01785587-15 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2457: finishing write tid 2 to smithi01785587-15 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2457: finishing write tid 3 to smithi01785587-15 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2457: finishing write tid 4 to smithi01785587-15 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2457: finishing write tid 5 to smithi01785587-15 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 378 (ObjNum 928 snap 314 seq_num 928) dirty exists 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2457: left oid 15 (ObjNum 928 snap 314 seq_num 928) 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2457: done (1 left) 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2458: done (0 left) 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2459: read oid 7 snap -1 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2459: expect (ObjNum 860 snap 291 seq_num 860) 2024-09-21T09:48:36.144 INFO:tasks.rados.rados.0.smithi017.stdout:2460: delete oid 36 current snap is 314 2024-09-21T09:48:36.151 INFO:tasks.rados.rados.0.smithi017.stdout:2459: expect (ObjNum 860 snap 291 seq_num 860) 2024-09-21T09:48:36.372 INFO:tasks.rados.rados.0.smithi017.stdout:2459: done (1 left) 2024-09-21T09:48:36.373 INFO:tasks.rados.rados.0.smithi017.stdout:2460: done (0 left) 2024-09-21T09:48:36.373 INFO:tasks.rados.rados.0.smithi017.stdout:2461: write oid 2 current snap is 314 2024-09-21T09:48:36.373 INFO:tasks.rados.rados.0.smithi017.stdout:2461: seq_num 930 ranges {430510=576636,1654842=768318,2922795=502619} 2024-09-21T09:48:36.378 INFO:tasks.rados.rados.0.smithi017.stdout:2461: writing smithi01785587-2 from 430510 to 1007146 tid 1 2024-09-21T09:48:36.386 INFO:tasks.rados.rados.0.smithi017.stdout:2461: writing smithi01785587-2 from 1654842 to 2423160 tid 2 2024-09-21T09:48:36.392 INFO:tasks.rados.rados.0.smithi017.stdout:2461: writing smithi01785587-2 from 2922795 to 3425414 tid 3 2024-09-21T09:48:36.392 INFO:tasks.rados.rados.0.smithi017.stdout:2462: delete oid 4 current snap is 314 2024-09-21T09:48:36.392 INFO:tasks.rados.rados.0.smithi017.stdout:2461: finishing write tid 1 to smithi01785587-2 2024-09-21T09:48:36.392 INFO:tasks.rados.rados.0.smithi017.stdout:2461: finishing write tid 2 to smithi01785587-2 2024-09-21T09:48:36.394 INFO:tasks.rados.rados.0.smithi017.stdout:2461: finishing write tid 3 to smithi01785587-2 2024-09-21T09:48:36.394 INFO:tasks.rados.rados.0.smithi017.stdout:2461: finishing write tid 4 to smithi01785587-2 2024-09-21T09:48:36.394 INFO:tasks.rados.rados.0.smithi017.stdout:2462: done (1 left) 2024-09-21T09:48:36.394 INFO:tasks.rados.rados.0.smithi017.stdout:2463: read oid 38 snap -1 2024-09-21T09:48:36.394 INFO:tasks.rados.rados.0.smithi017.stdout:2463: expect (ObjNum 916 snap 312 seq_num 916) 2024-09-21T09:48:36.394 INFO:tasks.rados.rados.0.smithi017.stdout:2464: delete oid 33 current snap is 314 2024-09-21T09:48:36.394 INFO:tasks.rados.rados.0.smithi017.stdout:2461: finishing write tid 5 to smithi01785587-2 2024-09-21T09:48:36.394 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 496 (ObjNum 930 snap 314 seq_num 930) dirty exists 2024-09-21T09:48:36.394 INFO:tasks.rados.rados.0.smithi017.stdout:2461: left oid 2 (ObjNum 930 snap 314 seq_num 930) 2024-09-21T09:48:36.395 INFO:tasks.rados.rados.0.smithi017.stdout:2461: done (2 left) 2024-09-21T09:48:36.395 INFO:tasks.rados.rados.0.smithi017.stdout:2464: done (1 left) 2024-09-21T09:48:36.395 INFO:tasks.rados.rados.0.smithi017.stdout:2465: delete oid 45 current snap is 314 2024-09-21T09:48:36.398 INFO:tasks.rados.rados.0.smithi017.stdout:2465: done (1 left) 2024-09-21T09:48:36.398 INFO:tasks.rados.rados.0.smithi017.stdout:2466: rollback oid 5 current snap is 314 2024-09-21T09:48:36.398 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 295 2024-09-21T09:48:36.398 INFO:tasks.rados.rados.0.smithi017.stdout:2467: delete oid 29 current snap is 314 2024-09-21T09:48:36.398 INFO:tasks.rados.rados.0.smithi017.stdout:2463: expect (ObjNum 916 snap 312 seq_num 916) 2024-09-21T09:48:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:36 smithi017 ceph-mon[102913]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T09:48:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:36 smithi017 ceph-mon[105151]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T09:48:36.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:36 smithi134 ceph-mon[90085]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T09:48:36.587 INFO:tasks.rados.rados.0.smithi017.stdout:2466: finishing rollback tid 0 to smithi01785587-5 2024-09-21T09:48:36.587 INFO:tasks.rados.rados.0.smithi017.stdout:2466: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:48:36.587 INFO:tasks.rados.rados.0.smithi017.stdout:2463: done (2 left) 2024-09-21T09:48:36.587 INFO:tasks.rados.rados.0.smithi017.stdout:2467: done (1 left) 2024-09-21T09:48:36.587 INFO:tasks.rados.rados.0.smithi017.stdout:2468: write oid 8 current snap is 314 2024-09-21T09:48:36.587 INFO:tasks.rados.rados.0.smithi017.stdout:2466: finishing rollback tid 2 to smithi01785587-5 2024-09-21T09:48:36.587 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1192 (ObjNum 844 snap 289 seq_num 844) dirty exists 2024-09-21T09:48:36.587 INFO:tasks.rados.rados.0.smithi017.stdout:2468: seq_num 935 ranges {514545=552881,1830899=519314,2943557=475332} 2024-09-21T09:48:36.592 INFO:tasks.rados.rados.0.smithi017.stdout:2468: writing smithi01785587-8 from 514545 to 1067426 tid 1 2024-09-21T09:48:36.599 INFO:tasks.rados.rados.0.smithi017.stdout:2468: writing smithi01785587-8 from 1830899 to 2350213 tid 2 2024-09-21T09:48:36.605 INFO:tasks.rados.rados.0.smithi017.stdout:2468: writing smithi01785587-8 from 2943557 to 3418889 tid 3 2024-09-21T09:48:36.605 INFO:tasks.rados.rados.0.smithi017.stdout:2466: done (1 left) 2024-09-21T09:48:36.605 INFO:tasks.rados.rados.0.smithi017.stdout:2469: delete oid 39 current snap is 314 2024-09-21T09:48:36.605 INFO:tasks.rados.rados.0.smithi017.stdout:2468: finishing write tid 1 to smithi01785587-8 2024-09-21T09:48:36.605 INFO:tasks.rados.rados.0.smithi017.stdout:2468: finishing write tid 2 to smithi01785587-8 2024-09-21T09:48:36.607 INFO:tasks.rados.rados.0.smithi017.stdout:2469: done (1 left) 2024-09-21T09:48:36.608 INFO:tasks.rados.rados.0.smithi017.stdout:2470: rollback oid 21 current snap is 314 2024-09-21T09:48:36.608 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 286 2024-09-21T09:48:36.608 INFO:tasks.rados.rados.0.smithi017.stdout:2471: read oid 20 snap 312 2024-09-21T09:48:36.608 INFO:tasks.rados.rados.0.smithi017.stdout:2471: expect (ObjNum 807 snap 279 seq_num 807) 2024-09-21T09:48:36.608 INFO:tasks.rados.rados.0.smithi017.stdout:2472: read oid 17 snap 312 2024-09-21T09:48:36.608 INFO:tasks.rados.rados.0.smithi017.stdout:2472: expect deleted 2024-09-21T09:48:36.608 INFO:tasks.rados.rados.0.smithi017.stdout:2473: write oid 10 current snap is 314 2024-09-21T09:48:36.608 INFO:tasks.rados.rados.0.smithi017.stdout:2473: seq_num 937 ranges {548159=419732,1664216=522983,2714756=614229} 2024-09-21T09:48:36.613 INFO:tasks.rados.rados.0.smithi017.stdout:2473: writing smithi01785587-10 from 548159 to 967891 tid 1 2024-09-21T09:48:36.620 INFO:tasks.rados.rados.0.smithi017.stdout:2473: writing smithi01785587-10 from 1664216 to 2187199 tid 2 2024-09-21T09:48:36.627 INFO:tasks.rados.rados.0.smithi017.stdout:2473: writing smithi01785587-10 from 2714756 to 3328985 tid 3 2024-09-21T09:48:36.627 INFO:tasks.rados.rados.0.smithi017.stdout:2474: snap_remove snap 293 2024-09-21T09:48:37.144 INFO:tasks.rados.rados.0.smithi017.stdout:2474: done (5 left) 2024-09-21T09:48:37.144 INFO:tasks.rados.rados.0.smithi017.stdout:2475: snap_create 2024-09-21T09:48:37.144 INFO:tasks.rados.rados.0.smithi017.stdout:2468: finishing write tid 3 to smithi01785587-8 2024-09-21T09:48:37.144 INFO:tasks.rados.rados.0.smithi017.stdout:2468: finishing write tid 4 to smithi01785587-8 2024-09-21T09:48:37.144 INFO:tasks.rados.rados.0.smithi017.stdout:2468: finishing write tid 5 to smithi01785587-8 2024-09-21T09:48:37.144 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1114 (ObjNum 935 snap 314 seq_num 935) dirty exists 2024-09-21T09:48:37.145 INFO:tasks.rados.rados.0.smithi017.stdout:2468: left oid 8 (ObjNum 935 snap 314 seq_num 935) 2024-09-21T09:48:37.145 INFO:tasks.rados.rados.0.smithi017.stdout:2470: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:48:37.145 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1219 (ObjNum 33 snap 0 seq_num 0) dirty dne 2024-09-21T09:48:37.145 INFO:tasks.rados.rados.0.smithi017.stdout:2473: finishing write tid 1 to smithi01785587-10 2024-09-21T09:48:37.145 INFO:tasks.rados.rados.0.smithi017.stdout:2471: expect (ObjNum 807 snap 279 seq_num 807) 2024-09-21T09:48:37.318 INFO:tasks.rados.rados.0.smithi017.stdout:2473: finishing write tid 2 to smithi01785587-10 2024-09-21T09:48:37.318 INFO:tasks.rados.rados.0.smithi017.stdout:2473: finishing write tid 3 to smithi01785587-10 2024-09-21T09:48:37.318 INFO:tasks.rados.rados.0.smithi017.stdout:2473: finishing write tid 4 to smithi01785587-10 2024-09-21T09:48:37.318 INFO:tasks.rados.rados.0.smithi017.stdout:2473: finishing write tid 5 to smithi01785587-10 2024-09-21T09:48:37.319 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 188 (ObjNum 937 snap 314 seq_num 937) dirty exists 2024-09-21T09:48:37.319 INFO:tasks.rados.rados.0.smithi017.stdout:2473: left oid 10 (ObjNum 937 snap 314 seq_num 937) 2024-09-21T09:48:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:37 smithi017 ceph-mon[102913]: pgmap v615: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.3 MiB/s wr, 22 op/s 2024-09-21T09:48:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:37 smithi017 ceph-mon[102913]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T09:48:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:37 smithi017 ceph-mon[105151]: pgmap v615: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.3 MiB/s wr, 22 op/s 2024-09-21T09:48:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:37 smithi017 ceph-mon[105151]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T09:48:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:37 smithi134 ceph-mon[90085]: pgmap v615: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.3 MiB/s wr, 22 op/s 2024-09-21T09:48:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:37 smithi134 ceph-mon[90085]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T09:48:38.130 INFO:tasks.rados.rados.0.smithi017.stdout:2468: done (5 left) 2024-09-21T09:48:38.130 INFO:tasks.rados.rados.0.smithi017.stdout:2470: done (4 left) 2024-09-21T09:48:38.131 INFO:tasks.rados.rados.0.smithi017.stdout:2471: done (3 left) 2024-09-21T09:48:38.131 INFO:tasks.rados.rados.0.smithi017.stdout:2472: done (2 left) 2024-09-21T09:48:38.131 INFO:tasks.rados.rados.0.smithi017.stdout:2473: done (1 left) 2024-09-21T09:48:38.131 INFO:tasks.rados.rados.0.smithi017.stdout:2475: done (0 left) 2024-09-21T09:48:38.131 INFO:tasks.rados.rados.0.smithi017.stdout:2476: read oid 7 snap -1 2024-09-21T09:48:38.131 INFO:tasks.rados.rados.0.smithi017.stdout:2476: expect (ObjNum 860 snap 291 seq_num 860) 2024-09-21T09:48:38.131 INFO:tasks.rados.rados.0.smithi017.stdout:2477: rollback oid 40 current snap is 315 2024-09-21T09:48:38.131 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 314 2024-09-21T09:48:38.131 INFO:tasks.rados.rados.0.smithi017.stdout:2478: snap_remove snap 312 2024-09-21T09:48:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:38 smithi017 ceph-mon[102913]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T09:48:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:38 smithi017 ceph-mon[102913]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T09:48:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:38 smithi017 ceph-mon[105151]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T09:48:38.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:38 smithi017 ceph-mon[105151]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T09:48:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:38 smithi134 ceph-mon[90085]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T09:48:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:38 smithi134 ceph-mon[90085]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T09:48:39.132 INFO:tasks.rados.rados.0.smithi017.stdout:2478: done (2 left) 2024-09-21T09:48:39.132 INFO:tasks.rados.rados.0.smithi017.stdout:2479: snap_remove snap 303 2024-09-21T09:48:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:39 smithi017 ceph-mon[102913]: pgmap v618: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-09-21T09:48:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:39 smithi017 ceph-mon[102913]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T09:48:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:39 smithi017 ceph-mon[105151]: pgmap v618: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-09-21T09:48:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:39 smithi017 ceph-mon[105151]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T09:48:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:39 smithi134 ceph-mon[90085]: pgmap v618: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-09-21T09:48:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:39 smithi134 ceph-mon[90085]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T09:48:40.133 INFO:tasks.rados.rados.0.smithi017.stdout:2479: done (2 left) 2024-09-21T09:48:40.133 INFO:tasks.rados.rados.0.smithi017.stdout:2480: rollback oid 41 current snap is 315 2024-09-21T09:48:40.133 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 313 2024-09-21T09:48:40.133 INFO:tasks.rados.rados.0.smithi017.stdout:2476: expect (ObjNum 860 snap 291 seq_num 860) 2024-09-21T09:48:40.357 INFO:tasks.rados.rados.0.smithi017.stdout:2477: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:48:40.357 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1264 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:48:40.357 INFO:tasks.rados.rados.0.smithi017.stdout:2480: finishing rollback tid 0 to smithi01785587-41 2024-09-21T09:48:40.357 INFO:tasks.rados.rados.0.smithi017.stdout:2480: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:48:40.357 INFO:tasks.rados.rados.0.smithi017.stdout:2480: finishing rollback tid 2 to smithi01785587-41 2024-09-21T09:48:40.357 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 388 (ObjNum 920 snap 313 seq_num 920) dirty exists 2024-09-21T09:48:40.358 INFO:tasks.rados.rados.0.smithi017.stdout:2476: done (2 left) 2024-09-21T09:48:40.358 INFO:tasks.rados.rados.0.smithi017.stdout:2477: done (1 left) 2024-09-21T09:48:40.358 INFO:tasks.rados.rados.0.smithi017.stdout:2480: done (0 left) 2024-09-21T09:48:40.358 INFO:tasks.rados.rados.0.smithi017.stdout:2481: read oid 20 snap -1 2024-09-21T09:48:40.358 INFO:tasks.rados.rados.0.smithi017.stdout:2481: expect (ObjNum 807 snap 279 seq_num 807) 2024-09-21T09:48:40.358 INFO:tasks.rados.rados.0.smithi017.stdout:2482: snap_remove snap 286 2024-09-21T09:48:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:48:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:48:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:41.154 INFO:tasks.rados.rados.0.smithi017.stdout:2482: done (1 left) 2024-09-21T09:48:41.154 INFO:tasks.rados.rados.0.smithi017.stdout:2483: snap_create 2024-09-21T09:48:41.155 INFO:tasks.rados.rados.0.smithi017.stdout:2481: expect (ObjNum 807 snap 279 seq_num 807) 2024-09-21T09:48:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:41 smithi017 ceph-mon[102913]: pgmap v621: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 451 KiB/s wr, 7 op/s 2024-09-21T09:48:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:41 smithi017 ceph-mon[102913]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T09:48:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:41.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:41 smithi017 ceph-mon[105151]: pgmap v621: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 451 KiB/s wr, 7 op/s 2024-09-21T09:48:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:41 smithi017 ceph-mon[105151]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T09:48:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:41 smithi134 ceph-mon[90085]: pgmap v621: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 451 KiB/s wr, 7 op/s 2024-09-21T09:48:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:41 smithi134 ceph-mon[90085]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T09:48:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:42.159 INFO:tasks.rados.rados.0.smithi017.stdout:2481: done (1 left) 2024-09-21T09:48:42.159 INFO:tasks.rados.rados.0.smithi017.stdout:2483: done (0 left) 2024-09-21T09:48:42.159 INFO:tasks.rados.rados.0.smithi017.stdout:2484: write oid 39 current snap is 316 2024-09-21T09:48:42.160 INFO:tasks.rados.rados.0.smithi017.stdout:2484: seq_num 938 ranges {564966=494981,1606345=782994,3025826=425501} 2024-09-21T09:48:42.164 INFO:tasks.rados.rados.0.smithi017.stdout:2484: writing smithi01785587-39 from 564966 to 1059947 tid 1 2024-09-21T09:48:42.172 INFO:tasks.rados.rados.0.smithi017.stdout:2484: writing smithi01785587-39 from 1606345 to 2389339 tid 2 2024-09-21T09:48:42.178 INFO:tasks.rados.rados.0.smithi017.stdout:2484: writing smithi01785587-39 from 3025826 to 3451327 tid 3 2024-09-21T09:48:42.178 INFO:tasks.rados.rados.0.smithi017.stdout:2485: snap_remove snap 311 2024-09-21T09:48:42.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:42 smithi017 ceph-mon[102913]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T09:48:42.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:42 smithi017 ceph-mon[105151]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T09:48:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:42 smithi134 ceph-mon[90085]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T09:48:43.132 INFO:tasks.rados.rados.0.smithi017.stdout:2485: done (1 left) 2024-09-21T09:48:43.132 INFO:tasks.rados.rados.0.smithi017.stdout:2486: snap_create 2024-09-21T09:48:43.132 INFO:tasks.rados.rados.0.smithi017.stdout:2484: finishing write tid 1 to smithi01785587-39 2024-09-21T09:48:43.133 INFO:tasks.rados.rados.0.smithi017.stdout:2484: finishing write tid 2 to smithi01785587-39 2024-09-21T09:48:43.133 INFO:tasks.rados.rados.0.smithi017.stdout:2484: finishing write tid 3 to smithi01785587-39 2024-09-21T09:48:43.133 INFO:tasks.rados.rados.0.smithi017.stdout:2484: finishing write tid 4 to smithi01785587-39 2024-09-21T09:48:43.133 INFO:tasks.rados.rados.0.smithi017.stdout:2484: finishing write tid 5 to smithi01785587-39 2024-09-21T09:48:43.133 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1321 (ObjNum 938 snap 316 seq_num 938) dirty exists 2024-09-21T09:48:43.133 INFO:tasks.rados.rados.0.smithi017.stdout:2484: left oid 39 (ObjNum 938 snap 316 seq_num 938) 2024-09-21T09:48:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:43 smithi017 ceph-mon[102913]: pgmap v624: 89 pgs: 89 active+clean; 244 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 758 KiB/s wr, 14 op/s 2024-09-21T09:48:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:43 smithi017 ceph-mon[102913]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T09:48:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:43 smithi017 ceph-mon[102913]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T09:48:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:43 smithi017 ceph-mon[105151]: pgmap v624: 89 pgs: 89 active+clean; 244 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 758 KiB/s wr, 14 op/s 2024-09-21T09:48:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:43 smithi017 ceph-mon[105151]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T09:48:43.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:43 smithi017 ceph-mon[105151]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T09:48:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:43 smithi134 ceph-mon[90085]: pgmap v624: 89 pgs: 89 active+clean; 244 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 758 KiB/s wr, 14 op/s 2024-09-21T09:48:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:43 smithi134 ceph-mon[90085]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T09:48:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:43 smithi134 ceph-mon[90085]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T09:48:44.133 INFO:tasks.rados.rados.0.smithi017.stdout:2484: done (1 left) 2024-09-21T09:48:44.133 INFO:tasks.rados.rados.0.smithi017.stdout:2486: done (0 left) 2024-09-21T09:48:44.133 INFO:tasks.rados.rados.0.smithi017.stdout:2487: write oid 13 current snap is 317 2024-09-21T09:48:44.133 INFO:tasks.rados.rados.0.smithi017.stdout:2487: seq_num 939 ranges {581773=570230,1664827=526652,2653249=636773} 2024-09-21T09:48:44.139 INFO:tasks.rados.rados.0.smithi017.stdout:2487: writing smithi01785587-13 from 581773 to 1152003 tid 1 2024-09-21T09:48:44.144 INFO:tasks.rados.rados.0.smithi017.stdout:2487: writing smithi01785587-13 from 1664827 to 2191479 tid 2 2024-09-21T09:48:44.151 INFO:tasks.rados.rados.0.smithi017.stdout:2487: writing smithi01785587-13 from 2653249 to 3290022 tid 3 2024-09-21T09:48:44.151 INFO:tasks.rados.rados.0.smithi017.stdout:2488: delete oid 35 current snap is 317 2024-09-21T09:48:44.151 INFO:tasks.rados.rados.0.smithi017.stdout:2487: finishing write tid 1 to smithi01785587-13 2024-09-21T09:48:44.151 INFO:tasks.rados.rados.0.smithi017.stdout:2487: finishing write tid 2 to smithi01785587-13 2024-09-21T09:48:44.153 INFO:tasks.rados.rados.0.smithi017.stdout:2488: done (1 left) 2024-09-21T09:48:44.153 INFO:tasks.rados.rados.0.smithi017.stdout:2489: snap_create 2024-09-21T09:48:44.155 INFO:tasks.rados.rados.0.smithi017.stdout:2487: finishing write tid 3 to smithi01785587-13 2024-09-21T09:48:44.155 INFO:tasks.rados.rados.0.smithi017.stdout:2487: finishing write tid 4 to smithi01785587-13 2024-09-21T09:48:44.155 INFO:tasks.rados.rados.0.smithi017.stdout:2487: finishing write tid 5 to smithi01785587-13 2024-09-21T09:48:44.155 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1044 (ObjNum 939 snap 317 seq_num 939) dirty exists 2024-09-21T09:48:44.155 INFO:tasks.rados.rados.0.smithi017.stdout:2487: left oid 13 (ObjNum 939 snap 317 seq_num 939) 2024-09-21T09:48:45.134 INFO:tasks.rados.rados.0.smithi017.stdout:2487: done (1 left) 2024-09-21T09:48:45.135 INFO:tasks.rados.rados.0.smithi017.stdout:2489: done (0 left) 2024-09-21T09:48:45.135 INFO:tasks.rados.rados.0.smithi017.stdout:2490: snap_remove snap 315 2024-09-21T09:48:45.135 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:48:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:48:44] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.51.0" 2024-09-21T09:48:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:45 smithi017 ceph-mon[102913]: pgmap v627: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 14 op/s 2024-09-21T09:48:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:45 smithi017 ceph-mon[102913]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T09:48:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:45 smithi017 ceph-mon[105151]: pgmap v627: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 14 op/s 2024-09-21T09:48:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:45 smithi017 ceph-mon[105151]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T09:48:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:45 smithi134 ceph-mon[90085]: pgmap v627: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 14 op/s 2024-09-21T09:48:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:45 smithi134 ceph-mon[90085]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T09:48:46.145 INFO:tasks.rados.rados.0.smithi017.stdout:2490: done (0 left) 2024-09-21T09:48:46.145 INFO:tasks.rados.rados.0.smithi017.stdout:2491: read oid 30 snap -1 2024-09-21T09:48:46.145 INFO:tasks.rados.rados.0.smithi017.stdout:2491: expect (ObjNum 843 snap 288 seq_num 843) 2024-09-21T09:48:46.145 INFO:tasks.rados.rados.0.smithi017.stdout:2492: read oid 44 snap -1 2024-09-21T09:48:46.145 INFO:tasks.rados.rados.0.smithi017.stdout:2492: expect deleted 2024-09-21T09:48:46.145 INFO:tasks.rados.rados.0.smithi017.stdout:2493: rollback oid 20 current snap is 318 2024-09-21T09:48:46.145 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 304 2024-09-21T09:48:46.145 INFO:tasks.rados.rados.0.smithi017.stdout:2494: delete oid 23 current snap is 318 2024-09-21T09:48:46.150 INFO:tasks.rados.rados.0.smithi017.stdout:2493: finishing rollback tid 0 to smithi01785587-20 2024-09-21T09:48:46.151 INFO:tasks.rados.rados.0.smithi017.stdout:2493: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:48:46.151 INFO:tasks.rados.rados.0.smithi017.stdout:2493: finishing rollback tid 2 to smithi01785587-20 2024-09-21T09:48:46.151 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 646 (ObjNum 807 snap 279 seq_num 807) dirty exists 2024-09-21T09:48:46.152 INFO:tasks.rados.rados.0.smithi017.stdout:2492: done (3 left) 2024-09-21T09:48:46.152 INFO:tasks.rados.rados.0.smithi017.stdout:2493: done (2 left) 2024-09-21T09:48:46.152 INFO:tasks.rados.rados.0.smithi017.stdout:2494: done (1 left) 2024-09-21T09:48:46.152 INFO:tasks.rados.rados.0.smithi017.stdout:2495: read oid 25 snap 272 2024-09-21T09:48:46.152 INFO:tasks.rados.rados.0.smithi017.stdout:2495: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:48:46.152 INFO:tasks.rados.rados.0.smithi017.stdout:2496: snap_create 2024-09-21T09:48:46.156 INFO:tasks.rados.rados.0.smithi017.stdout:2491: expect (ObjNum 843 snap 288 seq_num 843) 2024-09-21T09:48:46.291 INFO:tasks.rados.rados.0.smithi017.stdout:2495: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:48:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:46 smithi017 ceph-mon[102913]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T09:48:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:46 smithi017 ceph-mon[105151]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T09:48:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:46 smithi134 ceph-mon[90085]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T09:48:47.146 INFO:tasks.rados.rados.0.smithi017.stdout:2491: done (2 left) 2024-09-21T09:48:47.146 INFO:tasks.rados.rados.0.smithi017.stdout:2495: done (1 left) 2024-09-21T09:48:47.146 INFO:tasks.rados.rados.0.smithi017.stdout:2496: done (0 left) 2024-09-21T09:48:47.146 INFO:tasks.rados.rados.0.smithi017.stdout:2497: write oid 21 current snap is 319 2024-09-21T09:48:47.146 INFO:tasks.rados.rados.0.smithi017.stdout:2497: seq_num 942 ranges {632194=795977,2123920=790332,3419165=413029} 2024-09-21T09:48:47.154 INFO:tasks.rados.rados.0.smithi017.stdout:2497: writing smithi01785587-21 from 632194 to 1428171 tid 1 2024-09-21T09:48:47.162 INFO:tasks.rados.rados.0.smithi017.stdout:2497: writing smithi01785587-21 from 2123920 to 2914252 tid 2 2024-09-21T09:48:47.167 INFO:tasks.rados.rados.0.smithi017.stdout:2497: writing smithi01785587-21 from 3419165 to 3832194 tid 3 2024-09-21T09:48:47.167 INFO:tasks.rados.rados.0.smithi017.stdout:2498: write oid 30 current snap is 319 2024-09-21T09:48:47.167 INFO:tasks.rados.rados.0.smithi017.stdout:2497: finishing write tid 1 to smithi01785587-21 2024-09-21T09:48:47.167 INFO:tasks.rados.rados.0.smithi017.stdout:2497: finishing write tid 2 to smithi01785587-21 2024-09-21T09:48:47.167 INFO:tasks.rados.rados.0.smithi017.stdout:2498: seq_num 943 ranges {649001=587579,1898755=533990,3162941=686060} 2024-09-21T09:48:47.174 INFO:tasks.rados.rados.0.smithi017.stdout:2498: writing smithi01785587-30 from 649001 to 1236580 tid 1 2024-09-21T09:48:47.180 INFO:tasks.rados.rados.0.smithi017.stdout:2498: writing smithi01785587-30 from 1898755 to 2432745 tid 2 2024-09-21T09:48:47.188 INFO:tasks.rados.rados.0.smithi017.stdout:2498: writing smithi01785587-30 from 3162941 to 3849001 tid 3 2024-09-21T09:48:47.188 INFO:tasks.rados.rados.0.smithi017.stdout:2499: write oid 17 current snap is 319 2024-09-21T09:48:47.188 INFO:tasks.rados.rados.0.smithi017.stdout:2499: seq_num 944 ranges {665808=662828,1957237=677648,3074011=609486} 2024-09-21T09:48:47.195 INFO:tasks.rados.rados.0.smithi017.stdout:2499: writing smithi01785587-17 from 665808 to 1328636 tid 1 2024-09-21T09:48:47.203 INFO:tasks.rados.rados.0.smithi017.stdout:2499: writing smithi01785587-17 from 1957237 to 2634885 tid 2 2024-09-21T09:48:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2499: writing smithi01785587-17 from 3074011 to 3683497 tid 3 2024-09-21T09:48:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:2500: rollback oid 46 current snap is 319 2024-09-21T09:48:47.209 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 316 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2497: finishing write tid 3 to smithi01785587-21 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2497: finishing write tid 4 to smithi01785587-21 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2497: finishing write tid 5 to smithi01785587-21 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1279 (ObjNum 942 snap 319 seq_num 942) dirty exists 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2497: left oid 21 (ObjNum 942 snap 319 seq_num 942) 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2498: finishing write tid 1 to smithi01785587-30 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2498: finishing write tid 2 to smithi01785587-30 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2498: finishing write tid 3 to smithi01785587-30 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2498: finishing write tid 4 to smithi01785587-30 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2498: finishing write tid 5 to smithi01785587-30 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1051 (ObjNum 943 snap 319 seq_num 943) dirty exists 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2498: left oid 30 (ObjNum 943 snap 319 seq_num 943) 2024-09-21T09:48:47.210 INFO:tasks.rados.rados.0.smithi017.stdout:2499: finishing write tid 1 to smithi01785587-17 2024-09-21T09:48:47.211 INFO:tasks.rados.rados.0.smithi017.stdout:2499: finishing write tid 2 to smithi01785587-17 2024-09-21T09:48:47.211 INFO:tasks.rados.rados.0.smithi017.stdout:2497: done (3 left) 2024-09-21T09:48:47.211 INFO:tasks.rados.rados.0.smithi017.stdout:2498: done (2 left) 2024-09-21T09:48:47.211 INFO:tasks.rados.rados.0.smithi017.stdout:2501: snap_create 2024-09-21T09:48:47.213 INFO:tasks.rados.rados.0.smithi017.stdout:2500: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:48:47.213 INFO:tasks.rados.rados.0.smithi017.stdout:2499: finishing write tid 3 to smithi01785587-17 2024-09-21T09:48:47.213 INFO:tasks.rados.rados.0.smithi017.stdout:2499: finishing write tid 4 to smithi01785587-17 2024-09-21T09:48:47.213 INFO:tasks.rados.rados.0.smithi017.stdout:2500: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:48:47.213 INFO:tasks.rados.rados.0.smithi017.stdout:2500: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:48:47.213 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1156 (ObjNum 791 snap 271 seq_num 791) dirty exists 2024-09-21T09:48:47.213 INFO:tasks.rados.rados.0.smithi017.stdout:2499: finishing write tid 5 to smithi01785587-17 2024-09-21T09:48:47.213 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 794 (ObjNum 944 snap 319 seq_num 944) dirty exists 2024-09-21T09:48:47.213 INFO:tasks.rados.rados.0.smithi017.stdout:2499: left oid 17 (ObjNum 944 snap 319 seq_num 944) 2024-09-21T09:48:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:47 smithi017 ceph-mon[102913]: pgmap v630: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 840 KiB/s wr, 2 op/s 2024-09-21T09:48:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:47 smithi017 ceph-mon[102913]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T09:48:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:47 smithi017 ceph-mon[105151]: pgmap v630: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 840 KiB/s wr, 2 op/s 2024-09-21T09:48:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:47 smithi017 ceph-mon[105151]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T09:48:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:47 smithi134 ceph-mon[90085]: pgmap v630: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 840 KiB/s wr, 2 op/s 2024-09-21T09:48:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:47 smithi134 ceph-mon[90085]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T09:48:48.132 INFO:tasks.rados.rados.0.smithi017.stdout:2499: done (2 left) 2024-09-21T09:48:48.132 INFO:tasks.rados.rados.0.smithi017.stdout:2500: done (1 left) 2024-09-21T09:48:48.132 INFO:tasks.rados.rados.0.smithi017.stdout:2501: done (0 left) 2024-09-21T09:48:48.132 INFO:tasks.rados.rados.0.smithi017.stdout:2502: snap_create 2024-09-21T09:48:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:48 smithi017 ceph-mon[102913]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T09:48:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:48 smithi017 ceph-mon[102913]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T09:48:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:48 smithi017 ceph-mon[105151]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T09:48:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:48 smithi017 ceph-mon[105151]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T09:48:48.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:48 smithi134 ceph-mon[90085]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T09:48:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:48 smithi134 ceph-mon[90085]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T09:48:49.135 INFO:tasks.rados.rados.0.smithi017.stdout:2502: done (0 left) 2024-09-21T09:48:49.135 INFO:tasks.rados.rados.0.smithi017.stdout:2503: write oid 35 current snap is 321 2024-09-21T09:48:49.135 INFO:tasks.rados.rados.0.smithi017.stdout:2503: seq_num 945 ranges {682615=738077,2015719=537659,3217787=420758} 2024-09-21T09:48:49.142 INFO:tasks.rados.rados.0.smithi017.stdout:2503: writing smithi01785587-35 from 682615 to 1420692 tid 1 2024-09-21T09:48:49.148 INFO:tasks.rados.rados.0.smithi017.stdout:2503: writing smithi01785587-35 from 2015719 to 2553378 tid 2 2024-09-21T09:48:49.155 INFO:tasks.rados.rados.0.smithi017.stdout:2503: writing smithi01785587-35 from 3217787 to 3638545 tid 3 2024-09-21T09:48:49.155 INFO:tasks.rados.rados.0.smithi017.stdout:2504: write oid 26 current snap is 321 2024-09-21T09:48:49.155 INFO:tasks.rados.rados.0.smithi017.stdout:2503: finishing write tid 1 to smithi01785587-35 2024-09-21T09:48:49.155 INFO:tasks.rados.rados.0.smithi017.stdout:2503: finishing write tid 2 to smithi01785587-35 2024-09-21T09:48:49.155 INFO:tasks.rados.rados.0.smithi017.stdout:2504: seq_num 946 ranges {699422=413326,1557848=681317,3012504=748383} 2024-09-21T09:48:49.161 INFO:tasks.rados.rados.0.smithi017.stdout:2504: writing smithi01785587-26 from 699422 to 1112748 tid 1 2024-09-21T09:48:49.167 INFO:tasks.rados.rados.0.smithi017.stdout:2504: writing smithi01785587-26 from 1557848 to 2239165 tid 2 2024-09-21T09:48:49.176 INFO:tasks.rados.rados.0.smithi017.stdout:2504: writing smithi01785587-26 from 3012504 to 3760887 tid 3 2024-09-21T09:48:49.176 INFO:tasks.rados.rados.0.smithi017.stdout:2505: read oid 34 snap 300 2024-09-21T09:48:49.176 INFO:tasks.rados.rados.0.smithi017.stdout:2505: expect (ObjNum 836 snap 287 seq_num 836) 2024-09-21T09:48:49.176 INFO:tasks.rados.rados.0.smithi017.stdout:2506: snap_remove snap 319 2024-09-21T09:48:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:49 smithi017 ceph-mon[102913]: pgmap v633: 89 pgs: 89 active+clean; 238 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 423 KiB/s wr, 1 op/s 2024-09-21T09:48:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:49 smithi017 ceph-mon[102913]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T09:48:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:49 smithi017 ceph-mon[105151]: pgmap v633: 89 pgs: 89 active+clean; 238 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 423 KiB/s wr, 1 op/s 2024-09-21T09:48:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:49 smithi017 ceph-mon[105151]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T09:48:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:49 smithi134 ceph-mon[90085]: pgmap v633: 89 pgs: 89 active+clean; 238 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 423 KiB/s wr, 1 op/s 2024-09-21T09:48:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:49 smithi134 ceph-mon[90085]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T09:48:50.162 INFO:tasks.rados.rados.0.smithi017.stdout:2506: done (3 left) 2024-09-21T09:48:50.163 INFO:tasks.rados.rados.0.smithi017.stdout:2507: rollback oid 14 current snap is 321 2024-09-21T09:48:50.163 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 316 2024-09-21T09:48:50.163 INFO:tasks.rados.rados.0.smithi017.stdout:2508: read oid 10 snap -1 2024-09-21T09:48:50.163 INFO:tasks.rados.rados.0.smithi017.stdout:2508: expect (ObjNum 937 snap 314 seq_num 937) 2024-09-21T09:48:50.163 INFO:tasks.rados.rados.0.smithi017.stdout:2509: rollback oid 16 current snap is 321 2024-09-21T09:48:50.163 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 16 to 313 2024-09-21T09:48:50.163 INFO:tasks.rados.rados.0.smithi017.stdout:2503: finishing write tid 3 to smithi01785587-35 2024-09-21T09:48:50.164 INFO:tasks.rados.rados.0.smithi017.stdout:2503: finishing write tid 4 to smithi01785587-35 2024-09-21T09:48:50.164 INFO:tasks.rados.rados.0.smithi017.stdout:2503: finishing write tid 5 to smithi01785587-35 2024-09-21T09:48:50.164 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 275 (ObjNum 945 snap 321 seq_num 945) dirty exists 2024-09-21T09:48:50.164 INFO:tasks.rados.rados.0.smithi017.stdout:2503: left oid 35 (ObjNum 945 snap 321 seq_num 945) 2024-09-21T09:48:50.164 INFO:tasks.rados.rados.0.smithi017.stdout:2504: finishing write tid 1 to smithi01785587-26 2024-09-21T09:48:50.164 INFO:tasks.rados.rados.0.smithi017.stdout:2504: finishing write tid 2 to smithi01785587-26 2024-09-21T09:48:50.164 INFO:tasks.rados.rados.0.smithi017.stdout:2505: expect (ObjNum 836 snap 287 seq_num 836) 2024-09-21T09:48:50.231 INFO:tasks.rados.rados.0.smithi017.stdout:2504: finishing write tid 3 to smithi01785587-26 2024-09-21T09:48:50.231 INFO:tasks.rados.rados.0.smithi017.stdout:2504: finishing write tid 4 to smithi01785587-26 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:2504: finishing write tid 5 to smithi01785587-26 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1242 (ObjNum 946 snap 321 seq_num 946) dirty exists 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:2504: left oid 26 (ObjNum 946 snap 321 seq_num 946) 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:2509: finishing rollback tid 1 to smithi01785587-16 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1116 (ObjNum 129 snap 0 seq_num 2398090480) dirty dne 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:2507: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:2507: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:2503: done (5 left) 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:2504: done (4 left) 2024-09-21T09:48:50.232 INFO:tasks.rados.rados.0.smithi017.stdout:2505: done (3 left) 2024-09-21T09:48:50.233 INFO:tasks.rados.rados.0.smithi017.stdout:2509: done (2 left) 2024-09-21T09:48:50.233 INFO:tasks.rados.rados.0.smithi017.stdout:2510: write oid 39 current snap is 321 2024-09-21T09:48:50.233 INFO:tasks.rados.rados.0.smithi017.stdout:2507: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:48:50.233 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1234 (ObjNum 926 snap 314 seq_num 926) dirty exists 2024-09-21T09:48:50.233 INFO:tasks.rados.rados.0.smithi017.stdout:2508: expect (ObjNum 937 snap 314 seq_num 937) 2024-09-21T09:48:50.444 INFO:tasks.rados.rados.0.smithi017.stdout:2510: seq_num 947 ranges {716229=488575,1616330=541328,2756280=559655} 2024-09-21T09:48:50.450 INFO:tasks.rados.rados.0.smithi017.stdout:2510: writing smithi01785587-39 from 716229 to 1204804 tid 1 2024-09-21T09:48:50.456 INFO:tasks.rados.rados.0.smithi017.stdout:2510: writing smithi01785587-39 from 1616330 to 2157658 tid 2 2024-09-21T09:48:50.462 INFO:tasks.rados.rados.0.smithi017.stdout:2510: writing smithi01785587-39 from 2756280 to 3315935 tid 3 2024-09-21T09:48:50.463 INFO:tasks.rados.rados.0.smithi017.stdout:2507: done (2 left) 2024-09-21T09:48:50.463 INFO:tasks.rados.rados.0.smithi017.stdout:2508: done (1 left) 2024-09-21T09:48:50.463 INFO:tasks.rados.rados.0.smithi017.stdout:2511: read oid 45 snap -1 2024-09-21T09:48:50.463 INFO:tasks.rados.rados.0.smithi017.stdout:2511: expect deleted 2024-09-21T09:48:50.463 INFO:tasks.rados.rados.0.smithi017.stdout:2512: read oid 42 snap -1 2024-09-21T09:48:50.463 INFO:tasks.rados.rados.0.smithi017.stdout:2512: expect deleted 2024-09-21T09:48:50.463 INFO:tasks.rados.rados.0.smithi017.stdout:2513: delete oid 40 current snap is 321 2024-09-21T09:48:50.463 INFO:tasks.rados.rados.0.smithi017.stdout:2510: finishing write tid 1 to smithi01785587-39 2024-09-21T09:48:50.463 INFO:tasks.rados.rados.0.smithi017.stdout:2510: finishing write tid 2 to smithi01785587-39 2024-09-21T09:48:50.465 INFO:tasks.rados.rados.0.smithi017.stdout:2510: finishing write tid 3 to smithi01785587-39 2024-09-21T09:48:50.465 INFO:tasks.rados.rados.0.smithi017.stdout:2510: finishing write tid 4 to smithi01785587-39 2024-09-21T09:48:50.466 INFO:tasks.rados.rados.0.smithi017.stdout:2510: finishing write tid 5 to smithi01785587-39 2024-09-21T09:48:50.466 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1332 (ObjNum 947 snap 321 seq_num 947) dirty exists 2024-09-21T09:48:50.466 INFO:tasks.rados.rados.0.smithi017.stdout:2510: left oid 39 (ObjNum 947 snap 321 seq_num 947) 2024-09-21T09:48:50.466 INFO:tasks.rados.rados.0.smithi017.stdout:2510: done (3 left) 2024-09-21T09:48:50.466 INFO:tasks.rados.rados.0.smithi017.stdout:2511: done (2 left) 2024-09-21T09:48:50.466 INFO:tasks.rados.rados.0.smithi017.stdout:2512: done (1 left) 2024-09-21T09:48:50.466 INFO:tasks.rados.rados.0.smithi017.stdout:2513: done (0 left) 2024-09-21T09:48:50.466 INFO:tasks.rados.rados.0.smithi017.stdout:2514: write oid 5 current snap is 321 2024-09-21T09:48:50.467 INFO:tasks.rados.rados.0.smithi017.stdout:2514: seq_num 949 ranges {749843=639073,2133294=544997,3211126=582199} 2024-09-21T09:48:50.473 INFO:tasks.rados.rados.0.smithi017.stdout:2514: writing smithi01785587-5 from 749843 to 1388916 tid 1 2024-09-21T09:48:50.480 INFO:tasks.rados.rados.0.smithi017.stdout:2514: writing smithi01785587-5 from 2133294 to 2678291 tid 2 2024-09-21T09:48:50.487 INFO:tasks.rados.rados.0.smithi017.stdout:2514: writing smithi01785587-5 from 3211126 to 3793325 tid 3 2024-09-21T09:48:50.487 INFO:tasks.rados.rados.0.smithi017.stdout:2515: delete oid 18 current snap is 321 2024-09-21T09:48:50.487 INFO:tasks.rados.rados.0.smithi017.stdout:2514: finishing write tid 1 to smithi01785587-5 2024-09-21T09:48:50.487 INFO:tasks.rados.rados.0.smithi017.stdout:2514: finishing write tid 2 to smithi01785587-5 2024-09-21T09:48:50.489 INFO:tasks.rados.rados.0.smithi017.stdout:2515: done (1 left) 2024-09-21T09:48:50.489 INFO:tasks.rados.rados.0.smithi017.stdout:2516: delete oid 50 current snap is 321 2024-09-21T09:48:50.489 INFO:tasks.rados.rados.0.smithi017.stdout:2514: finishing write tid 3 to smithi01785587-5 2024-09-21T09:48:50.489 INFO:tasks.rados.rados.0.smithi017.stdout:2514: finishing write tid 4 to smithi01785587-5 2024-09-21T09:48:50.489 INFO:tasks.rados.rados.0.smithi017.stdout:2514: finishing write tid 5 to smithi01785587-5 2024-09-21T09:48:50.490 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1239 (ObjNum 949 snap 321 seq_num 949) dirty exists 2024-09-21T09:48:50.490 INFO:tasks.rados.rados.0.smithi017.stdout:2514: left oid 5 (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:48:50.491 INFO:tasks.rados.rados.0.smithi017.stdout:2514: done (1 left) 2024-09-21T09:48:50.491 INFO:tasks.rados.rados.0.smithi017.stdout:2516: done (0 left) 2024-09-21T09:48:50.491 INFO:tasks.rados.rados.0.smithi017.stdout:2517: write oid 36 current snap is 321 2024-09-21T09:48:50.491 INFO:tasks.rados.rados.0.smithi017.stdout:2517: seq_num 952 ranges {400264=581173,1508740=491524} 2024-09-21T09:48:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:48:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:48:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:48:50.496 INFO:tasks.rados.rados.0.smithi017.stdout:2517: writing smithi01785587-36 from 400264 to 981437 tid 1 2024-09-21T09:48:50.502 INFO:tasks.rados.rados.0.smithi017.stdout:2517: writing smithi01785587-36 from 1508740 to 2000264 tid 2 2024-09-21T09:48:50.502 INFO:tasks.rados.rados.0.smithi017.stdout:2518: delete oid 13 current snap is 321 2024-09-21T09:48:50.502 INFO:tasks.rados.rados.0.smithi017.stdout:2517: finishing write tid 1 to smithi01785587-36 2024-09-21T09:48:50.504 INFO:tasks.rados.rados.0.smithi017.stdout:2518: done (1 left) 2024-09-21T09:48:50.504 INFO:tasks.rados.rados.0.smithi017.stdout:2519: read oid 45 snap -1 2024-09-21T09:48:50.504 INFO:tasks.rados.rados.0.smithi017.stdout:2519: expect deleted 2024-09-21T09:48:50.505 INFO:tasks.rados.rados.0.smithi017.stdout:2520: snap_remove snap 295 2024-09-21T09:48:51.165 INFO:tasks.rados.rados.0.smithi017.stdout:2520: done (2 left) 2024-09-21T09:48:51.165 INFO:tasks.rados.rados.0.smithi017.stdout:2521: delete oid 34 current snap is 321 2024-09-21T09:48:51.165 INFO:tasks.rados.rados.0.smithi017.stdout:2517: finishing write tid 2 to smithi01785587-36 2024-09-21T09:48:51.165 INFO:tasks.rados.rados.0.smithi017.stdout:2517: finishing write tid 3 to smithi01785587-36 2024-09-21T09:48:51.165 INFO:tasks.rados.rados.0.smithi017.stdout:2517: finishing write tid 4 to smithi01785587-36 2024-09-21T09:48:51.165 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1336 (ObjNum 952 snap 321 seq_num 952) dirty exists 2024-09-21T09:48:51.166 INFO:tasks.rados.rados.0.smithi017.stdout:2517: left oid 36 (ObjNum 952 snap 321 seq_num 952) 2024-09-21T09:48:51.172 INFO:tasks.rados.rados.0.smithi017.stdout:2517: done (2 left) 2024-09-21T09:48:51.172 INFO:tasks.rados.rados.0.smithi017.stdout:2519: done (1 left) 2024-09-21T09:48:51.172 INFO:tasks.rados.rados.0.smithi017.stdout:2521: done (0 left) 2024-09-21T09:48:51.172 INFO:tasks.rados.rados.0.smithi017.stdout:2522: snap_create 2024-09-21T09:48:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:51 smithi017 ceph-mon[102913]: pgmap v636: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 476 KiB/s wr, 1 op/s 2024-09-21T09:48:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:51 smithi017 ceph-mon[102913]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T09:48:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:51 smithi017 ceph-mon[105151]: pgmap v636: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 476 KiB/s wr, 1 op/s 2024-09-21T09:48:51.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:51 smithi017 ceph-mon[105151]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T09:48:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:51 smithi134 ceph-mon[90085]: pgmap v636: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 476 KiB/s wr, 1 op/s 2024-09-21T09:48:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:51 smithi134 ceph-mon[90085]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T09:48:52.175 INFO:tasks.rados.rados.0.smithi017.stdout:2522: done (0 left) 2024-09-21T09:48:52.175 INFO:tasks.rados.rados.0.smithi017.stdout:2523: delete oid 11 current snap is 322 2024-09-21T09:48:52.177 INFO:tasks.rados.rados.0.smithi017.stdout:2523: done (0 left) 2024-09-21T09:48:52.177 INFO:tasks.rados.rados.0.smithi017.stdout:2524: read oid 13 snap -1 2024-09-21T09:48:52.177 INFO:tasks.rados.rados.0.smithi017.stdout:2524: expect deleted 2024-09-21T09:48:52.177 INFO:tasks.rados.rados.0.smithi017.stdout:2525: snap_create 2024-09-21T09:48:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:52 smithi017 ceph-mon[102913]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T09:48:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:52 smithi017 ceph-mon[105151]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T09:48:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:52 smithi134 ceph-mon[90085]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T09:48:53.134 INFO:tasks.rados.rados.0.smithi017.stdout:2524: done (1 left) 2024-09-21T09:48:53.134 INFO:tasks.rados.rados.0.smithi017.stdout:2525: done (0 left) 2024-09-21T09:48:53.134 INFO:tasks.rados.rados.0.smithi017.stdout:2526: write oid 7 current snap is 323 2024-09-21T09:48:53.134 INFO:tasks.rados.rados.0.smithi017.stdout:2526: seq_num 956 ranges {467492=482169,1626315=441177} 2024-09-21T09:48:53.139 INFO:tasks.rados.rados.0.smithi017.stdout:2526: writing smithi01785587-7 from 467492 to 949661 tid 1 2024-09-21T09:48:53.146 INFO:tasks.rados.rados.0.smithi017.stdout:2526: writing smithi01785587-7 from 1626315 to 2067492 tid 2 2024-09-21T09:48:53.146 INFO:tasks.rados.rados.0.smithi017.stdout:2527: read oid 10 snap -1 2024-09-21T09:48:53.146 INFO:tasks.rados.rados.0.smithi017.stdout:2527: expect (ObjNum 937 snap 314 seq_num 937) 2024-09-21T09:48:53.146 INFO:tasks.rados.rados.0.smithi017.stdout:2528: snap_remove snap 318 2024-09-21T09:48:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:53 smithi017 ceph-mon[102913]: pgmap v639: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:48:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:53 smithi017 ceph-mon[102913]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T09:48:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:53 smithi017 ceph-mon[102913]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T09:48:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:53 smithi017 ceph-mon[105151]: pgmap v639: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:48:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:53 smithi017 ceph-mon[105151]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T09:48:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:53 smithi017 ceph-mon[105151]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T09:48:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:53 smithi134 ceph-mon[90085]: pgmap v639: 89 pgs: 89 active+clean; 250 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:48:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:53 smithi134 ceph-mon[90085]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T09:48:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:53 smithi134 ceph-mon[90085]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T09:48:54.137 INFO:tasks.rados.rados.0.smithi017.stdout:2528: done (2 left) 2024-09-21T09:48:54.137 INFO:tasks.rados.rados.0.smithi017.stdout:2529: snap_remove snap 314 2024-09-21T09:48:55.133 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:48:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:48:54] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.51.0" 2024-09-21T09:48:55.141 INFO:tasks.rados.rados.0.smithi017.stdout:2529: done (2 left) 2024-09-21T09:48:55.142 INFO:tasks.rados.rados.0.smithi017.stdout:2530: write oid 34 current snap is 323 2024-09-21T09:48:55.142 INFO:tasks.rados.rados.0.smithi017.stdout:2526: finishing write tid 1 to smithi01785587-7 2024-09-21T09:48:55.142 INFO:tasks.rados.rados.0.smithi017.stdout:2530: seq_num 957 ranges {484299=557418,1684797=399502} 2024-09-21T09:48:55.148 INFO:tasks.rados.rados.0.smithi017.stdout:2530: writing smithi01785587-34 from 484299 to 1041717 tid 1 2024-09-21T09:48:55.154 INFO:tasks.rados.rados.0.smithi017.stdout:2530: writing smithi01785587-34 from 1684797 to 2084299 tid 2 2024-09-21T09:48:55.155 INFO:tasks.rados.rados.0.smithi017.stdout:2531: write oid 20 current snap is 323 2024-09-21T09:48:55.155 INFO:tasks.rados.rados.0.smithi017.stdout:2526: finishing write tid 2 to smithi01785587-7 2024-09-21T09:48:55.155 INFO:tasks.rados.rados.0.smithi017.stdout:2526: finishing write tid 3 to smithi01785587-7 2024-09-21T09:48:55.155 INFO:tasks.rados.rados.0.smithi017.stdout:2526: finishing write tid 4 to smithi01785587-7 2024-09-21T09:48:55.155 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1287 (ObjNum 956 snap 323 seq_num 956) dirty exists 2024-09-21T09:48:55.155 INFO:tasks.rados.rados.0.smithi017.stdout:2526: left oid 7 (ObjNum 956 snap 323 seq_num 956) 2024-09-21T09:48:55.155 INFO:tasks.rados.rados.0.smithi017.stdout:2527: expect (ObjNum 937 snap 314 seq_num 937) 2024-09-21T09:48:55.368 INFO:tasks.rados.rados.0.smithi017.stdout:2530: finishing write tid 1 to smithi01785587-34 2024-09-21T09:48:55.368 INFO:tasks.rados.rados.0.smithi017.stdout:2530: finishing write tid 2 to smithi01785587-34 2024-09-21T09:48:55.368 INFO:tasks.rados.rados.0.smithi017.stdout:2531: seq_num 958 ranges {501106=749020,1743279=357827} 2024-09-21T09:48:55.375 INFO:tasks.rados.rados.0.smithi017.stdout:2531: writing smithi01785587-20 from 501106 to 1250126 tid 1 2024-09-21T09:48:55.379 INFO:tasks.rados.rados.0.smithi017.stdout:2531: writing smithi01785587-20 from 1743279 to 2101106 tid 2 2024-09-21T09:48:55.380 INFO:tasks.rados.rados.0.smithi017.stdout:2526: done (3 left) 2024-09-21T09:48:55.380 INFO:tasks.rados.rados.0.smithi017.stdout:2527: done (2 left) 2024-09-21T09:48:55.380 INFO:tasks.rados.rados.0.smithi017.stdout:2532: read oid 18 snap -1 2024-09-21T09:48:55.380 INFO:tasks.rados.rados.0.smithi017.stdout:2532: expect deleted 2024-09-21T09:48:55.380 INFO:tasks.rados.rados.0.smithi017.stdout:2533: snap_remove snap 320 2024-09-21T09:48:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:55 smithi017 ceph-mon[105151]: pgmap v642: 89 pgs: 89 active+clean; 254 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-21T09:48:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:55 smithi017 ceph-mon[105151]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T09:48:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:55 smithi017 ceph-mon[102913]: pgmap v642: 89 pgs: 89 active+clean; 254 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-21T09:48:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:55 smithi017 ceph-mon[102913]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T09:48:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:55 smithi134 ceph-mon[90085]: pgmap v642: 89 pgs: 89 active+clean; 254 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-21T09:48:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:55 smithi134 ceph-mon[90085]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T09:48:56.148 INFO:tasks.rados.rados.0.smithi017.stdout:2533: done (3 left) 2024-09-21T09:48:56.148 INFO:tasks.rados.rados.0.smithi017.stdout:2534: write oid 15 current snap is 323 2024-09-21T09:48:56.148 INFO:tasks.rados.rados.0.smithi017.stdout:2534: seq_num 959 ranges {517913=424269,1401761=446989} 2024-09-21T09:48:56.153 INFO:tasks.rados.rados.0.smithi017.stdout:2534: writing smithi01785587-15 from 517913 to 942182 tid 1 2024-09-21T09:48:56.159 INFO:tasks.rados.rados.0.smithi017.stdout:2534: writing smithi01785587-15 from 1401761 to 1848750 tid 2 2024-09-21T09:48:56.159 INFO:tasks.rados.rados.0.smithi017.stdout:2535: write oid 25 current snap is 323 2024-09-21T09:48:56.159 INFO:tasks.rados.rados.0.smithi017.stdout:2530: finishing write tid 3 to smithi01785587-34 2024-09-21T09:48:56.159 INFO:tasks.rados.rados.0.smithi017.stdout:2530: finishing write tid 4 to smithi01785587-34 2024-09-21T09:48:56.159 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1298 (ObjNum 957 snap 323 seq_num 957) dirty exists 2024-09-21T09:48:56.159 INFO:tasks.rados.rados.0.smithi017.stdout:2530: left oid 34 (ObjNum 957 snap 323 seq_num 957) 2024-09-21T09:48:56.159 INFO:tasks.rados.rados.0.smithi017.stdout:2531: finishing write tid 1 to smithi01785587-20 2024-09-21T09:48:56.160 INFO:tasks.rados.rados.0.smithi017.stdout:2535: seq_num 960 ranges {534720=499518,1460243=674477} 2024-09-21T09:48:56.164 INFO:tasks.rados.rados.0.smithi017.stdout:2535: writing smithi01785587-25 from 534720 to 1034238 tid 1 2024-09-21T09:48:56.171 INFO:tasks.rados.rados.0.smithi017.stdout:2535: writing smithi01785587-25 from 1460243 to 2134720 tid 2 2024-09-21T09:48:56.171 INFO:tasks.rados.rados.0.smithi017.stdout:2530: done (4 left) 2024-09-21T09:48:56.171 INFO:tasks.rados.rados.0.smithi017.stdout:2532: done (3 left) 2024-09-21T09:48:56.171 INFO:tasks.rados.rados.0.smithi017.stdout:2536: snap_remove snap 316 2024-09-21T09:48:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:56 smithi017 ceph-mon[102913]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T09:48:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:56 smithi017 ceph-mon[105151]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T09:48:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:56.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:56 smithi134 ceph-mon[90085]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T09:48:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:57.155 INFO:tasks.rados.rados.0.smithi017.stdout:2536: done (3 left) 2024-09-21T09:48:57.155 INFO:tasks.rados.rados.0.smithi017.stdout:2537: read oid 28 snap -1 2024-09-21T09:48:57.155 INFO:tasks.rados.rados.0.smithi017.stdout:2537: expect deleted 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2538: read oid 6 snap 304 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2538: expect (ObjNum 763 snap 257 seq_num 763) 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2539: write oid 27 current snap is 323 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2531: finishing write tid 2 to smithi01785587-20 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2531: finishing write tid 3 to smithi01785587-20 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2531: finishing write tid 4 to smithi01785587-20 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 658 (ObjNum 958 snap 323 seq_num 958) dirty exists 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2531: left oid 20 (ObjNum 958 snap 323 seq_num 958) 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2534: finishing write tid 1 to smithi01785587-15 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2534: finishing write tid 2 to smithi01785587-15 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2534: finishing write tid 3 to smithi01785587-15 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2534: finishing write tid 4 to smithi01785587-15 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 396 (ObjNum 959 snap 323 seq_num 959) dirty exists 2024-09-21T09:48:57.156 INFO:tasks.rados.rados.0.smithi017.stdout:2534: left oid 15 (ObjNum 959 snap 323 seq_num 959) 2024-09-21T09:48:57.157 INFO:tasks.rados.rados.0.smithi017.stdout:2535: finishing write tid 1 to smithi01785587-25 2024-09-21T09:48:57.157 INFO:tasks.rados.rados.0.smithi017.stdout:2535: finishing write tid 2 to smithi01785587-25 2024-09-21T09:48:57.157 INFO:tasks.rados.rados.0.smithi017.stdout:2535: finishing write tid 3 to smithi01785587-25 2024-09-21T09:48:57.157 INFO:tasks.rados.rados.0.smithi017.stdout:2535: finishing write tid 4 to smithi01785587-25 2024-09-21T09:48:57.157 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 663 (ObjNum 960 snap 323 seq_num 960) dirty exists 2024-09-21T09:48:57.157 INFO:tasks.rados.rados.0.smithi017.stdout:2535: left oid 25 (ObjNum 960 snap 323 seq_num 960) 2024-09-21T09:48:57.157 INFO:tasks.rados.rados.0.smithi017.stdout:2539: seq_num 961 ranges {551527=574767,1918725=232802} 2024-09-21T09:48:57.162 INFO:tasks.rados.rados.0.smithi017.stdout:2539: writing smithi01785587-27 from 551527 to 1126294 tid 1 2024-09-21T09:48:57.168 INFO:tasks.rados.rados.0.smithi017.stdout:2539: writing smithi01785587-27 from 1918725 to 2151527 tid 2 2024-09-21T09:48:57.168 INFO:tasks.rados.rados.0.smithi017.stdout:2531: done (5 left) 2024-09-21T09:48:57.168 INFO:tasks.rados.rados.0.smithi017.stdout:2534: done (4 left) 2024-09-21T09:48:57.168 INFO:tasks.rados.rados.0.smithi017.stdout:2535: done (3 left) 2024-09-21T09:48:57.169 INFO:tasks.rados.rados.0.smithi017.stdout:2540: write oid 1 current snap is 323 2024-09-21T09:48:57.169 INFO:tasks.rados.rados.0.smithi017.stdout:2540: seq_num 962 ranges {568334=650016,1860854=307480} 2024-09-21T09:48:57.179 INFO:tasks.rados.rados.0.smithi017.stdout:2540: writing smithi01785587-1 from 568334 to 1218350 tid 1 2024-09-21T09:48:57.181 INFO:tasks.rados.rados.0.smithi017.stdout:2540: writing smithi01785587-1 from 1860854 to 2168334 tid 2 2024-09-21T09:48:57.181 INFO:tasks.rados.rados.0.smithi017.stdout:2541: delete oid 8 current snap is 323 2024-09-21T09:48:57.181 INFO:tasks.rados.rados.0.smithi017.stdout:2539: finishing write tid 1 to smithi01785587-27 2024-09-21T09:48:57.181 INFO:tasks.rados.rados.0.smithi017.stdout:2539: finishing write tid 2 to smithi01785587-27 2024-09-21T09:48:57.181 INFO:tasks.rados.rados.0.smithi017.stdout:2539: finishing write tid 3 to smithi01785587-27 2024-09-21T09:48:57.182 INFO:tasks.rados.rados.0.smithi017.stdout:2539: finishing write tid 4 to smithi01785587-27 2024-09-21T09:48:57.182 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 806 (ObjNum 961 snap 323 seq_num 961) dirty exists 2024-09-21T09:48:57.182 INFO:tasks.rados.rados.0.smithi017.stdout:2539: left oid 27 (ObjNum 961 snap 323 seq_num 961) 2024-09-21T09:48:57.182 INFO:tasks.rados.rados.0.smithi017.stdout:2538: expect (ObjNum 763 snap 257 seq_num 763) 2024-09-21T09:48:57.334 INFO:tasks.rados.rados.0.smithi017.stdout:2540: finishing write tid 1 to smithi01785587-1 2024-09-21T09:48:57.334 INFO:tasks.rados.rados.0.smithi017.stdout:2540: finishing write tid 2 to smithi01785587-1 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2540: finishing write tid 3 to smithi01785587-1 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2540: finishing write tid 4 to smithi01785587-1 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 798 (ObjNum 962 snap 323 seq_num 962) dirty exists 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2540: left oid 1 (ObjNum 962 snap 323 seq_num 962) 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2537: done (4 left) 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2538: done (3 left) 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2539: done (2 left) 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2540: done (1 left) 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2541: done (0 left) 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2542: read oid 5 snap 317 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2542: expect (ObjNum 844 snap 289 seq_num 844) 2024-09-21T09:48:57.335 INFO:tasks.rados.rados.0.smithi017.stdout:2543: delete oid 1 current snap is 323 2024-09-21T09:48:57.336 INFO:tasks.rados.rados.0.smithi017.stdout:2543: done (1 left) 2024-09-21T09:48:57.336 INFO:tasks.rados.rados.0.smithi017.stdout:2544: read oid 25 snap 282 2024-09-21T09:48:57.336 INFO:tasks.rados.rados.0.smithi017.stdout:2544: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:48:57.336 INFO:tasks.rados.rados.0.smithi017.stdout:2545: snap_remove snap 304 2024-09-21T09:48:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:57 smithi017 ceph-mon[102913]: pgmap v645: 89 pgs: 89 active+clean; 257 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-21T09:48:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:57 smithi017 ceph-mon[102913]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T09:48:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:57 smithi017 ceph-mon[105151]: pgmap v645: 89 pgs: 89 active+clean; 257 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-21T09:48:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:57 smithi017 ceph-mon[105151]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T09:48:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:57 smithi134 ceph-mon[90085]: pgmap v645: 89 pgs: 89 active+clean; 257 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 MiB/s wr, 5 op/s 2024-09-21T09:48:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:57 smithi134 ceph-mon[90085]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T09:48:58.138 INFO:tasks.rados.rados.0.smithi017.stdout:2542: expect (ObjNum 844 snap 289 seq_num 844) 2024-09-21T09:48:58.281 INFO:tasks.rados.rados.0.smithi017.stdout:2544: expect (ObjNum 765 snap 259 seq_num 765) 2024-09-21T09:48:58.432 INFO:tasks.rados.rados.0.smithi017.stdout:2542: done (2 left) 2024-09-21T09:48:58.432 INFO:tasks.rados.rados.0.smithi017.stdout:2544: done (1 left) 2024-09-21T09:48:58.432 INFO:tasks.rados.rados.0.smithi017.stdout:2545: done (0 left) 2024-09-21T09:48:58.432 INFO:tasks.rados.rados.0.smithi017.stdout:2546: snap_create 2024-09-21T09:48:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:58 smithi017 ceph-mon[102913]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T09:48:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:58 smithi017 ceph-mon[102913]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T09:48:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:58 smithi017 ceph-mon[105151]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T09:48:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:58 smithi017 ceph-mon[105151]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T09:48:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:58 smithi134 ceph-mon[90085]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T09:48:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:58 smithi134 ceph-mon[90085]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T09:48:59.165 INFO:tasks.rados.rados.0.smithi017.stdout:2546: done (0 left) 2024-09-21T09:48:59.165 INFO:tasks.rados.rados.0.smithi017.stdout:2547: snap_create 2024-09-21T09:48:59.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:48:59 smithi017 ceph-mon[102913]: pgmap v648: 89 pgs: 89 active+clean; 260 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:48:59.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:48:59 smithi017 ceph-mon[105151]: pgmap v648: 89 pgs: 89 active+clean; 260 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:48:59.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:48:59 smithi134 ceph-mon[90085]: pgmap v648: 89 pgs: 89 active+clean; 260 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-09-21T09:49:00.171 INFO:tasks.rados.rados.0.smithi017.stdout:2547: done (0 left) 2024-09-21T09:49:00.171 INFO:tasks.rados.rados.0.smithi017.stdout:2548: snap_remove snap 282 2024-09-21T09:49:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:00 smithi017 ceph-mon[102913]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T09:49:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:00 smithi017 ceph-mon[105151]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T09:49:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:00 smithi134 ceph-mon[90085]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T09:49:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:49:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:49:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi017.stdout:2548: done (0 left) 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi017.stdout:2549: delete oid 4 current snap is 325 2024-09-21T09:49:01.183 INFO:tasks.rados.rados.0.smithi017.stdout:2549: done (0 left) 2024-09-21T09:49:01.183 INFO:tasks.rados.rados.0.smithi017.stdout:2550: snap_remove snap 322 2024-09-21T09:49:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:01 smithi017 ceph-mon[102913]: pgmap v651: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 261 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 657 KiB/s wr, 5 op/s 2024-09-21T09:49:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:01 smithi017 ceph-mon[102913]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T09:49:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:01 smithi017 ceph-mon[105151]: pgmap v651: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 261 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 657 KiB/s wr, 5 op/s 2024-09-21T09:49:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:01 smithi017 ceph-mon[105151]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T09:49:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:01 smithi134 ceph-mon[90085]: pgmap v651: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 261 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 657 KiB/s wr, 5 op/s 2024-09-21T09:49:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:01 smithi134 ceph-mon[90085]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T09:49:02.185 INFO:tasks.rados.rados.0.smithi017.stdout:2550: done (0 left) 2024-09-21T09:49:02.185 INFO:tasks.rados.rados.0.smithi017.stdout:2551: rollback oid 35 current snap is 325 2024-09-21T09:49:02.185 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 300 2024-09-21T09:49:02.185 INFO:tasks.rados.rados.0.smithi017.stdout:2552: read oid 29 snap -1 2024-09-21T09:49:02.186 INFO:tasks.rados.rados.0.smithi017.stdout:2552: expect deleted 2024-09-21T09:49:02.186 INFO:tasks.rados.rados.0.smithi017.stdout:2553: write oid 44 current snap is 325 2024-09-21T09:49:02.186 INFO:tasks.rados.rados.0.smithi017.stdout:2553: seq_num 966 ranges {635562=667365,2094782=140780} 2024-09-21T09:49:02.192 INFO:tasks.rados.rados.0.smithi017.stdout:2553: writing smithi01785587-44 from 635562 to 1302927 tid 1 2024-09-21T09:49:02.198 INFO:tasks.rados.rados.0.smithi017.stdout:2553: writing smithi01785587-44 from 2094782 to 2235562 tid 2 2024-09-21T09:49:02.198 INFO:tasks.rados.rados.0.smithi017.stdout:2554: write oid 7 current snap is 325 2024-09-21T09:49:02.198 INFO:tasks.rados.rados.0.smithi017.stdout:2554: seq_num 967 ranges {652369=742614,2153264=99105} 2024-09-21T09:49:02.206 INFO:tasks.rados.rados.0.smithi017.stdout:2554: writing smithi01785587-7 from 652369 to 1394983 tid 1 2024-09-21T09:49:02.211 INFO:tasks.rados.rados.0.smithi017.stdout:2554: writing smithi01785587-7 from 2153264 to 2252369 tid 2 2024-09-21T09:49:02.211 INFO:tasks.rados.rados.0.smithi017.stdout:2555: delete oid 15 current snap is 325 2024-09-21T09:49:02.211 INFO:tasks.rados.rados.0.smithi017.stdout:2551: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:49:02.211 INFO:tasks.rados.rados.0.smithi017.stdout:2551: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:49:02.211 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 278 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:02.211 INFO:tasks.rados.rados.0.smithi017.stdout:2553: finishing write tid 1 to smithi01785587-44 2024-09-21T09:49:02.212 INFO:tasks.rados.rados.0.smithi017.stdout:2553: finishing write tid 2 to smithi01785587-44 2024-09-21T09:49:02.212 INFO:tasks.rados.rados.0.smithi017.stdout:2553: finishing write tid 3 to smithi01785587-44 2024-09-21T09:49:02.212 INFO:tasks.rados.rados.0.smithi017.stdout:2553: finishing write tid 4 to smithi01785587-44 2024-09-21T09:49:02.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1282 (ObjNum 966 snap 325 seq_num 966) dirty exists 2024-09-21T09:49:02.212 INFO:tasks.rados.rados.0.smithi017.stdout:2553: left oid 44 (ObjNum 966 snap 325 seq_num 966) 2024-09-21T09:49:02.212 INFO:tasks.rados.rados.0.smithi017.stdout:2554: finishing write tid 1 to smithi01785587-7 2024-09-21T09:49:02.213 INFO:tasks.rados.rados.0.smithi017.stdout:2551: done (4 left) 2024-09-21T09:49:02.213 INFO:tasks.rados.rados.0.smithi017.stdout:2552: done (3 left) 2024-09-21T09:49:02.213 INFO:tasks.rados.rados.0.smithi017.stdout:2553: done (2 left) 2024-09-21T09:49:02.213 INFO:tasks.rados.rados.0.smithi017.stdout:2555: done (1 left) 2024-09-21T09:49:02.213 INFO:tasks.rados.rados.0.smithi017.stdout:2556: snap_create 2024-09-21T09:49:02.213 INFO:tasks.rados.rados.0.smithi017.stdout:2554: finishing write tid 2 to smithi01785587-7 2024-09-21T09:49:02.217 INFO:tasks.rados.rados.0.smithi017.stdout:2554: finishing write tid 3 to smithi01785587-7 2024-09-21T09:49:02.218 INFO:tasks.rados.rados.0.smithi017.stdout:2554: finishing write tid 4 to smithi01785587-7 2024-09-21T09:49:02.218 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1320 (ObjNum 967 snap 325 seq_num 967) dirty exists 2024-09-21T09:49:02.218 INFO:tasks.rados.rados.0.smithi017.stdout:2554: left oid 7 (ObjNum 967 snap 325 seq_num 967) 2024-09-21T09:49:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:02 smithi017 ceph-mon[102913]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T09:49:02.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:02 smithi017 ceph-mon[105151]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T09:49:02.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:02 smithi134 ceph-mon[90085]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T09:49:03.136 INFO:tasks.rados.rados.0.smithi017.stdout:2554: done (1 left) 2024-09-21T09:49:03.137 INFO:tasks.rados.rados.0.smithi017.stdout:2556: done (0 left) 2024-09-21T09:49:03.137 INFO:tasks.rados.rados.0.smithi017.stdout:2557: write oid 4 current snap is 326 2024-09-21T09:49:03.137 INFO:tasks.rados.rados.0.smithi017.stdout:2557: seq_num 969 ranges {685983=493112,1870228=415755} 2024-09-21T09:49:03.143 INFO:tasks.rados.rados.0.smithi017.stdout:2557: writing smithi01785587-4 from 685983 to 1179095 tid 1 2024-09-21T09:49:03.149 INFO:tasks.rados.rados.0.smithi017.stdout:2557: writing smithi01785587-4 from 1870228 to 2285983 tid 2 2024-09-21T09:49:03.150 INFO:tasks.rados.rados.0.smithi017.stdout:2558: snap_remove snap 324 2024-09-21T09:49:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:03 smithi017 ceph-mon[102913]: pgmap v654: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 263 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 931 KiB/s wr, 11 op/s 2024-09-21T09:49:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:03 smithi017 ceph-mon[102913]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T09:49:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:03 smithi017 ceph-mon[102913]: osdmap e718: 8 total, 8 up, 8 in 2024-09-21T09:49:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:03 smithi017 ceph-mon[105151]: pgmap v654: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 263 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 931 KiB/s wr, 11 op/s 2024-09-21T09:49:03.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:03 smithi017 ceph-mon[105151]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T09:49:03.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:03 smithi017 ceph-mon[105151]: osdmap e718: 8 total, 8 up, 8 in 2024-09-21T09:49:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:03 smithi134 ceph-mon[90085]: pgmap v654: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 263 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 931 KiB/s wr, 11 op/s 2024-09-21T09:49:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:03 smithi134 ceph-mon[90085]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T09:49:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:03 smithi134 ceph-mon[90085]: osdmap e718: 8 total, 8 up, 8 in 2024-09-21T09:49:04.139 INFO:tasks.rados.rados.0.smithi017.stdout:2558: done (1 left) 2024-09-21T09:49:04.139 INFO:tasks.rados.rados.0.smithi017.stdout:2559: snap_remove snap 321 2024-09-21T09:49:05.142 INFO:tasks.rados.rados.0.smithi017.stdout:2559: done (1 left) 2024-09-21T09:49:05.142 INFO:tasks.rados.rados.0.smithi017.stdout:2560: snap_remove snap 313 2024-09-21T09:49:05.145 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:49:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:49:04] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.51.0" 2024-09-21T09:49:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:05 smithi017 ceph-mon[102913]: pgmap v657: 89 pgs: 89 active+clean; 259 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 734 KiB/s wr, 10 op/s 2024-09-21T09:49:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:05 smithi017 ceph-mon[102913]: osdmap e719: 8 total, 8 up, 8 in 2024-09-21T09:49:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:05 smithi017 ceph-mon[105151]: pgmap v657: 89 pgs: 89 active+clean; 259 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 734 KiB/s wr, 10 op/s 2024-09-21T09:49:05.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:05 smithi017 ceph-mon[105151]: osdmap e719: 8 total, 8 up, 8 in 2024-09-21T09:49:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:05 smithi134 ceph-mon[90085]: pgmap v657: 89 pgs: 89 active+clean; 259 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 734 KiB/s wr, 10 op/s 2024-09-21T09:49:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:05 smithi134 ceph-mon[90085]: osdmap e719: 8 total, 8 up, 8 in 2024-09-21T09:49:06.157 INFO:tasks.rados.rados.0.smithi017.stdout:2560: done (1 left) 2024-09-21T09:49:06.157 INFO:tasks.rados.rados.0.smithi017.stdout:2561: write oid 45 current snap is 326 2024-09-21T09:49:06.157 INFO:tasks.rados.rados.0.smithi017.stdout:2561: seq_num 970 ranges {702790=568361,1812357=490433} 2024-09-21T09:49:06.164 INFO:tasks.rados.rados.0.smithi017.stdout:2561: writing smithi01785587-45 from 702790 to 1271151 tid 1 2024-09-21T09:49:06.170 INFO:tasks.rados.rados.0.smithi017.stdout:2561: writing smithi01785587-45 from 1812357 to 2302790 tid 2 2024-09-21T09:49:06.170 INFO:tasks.rados.rados.0.smithi017.stdout:2562: snap_create 2024-09-21T09:49:06.170 INFO:tasks.rados.rados.0.smithi017.stdout:2557: finishing write tid 1 to smithi01785587-4 2024-09-21T09:49:06.170 INFO:tasks.rados.rados.0.smithi017.stdout:2557: finishing write tid 2 to smithi01785587-4 2024-09-21T09:49:06.170 INFO:tasks.rados.rados.0.smithi017.stdout:2557: finishing write tid 3 to smithi01785587-4 2024-09-21T09:49:06.170 INFO:tasks.rados.rados.0.smithi017.stdout:2557: finishing write tid 4 to smithi01785587-4 2024-09-21T09:49:06.171 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1275 (ObjNum 969 snap 326 seq_num 969) dirty exists 2024-09-21T09:49:06.171 INFO:tasks.rados.rados.0.smithi017.stdout:2557: left oid 4 (ObjNum 969 snap 326 seq_num 969) 2024-09-21T09:49:06.171 INFO:tasks.rados.rados.0.smithi017.stdout:2561: finishing write tid 1 to smithi01785587-45 2024-09-21T09:49:06.173 INFO:tasks.rados.rados.0.smithi017.stdout:2561: finishing write tid 2 to smithi01785587-45 2024-09-21T09:49:06.173 INFO:tasks.rados.rados.0.smithi017.stdout:2561: finishing write tid 3 to smithi01785587-45 2024-09-21T09:49:06.173 INFO:tasks.rados.rados.0.smithi017.stdout:2561: finishing write tid 4 to smithi01785587-45 2024-09-21T09:49:06.173 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 825 (ObjNum 970 snap 326 seq_num 970) dirty exists 2024-09-21T09:49:06.174 INFO:tasks.rados.rados.0.smithi017.stdout:2561: left oid 45 (ObjNum 970 snap 326 seq_num 970) 2024-09-21T09:49:06.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:06 smithi017 ceph-mon[102913]: osdmap e720: 8 total, 8 up, 8 in 2024-09-21T09:49:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:06 smithi017 ceph-mon[105151]: osdmap e720: 8 total, 8 up, 8 in 2024-09-21T09:49:06.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:06 smithi134 ceph-mon[90085]: osdmap e720: 8 total, 8 up, 8 in 2024-09-21T09:49:07.156 INFO:tasks.rados.rados.0.smithi017.stdout:2557: done (2 left) 2024-09-21T09:49:07.156 INFO:tasks.rados.rados.0.smithi017.stdout:2561: done (1 left) 2024-09-21T09:49:07.156 INFO:tasks.rados.rados.0.smithi017.stdout:2562: done (0 left) 2024-09-21T09:49:07.156 INFO:tasks.rados.rados.0.smithi017.stdout:2563: delete oid 44 current snap is 327 2024-09-21T09:49:07.162 INFO:tasks.rados.rados.0.smithi017.stdout:2563: done (0 left) 2024-09-21T09:49:07.162 INFO:tasks.rados.rados.0.smithi017.stdout:2564: write oid 25 current snap is 327 2024-09-21T09:49:07.162 INFO:tasks.rados.rados.0.smithi017.stdout:2564: seq_num 972 ranges {736404=718859,1929321=407083} 2024-09-21T09:49:07.169 INFO:tasks.rados.rados.0.smithi017.stdout:2564: writing smithi01785587-25 from 736404 to 1455263 tid 1 2024-09-21T09:49:07.175 INFO:tasks.rados.rados.0.smithi017.stdout:2564: writing smithi01785587-25 from 1929321 to 2336404 tid 2 2024-09-21T09:49:07.175 INFO:tasks.rados.rados.0.smithi017.stdout:2565: read oid 49 snap -1 2024-09-21T09:49:07.175 INFO:tasks.rados.rados.0.smithi017.stdout:2565: expect deleted 2024-09-21T09:49:07.175 INFO:tasks.rados.rados.0.smithi017.stdout:2566: snap_create 2024-09-21T09:49:07.177 INFO:tasks.rados.rados.0.smithi017.stdout:2564: finishing write tid 1 to smithi01785587-25 2024-09-21T09:49:07.178 INFO:tasks.rados.rados.0.smithi017.stdout:2564: finishing write tid 2 to smithi01785587-25 2024-09-21T09:49:07.178 INFO:tasks.rados.rados.0.smithi017.stdout:2564: finishing write tid 3 to smithi01785587-25 2024-09-21T09:49:07.178 INFO:tasks.rados.rados.0.smithi017.stdout:2564: finishing write tid 4 to smithi01785587-25 2024-09-21T09:49:07.179 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 689 (ObjNum 972 snap 327 seq_num 972) dirty exists 2024-09-21T09:49:07.179 INFO:tasks.rados.rados.0.smithi017.stdout:2564: left oid 25 (ObjNum 972 snap 327 seq_num 972) 2024-09-21T09:49:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:07 smithi017 ceph-mon[102913]: pgmap v660: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 420 KiB/s wr, 3 op/s 2024-09-21T09:49:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:07 smithi017 ceph-mon[102913]: osdmap e721: 8 total, 8 up, 8 in 2024-09-21T09:49:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:07 smithi017 ceph-mon[105151]: pgmap v660: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 420 KiB/s wr, 3 op/s 2024-09-21T09:49:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:07 smithi017 ceph-mon[105151]: osdmap e721: 8 total, 8 up, 8 in 2024-09-21T09:49:07.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:07 smithi134 ceph-mon[90085]: pgmap v660: 89 pgs: 89 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 420 KiB/s wr, 3 op/s 2024-09-21T09:49:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:07 smithi134 ceph-mon[90085]: osdmap e721: 8 total, 8 up, 8 in 2024-09-21T09:49:08.138 INFO:tasks.rados.rados.0.smithi017.stdout:2564: done (2 left) 2024-09-21T09:49:08.138 INFO:tasks.rados.rados.0.smithi017.stdout:2565: done (1 left) 2024-09-21T09:49:08.138 INFO:tasks.rados.rados.0.smithi017.stdout:2566: done (0 left) 2024-09-21T09:49:08.139 INFO:tasks.rados.rados.0.smithi017.stdout:2567: snap_create 2024-09-21T09:49:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:08 smithi134 ceph-mon[90085]: osdmap e722: 8 total, 8 up, 8 in 2024-09-21T09:49:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:08 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:08 smithi134 ceph-mon[90085]: osdmap e723: 8 total, 8 up, 8 in 2024-09-21T09:49:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:08 smithi017 ceph-mon[102913]: osdmap e722: 8 total, 8 up, 8 in 2024-09-21T09:49:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:08 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:08 smithi017 ceph-mon[102913]: osdmap e723: 8 total, 8 up, 8 in 2024-09-21T09:49:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:08 smithi017 ceph-mon[105151]: osdmap e722: 8 total, 8 up, 8 in 2024-09-21T09:49:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:08 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:08 smithi017 ceph-mon[105151]: osdmap e723: 8 total, 8 up, 8 in 2024-09-21T09:49:09.138 INFO:tasks.rados.rados.0.smithi017.stdout:2567: done (0 left) 2024-09-21T09:49:09.138 INFO:tasks.rados.rados.0.smithi017.stdout:2568: snap_create 2024-09-21T09:49:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:09 smithi017 ceph-mon[102913]: pgmap v663: 89 pgs: 89 active+clean; 235 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 625 KiB/s wr, 4 op/s 2024-09-21T09:49:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:09 smithi017 ceph-mon[102913]: osdmap e724: 8 total, 8 up, 8 in 2024-09-21T09:49:09.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:09 smithi017 ceph-mon[105151]: pgmap v663: 89 pgs: 89 active+clean; 235 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 625 KiB/s wr, 4 op/s 2024-09-21T09:49:09.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:09 smithi017 ceph-mon[105151]: osdmap e724: 8 total, 8 up, 8 in 2024-09-21T09:49:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:09 smithi134 ceph-mon[90085]: pgmap v663: 89 pgs: 89 active+clean; 235 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 625 KiB/s wr, 4 op/s 2024-09-21T09:49:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:09 smithi134 ceph-mon[90085]: osdmap e724: 8 total, 8 up, 8 in 2024-09-21T09:49:10.139 INFO:tasks.rados.rados.0.smithi017.stdout:2568: done (0 left) 2024-09-21T09:49:10.140 INFO:tasks.rados.rados.0.smithi017.stdout:2569: read oid 32 snap -1 2024-09-21T09:49:10.140 INFO:tasks.rados.rados.0.smithi017.stdout:2569: expect deleted 2024-09-21T09:49:10.140 INFO:tasks.rados.rados.0.smithi017.stdout:2570: write oid 41 current snap is 330 2024-09-21T09:49:10.140 INFO:tasks.rados.rados.0.smithi017.stdout:2570: seq_num 973 ranges {753211=794108,1987803=365408} 2024-09-21T09:49:10.148 INFO:tasks.rados.rados.0.smithi017.stdout:2570: writing smithi01785587-41 from 753211 to 1547319 tid 1 2024-09-21T09:49:10.153 INFO:tasks.rados.rados.0.smithi017.stdout:2570: writing smithi01785587-41 from 1987803 to 2353211 tid 2 2024-09-21T09:49:10.153 INFO:tasks.rados.rados.0.smithi017.stdout:2571: snap_create 2024-09-21T09:49:10.155 INFO:tasks.rados.rados.0.smithi017.stdout:2570: finishing write tid 1 to smithi01785587-41 2024-09-21T09:49:10.156 INFO:tasks.rados.rados.0.smithi017.stdout:2570: finishing write tid 2 to smithi01785587-41 2024-09-21T09:49:10.156 INFO:tasks.rados.rados.0.smithi017.stdout:2570: finishing write tid 3 to smithi01785587-41 2024-09-21T09:49:10.156 INFO:tasks.rados.rados.0.smithi017.stdout:2570: finishing write tid 4 to smithi01785587-41 2024-09-21T09:49:10.156 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 414 (ObjNum 973 snap 330 seq_num 973) dirty exists 2024-09-21T09:49:10.156 INFO:tasks.rados.rados.0.smithi017.stdout:2570: left oid 41 (ObjNum 973 snap 330 seq_num 973) 2024-09-21T09:49:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:49:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:49:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:11.144 INFO:tasks.rados.rados.0.smithi017.stdout:2569: done (2 left) 2024-09-21T09:49:11.144 INFO:tasks.rados.rados.0.smithi017.stdout:2570: done (1 left) 2024-09-21T09:49:11.144 INFO:tasks.rados.rados.0.smithi017.stdout:2571: done (0 left) 2024-09-21T09:49:11.144 INFO:tasks.rados.rados.0.smithi017.stdout:2572: write oid 16 current snap is 331 2024-09-21T09:49:11.144 INFO:tasks.rados.rados.0.smithi017.stdout:2572: seq_num 974 ranges {770018=585710,2046285=323733} 2024-09-21T09:49:11.151 INFO:tasks.rados.rados.0.smithi017.stdout:2572: writing smithi01785587-16 from 770018 to 1355728 tid 1 2024-09-21T09:49:11.157 INFO:tasks.rados.rados.0.smithi017.stdout:2572: writing smithi01785587-16 from 2046285 to 2370018 tid 2 2024-09-21T09:49:11.157 INFO:tasks.rados.rados.0.smithi017.stdout:2573: read oid 5 snap -1 2024-09-21T09:49:11.157 INFO:tasks.rados.rados.0.smithi017.stdout:2573: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:49:11.157 INFO:tasks.rados.rados.0.smithi017.stdout:2574: snap_remove snap 272 2024-09-21T09:49:11.422 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:11 smithi134 ceph-mon[90085]: pgmap v666: 89 pgs: 89 active+clean; 233 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 464 KiB/s wr, 2 op/s 2024-09-21T09:49:11.422 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:11 smithi134 ceph-mon[90085]: osdmap e725: 8 total, 8 up, 8 in 2024-09-21T09:49:11.422 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:11.422 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:11.422 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[105151]: pgmap v666: 89 pgs: 89 active+clean; 233 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 464 KiB/s wr, 2 op/s 2024-09-21T09:49:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[105151]: osdmap e725: 8 total, 8 up, 8 in 2024-09-21T09:49:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[102913]: pgmap v666: 89 pgs: 89 active+clean; 233 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 464 KiB/s wr, 2 op/s 2024-09-21T09:49:11.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[102913]: osdmap e725: 8 total, 8 up, 8 in 2024-09-21T09:49:11.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:11.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:11.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:12.150 INFO:tasks.rados.rados.0.smithi017.stdout:2574: done (2 left) 2024-09-21T09:49:12.151 INFO:tasks.rados.rados.0.smithi017.stdout:2575: write oid 34 current snap is 331 2024-09-21T09:49:12.151 INFO:tasks.rados.rados.0.smithi017.stdout:2575: seq_num 975 ranges {786825=660959,2104767=282058} 2024-09-21T09:49:12.160 INFO:tasks.rados.rados.0.smithi017.stdout:2575: writing smithi01785587-34 from 786825 to 1447784 tid 1 2024-09-21T09:49:12.166 INFO:tasks.rados.rados.0.smithi017.stdout:2575: writing smithi01785587-34 from 2104767 to 2386825 tid 2 2024-09-21T09:49:12.166 INFO:tasks.rados.rados.0.smithi017.stdout:2576: snap_create 2024-09-21T09:49:12.166 INFO:tasks.rados.rados.0.smithi017.stdout:2572: finishing write tid 1 to smithi01785587-16 2024-09-21T09:49:12.166 INFO:tasks.rados.rados.0.smithi017.stdout:2572: finishing write tid 2 to smithi01785587-16 2024-09-21T09:49:12.166 INFO:tasks.rados.rados.0.smithi017.stdout:2572: finishing write tid 3 to smithi01785587-16 2024-09-21T09:49:12.166 INFO:tasks.rados.rados.0.smithi017.stdout:2572: finishing write tid 4 to smithi01785587-16 2024-09-21T09:49:12.167 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1282 (ObjNum 974 snap 331 seq_num 974) dirty exists 2024-09-21T09:49:12.167 INFO:tasks.rados.rados.0.smithi017.stdout:2572: left oid 16 (ObjNum 974 snap 331 seq_num 974) 2024-09-21T09:49:12.167 INFO:tasks.rados.rados.0.smithi017.stdout:2573: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:49:12.424 INFO:tasks.rados.rados.0.smithi017.stdout:2575: finishing write tid 1 to smithi01785587-34 2024-09-21T09:49:12.424 INFO:tasks.rados.rados.0.smithi017.stdout:2575: finishing write tid 2 to smithi01785587-34 2024-09-21T09:49:12.424 INFO:tasks.rados.rados.0.smithi017.stdout:2575: finishing write tid 3 to smithi01785587-34 2024-09-21T09:49:12.424 INFO:tasks.rados.rados.0.smithi017.stdout:2575: finishing write tid 4 to smithi01785587-34 2024-09-21T09:49:12.424 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1336 (ObjNum 975 snap 331 seq_num 975) dirty exists 2024-09-21T09:49:12.424 INFO:tasks.rados.rados.0.smithi017.stdout:2575: left oid 34 (ObjNum 975 snap 331 seq_num 975) 2024-09-21T09:49:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[102913]: osdmap e726: 8 total, 8 up, 8 in 2024-09-21T09:49:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:12.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[105151]: osdmap e726: 8 total, 8 up, 8 in 2024-09-21T09:49:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:12.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:12 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:12.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:12 smithi134 ceph-mon[90085]: osdmap e726: 8 total, 8 up, 8 in 2024-09-21T09:49:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:12 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:12 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:12 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:12 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:12 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:49:13.139 INFO:tasks.rados.rados.0.smithi017.stdout:2572: done (3 left) 2024-09-21T09:49:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:2573: done (2 left) 2024-09-21T09:49:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:2575: done (1 left) 2024-09-21T09:49:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:2576: done (0 left) 2024-09-21T09:49:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:2577: write oid 33 current snap is 332 2024-09-21T09:49:13.140 INFO:tasks.rados.rados.0.smithi017.stdout:2577: seq_num 976 ranges {403632=736208,1763249=619999} 2024-09-21T09:49:13.145 INFO:tasks.rados.rados.0.smithi017.stdout:2577: writing smithi01785587-33 from 403632 to 1139840 tid 1 2024-09-21T09:49:13.153 INFO:tasks.rados.rados.0.smithi017.stdout:2577: writing smithi01785587-33 from 1763249 to 2383248 tid 2 2024-09-21T09:49:13.153 INFO:tasks.rados.rados.0.smithi017.stdout:2578: delete oid 20 current snap is 332 2024-09-21T09:49:13.153 INFO:tasks.rados.rados.0.smithi017.stdout:2577: finishing write tid 1 to smithi01785587-33 2024-09-21T09:49:13.155 INFO:tasks.rados.rados.0.smithi017.stdout:2578: done (1 left) 2024-09-21T09:49:13.155 INFO:tasks.rados.rados.0.smithi017.stdout:2579: snap_create 2024-09-21T09:49:13.156 INFO:tasks.rados.rados.0.smithi017.stdout:2577: finishing write tid 2 to smithi01785587-33 2024-09-21T09:49:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:2577: finishing write tid 3 to smithi01785587-33 2024-09-21T09:49:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:2577: finishing write tid 4 to smithi01785587-33 2024-09-21T09:49:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1293 (ObjNum 976 snap 332 seq_num 976) dirty exists 2024-09-21T09:49:13.157 INFO:tasks.rados.rados.0.smithi017.stdout:2577: left oid 33 (ObjNum 976 snap 332 seq_num 976) 2024-09-21T09:49:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:13 smithi017 ceph-mon[102913]: pgmap v669: 89 pgs: 89 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 817 KiB/s wr, 3 op/s 2024-09-21T09:49:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:13 smithi017 ceph-mon[102913]: osdmap e727: 8 total, 8 up, 8 in 2024-09-21T09:49:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:13 smithi017 ceph-mon[102913]: osdmap e728: 8 total, 8 up, 8 in 2024-09-21T09:49:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:13 smithi017 ceph-mon[105151]: pgmap v669: 89 pgs: 89 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 817 KiB/s wr, 3 op/s 2024-09-21T09:49:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:13 smithi017 ceph-mon[105151]: osdmap e727: 8 total, 8 up, 8 in 2024-09-21T09:49:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:13 smithi017 ceph-mon[105151]: osdmap e728: 8 total, 8 up, 8 in 2024-09-21T09:49:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:13 smithi134 ceph-mon[90085]: pgmap v669: 89 pgs: 89 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 817 KiB/s wr, 3 op/s 2024-09-21T09:49:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:13 smithi134 ceph-mon[90085]: osdmap e727: 8 total, 8 up, 8 in 2024-09-21T09:49:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:13 smithi134 ceph-mon[90085]: osdmap e728: 8 total, 8 up, 8 in 2024-09-21T09:49:14.157 INFO:tasks.rados.rados.0.smithi017.stdout:2577: done (1 left) 2024-09-21T09:49:14.157 INFO:tasks.rados.rados.0.smithi017.stdout:2579: done (0 left) 2024-09-21T09:49:14.157 INFO:tasks.rados.rados.0.smithi017.stdout:2580: rollback oid 15 current snap is 333 2024-09-21T09:49:14.157 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 332 2024-09-21T09:49:14.157 INFO:tasks.rados.rados.0.smithi017.stdout:2581: read oid 28 snap -1 2024-09-21T09:49:14.158 INFO:tasks.rados.rados.0.smithi017.stdout:2581: expect deleted 2024-09-21T09:49:14.158 INFO:tasks.rados.rados.0.smithi017.stdout:2582: read oid 11 snap -1 2024-09-21T09:49:14.158 INFO:tasks.rados.rados.0.smithi017.stdout:2582: expect deleted 2024-09-21T09:49:14.158 INFO:tasks.rados.rados.0.smithi017.stdout:2583: snap_create 2024-09-21T09:49:14.160 INFO:tasks.rados.rados.0.smithi017.stdout:2580: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:49:14.161 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 402 (ObjNum 113 snap 0 seq_num 2399821488) dirty dne 2024-09-21T09:49:15.157 INFO:tasks.rados.rados.0.smithi017.stdout:2580: done (3 left) 2024-09-21T09:49:15.157 INFO:tasks.rados.rados.0.smithi017.stdout:2581: done (2 left) 2024-09-21T09:49:15.157 INFO:tasks.rados.rados.0.smithi017.stdout:2582: done (1 left) 2024-09-21T09:49:15.158 INFO:tasks.rados.rados.0.smithi017.stdout:2583: done (0 left) 2024-09-21T09:49:15.158 INFO:tasks.rados.rados.0.smithi017.stdout:2584: snap_create 2024-09-21T09:49:15.158 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:49:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:49:14] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:49:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:15 smithi017 ceph-mon[102913]: pgmap v672: 89 pgs: 89 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 558 KiB/s wr, 2 op/s 2024-09-21T09:49:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:15 smithi017 ceph-mon[102913]: osdmap e729: 8 total, 8 up, 8 in 2024-09-21T09:49:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:15 smithi017 ceph-mon[105151]: pgmap v672: 89 pgs: 89 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 558 KiB/s wr, 2 op/s 2024-09-21T09:49:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:15 smithi017 ceph-mon[105151]: osdmap e729: 8 total, 8 up, 8 in 2024-09-21T09:49:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:15 smithi134 ceph-mon[90085]: pgmap v672: 89 pgs: 89 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 558 KiB/s wr, 2 op/s 2024-09-21T09:49:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:15 smithi134 ceph-mon[90085]: osdmap e729: 8 total, 8 up, 8 in 2024-09-21T09:49:16.169 INFO:tasks.rados.rados.0.smithi017.stdout:2584: done (0 left) 2024-09-21T09:49:16.169 INFO:tasks.rados.rados.0.smithi017.stdout:2585: read oid 7 snap -1 2024-09-21T09:49:16.169 INFO:tasks.rados.rados.0.smithi017.stdout:2585: expect (ObjNum 967 snap 325 seq_num 967) 2024-09-21T09:49:16.169 INFO:tasks.rados.rados.0.smithi017.stdout:2586: read oid 5 snap -1 2024-09-21T09:49:16.169 INFO:tasks.rados.rados.0.smithi017.stdout:2586: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:49:16.169 INFO:tasks.rados.rados.0.smithi017.stdout:2587: write oid 35 current snap is 335 2024-09-21T09:49:16.170 INFO:tasks.rados.rados.0.smithi017.stdout:2587: seq_num 978 ranges {437246=486706,1363860=623668} 2024-09-21T09:49:16.175 INFO:tasks.rados.rados.0.smithi017.stdout:2587: writing smithi01785587-35 from 437246 to 923952 tid 1 2024-09-21T09:49:16.182 INFO:tasks.rados.rados.0.smithi017.stdout:2587: writing smithi01785587-35 from 1363860 to 1987528 tid 2 2024-09-21T09:49:16.182 INFO:tasks.rados.rados.0.smithi017.stdout:2588: write oid 17 current snap is 335 2024-09-21T09:49:16.182 INFO:tasks.rados.rados.0.smithi017.stdout:2588: seq_num 979 ranges {454053=561955,1422342=483679,2368404=85649} 2024-09-21T09:49:16.187 INFO:tasks.rados.rados.0.smithi017.stdout:2588: writing smithi01785587-17 from 454053 to 1016008 tid 1 2024-09-21T09:49:16.192 INFO:tasks.rados.rados.0.smithi017.stdout:2588: writing smithi01785587-17 from 1422342 to 1906021 tid 2 2024-09-21T09:49:16.195 INFO:tasks.rados.rados.0.smithi017.stdout:2588: writing smithi01785587-17 from 2368404 to 2454053 tid 3 2024-09-21T09:49:16.196 INFO:tasks.rados.rados.0.smithi017.stdout:2589: delete oid 33 current snap is 335 2024-09-21T09:49:16.196 INFO:tasks.rados.rados.0.smithi017.stdout:2585: expect (ObjNum 967 snap 325 seq_num 967) 2024-09-21T09:49:16.330 INFO:tasks.rados.rados.0.smithi017.stdout:2586: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:49:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:16 smithi017 ceph-mon[102913]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T09:49:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:16 smithi017 ceph-mon[105151]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T09:49:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:16 smithi134 ceph-mon[90085]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T09:49:16.586 INFO:tasks.rados.rados.0.smithi017.stdout:2587: finishing write tid 1 to smithi01785587-35 2024-09-21T09:49:16.586 INFO:tasks.rados.rados.0.smithi017.stdout:2587: finishing write tid 2 to smithi01785587-35 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2587: finishing write tid 3 to smithi01785587-35 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2587: finishing write tid 4 to smithi01785587-35 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 285 (ObjNum 978 snap 335 seq_num 978) dirty exists 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2587: left oid 35 (ObjNum 978 snap 335 seq_num 978) 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2588: finishing write tid 1 to smithi01785587-17 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2588: finishing write tid 2 to smithi01785587-17 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2585: done (4 left) 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2586: done (3 left) 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2587: done (2 left) 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2589: done (1 left) 2024-09-21T09:49:16.587 INFO:tasks.rados.rados.0.smithi017.stdout:2590: read oid 44 snap -1 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2590: expect deleted 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2591: read oid 24 snap -1 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2591: expect deleted 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2592: read oid 19 snap -1 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2592: expect (ObjNum 912 snap 311 seq_num 912) 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2593: read oid 47 snap 300 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2593: expect deleted 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2594: read oid 34 snap 325 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2594: expect (ObjNum 957 snap 323 seq_num 957) 2024-09-21T09:49:16.588 INFO:tasks.rados.rados.0.smithi017.stdout:2595: read oid 21 snap -1 2024-09-21T09:49:16.589 INFO:tasks.rados.rados.0.smithi017.stdout:2595: expect (ObjNum 942 snap 319 seq_num 942) 2024-09-21T09:49:16.589 INFO:tasks.rados.rados.0.smithi017.stdout:2596: snap_remove snap 329 2024-09-21T09:49:17.174 INFO:tasks.rados.rados.0.smithi017.stdout:2596: done (7 left) 2024-09-21T09:49:17.175 INFO:tasks.rados.rados.0.smithi017.stdout:2597: write oid 12 current snap is 335 2024-09-21T09:49:17.175 INFO:tasks.rados.rados.0.smithi017.stdout:2597: seq_num 981 ranges {487667=428806,1655659=770995} 2024-09-21T09:49:17.180 INFO:tasks.rados.rados.0.smithi017.stdout:2597: writing smithi01785587-12 from 487667 to 916473 tid 1 2024-09-21T09:49:17.189 INFO:tasks.rados.rados.0.smithi017.stdout:2597: writing smithi01785587-12 from 1655659 to 2426654 tid 2 2024-09-21T09:49:17.189 INFO:tasks.rados.rados.0.smithi017.stdout:2598: rollback oid 15 current snap is 335 2024-09-21T09:49:17.189 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 327 2024-09-21T09:49:17.189 INFO:tasks.rados.rados.0.smithi017.stdout:2588: finishing write tid 3 to smithi01785587-17 2024-09-21T09:49:17.189 INFO:tasks.rados.rados.0.smithi017.stdout:2588: finishing write tid 4 to smithi01785587-17 2024-09-21T09:49:17.189 INFO:tasks.rados.rados.0.smithi017.stdout:2588: finishing write tid 5 to smithi01785587-17 2024-09-21T09:49:17.189 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 836 (ObjNum 979 snap 335 seq_num 979) dirty exists 2024-09-21T09:49:17.190 INFO:tasks.rados.rados.0.smithi017.stdout:2588: left oid 17 (ObjNum 979 snap 335 seq_num 979) 2024-09-21T09:49:17.190 INFO:tasks.rados.rados.0.smithi017.stdout:2588: done (8 left) 2024-09-21T09:49:17.190 INFO:tasks.rados.rados.0.smithi017.stdout:2590: done (7 left) 2024-09-21T09:49:17.190 INFO:tasks.rados.rados.0.smithi017.stdout:2599: write oid 32 current snap is 335 2024-09-21T09:49:17.190 INFO:tasks.rados.rados.0.smithi017.stdout:2599: seq_num 982 ranges {504474=504055,1714141=631006} 2024-09-21T09:49:17.194 INFO:tasks.rados.rados.0.smithi017.stdout:2599: writing smithi01785587-32 from 504474 to 1008529 tid 1 2024-09-21T09:49:17.202 INFO:tasks.rados.rados.0.smithi017.stdout:2599: writing smithi01785587-32 from 1714141 to 2345147 tid 2 2024-09-21T09:49:17.202 INFO:tasks.rados.rados.0.smithi017.stdout:2591: done (7 left) 2024-09-21T09:49:17.202 INFO:tasks.rados.rados.0.smithi017.stdout:2600: write oid 33 current snap is 335 2024-09-21T09:49:17.202 INFO:tasks.rados.rados.0.smithi017.stdout:2594: expect (ObjNum 957 snap 323 seq_num 957) 2024-09-21T09:49:17.273 INFO:tasks.rados.rados.0.smithi017.stdout:2595: expect (ObjNum 942 snap 319 seq_num 942) 2024-09-21T09:49:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:2592: expect (ObjNum 912 snap 311 seq_num 912) 2024-09-21T09:49:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:17 smithi017 ceph-mon[102913]: pgmap v675: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 553 KiB/s wr, 4 op/s 2024-09-21T09:49:17.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:17 smithi017 ceph-mon[102913]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T09:49:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:17 smithi017 ceph-mon[105151]: pgmap v675: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 553 KiB/s wr, 4 op/s 2024-09-21T09:49:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:17 smithi017 ceph-mon[105151]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T09:49:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:17 smithi134 ceph-mon[90085]: pgmap v675: 89 pgs: 89 active+clean; 193 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 553 KiB/s wr, 4 op/s 2024-09-21T09:49:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:17 smithi134 ceph-mon[90085]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T09:49:17.596 INFO:tasks.rados.rados.0.smithi017.stdout:2597: finishing write tid 1 to smithi01785587-12 2024-09-21T09:49:17.596 INFO:tasks.rados.rados.0.smithi017.stdout:2598: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:49:17.596 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 402 (ObjNum -1895179296 snap 21949 seq_num 304) dirty dne 2024-09-21T09:49:17.596 INFO:tasks.rados.rados.0.smithi017.stdout:2597: finishing write tid 2 to smithi01785587-12 2024-09-21T09:49:17.597 INFO:tasks.rados.rados.0.smithi017.stdout:2597: finishing write tid 3 to smithi01785587-12 2024-09-21T09:49:17.597 INFO:tasks.rados.rados.0.smithi017.stdout:2597: finishing write tid 4 to smithi01785587-12 2024-09-21T09:49:17.597 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 890 (ObjNum 981 snap 335 seq_num 981) dirty exists 2024-09-21T09:49:17.597 INFO:tasks.rados.rados.0.smithi017.stdout:2597: left oid 12 (ObjNum 981 snap 335 seq_num 981) 2024-09-21T09:49:17.597 INFO:tasks.rados.rados.0.smithi017.stdout:2599: finishing write tid 1 to smithi01785587-32 2024-09-21T09:49:17.597 INFO:tasks.rados.rados.0.smithi017.stdout:2600: seq_num 983 ranges {521281=579304,1656270=774664} 2024-09-21T09:49:17.602 INFO:tasks.rados.rados.0.smithi017.stdout:2600: writing smithi01785587-33 from 521281 to 1100585 tid 1 2024-09-21T09:49:17.610 INFO:tasks.rados.rados.0.smithi017.stdout:2600: writing smithi01785587-33 from 1656270 to 2430934 tid 2 2024-09-21T09:49:17.610 INFO:tasks.rados.rados.0.smithi017.stdout:2592: done (7 left) 2024-09-21T09:49:17.610 INFO:tasks.rados.rados.0.smithi017.stdout:2593: done (6 left) 2024-09-21T09:49:17.610 INFO:tasks.rados.rados.0.smithi017.stdout:2594: done (5 left) 2024-09-21T09:49:17.610 INFO:tasks.rados.rados.0.smithi017.stdout:2595: done (4 left) 2024-09-21T09:49:17.610 INFO:tasks.rados.rados.0.smithi017.stdout:2597: done (3 left) 2024-09-21T09:49:17.610 INFO:tasks.rados.rados.0.smithi017.stdout:2598: done (2 left) 2024-09-21T09:49:17.610 INFO:tasks.rados.rados.0.smithi017.stdout:2601: read oid 5 snap -1 2024-09-21T09:49:17.611 INFO:tasks.rados.rados.0.smithi017.stdout:2601: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:49:17.611 INFO:tasks.rados.rados.0.smithi017.stdout:2602: rollback oid 47 current snap is 335 2024-09-21T09:49:17.611 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 47 to 305 2024-09-21T09:49:17.611 INFO:tasks.rados.rados.0.smithi017.stdout:2603: snap_remove snap 326 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2603: done (4 left) 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2604: read oid 20 snap 327 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2604: expect (ObjNum 958 snap 323 seq_num 958) 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2605: rollback oid 37 current snap is 335 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 334 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2599: finishing write tid 2 to smithi01785587-32 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2599: finishing write tid 3 to smithi01785587-32 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2599: finishing write tid 4 to smithi01785587-32 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1216 (ObjNum 982 snap 335 seq_num 982) dirty exists 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2599: left oid 32 (ObjNum 982 snap 335 seq_num 982) 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2600: finishing write tid 1 to smithi01785587-33 2024-09-21T09:49:18.182 INFO:tasks.rados.rados.0.smithi017.stdout:2602: finishing rollback tid 1 to smithi01785587-47 2024-09-21T09:49:18.183 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 0 (ObjNum 305 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:18.183 INFO:tasks.rados.rados.0.smithi017.stdout:2601: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:49:18.438 INFO:tasks.rados.rados.0.smithi017.stdout:2600: finishing write tid 2 to smithi01785587-33 2024-09-21T09:49:18.438 INFO:tasks.rados.rados.0.smithi017.stdout:2600: finishing write tid 3 to smithi01785587-33 2024-09-21T09:49:18.438 INFO:tasks.rados.rados.0.smithi017.stdout:2600: finishing write tid 4 to smithi01785587-33 2024-09-21T09:49:18.438 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1298 (ObjNum 983 snap 335 seq_num 983) dirty exists 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2600: left oid 33 (ObjNum 983 snap 335 seq_num 983) 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2599: done (5 left) 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2600: done (4 left) 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2601: done (3 left) 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2602: done (2 left) 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2606: read oid 28 snap -1 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2606: expect deleted 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2607: read oid 44 snap -1 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2607: expect deleted 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2608: rollback oid 21 current snap is 335 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 300 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2605: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:49:18.439 INFO:tasks.rados.rados.0.smithi017.stdout:2604: expect (ObjNum 958 snap 323 seq_num 958) 2024-09-21T09:49:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:18 smithi017 ceph-mon[102913]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T09:49:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:18 smithi017 ceph-mon[105151]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T09:49:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:18 smithi134 ceph-mon[90085]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T09:49:18.527 INFO:tasks.rados.rados.0.smithi017.stdout:2605: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:49:18.527 INFO:tasks.rados.rados.0.smithi017.stdout:2605: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:49:18.527 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1374 (ObjNum 698 snap 221 seq_num 698) dirty exists 2024-09-21T09:49:18.528 INFO:tasks.rados.rados.0.smithi017.stdout:2604: done (4 left) 2024-09-21T09:49:18.528 INFO:tasks.rados.rados.0.smithi017.stdout:2605: done (3 left) 2024-09-21T09:49:18.528 INFO:tasks.rados.rados.0.smithi017.stdout:2609: write oid 26 current snap is 335 2024-09-21T09:49:18.528 INFO:tasks.rados.rados.0.smithi017.stdout:2609: seq_num 984 ranges {538088=654553,1714752=634675} 2024-09-21T09:49:18.534 INFO:tasks.rados.rados.0.smithi017.stdout:2609: writing smithi01785587-26 from 538088 to 1192641 tid 1 2024-09-21T09:49:18.540 INFO:tasks.rados.rados.0.smithi017.stdout:2609: writing smithi01785587-26 from 1714752 to 2349427 tid 2 2024-09-21T09:49:18.540 INFO:tasks.rados.rados.0.smithi017.stdout:2606: done (3 left) 2024-09-21T09:49:18.540 INFO:tasks.rados.rados.0.smithi017.stdout:2607: done (2 left) 2024-09-21T09:49:18.540 INFO:tasks.rados.rados.0.smithi017.stdout:2610: snap_create 2024-09-21T09:49:18.540 INFO:tasks.rados.rados.0.smithi017.stdout:2608: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:49:18.540 INFO:tasks.rados.rados.0.smithi017.stdout:2608: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:49:18.541 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1343 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:18.541 INFO:tasks.rados.rados.0.smithi017.stdout:2609: finishing write tid 1 to smithi01785587-26 2024-09-21T09:49:18.544 INFO:tasks.rados.rados.0.smithi017.stdout:2609: finishing write tid 2 to smithi01785587-26 2024-09-21T09:49:18.544 INFO:tasks.rados.rados.0.smithi017.stdout:2609: finishing write tid 3 to smithi01785587-26 2024-09-21T09:49:18.544 INFO:tasks.rados.rados.0.smithi017.stdout:2609: finishing write tid 4 to smithi01785587-26 2024-09-21T09:49:18.544 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1302 (ObjNum 984 snap 335 seq_num 984) dirty exists 2024-09-21T09:49:18.544 INFO:tasks.rados.rados.0.smithi017.stdout:2609: left oid 26 (ObjNum 984 snap 335 seq_num 984) 2024-09-21T09:49:19.181 INFO:tasks.rados.rados.0.smithi017.stdout:2608: done (2 left) 2024-09-21T09:49:19.181 INFO:tasks.rados.rados.0.smithi017.stdout:2609: done (1 left) 2024-09-21T09:49:19.181 INFO:tasks.rados.rados.0.smithi017.stdout:2610: done (0 left) 2024-09-21T09:49:19.181 INFO:tasks.rados.rados.0.smithi017.stdout:2611: write oid 10 current snap is 336 2024-09-21T09:49:19.181 INFO:tasks.rados.rados.0.smithi017.stdout:2611: seq_num 985 ranges {554895=729802,1773234=778333} 2024-09-21T09:49:19.188 INFO:tasks.rados.rados.0.smithi017.stdout:2611: writing smithi01785587-10 from 554895 to 1284697 tid 1 2024-09-21T09:49:19.196 INFO:tasks.rados.rados.0.smithi017.stdout:2611: writing smithi01785587-10 from 1773234 to 2551567 tid 2 2024-09-21T09:49:19.196 INFO:tasks.rados.rados.0.smithi017.stdout:2612: read oid 7 snap 317 2024-09-21T09:49:19.196 INFO:tasks.rados.rados.0.smithi017.stdout:2612: expect (ObjNum 860 snap 291 seq_num 860) 2024-09-21T09:49:19.196 INFO:tasks.rados.rados.0.smithi017.stdout:2613: rollback oid 15 current snap is 336 2024-09-21T09:49:19.196 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 330 2024-09-21T09:49:19.196 INFO:tasks.rados.rados.0.smithi017.stdout:2611: finishing write tid 1 to smithi01785587-10 2024-09-21T09:49:19.196 INFO:tasks.rados.rados.0.smithi017.stdout:2614: read oid 29 snap 330 2024-09-21T09:49:19.196 INFO:tasks.rados.rados.0.smithi017.stdout:2614: expect deleted 2024-09-21T09:49:19.197 INFO:tasks.rados.rados.0.smithi017.stdout:2615: read oid 35 snap 334 2024-09-21T09:49:19.197 INFO:tasks.rados.rados.0.smithi017.stdout:2615: expect deleted 2024-09-21T09:49:19.197 INFO:tasks.rados.rados.0.smithi017.stdout:2616: delete oid 13 current snap is 336 2024-09-21T09:49:19.198 INFO:tasks.rados.rados.0.smithi017.stdout:2613: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:49:19.198 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 402 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:49:19.200 INFO:tasks.rados.rados.0.smithi017.stdout:2611: finishing write tid 2 to smithi01785587-10 2024-09-21T09:49:19.200 INFO:tasks.rados.rados.0.smithi017.stdout:2613: done (5 left) 2024-09-21T09:49:19.200 INFO:tasks.rados.rados.0.smithi017.stdout:2614: done (4 left) 2024-09-21T09:49:19.200 INFO:tasks.rados.rados.0.smithi017.stdout:2615: done (3 left) 2024-09-21T09:49:19.201 INFO:tasks.rados.rados.0.smithi017.stdout:2616: done (2 left) 2024-09-21T09:49:19.201 INFO:tasks.rados.rados.0.smithi017.stdout:2617: snap_remove snap 305 2024-09-21T09:49:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:19 smithi017 ceph-mon[102913]: pgmap v678: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 784 KiB/s wr, 8 op/s 2024-09-21T09:49:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:19 smithi017 ceph-mon[102913]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T09:49:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:19 smithi017 ceph-mon[105151]: pgmap v678: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 784 KiB/s wr, 8 op/s 2024-09-21T09:49:19.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:19 smithi017 ceph-mon[105151]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T09:49:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:19 smithi134 ceph-mon[90085]: pgmap v678: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 784 KiB/s wr, 8 op/s 2024-09-21T09:49:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:19 smithi134 ceph-mon[90085]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T09:49:20.192 INFO:tasks.rados.rados.0.smithi017.stdout:2617: done (2 left) 2024-09-21T09:49:20.193 INFO:tasks.rados.rados.0.smithi017.stdout:2618: rollback oid 33 current snap is 336 2024-09-21T09:49:20.193 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 332 2024-09-21T09:49:20.193 INFO:tasks.rados.rados.0.smithi017.stdout:2612: expect (ObjNum 860 snap 291 seq_num 860) 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:2611: finishing write tid 3 to smithi01785587-10 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:2611: finishing write tid 4 to smithi01785587-10 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 212 (ObjNum 985 snap 336 seq_num 985) dirty exists 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:2611: left oid 10 (ObjNum 985 snap 336 seq_num 985) 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:2618: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:2618: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:2618: finishing rollback tid 2 to smithi01785587-33 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1306 (ObjNum 976 snap 332 seq_num 976) dirty exists 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:2611: done (2 left) 2024-09-21T09:49:20.417 INFO:tasks.rados.rados.0.smithi017.stdout:2612: done (1 left) 2024-09-21T09:49:20.418 INFO:tasks.rados.rados.0.smithi017.stdout:2618: done (0 left) 2024-09-21T09:49:20.418 INFO:tasks.rados.rados.0.smithi017.stdout:2619: delete oid 19 current snap is 336 2024-09-21T09:49:20.419 INFO:tasks.rados.rados.0.smithi017.stdout:2619: done (0 left) 2024-09-21T09:49:20.419 INFO:tasks.rados.rados.0.smithi017.stdout:2620: snap_remove snap 317 2024-09-21T09:49:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:20 smithi017 ceph-mon[102913]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T09:49:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:20 smithi017 ceph-mon[105151]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T09:49:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:20 smithi134 ceph-mon[90085]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T09:49:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:49:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:49:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:21.194 INFO:tasks.rados.rados.0.smithi017.stdout:2620: done (0 left) 2024-09-21T09:49:21.194 INFO:tasks.rados.rados.0.smithi017.stdout:2621: delete oid 38 current snap is 336 2024-09-21T09:49:21.199 INFO:tasks.rados.rados.0.smithi017.stdout:2621: done (0 left) 2024-09-21T09:49:21.199 INFO:tasks.rados.rados.0.smithi017.stdout:2622: delete oid 32 current snap is 336 2024-09-21T09:49:21.208 INFO:tasks.rados.rados.0.smithi017.stdout:2622: done (0 left) 2024-09-21T09:49:21.208 INFO:tasks.rados.rados.0.smithi017.stdout:2623: snap_remove snap 330 2024-09-21T09:49:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:21 smithi017 ceph-mon[105151]: pgmap v681: 89 pgs: 89 active+clean; 178 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 507 KiB/s wr, 4 op/s 2024-09-21T09:49:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:21 smithi017 ceph-mon[105151]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T09:49:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:21 smithi017 ceph-mon[102913]: pgmap v681: 89 pgs: 89 active+clean; 178 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 507 KiB/s wr, 4 op/s 2024-09-21T09:49:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:21 smithi017 ceph-mon[102913]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T09:49:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:21 smithi134 ceph-mon[90085]: pgmap v681: 89 pgs: 89 active+clean; 178 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 507 KiB/s wr, 4 op/s 2024-09-21T09:49:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:21 smithi134 ceph-mon[90085]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T09:49:22.208 INFO:tasks.rados.rados.0.smithi017.stdout:2623: done (0 left) 2024-09-21T09:49:22.208 INFO:tasks.rados.rados.0.smithi017.stdout:2624: snap_create 2024-09-21T09:49:22.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:22 smithi134 ceph-mon[90085]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T09:49:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:22 smithi017 ceph-mon[102913]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T09:49:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:22 smithi017 ceph-mon[105151]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T09:49:23.143 INFO:tasks.rados.rados.0.smithi017.stdout:2624: done (0 left) 2024-09-21T09:49:23.143 INFO:tasks.rados.rados.0.smithi017.stdout:2625: snap_remove snap 331 2024-09-21T09:49:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:23 smithi134 ceph-mon[90085]: pgmap v684: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 2.1 MiB/s wr, 25 op/s 2024-09-21T09:49:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:23 smithi134 ceph-mon[90085]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T09:49:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:23 smithi134 ceph-mon[90085]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T09:49:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:23 smithi017 ceph-mon[105151]: pgmap v684: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 2.1 MiB/s wr, 25 op/s 2024-09-21T09:49:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:23 smithi017 ceph-mon[105151]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T09:49:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:23 smithi017 ceph-mon[105151]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T09:49:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:23 smithi017 ceph-mon[102913]: pgmap v684: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 2.1 MiB/s wr, 25 op/s 2024-09-21T09:49:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:23 smithi017 ceph-mon[102913]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T09:49:23.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:23 smithi017 ceph-mon[102913]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T09:49:24.146 INFO:tasks.rados.rados.0.smithi017.stdout:2625: done (0 left) 2024-09-21T09:49:24.146 INFO:tasks.rados.rados.0.smithi017.stdout:2626: rollback oid 42 current snap is 337 2024-09-21T09:49:24.146 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 333 2024-09-21T09:49:24.146 INFO:tasks.rados.rados.0.smithi017.stdout:2627: rollback oid 5 current snap is 337 2024-09-21T09:49:24.146 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 328 2024-09-21T09:49:24.146 INFO:tasks.rados.rados.0.smithi017.stdout:2628: delete oid 47 current snap is 337 2024-09-21T09:49:24.147 INFO:tasks.rados.rados.0.smithi017.stdout:2626: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:49:24.147 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:24.152 INFO:tasks.rados.rados.0.smithi017.stdout:2627: finishing rollback tid 0 to smithi01785587-5 2024-09-21T09:49:24.154 INFO:tasks.rados.rados.0.smithi017.stdout:2627: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:49:24.154 INFO:tasks.rados.rados.0.smithi017.stdout:2627: finishing rollback tid 2 to smithi01785587-5 2024-09-21T09:49:24.154 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1310 (ObjNum 949 snap 321 seq_num 949) dirty exists 2024-09-21T09:49:24.154 INFO:tasks.rados.rados.0.smithi017.stdout:2626: done (2 left) 2024-09-21T09:49:24.154 INFO:tasks.rados.rados.0.smithi017.stdout:2627: done (1 left) 2024-09-21T09:49:24.154 INFO:tasks.rados.rados.0.smithi017.stdout:2628: done (0 left) 2024-09-21T09:49:24.155 INFO:tasks.rados.rados.0.smithi017.stdout:2629: write oid 1 current snap is 337 2024-09-21T09:49:24.155 INFO:tasks.rados.rados.0.smithi017.stdout:2629: seq_num 991 ranges {655737=497649,1607773=789340} 2024-09-21T09:49:24.161 INFO:tasks.rados.rados.0.smithi017.stdout:2629: writing smithi01785587-1 from 655737 to 1153386 tid 1 2024-09-21T09:49:24.168 INFO:tasks.rados.rados.0.smithi017.stdout:2629: writing smithi01785587-1 from 1607773 to 2397113 tid 2 2024-09-21T09:49:24.168 INFO:tasks.rados.rados.0.smithi017.stdout:2630: rollback oid 35 current snap is 337 2024-09-21T09:49:24.168 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 336 2024-09-21T09:49:24.168 INFO:tasks.rados.rados.0.smithi017.stdout:2629: finishing write tid 1 to smithi01785587-1 2024-09-21T09:49:24.168 INFO:tasks.rados.rados.0.smithi017.stdout:2631: rollback oid 14 current snap is 337 2024-09-21T09:49:24.168 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 323 2024-09-21T09:49:24.168 INFO:tasks.rados.rados.0.smithi017.stdout:2632: delete oid 16 current snap is 337 2024-09-21T09:49:24.171 INFO:tasks.rados.rados.0.smithi017.stdout:2630: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:49:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2632: done (3 left) 2024-09-21T09:49:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2633: read oid 31 snap -1 2024-09-21T09:49:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2633: expect deleted 2024-09-21T09:49:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2634: write oid 44 current snap is 337 2024-09-21T09:49:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2631: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:49:24.172 INFO:tasks.rados.rados.0.smithi017.stdout:2634: seq_num 993 ranges {689351=648147,2124737=564614} 2024-09-21T09:49:24.179 INFO:tasks.rados.rados.0.smithi017.stdout:2634: writing smithi01785587-44 from 689351 to 1337498 tid 1 2024-09-21T09:49:24.187 INFO:tasks.rados.rados.0.smithi017.stdout:2634: writing smithi01785587-44 from 2124737 to 2689351 tid 2 2024-09-21T09:49:24.187 INFO:tasks.rados.rados.0.smithi017.stdout:2635: read oid 22 snap -1 2024-09-21T09:49:24.187 INFO:tasks.rados.rados.0.smithi017.stdout:2635: expect (ObjNum 898 snap 309 seq_num 898) 2024-09-21T09:49:24.187 INFO:tasks.rados.rados.0.smithi017.stdout:2636: read oid 46 snap -1 2024-09-21T09:49:24.187 INFO:tasks.rados.rados.0.smithi017.stdout:2636: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:49:24.187 INFO:tasks.rados.rados.0.smithi017.stdout:2637: snap_create 2024-09-21T09:49:24.187 INFO:tasks.rados.rados.0.smithi017.stdout:2631: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:2630: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:2629: finishing write tid 2 to smithi01785587-1 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:2629: finishing write tid 3 to smithi01785587-1 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:2630: finishing rollback tid 2 to smithi01785587-35 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 289 (ObjNum 978 snap 335 seq_num 978) dirty exists 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:2631: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1314 (ObjNum 926 snap 314 seq_num 926) dirty exists 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:2629: finishing write tid 4 to smithi01785587-1 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 822 (ObjNum 991 snap 337 seq_num 991) dirty exists 2024-09-21T09:49:24.188 INFO:tasks.rados.rados.0.smithi017.stdout:2629: left oid 1 (ObjNum 991 snap 337 seq_num 991) 2024-09-21T09:49:24.189 INFO:tasks.rados.rados.0.smithi017.stdout:2634: finishing write tid 1 to smithi01785587-44 2024-09-21T09:49:24.190 INFO:tasks.rados.rados.0.smithi017.stdout:2636: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:49:24.301 INFO:tasks.rados.rados.0.smithi017.stdout:2634: finishing write tid 2 to smithi01785587-44 2024-09-21T09:49:24.301 INFO:tasks.rados.rados.0.smithi017.stdout:2634: finishing write tid 3 to smithi01785587-44 2024-09-21T09:49:24.301 INFO:tasks.rados.rados.0.smithi017.stdout:2634: finishing write tid 4 to smithi01785587-44 2024-09-21T09:49:24.302 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1317 (ObjNum 993 snap 337 seq_num 993) dirty exists 2024-09-21T09:49:24.302 INFO:tasks.rados.rados.0.smithi017.stdout:2634: left oid 44 (ObjNum 993 snap 337 seq_num 993) 2024-09-21T09:49:24.302 INFO:tasks.rados.rados.0.smithi017.stdout:2635: expect (ObjNum 898 snap 309 seq_num 898) 2024-09-21T09:49:25.141 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:49:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:49:24] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2629: done (7 left) 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2630: done (6 left) 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2631: done (5 left) 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2633: done (4 left) 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2634: done (3 left) 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2635: done (2 left) 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2636: done (1 left) 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2637: done (0 left) 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2638: write oid 20 current snap is 338 2024-09-21T09:49:25.150 INFO:tasks.rados.rados.0.smithi017.stdout:2638: seq_num 994 ranges {706158=723396,2183219=522939} 2024-09-21T09:49:25.157 INFO:tasks.rados.rados.0.smithi017.stdout:2638: writing smithi01785587-20 from 706158 to 1429554 tid 1 2024-09-21T09:49:25.165 INFO:tasks.rados.rados.0.smithi017.stdout:2638: writing smithi01785587-20 from 2183219 to 2706158 tid 2 2024-09-21T09:49:25.165 INFO:tasks.rados.rados.0.smithi017.stdout:2639: rollback oid 39 current snap is 338 2024-09-21T09:49:25.165 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 336 2024-09-21T09:49:25.165 INFO:tasks.rados.rados.0.smithi017.stdout:2640: delete oid 9 current snap is 338 2024-09-21T09:49:25.165 INFO:tasks.rados.rados.0.smithi017.stdout:2638: finishing write tid 1 to smithi01785587-20 2024-09-21T09:49:25.168 INFO:tasks.rados.rados.0.smithi017.stdout:2638: finishing write tid 2 to smithi01785587-20 2024-09-21T09:49:25.168 INFO:tasks.rados.rados.0.smithi017.stdout:2638: finishing write tid 3 to smithi01785587-20 2024-09-21T09:49:25.168 INFO:tasks.rados.rados.0.smithi017.stdout:2638: finishing write tid 4 to smithi01785587-20 2024-09-21T09:49:25.168 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 716 (ObjNum 994 snap 338 seq_num 994) dirty exists 2024-09-21T09:49:25.168 INFO:tasks.rados.rados.0.smithi017.stdout:2638: left oid 20 (ObjNum 994 snap 338 seq_num 994) 2024-09-21T09:49:25.169 INFO:tasks.rados.rados.0.smithi017.stdout:2638: done (2 left) 2024-09-21T09:49:25.170 INFO:tasks.rados.rados.0.smithi017.stdout:2640: done (1 left) 2024-09-21T09:49:25.170 INFO:tasks.rados.rados.0.smithi017.stdout:2641: read oid 12 snap -1 2024-09-21T09:49:25.170 INFO:tasks.rados.rados.0.smithi017.stdout:2641: expect (ObjNum 981 snap 335 seq_num 981) 2024-09-21T09:49:25.170 INFO:tasks.rados.rados.0.smithi017.stdout:2642: write oid 30 current snap is 338 2024-09-21T09:49:25.170 INFO:tasks.rados.rados.0.smithi017.stdout:2642: seq_num 996 ranges {739772=590247,1900183=540336} 2024-09-21T09:49:25.177 INFO:tasks.rados.rados.0.smithi017.stdout:2642: writing smithi01785587-30 from 739772 to 1330019 tid 1 2024-09-21T09:49:25.183 INFO:tasks.rados.rados.0.smithi017.stdout:2642: writing smithi01785587-30 from 1900183 to 2440519 tid 2 2024-09-21T09:49:25.183 INFO:tasks.rados.rados.0.smithi017.stdout:2643: read oid 3 snap -1 2024-09-21T09:49:25.183 INFO:tasks.rados.rados.0.smithi017.stdout:2643: expect (ObjNum 921 snap 313 seq_num 921) 2024-09-21T09:49:25.183 INFO:tasks.rados.rados.0.smithi017.stdout:2644: snap_remove snap 300 2024-09-21T09:49:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:25 smithi017 ceph-mon[102913]: pgmap v687: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.8 MiB/s wr, 24 op/s 2024-09-21T09:49:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:25 smithi017 ceph-mon[102913]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T09:49:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:25 smithi017 ceph-mon[105151]: pgmap v687: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.8 MiB/s wr, 24 op/s 2024-09-21T09:49:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:25 smithi017 ceph-mon[105151]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T09:49:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:25 smithi134 ceph-mon[90085]: pgmap v687: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.8 MiB/s wr, 24 op/s 2024-09-21T09:49:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:25 smithi134 ceph-mon[90085]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T09:49:26.156 INFO:tasks.rados.rados.0.smithi017.stdout:2644: done (4 left) 2024-09-21T09:49:26.156 INFO:tasks.rados.rados.0.smithi017.stdout:2645: write oid 4 current snap is 338 2024-09-21T09:49:26.156 INFO:tasks.rados.rados.0.smithi017.stdout:2645: seq_num 997 ranges {756579=665496,1958665=400347} 2024-09-21T09:49:26.164 INFO:tasks.rados.rados.0.smithi017.stdout:2645: writing smithi01785587-4 from 756579 to 1422075 tid 1 2024-09-21T09:49:26.170 INFO:tasks.rados.rados.0.smithi017.stdout:2645: writing smithi01785587-4 from 1958665 to 2359012 tid 2 2024-09-21T09:49:26.170 INFO:tasks.rados.rados.0.smithi017.stdout:2646: write oid 25 current snap is 338 2024-09-21T09:49:26.170 INFO:tasks.rados.rados.0.smithi017.stdout:2639: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:49:26.170 INFO:tasks.rados.rados.0.smithi017.stdout:2646: seq_num 998 ranges {773386=740745,2017147=544005} 2024-09-21T09:49:26.178 INFO:tasks.rados.rados.0.smithi017.stdout:2646: writing smithi01785587-25 from 773386 to 1514131 tid 1 2024-09-21T09:49:26.184 INFO:tasks.rados.rados.0.smithi017.stdout:2646: writing smithi01785587-25 from 2017147 to 2561152 tid 2 2024-09-21T09:49:26.184 INFO:tasks.rados.rados.0.smithi017.stdout:2647: snap_create 2024-09-21T09:49:26.184 INFO:tasks.rados.rados.0.smithi017.stdout:2639: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:49:26.184 INFO:tasks.rados.rados.0.smithi017.stdout:2639: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:49:26.184 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1392 (ObjNum 947 snap 321 seq_num 947) dirty exists 2024-09-21T09:49:26.185 INFO:tasks.rados.rados.0.smithi017.stdout:2641: expect (ObjNum 981 snap 335 seq_num 981) 2024-09-21T09:49:26.290 INFO:tasks.rados.rados.0.smithi017.stdout:2642: finishing write tid 1 to smithi01785587-30 2024-09-21T09:49:26.290 INFO:tasks.rados.rados.0.smithi017.stdout:2642: finishing write tid 2 to smithi01785587-30 2024-09-21T09:49:26.290 INFO:tasks.rados.rados.0.smithi017.stdout:2642: finishing write tid 3 to smithi01785587-30 2024-09-21T09:49:26.290 INFO:tasks.rados.rados.0.smithi017.stdout:2642: finishing write tid 4 to smithi01785587-30 2024-09-21T09:49:26.290 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1116 (ObjNum 996 snap 338 seq_num 996) dirty exists 2024-09-21T09:49:26.290 INFO:tasks.rados.rados.0.smithi017.stdout:2642: left oid 30 (ObjNum 996 snap 338 seq_num 996) 2024-09-21T09:49:26.291 INFO:tasks.rados.rados.0.smithi017.stdout:2643: expect (ObjNum 921 snap 313 seq_num 921) 2024-09-21T09:49:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:26 smithi017 ceph-mon[102913]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T09:49:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:26 smithi017 ceph-mon[105151]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T09:49:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:2645: finishing write tid 1 to smithi01785587-4 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:2645: finishing write tid 2 to smithi01785587-4 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:2645: finishing write tid 3 to smithi01785587-4 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:2645: finishing write tid 4 to smithi01785587-4 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1326 (ObjNum 997 snap 338 seq_num 997) dirty exists 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:2645: left oid 4 (ObjNum 997 snap 338 seq_num 997) 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:2646: finishing write tid 1 to smithi01785587-25 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:2646: finishing write tid 2 to smithi01785587-25 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:2646: finishing write tid 3 to smithi01785587-25 2024-09-21T09:49:26.491 INFO:tasks.rados.rados.0.smithi017.stdout:2646: finishing write tid 4 to smithi01785587-25 2024-09-21T09:49:26.492 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 726 (ObjNum 998 snap 338 seq_num 998) dirty exists 2024-09-21T09:49:26.492 INFO:tasks.rados.rados.0.smithi017.stdout:2646: left oid 25 (ObjNum 998 snap 338 seq_num 998) 2024-09-21T09:49:26.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:26 smithi134 ceph-mon[90085]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T09:49:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:27.161 INFO:tasks.rados.rados.0.smithi017.stdout:2639: done (6 left) 2024-09-21T09:49:27.161 INFO:tasks.rados.rados.0.smithi017.stdout:2641: done (5 left) 2024-09-21T09:49:27.161 INFO:tasks.rados.rados.0.smithi017.stdout:2642: done (4 left) 2024-09-21T09:49:27.161 INFO:tasks.rados.rados.0.smithi017.stdout:2643: done (3 left) 2024-09-21T09:49:27.161 INFO:tasks.rados.rados.0.smithi017.stdout:2645: done (2 left) 2024-09-21T09:49:27.161 INFO:tasks.rados.rados.0.smithi017.stdout:2646: done (1 left) 2024-09-21T09:49:27.162 INFO:tasks.rados.rados.0.smithi017.stdout:2647: done (0 left) 2024-09-21T09:49:27.162 INFO:tasks.rados.rados.0.smithi017.stdout:2648: delete oid 37 current snap is 339 2024-09-21T09:49:27.166 INFO:tasks.rados.rados.0.smithi017.stdout:2648: done (0 left) 2024-09-21T09:49:27.166 INFO:tasks.rados.rados.0.smithi017.stdout:2649: snap_create 2024-09-21T09:49:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:27 smithi017 ceph-mon[105151]: pgmap v690: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:49:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:27 smithi017 ceph-mon[105151]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T09:49:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:27 smithi017 ceph-mon[102913]: pgmap v690: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:49:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:27 smithi017 ceph-mon[102913]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T09:49:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:27 smithi134 ceph-mon[90085]: pgmap v690: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:49:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:27 smithi134 ceph-mon[90085]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T09:49:28.145 INFO:tasks.rados.rados.0.smithi017.stdout:2649: done (0 left) 2024-09-21T09:49:28.145 INFO:tasks.rados.rados.0.smithi017.stdout:2650: write oid 43 current snap is 340 2024-09-21T09:49:28.145 INFO:tasks.rados.rados.0.smithi017.stdout:2650: seq_num 1000 ranges {407000=491243,1617758=547674,2595228=211772} 2024-09-21T09:49:28.149 INFO:tasks.rados.rados.0.smithi017.stdout:2650: writing smithi01785587-43 from 407000 to 898243 tid 1 2024-09-21T09:49:28.157 INFO:tasks.rados.rados.0.smithi017.stdout:2650: writing smithi01785587-43 from 1617758 to 2165432 tid 2 2024-09-21T09:49:28.161 INFO:tasks.rados.rados.0.smithi017.stdout:2650: writing smithi01785587-43 from 2595228 to 2807000 tid 3 2024-09-21T09:49:28.162 INFO:tasks.rados.rados.0.smithi017.stdout:2651: delete oid 14 current snap is 340 2024-09-21T09:49:28.162 INFO:tasks.rados.rados.0.smithi017.stdout:2650: finishing write tid 1 to smithi01785587-43 2024-09-21T09:49:28.162 INFO:tasks.rados.rados.0.smithi017.stdout:2650: finishing write tid 2 to smithi01785587-43 2024-09-21T09:49:28.164 INFO:tasks.rados.rados.0.smithi017.stdout:2650: finishing write tid 3 to smithi01785587-43 2024-09-21T09:49:28.169 INFO:tasks.rados.rados.0.smithi017.stdout:2650: finishing write tid 4 to smithi01785587-43 2024-09-21T09:49:28.169 INFO:tasks.rados.rados.0.smithi017.stdout:2651: done (1 left) 2024-09-21T09:49:28.169 INFO:tasks.rados.rados.0.smithi017.stdout:2652: read oid 49 snap -1 2024-09-21T09:49:28.169 INFO:tasks.rados.rados.0.smithi017.stdout:2652: expect deleted 2024-09-21T09:49:28.169 INFO:tasks.rados.rados.0.smithi017.stdout:2653: read oid 27 snap -1 2024-09-21T09:49:28.169 INFO:tasks.rados.rados.0.smithi017.stdout:2653: expect (ObjNum 961 snap 323 seq_num 961) 2024-09-21T09:49:28.170 INFO:tasks.rados.rados.0.smithi017.stdout:2654: write oid 3 current snap is 340 2024-09-21T09:49:28.170 INFO:tasks.rados.rados.0.smithi017.stdout:2654: seq_num 1002 ranges {440614=641741,1734722=551343} 2024-09-21T09:49:28.175 INFO:tasks.rados.rados.0.smithi017.stdout:2654: writing smithi01785587-3 from 440614 to 1082355 tid 1 2024-09-21T09:49:28.182 INFO:tasks.rados.rados.0.smithi017.stdout:2654: writing smithi01785587-3 from 1734722 to 2286065 tid 2 2024-09-21T09:49:28.182 INFO:tasks.rados.rados.0.smithi017.stdout:2655: delete oid 12 current snap is 340 2024-09-21T09:49:28.182 INFO:tasks.rados.rados.0.smithi017.stdout:2650: finishing write tid 5 to smithi01785587-43 2024-09-21T09:49:28.182 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 830 (ObjNum 1000 snap 340 seq_num 1000) dirty exists 2024-09-21T09:49:28.182 INFO:tasks.rados.rados.0.smithi017.stdout:2650: left oid 43 (ObjNum 1000 snap 340 seq_num 1000) 2024-09-21T09:49:28.182 INFO:tasks.rados.rados.0.smithi017.stdout:2653: expect (ObjNum 961 snap 323 seq_num 961) 2024-09-21T09:49:28.317 INFO:tasks.rados.rados.0.smithi017.stdout:2654: finishing write tid 1 to smithi01785587-3 2024-09-21T09:49:28.317 INFO:tasks.rados.rados.0.smithi017.stdout:2654: finishing write tid 2 to smithi01785587-3 2024-09-21T09:49:28.317 INFO:tasks.rados.rados.0.smithi017.stdout:2654: finishing write tid 3 to smithi01785587-3 2024-09-21T09:49:28.317 INFO:tasks.rados.rados.0.smithi017.stdout:2654: finishing write tid 4 to smithi01785587-3 2024-09-21T09:49:28.317 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1331 (ObjNum 1002 snap 340 seq_num 1002) dirty exists 2024-09-21T09:49:28.318 INFO:tasks.rados.rados.0.smithi017.stdout:2654: left oid 3 (ObjNum 1002 snap 340 seq_num 1002) 2024-09-21T09:49:28.318 INFO:tasks.rados.rados.0.smithi017.stdout:2650: done (4 left) 2024-09-21T09:49:28.318 INFO:tasks.rados.rados.0.smithi017.stdout:2652: done (3 left) 2024-09-21T09:49:28.318 INFO:tasks.rados.rados.0.smithi017.stdout:2653: done (2 left) 2024-09-21T09:49:28.318 INFO:tasks.rados.rados.0.smithi017.stdout:2654: done (1 left) 2024-09-21T09:49:28.318 INFO:tasks.rados.rados.0.smithi017.stdout:2655: done (0 left) 2024-09-21T09:49:28.318 INFO:tasks.rados.rados.0.smithi017.stdout:2656: rollback oid 5 current snap is 340 2024-09-21T09:49:28.318 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 323 2024-09-21T09:49:28.318 INFO:tasks.rados.rados.0.smithi017.stdout:2657: snap_create 2024-09-21T09:49:28.320 INFO:tasks.rados.rados.0.smithi017.stdout:2656: finishing rollback tid 0 to smithi01785587-5 2024-09-21T09:49:28.320 INFO:tasks.rados.rados.0.smithi017.stdout:2656: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:49:28.320 INFO:tasks.rados.rados.0.smithi017.stdout:2656: finishing rollback tid 2 to smithi01785587-5 2024-09-21T09:49:28.320 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1335 (ObjNum 949 snap 321 seq_num 949) dirty exists 2024-09-21T09:49:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:28 smithi017 ceph-mon[102913]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T09:49:28.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:28 smithi017 ceph-mon[102913]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T09:49:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:28 smithi017 ceph-mon[105151]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T09:49:28.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:28 smithi017 ceph-mon[105151]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T09:49:28.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:28 smithi134 ceph-mon[90085]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T09:49:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:28 smithi134 ceph-mon[90085]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T09:49:29.171 INFO:tasks.rados.rados.0.smithi017.stdout:2656: done (1 left) 2024-09-21T09:49:29.171 INFO:tasks.rados.rados.0.smithi017.stdout:2657: done (0 left) 2024-09-21T09:49:29.172 INFO:tasks.rados.rados.0.smithi017.stdout:2658: read oid 46 snap -1 2024-09-21T09:49:29.172 INFO:tasks.rados.rados.0.smithi017.stdout:2658: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:49:29.172 INFO:tasks.rados.rados.0.smithi017.stdout:2659: delete oid 17 current snap is 341 2024-09-21T09:49:29.176 INFO:tasks.rados.rados.0.smithi017.stdout:2659: done (1 left) 2024-09-21T09:49:29.176 INFO:tasks.rados.rados.0.smithi017.stdout:2660: write oid 27 current snap is 341 2024-09-21T09:49:29.176 INFO:tasks.rados.rados.0.smithi017.stdout:2660: seq_num 1005 ranges {491035=583841,1510168=698670,2732343=158692} 2024-09-21T09:49:29.182 INFO:tasks.rados.rados.0.smithi017.stdout:2660: writing smithi01785587-27 from 491035 to 1074876 tid 1 2024-09-21T09:49:29.188 INFO:tasks.rados.rados.0.smithi017.stdout:2660: writing smithi01785587-27 from 1510168 to 2208838 tid 2 2024-09-21T09:49:29.192 INFO:tasks.rados.rados.0.smithi017.stdout:2660: writing smithi01785587-27 from 2732343 to 2891035 tid 3 2024-09-21T09:49:29.192 INFO:tasks.rados.rados.0.smithi017.stdout:2661: write oid 16 current snap is 341 2024-09-21T09:49:29.192 INFO:tasks.rados.rados.0.smithi017.stdout:2661: seq_num 1006 ranges {507842=659090,1568650=558681,2759766=148076} 2024-09-21T09:49:29.198 INFO:tasks.rados.rados.0.smithi017.stdout:2661: writing smithi01785587-16 from 507842 to 1166932 tid 1 2024-09-21T09:49:29.204 INFO:tasks.rados.rados.0.smithi017.stdout:2661: writing smithi01785587-16 from 1568650 to 2127331 tid 2 2024-09-21T09:49:29.209 INFO:tasks.rados.rados.0.smithi017.stdout:2661: writing smithi01785587-16 from 2759766 to 2907842 tid 3 2024-09-21T09:49:29.209 INFO:tasks.rados.rados.0.smithi017.stdout:2662: rollback oid 26 current snap is 341 2024-09-21T09:49:29.209 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 26 to 325 2024-09-21T09:49:29.209 INFO:tasks.rados.rados.0.smithi017.stdout:2658: expect (ObjNum 791 snap 271 seq_num 791) 2024-09-21T09:49:29.321 INFO:tasks.rados.rados.0.smithi017.stdout:2660: finishing write tid 1 to smithi01785587-27 2024-09-21T09:49:29.321 INFO:tasks.rados.rados.0.smithi017.stdout:2660: finishing write tid 2 to smithi01785587-27 2024-09-21T09:49:29.321 INFO:tasks.rados.rados.0.smithi017.stdout:2660: finishing write tid 3 to smithi01785587-27 2024-09-21T09:49:29.321 INFO:tasks.rados.rados.0.smithi017.stdout:2660: finishing write tid 4 to smithi01785587-27 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2660: finishing write tid 5 to smithi01785587-27 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 863 (ObjNum 1005 snap 341 seq_num 1005) dirty exists 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2660: left oid 27 (ObjNum 1005 snap 341 seq_num 1005) 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2661: finishing write tid 1 to smithi01785587-16 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2658: done (3 left) 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2660: done (2 left) 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2663: read oid 28 snap 337 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2663: expect deleted 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2664: delete oid 1 current snap is 341 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2661: finishing write tid 2 to smithi01785587-16 2024-09-21T09:49:29.322 INFO:tasks.rados.rados.0.smithi017.stdout:2661: finishing write tid 3 to smithi01785587-16 2024-09-21T09:49:29.323 INFO:tasks.rados.rados.0.smithi017.stdout:2661: finishing write tid 4 to smithi01785587-16 2024-09-21T09:49:29.323 INFO:tasks.rados.rados.0.smithi017.stdout:2661: finishing write tid 5 to smithi01785587-16 2024-09-21T09:49:29.323 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1330 (ObjNum 1006 snap 341 seq_num 1006) dirty exists 2024-09-21T09:49:29.323 INFO:tasks.rados.rados.0.smithi017.stdout:2661: left oid 16 (ObjNum 1006 snap 341 seq_num 1006) 2024-09-21T09:49:29.323 INFO:tasks.rados.rados.0.smithi017.stdout:2662: finishing rollback tid 0 to smithi01785587-26 2024-09-21T09:49:29.323 INFO:tasks.rados.rados.0.smithi017.stdout:2662: finishing rollback tid 1 to smithi01785587-26 2024-09-21T09:49:29.323 INFO:tasks.rados.rados.0.smithi017.stdout:2662: finishing rollback tid 2 to smithi01785587-26 2024-09-21T09:49:29.323 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1334 (ObjNum 946 snap 321 seq_num 946) dirty exists 2024-09-21T09:49:29.324 INFO:tasks.rados.rados.0.smithi017.stdout:2661: done (3 left) 2024-09-21T09:49:29.324 INFO:tasks.rados.rados.0.smithi017.stdout:2662: done (2 left) 2024-09-21T09:49:29.324 INFO:tasks.rados.rados.0.smithi017.stdout:2663: done (1 left) 2024-09-21T09:49:29.324 INFO:tasks.rados.rados.0.smithi017.stdout:2664: done (0 left) 2024-09-21T09:49:29.324 INFO:tasks.rados.rados.0.smithi017.stdout:2665: read oid 18 snap -1 2024-09-21T09:49:29.324 INFO:tasks.rados.rados.0.smithi017.stdout:2665: expect deleted 2024-09-21T09:49:29.324 INFO:tasks.rados.rados.0.smithi017.stdout:2666: read oid 16 snap -1 2024-09-21T09:49:29.324 INFO:tasks.rados.rados.0.smithi017.stdout:2666: expect (ObjNum 1006 snap 341 seq_num 1006) 2024-09-21T09:49:29.324 INFO:tasks.rados.rados.0.smithi017.stdout:2667: write oid 45 current snap is 341 2024-09-21T09:49:29.325 INFO:tasks.rados.rados.0.smithi017.stdout:2667: seq_num 1008 ranges {541456=409588,1569261=562350,2814612=126844} 2024-09-21T09:49:29.328 INFO:tasks.rados.rados.0.smithi017.stdout:2667: writing smithi01785587-45 from 541456 to 951044 tid 1 2024-09-21T09:49:29.335 INFO:tasks.rados.rados.0.smithi017.stdout:2667: writing smithi01785587-45 from 1569261 to 2131611 tid 2 2024-09-21T09:49:29.339 INFO:tasks.rados.rados.0.smithi017.stdout:2667: writing smithi01785587-45 from 2814612 to 2941456 tid 3 2024-09-21T09:49:29.340 INFO:tasks.rados.rados.0.smithi017.stdout:2668: write oid 27 current snap is 341 2024-09-21T09:49:29.340 INFO:tasks.rados.rados.0.smithi017.stdout:2668: seq_num 1009 ranges {558263=484837,1627743=706008} 2024-09-21T09:49:29.345 INFO:tasks.rados.rados.0.smithi017.stdout:2668: writing smithi01785587-27 from 558263 to 1043100 tid 1 2024-09-21T09:49:29.352 INFO:tasks.rados.rados.0.smithi017.stdout:2668: writing smithi01785587-27 from 1627743 to 2333751 tid 2 2024-09-21T09:49:29.353 INFO:tasks.rados.rados.0.smithi017.stdout:2669: write oid 6 current snap is 341 2024-09-21T09:49:29.353 INFO:tasks.rados.rados.0.smithi017.stdout:2669: seq_num 1010 ranges {575070=560086,1686225=566019,2869458=105612} 2024-09-21T09:49:29.359 INFO:tasks.rados.rados.0.smithi017.stdout:2669: writing smithi01785587-6 from 575070 to 1135156 tid 1 2024-09-21T09:49:29.365 INFO:tasks.rados.rados.0.smithi017.stdout:2669: writing smithi01785587-6 from 1686225 to 2252244 tid 2 2024-09-21T09:49:29.369 INFO:tasks.rados.rados.0.smithi017.stdout:2669: writing smithi01785587-6 from 2869458 to 2975070 tid 3 2024-09-21T09:49:29.370 INFO:tasks.rados.rados.0.smithi017.stdout:2670: snap_remove snap 328 2024-09-21T09:49:29.370 INFO:tasks.rados.rados.0.smithi017.stdout:2666: expect (ObjNum 1006 snap 341 seq_num 1006) 2024-09-21T09:49:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:29 smithi017 ceph-mon[102913]: pgmap v693: 89 pgs: 89 active+clean; 173 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:49:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:29 smithi017 ceph-mon[105151]: pgmap v693: 89 pgs: 89 active+clean; 173 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:49:29.466 INFO:tasks.rados.rados.0.smithi017.stdout:2667: finishing write tid 1 to smithi01785587-45 2024-09-21T09:49:29.466 INFO:tasks.rados.rados.0.smithi017.stdout:2667: finishing write tid 2 to smithi01785587-45 2024-09-21T09:49:29.466 INFO:tasks.rados.rados.0.smithi017.stdout:2667: finishing write tid 3 to smithi01785587-45 2024-09-21T09:49:29.466 INFO:tasks.rados.rados.0.smithi017.stdout:2667: finishing write tid 4 to smithi01785587-45 2024-09-21T09:49:29.467 INFO:tasks.rados.rados.0.smithi017.stdout:2667: finishing write tid 5 to smithi01785587-45 2024-09-21T09:49:29.467 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 868 (ObjNum 1008 snap 341 seq_num 1008) dirty exists 2024-09-21T09:49:29.467 INFO:tasks.rados.rados.0.smithi017.stdout:2667: left oid 45 (ObjNum 1008 snap 341 seq_num 1008) 2024-09-21T09:49:29.467 INFO:tasks.rados.rados.0.smithi017.stdout:2668: finishing write tid 1 to smithi01785587-27 2024-09-21T09:49:29.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:29 smithi134 ceph-mon[90085]: pgmap v693: 89 pgs: 89 active+clean; 173 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:49:30.179 INFO:tasks.rados.rados.0.smithi017.stdout:2665: done (5 left) 2024-09-21T09:49:30.180 INFO:tasks.rados.rados.0.smithi017.stdout:2666: done (4 left) 2024-09-21T09:49:30.180 INFO:tasks.rados.rados.0.smithi017.stdout:2667: done (3 left) 2024-09-21T09:49:30.180 INFO:tasks.rados.rados.0.smithi017.stdout:2670: done (2 left) 2024-09-21T09:49:30.180 INFO:tasks.rados.rados.0.smithi017.stdout:2671: rollback oid 32 current snap is 341 2024-09-21T09:49:30.180 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 327 2024-09-21T09:49:30.180 INFO:tasks.rados.rados.0.smithi017.stdout:2672: write oid 34 current snap is 341 2024-09-21T09:49:30.180 INFO:tasks.rados.rados.0.smithi017.stdout:2672: seq_num 1011 ranges {591877=635335,1628354=709677} 2024-09-21T09:49:30.192 INFO:tasks.rados.rados.0.smithi017.stdout:2672: writing smithi01785587-34 from 591877 to 1227212 tid 1 2024-09-21T09:49:30.194 INFO:tasks.rados.rados.0.smithi017.stdout:2672: writing smithi01785587-34 from 1628354 to 2338031 tid 2 2024-09-21T09:49:30.194 INFO:tasks.rados.rados.0.smithi017.stdout:2673: snap_remove snap 340 2024-09-21T09:49:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:30 smithi017 ceph-mon[105151]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T09:49:30.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:30 smithi017 ceph-mon[102913]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T09:49:30.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:30 smithi134 ceph-mon[90085]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T09:49:30.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:49:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:49:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:31.179 INFO:tasks.rados.rados.0.smithi017.stdout:2673: done (4 left) 2024-09-21T09:49:31.179 INFO:tasks.rados.rados.0.smithi017.stdout:2674: read oid 38 snap -1 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2674: expect deleted 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2675: write oid 47 current snap is 341 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2668: finishing write tid 2 to smithi01785587-27 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2668: finishing write tid 3 to smithi01785587-27 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2668: finishing write tid 4 to smithi01785587-27 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 871 (ObjNum 1009 snap 341 seq_num 1009) dirty exists 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2668: left oid 27 (ObjNum 1009 snap 341 seq_num 1009) 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2669: finishing write tid 1 to smithi01785587-6 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2669: finishing write tid 2 to smithi01785587-6 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2669: finishing write tid 3 to smithi01785587-6 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2669: finishing write tid 4 to smithi01785587-6 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2669: finishing write tid 5 to smithi01785587-6 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 836 (ObjNum 1010 snap 341 seq_num 1010) dirty exists 2024-09-21T09:49:31.180 INFO:tasks.rados.rados.0.smithi017.stdout:2669: left oid 6 (ObjNum 1010 snap 341 seq_num 1010) 2024-09-21T09:49:31.181 INFO:tasks.rados.rados.0.smithi017.stdout:2671: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:49:31.181 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1230 (ObjNum 97 snap 0 seq_num 1) dirty dne 2024-09-21T09:49:31.181 INFO:tasks.rados.rados.0.smithi017.stdout:2672: finishing write tid 1 to smithi01785587-34 2024-09-21T09:49:31.181 INFO:tasks.rados.rados.0.smithi017.stdout:2672: finishing write tid 2 to smithi01785587-34 2024-09-21T09:49:31.181 INFO:tasks.rados.rados.0.smithi017.stdout:2672: finishing write tid 3 to smithi01785587-34 2024-09-21T09:49:31.181 INFO:tasks.rados.rados.0.smithi017.stdout:2672: finishing write tid 4 to smithi01785587-34 2024-09-21T09:49:31.181 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1369 (ObjNum 1011 snap 341 seq_num 1011) dirty exists 2024-09-21T09:49:31.181 INFO:tasks.rados.rados.0.smithi017.stdout:2672: left oid 34 (ObjNum 1011 snap 341 seq_num 1011) 2024-09-21T09:49:31.181 INFO:tasks.rados.rados.0.smithi017.stdout:2675: seq_num 1012 ranges {608684=426937,1803189=569688,2924304=84380} 2024-09-21T09:49:31.185 INFO:tasks.rados.rados.0.smithi017.stdout:2675: writing smithi01785587-47 from 608684 to 1035621 tid 1 2024-09-21T09:49:31.192 INFO:tasks.rados.rados.0.smithi017.stdout:2675: writing smithi01785587-47 from 1803189 to 2372877 tid 2 2024-09-21T09:49:31.197 INFO:tasks.rados.rados.0.smithi017.stdout:2675: writing smithi01785587-47 from 2924304 to 3008684 tid 3 2024-09-21T09:49:31.197 INFO:tasks.rados.rados.0.smithi017.stdout:2668: done (5 left) 2024-09-21T09:49:31.197 INFO:tasks.rados.rados.0.smithi017.stdout:2669: done (4 left) 2024-09-21T09:49:31.197 INFO:tasks.rados.rados.0.smithi017.stdout:2671: done (3 left) 2024-09-21T09:49:31.197 INFO:tasks.rados.rados.0.smithi017.stdout:2672: done (2 left) 2024-09-21T09:49:31.197 INFO:tasks.rados.rados.0.smithi017.stdout:2676: read oid 44 snap 327 2024-09-21T09:49:31.197 INFO:tasks.rados.rados.0.smithi017.stdout:2676: expect deleted 2024-09-21T09:49:31.197 INFO:tasks.rados.rados.0.smithi017.stdout:2677: write oid 13 current snap is 341 2024-09-21T09:49:31.197 INFO:tasks.rados.rados.0.smithi017.stdout:2677: seq_num 1013 ranges {625491=502186,1861671=713346} 2024-09-21T09:49:31.203 INFO:tasks.rados.rados.0.smithi017.stdout:2677: writing smithi01785587-13 from 625491 to 1127677 tid 1 2024-09-21T09:49:31.212 INFO:tasks.rados.rados.0.smithi017.stdout:2677: writing smithi01785587-13 from 1861671 to 2575017 tid 2 2024-09-21T09:49:31.212 INFO:tasks.rados.rados.0.smithi017.stdout:2678: snap_create 2024-09-21T09:49:31.212 INFO:tasks.rados.rados.0.smithi017.stdout:2675: finishing write tid 1 to smithi01785587-47 2024-09-21T09:49:31.212 INFO:tasks.rados.rados.0.smithi017.stdout:2675: finishing write tid 2 to smithi01785587-47 2024-09-21T09:49:31.212 INFO:tasks.rados.rados.0.smithi017.stdout:2675: finishing write tid 3 to smithi01785587-47 2024-09-21T09:49:31.212 INFO:tasks.rados.rados.0.smithi017.stdout:2675: finishing write tid 4 to smithi01785587-47 2024-09-21T09:49:31.212 INFO:tasks.rados.rados.0.smithi017.stdout:2675: finishing write tid 5 to smithi01785587-47 2024-09-21T09:49:31.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 907 (ObjNum 1012 snap 341 seq_num 1012) dirty exists 2024-09-21T09:49:31.213 INFO:tasks.rados.rados.0.smithi017.stdout:2675: left oid 47 (ObjNum 1012 snap 341 seq_num 1012) 2024-09-21T09:49:31.213 INFO:tasks.rados.rados.0.smithi017.stdout:2677: finishing write tid 1 to smithi01785587-13 2024-09-21T09:49:31.216 INFO:tasks.rados.rados.0.smithi017.stdout:2677: finishing write tid 2 to smithi01785587-13 2024-09-21T09:49:31.216 INFO:tasks.rados.rados.0.smithi017.stdout:2677: finishing write tid 3 to smithi01785587-13 2024-09-21T09:49:31.216 INFO:tasks.rados.rados.0.smithi017.stdout:2677: finishing write tid 4 to smithi01785587-13 2024-09-21T09:49:31.216 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1127 (ObjNum 1013 snap 341 seq_num 1013) dirty exists 2024-09-21T09:49:31.216 INFO:tasks.rados.rados.0.smithi017.stdout:2677: left oid 13 (ObjNum 1013 snap 341 seq_num 1013) 2024-09-21T09:49:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:31 smithi017 ceph-mon[105151]: pgmap v696: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:49:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:31 smithi017 ceph-mon[105151]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T09:49:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:31 smithi017 ceph-mon[102913]: pgmap v696: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:49:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:31 smithi017 ceph-mon[102913]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T09:49:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:31 smithi134 ceph-mon[90085]: pgmap v696: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:49:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:31 smithi134 ceph-mon[90085]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T09:49:32.188 INFO:tasks.rados.rados.0.smithi017.stdout:2674: done (4 left) 2024-09-21T09:49:32.189 INFO:tasks.rados.rados.0.smithi017.stdout:2675: done (3 left) 2024-09-21T09:49:32.189 INFO:tasks.rados.rados.0.smithi017.stdout:2676: done (2 left) 2024-09-21T09:49:32.189 INFO:tasks.rados.rados.0.smithi017.stdout:2677: done (1 left) 2024-09-21T09:49:32.189 INFO:tasks.rados.rados.0.smithi017.stdout:2678: done (0 left) 2024-09-21T09:49:32.189 INFO:tasks.rados.rados.0.smithi017.stdout:2679: delete oid 4 current snap is 342 2024-09-21T09:49:32.193 INFO:tasks.rados.rados.0.smithi017.stdout:2679: done (0 left) 2024-09-21T09:49:32.193 INFO:tasks.rados.rados.0.smithi017.stdout:2680: rollback oid 2 current snap is 342 2024-09-21T09:49:32.194 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 2 to 341 2024-09-21T09:49:32.194 INFO:tasks.rados.rados.0.smithi017.stdout:2681: snap_create 2024-09-21T09:49:32.198 INFO:tasks.rados.rados.0.smithi017.stdout:2680: finishing rollback tid 0 to smithi01785587-2 2024-09-21T09:49:32.198 INFO:tasks.rados.rados.0.smithi017.stdout:2680: finishing rollback tid 1 to smithi01785587-2 2024-09-21T09:49:32.199 INFO:tasks.rados.rados.0.smithi017.stdout:2680: finishing rollback tid 2 to smithi01785587-2 2024-09-21T09:49:32.199 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 526 (ObjNum 930 snap 314 seq_num 930) dirty exists 2024-09-21T09:49:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:32 smithi017 ceph-mon[102913]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T09:49:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:32 smithi017 ceph-mon[105151]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T09:49:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:32 smithi134 ceph-mon[90085]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T09:49:33.146 INFO:tasks.rados.rados.0.smithi017.stdout:2680: done (1 left) 2024-09-21T09:49:33.146 INFO:tasks.rados.rados.0.smithi017.stdout:2681: done (0 left) 2024-09-21T09:49:33.146 INFO:tasks.rados.rados.0.smithi017.stdout:2682: read oid 35 snap -1 2024-09-21T09:49:33.146 INFO:tasks.rados.rados.0.smithi017.stdout:2682: expect (ObjNum 978 snap 335 seq_num 978) 2024-09-21T09:49:33.146 INFO:tasks.rados.rados.0.smithi017.stdout:2683: write oid 26 current snap is 343 2024-09-21T09:49:33.146 INFO:tasks.rados.rados.0.smithi017.stdout:2683: seq_num 1015 ranges {659105=652684,1862282=717015} 2024-09-21T09:49:33.153 INFO:tasks.rados.rados.0.smithi017.stdout:2683: writing smithi01785587-26 from 659105 to 1311789 tid 1 2024-09-21T09:49:33.162 INFO:tasks.rados.rados.0.smithi017.stdout:2683: writing smithi01785587-26 from 1862282 to 2579297 tid 2 2024-09-21T09:49:33.162 INFO:tasks.rados.rados.0.smithi017.stdout:2684: write oid 41 current snap is 343 2024-09-21T09:49:33.162 INFO:tasks.rados.rados.0.smithi017.stdout:2684: seq_num 1016 ranges {675912=727933,1920764=460673,2801290=274622} 2024-09-21T09:49:33.169 INFO:tasks.rados.rados.0.smithi017.stdout:2684: writing smithi01785587-41 from 675912 to 1403845 tid 1 2024-09-21T09:49:33.175 INFO:tasks.rados.rados.0.smithi017.stdout:2684: writing smithi01785587-41 from 1920764 to 2381437 tid 2 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi017.stdout:2684: writing smithi01785587-41 from 2801290 to 3075912 tid 3 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi017.stdout:2685: read oid 32 snap 338 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi017.stdout:2685: expect deleted 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi017.stdout:2686: write oid 19 current snap is 343 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi017.stdout:2682: expect (ObjNum 978 snap 335 seq_num 978) 2024-09-21T09:49:33.262 INFO:tasks.rados.rados.0.smithi017.stdout:2683: finishing write tid 1 to smithi01785587-26 2024-09-21T09:49:33.262 INFO:tasks.rados.rados.0.smithi017.stdout:2683: finishing write tid 2 to smithi01785587-26 2024-09-21T09:49:33.262 INFO:tasks.rados.rados.0.smithi017.stdout:2683: finishing write tid 3 to smithi01785587-26 2024-09-21T09:49:33.262 INFO:tasks.rados.rados.0.smithi017.stdout:2683: finishing write tid 4 to smithi01785587-26 2024-09-21T09:49:33.262 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1346 (ObjNum 1015 snap 343 seq_num 1015) dirty exists 2024-09-21T09:49:33.262 INFO:tasks.rados.rados.0.smithi017.stdout:2683: left oid 26 (ObjNum 1015 snap 343 seq_num 1015) 2024-09-21T09:49:33.262 INFO:tasks.rados.rados.0.smithi017.stdout:2684: finishing write tid 1 to smithi01785587-41 2024-09-21T09:49:33.262 INFO:tasks.rados.rados.0.smithi017.stdout:2686: seq_num 1017 ranges {692719=403182,1579246=720684,2828713=264006} 2024-09-21T09:49:33.268 INFO:tasks.rados.rados.0.smithi017.stdout:2686: writing smithi01785587-19 from 692719 to 1095901 tid 1 2024-09-21T09:49:33.274 INFO:tasks.rados.rados.0.smithi017.stdout:2686: writing smithi01785587-19 from 1579246 to 2299930 tid 2 2024-09-21T09:49:33.280 INFO:tasks.rados.rados.0.smithi017.stdout:2686: writing smithi01785587-19 from 2828713 to 3092719 tid 3 2024-09-21T09:49:33.280 INFO:tasks.rados.rados.0.smithi017.stdout:2682: done (4 left) 2024-09-21T09:49:33.280 INFO:tasks.rados.rados.0.smithi017.stdout:2683: done (3 left) 2024-09-21T09:49:33.280 INFO:tasks.rados.rados.0.smithi017.stdout:2687: read oid 22 snap -1 2024-09-21T09:49:33.280 INFO:tasks.rados.rados.0.smithi017.stdout:2687: expect (ObjNum 898 snap 309 seq_num 898) 2024-09-21T09:49:33.280 INFO:tasks.rados.rados.0.smithi017.stdout:2688: rollback oid 35 current snap is 343 2024-09-21T09:49:33.280 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 323 2024-09-21T09:49:33.280 INFO:tasks.rados.rados.0.smithi017.stdout:2684: finishing write tid 2 to smithi01785587-41 2024-09-21T09:49:33.280 INFO:tasks.rados.rados.0.smithi017.stdout:2684: finishing write tid 3 to smithi01785587-41 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2684: finishing write tid 4 to smithi01785587-41 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2684: finishing write tid 5 to smithi01785587-41 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 432 (ObjNum 1016 snap 343 seq_num 1016) dirty exists 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2684: left oid 41 (ObjNum 1016 snap 343 seq_num 1016) 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2686: finishing write tid 1 to smithi01785587-19 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2684: done (4 left) 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2685: done (3 left) 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2689: read oid 20 snap -1 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2689: expect (ObjNum 994 snap 338 seq_num 994) 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2690: read oid 41 snap 323 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2690: expect (ObjNum 920 snap 313 seq_num 920) 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2691: read oid 9 snap -1 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2691: expect deleted 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2692: read oid 38 snap 338 2024-09-21T09:49:33.281 INFO:tasks.rados.rados.0.smithi017.stdout:2692: expect deleted 2024-09-21T09:49:33.282 INFO:tasks.rados.rados.0.smithi017.stdout:2693: write oid 25 current snap is 343 2024-09-21T09:49:33.282 INFO:tasks.rados.rados.0.smithi017.stdout:2686: finishing write tid 2 to smithi01785587-19 2024-09-21T09:49:33.282 INFO:tasks.rados.rados.0.smithi017.stdout:2693: seq_num 1018 ranges {709526=478431,1637728=464342,2856136=253390} 2024-09-21T09:49:33.286 INFO:tasks.rados.rados.0.smithi017.stdout:2693: writing smithi01785587-25 from 709526 to 1187957 tid 1 2024-09-21T09:49:33.292 INFO:tasks.rados.rados.0.smithi017.stdout:2693: writing smithi01785587-25 from 1637728 to 2102070 tid 2 2024-09-21T09:49:33.298 INFO:tasks.rados.rados.0.smithi017.stdout:2693: writing smithi01785587-25 from 2856136 to 3109526 tid 3 2024-09-21T09:49:33.298 INFO:tasks.rados.rados.0.smithi017.stdout:2694: snap_remove snap 334 2024-09-21T09:49:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:33 smithi017 ceph-mon[105151]: pgmap v699: 89 pgs: 89 active+clean; 140 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-21T09:49:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:33 smithi017 ceph-mon[105151]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T09:49:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:33 smithi017 ceph-mon[105151]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T09:49:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:33 smithi017 ceph-mon[102913]: pgmap v699: 89 pgs: 89 active+clean; 140 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-21T09:49:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:33 smithi017 ceph-mon[102913]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T09:49:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:33 smithi017 ceph-mon[102913]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T09:49:33.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:33 smithi134 ceph-mon[90085]: pgmap v699: 89 pgs: 89 active+clean; 140 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-21T09:49:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:33 smithi134 ceph-mon[90085]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T09:49:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:33 smithi134 ceph-mon[90085]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T09:49:34.195 INFO:tasks.rados.rados.0.smithi017.stdout:2694: done (8 left) 2024-09-21T09:49:34.195 INFO:tasks.rados.rados.0.smithi017.stdout:2695: read oid 18 snap -1 2024-09-21T09:49:34.195 INFO:tasks.rados.rados.0.smithi017.stdout:2695: expect deleted 2024-09-21T09:49:34.196 INFO:tasks.rados.rados.0.smithi017.stdout:2688: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:49:34.196 INFO:tasks.rados.rados.0.smithi017.stdout:2688: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:49:34.196 INFO:tasks.rados.rados.0.smithi017.stdout:2686: finishing write tid 3 to smithi01785587-19 2024-09-21T09:49:34.197 INFO:tasks.rados.rados.0.smithi017.stdout:2688: finishing rollback tid 2 to smithi01785587-35 2024-09-21T09:49:34.197 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 293 (ObjNum 945 snap 321 seq_num 945) dirty exists 2024-09-21T09:49:34.197 INFO:tasks.rados.rados.0.smithi017.stdout:2686: finishing write tid 4 to smithi01785587-19 2024-09-21T09:49:34.197 INFO:tasks.rados.rados.0.smithi017.stdout:2686: finishing write tid 5 to smithi01785587-19 2024-09-21T09:49:34.197 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 1131 (ObjNum 1017 snap 343 seq_num 1017) dirty exists 2024-09-21T09:49:34.197 INFO:tasks.rados.rados.0.smithi017.stdout:2686: left oid 19 (ObjNum 1017 snap 343 seq_num 1017) 2024-09-21T09:49:34.197 INFO:tasks.rados.rados.0.smithi017.stdout:2689: expect (ObjNum 994 snap 338 seq_num 994) 2024-09-21T09:49:34.287 INFO:tasks.rados.rados.0.smithi017.stdout:2687: expect (ObjNum 898 snap 309 seq_num 898) 2024-09-21T09:49:34.503 INFO:tasks.rados.rados.0.smithi017.stdout:2690: expect (ObjNum 920 snap 313 seq_num 920) 2024-09-21T09:49:34.615 INFO:tasks.rados.rados.0.smithi017.stdout:2693: finishing write tid 1 to smithi01785587-25 2024-09-21T09:49:34.615 INFO:tasks.rados.rados.0.smithi017.stdout:2693: finishing write tid 2 to smithi01785587-25 2024-09-21T09:49:34.615 INFO:tasks.rados.rados.0.smithi017.stdout:2693: finishing write tid 3 to smithi01785587-25 2024-09-21T09:49:34.615 INFO:tasks.rados.rados.0.smithi017.stdout:2693: finishing write tid 4 to smithi01785587-25 2024-09-21T09:49:34.615 INFO:tasks.rados.rados.0.smithi017.stdout:2693: finishing write tid 5 to smithi01785587-25 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 735 (ObjNum 1018 snap 343 seq_num 1018) dirty exists 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2693: left oid 25 (ObjNum 1018 snap 343 seq_num 1018) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2686: done (8 left) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2687: done (7 left) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2688: done (6 left) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2689: done (5 left) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2690: done (4 left) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2691: done (3 left) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2692: done (2 left) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2693: done (1 left) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2696: read oid 10 snap -1 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2696: expect (ObjNum 985 snap 336 seq_num 985) 2024-09-21T09:49:34.616 INFO:tasks.rados.rados.0.smithi017.stdout:2697: snap_create 2024-09-21T09:49:34.619 INFO:tasks.rados.rados.0.smithi017.stdout:2696: expect (ObjNum 985 snap 336 seq_num 985) 2024-09-21T09:49:35.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:49:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:49:34] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.51.0" 2024-09-21T09:49:35.199 INFO:tasks.rados.rados.0.smithi017.stdout:2695: done (2 left) 2024-09-21T09:49:35.199 INFO:tasks.rados.rados.0.smithi017.stdout:2696: done (1 left) 2024-09-21T09:49:35.199 INFO:tasks.rados.rados.0.smithi017.stdout:2697: done (0 left) 2024-09-21T09:49:35.199 INFO:tasks.rados.rados.0.smithi017.stdout:2698: write oid 44 current snap is 344 2024-09-21T09:49:35.199 INFO:tasks.rados.rados.0.smithi017.stdout:2698: seq_num 1019 ranges {726333=670033,2096210=724353} 2024-09-21T09:49:35.207 INFO:tasks.rados.rados.0.smithi017.stdout:2698: writing smithi01785587-44 from 726333 to 1396366 tid 1 2024-09-21T09:49:35.215 INFO:tasks.rados.rados.0.smithi017.stdout:2698: writing smithi01785587-44 from 2096210 to 2820563 tid 2 2024-09-21T09:49:35.215 INFO:tasks.rados.rados.0.smithi017.stdout:2699: write oid 47 current snap is 344 2024-09-21T09:49:35.215 INFO:tasks.rados.rados.0.smithi017.stdout:2698: finishing write tid 1 to smithi01785587-44 2024-09-21T09:49:35.216 INFO:tasks.rados.rados.0.smithi017.stdout:2699: seq_num 1020 ranges {743140=745282,2154692=468011} 2024-09-21T09:49:35.223 INFO:tasks.rados.rados.0.smithi017.stdout:2699: writing smithi01785587-47 from 743140 to 1488422 tid 1 2024-09-21T09:49:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:2699: writing smithi01785587-47 from 2154692 to 2622703 tid 2 2024-09-21T09:49:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:2700: read oid 12 snap -1 2024-09-21T09:49:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:2700: expect deleted 2024-09-21T09:49:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:2701: write oid 14 current snap is 344 2024-09-21T09:49:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:2701: seq_num 1021 ranges {759947=420531,1813174=728022} 2024-09-21T09:49:35.236 INFO:tasks.rados.rados.0.smithi017.stdout:2701: writing smithi01785587-14 from 759947 to 1180478 tid 1 2024-09-21T09:49:35.244 INFO:tasks.rados.rados.0.smithi017.stdout:2701: writing smithi01785587-14 from 1813174 to 2541196 tid 2 2024-09-21T09:49:35.244 INFO:tasks.rados.rados.0.smithi017.stdout:2702: rollback oid 16 current snap is 344 2024-09-21T09:49:35.244 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 16 to 333 2024-09-21T09:49:35.244 INFO:tasks.rados.rados.0.smithi017.stdout:2698: finishing write tid 2 to smithi01785587-44 2024-09-21T09:49:35.245 INFO:tasks.rados.rados.0.smithi017.stdout:2698: finishing write tid 3 to smithi01785587-44 2024-09-21T09:49:35.245 INFO:tasks.rados.rados.0.smithi017.stdout:2698: finishing write tid 4 to smithi01785587-44 2024-09-21T09:49:35.245 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1351 (ObjNum 1019 snap 344 seq_num 1019) dirty exists 2024-09-21T09:49:35.245 INFO:tasks.rados.rados.0.smithi017.stdout:2698: left oid 44 (ObjNum 1019 snap 344 seq_num 1019) 2024-09-21T09:49:35.245 INFO:tasks.rados.rados.0.smithi017.stdout:2699: finishing write tid 1 to smithi01785587-47 2024-09-21T09:49:35.245 INFO:tasks.rados.rados.0.smithi017.stdout:2698: done (4 left) 2024-09-21T09:49:35.245 INFO:tasks.rados.rados.0.smithi017.stdout:2703: snap_remove snap 332 2024-09-21T09:49:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:35 smithi017 ceph-mon[105151]: pgmap v702: 89 pgs: 89 active+clean; 143 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 3.2 MiB/s wr, 25 op/s 2024-09-21T09:49:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:35 smithi017 ceph-mon[105151]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T09:49:35.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:35 smithi017 ceph-mon[102913]: pgmap v702: 89 pgs: 89 active+clean; 143 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 3.2 MiB/s wr, 25 op/s 2024-09-21T09:49:35.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:35 smithi017 ceph-mon[102913]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T09:49:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:35 smithi134 ceph-mon[90085]: pgmap v702: 89 pgs: 89 active+clean; 143 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 3.2 MiB/s wr, 25 op/s 2024-09-21T09:49:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:35 smithi134 ceph-mon[90085]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T09:49:36.209 INFO:tasks.rados.rados.0.smithi017.stdout:2699: finishing write tid 2 to smithi01785587-47 2024-09-21T09:49:36.209 INFO:tasks.rados.rados.0.smithi017.stdout:2699: finishing write tid 3 to smithi01785587-47 2024-09-21T09:49:36.209 INFO:tasks.rados.rados.0.smithi017.stdout:2699: finishing write tid 4 to smithi01785587-47 2024-09-21T09:49:36.209 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 913 (ObjNum 1020 snap 344 seq_num 1020) dirty exists 2024-09-21T09:49:36.209 INFO:tasks.rados.rados.0.smithi017.stdout:2699: left oid 47 (ObjNum 1020 snap 344 seq_num 1020) 2024-09-21T09:49:36.209 INFO:tasks.rados.rados.0.smithi017.stdout:2701: finishing write tid 1 to smithi01785587-14 2024-09-21T09:49:36.209 INFO:tasks.rados.rados.0.smithi017.stdout:2701: finishing write tid 2 to smithi01785587-14 2024-09-21T09:49:36.209 INFO:tasks.rados.rados.0.smithi017.stdout:2701: finishing write tid 3 to smithi01785587-14 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2702: finishing rollback tid 0 to smithi01785587-16 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2702: finishing rollback tid 1 to smithi01785587-16 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2701: finishing write tid 4 to smithi01785587-14 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1354 (ObjNum 1021 snap 344 seq_num 1021) dirty exists 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2701: left oid 14 (ObjNum 1021 snap 344 seq_num 1021) 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2702: finishing rollback tid 2 to smithi01785587-16 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1358 (ObjNum 974 snap 331 seq_num 974) dirty exists 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2699: done (4 left) 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2700: done (3 left) 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2701: done (2 left) 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2702: done (1 left) 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2703: done (0 left) 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2704: rollback oid 16 current snap is 344 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 16 to 343 2024-09-21T09:49:36.210 INFO:tasks.rados.rados.0.smithi017.stdout:2705: read oid 12 snap -1 2024-09-21T09:49:36.211 INFO:tasks.rados.rados.0.smithi017.stdout:2705: expect deleted 2024-09-21T09:49:36.211 INFO:tasks.rados.rados.0.smithi017.stdout:2706: read oid 8 snap -1 2024-09-21T09:49:36.211 INFO:tasks.rados.rados.0.smithi017.stdout:2706: expect deleted 2024-09-21T09:49:36.211 INFO:tasks.rados.rados.0.smithi017.stdout:2707: rollback oid 27 current snap is 344 2024-09-21T09:49:36.211 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 342 2024-09-21T09:49:36.211 INFO:tasks.rados.rados.0.smithi017.stdout:2708: snap_remove snap 325 2024-09-21T09:49:36.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:36 smithi134 ceph-mon[90085]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T09:49:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:36 smithi017 ceph-mon[102913]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T09:49:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:36 smithi017 ceph-mon[105151]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T09:49:37.209 INFO:tasks.rados.rados.0.smithi017.stdout:2708: done (4 left) 2024-09-21T09:49:37.209 INFO:tasks.rados.rados.0.smithi017.stdout:2709: read oid 19 snap 335 2024-09-21T09:49:37.209 INFO:tasks.rados.rados.0.smithi017.stdout:2709: expect (ObjNum 912 snap 311 seq_num 912) 2024-09-21T09:49:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:2710: delete oid 50 current snap is 344 2024-09-21T09:49:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:2707: finishing rollback tid 0 to smithi01785587-27 2024-09-21T09:49:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:2707: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:49:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:2707: finishing rollback tid 2 to smithi01785587-27 2024-09-21T09:49:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 897 (ObjNum 1009 snap 341 seq_num 1009) dirty exists 2024-09-21T09:49:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:2704: finishing rollback tid 0 to smithi01785587-16 2024-09-21T09:49:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:2704: finishing rollback tid 1 to smithi01785587-16 2024-09-21T09:49:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:2704: finishing rollback tid 2 to smithi01785587-16 2024-09-21T09:49:37.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1365 (ObjNum 1006 snap 341 seq_num 1006) dirty exists 2024-09-21T09:49:37.219 INFO:tasks.rados.rados.0.smithi017.stdout:2704: done (5 left) 2024-09-21T09:49:37.219 INFO:tasks.rados.rados.0.smithi017.stdout:2705: done (4 left) 2024-09-21T09:49:37.219 INFO:tasks.rados.rados.0.smithi017.stdout:2706: done (3 left) 2024-09-21T09:49:37.219 INFO:tasks.rados.rados.0.smithi017.stdout:2707: done (2 left) 2024-09-21T09:49:37.219 INFO:tasks.rados.rados.0.smithi017.stdout:2710: done (1 left) 2024-09-21T09:49:37.219 INFO:tasks.rados.rados.0.smithi017.stdout:2711: write oid 31 current snap is 344 2024-09-21T09:49:37.219 INFO:tasks.rados.rados.0.smithi017.stdout:2711: seq_num 1023 ranges {793561=571029,1813785=731691,2993251=200310} 2024-09-21T09:49:37.224 INFO:tasks.rados.rados.0.smithi017.stdout:2711: writing smithi01785587-31 from 793561 to 1364590 tid 1 2024-09-21T09:49:37.230 INFO:tasks.rados.rados.0.smithi017.stdout:2711: writing smithi01785587-31 from 1813785 to 2545476 tid 2 2024-09-21T09:49:37.234 INFO:tasks.rados.rados.0.smithi017.stdout:2711: writing smithi01785587-31 from 2993251 to 3193561 tid 3 2024-09-21T09:49:37.234 INFO:tasks.rados.rados.0.smithi017.stdout:2712: snap_remove snap 336 2024-09-21T09:49:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:37 smithi134 ceph-mon[90085]: pgmap v705: 89 pgs: 89 active+clean; 159 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.5 MiB/s wr, 22 op/s 2024-09-21T09:49:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:37 smithi134 ceph-mon[90085]: osdmap e751: 8 total, 8 up, 8 in 2024-09-21T09:49:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:37 smithi017 ceph-mon[102913]: pgmap v705: 89 pgs: 89 active+clean; 159 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.5 MiB/s wr, 22 op/s 2024-09-21T09:49:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:37 smithi017 ceph-mon[102913]: osdmap e751: 8 total, 8 up, 8 in 2024-09-21T09:49:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:37 smithi017 ceph-mon[105151]: pgmap v705: 89 pgs: 89 active+clean; 159 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.5 MiB/s wr, 22 op/s 2024-09-21T09:49:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:37 smithi017 ceph-mon[105151]: osdmap e751: 8 total, 8 up, 8 in 2024-09-21T09:49:38.147 INFO:tasks.rados.rados.0.smithi017.stdout:2712: done (2 left) 2024-09-21T09:49:38.147 INFO:tasks.rados.rados.0.smithi017.stdout:2713: delete oid 50 current snap is 344 2024-09-21T09:49:38.147 INFO:tasks.rados.rados.0.smithi017.stdout:2709: expect (ObjNum 912 snap 311 seq_num 912) 2024-09-21T09:49:38.351 INFO:tasks.rados.rados.0.smithi017.stdout:2711: finishing write tid 1 to smithi01785587-31 2024-09-21T09:49:38.351 INFO:tasks.rados.rados.0.smithi017.stdout:2711: finishing write tid 2 to smithi01785587-31 2024-09-21T09:49:38.351 INFO:tasks.rados.rados.0.smithi017.stdout:2711: finishing write tid 3 to smithi01785587-31 2024-09-21T09:49:38.351 INFO:tasks.rados.rados.0.smithi017.stdout:2711: finishing write tid 4 to smithi01785587-31 2024-09-21T09:49:38.351 INFO:tasks.rados.rados.0.smithi017.stdout:2711: finishing write tid 5 to smithi01785587-31 2024-09-21T09:49:38.351 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1372 (ObjNum 1023 snap 344 seq_num 1023) dirty exists 2024-09-21T09:49:38.351 INFO:tasks.rados.rados.0.smithi017.stdout:2711: left oid 31 (ObjNum 1023 snap 344 seq_num 1023) 2024-09-21T09:49:38.351 INFO:tasks.rados.rados.0.smithi017.stdout:2709: done (2 left) 2024-09-21T09:49:38.351 INFO:tasks.rados.rados.0.smithi017.stdout:2711: done (1 left) 2024-09-21T09:49:38.352 INFO:tasks.rados.rados.0.smithi017.stdout:2713: done (0 left) 2024-09-21T09:49:38.352 INFO:tasks.rados.rados.0.smithi017.stdout:2714: read oid 38 snap 342 2024-09-21T09:49:38.352 INFO:tasks.rados.rados.0.smithi017.stdout:2714: expect deleted 2024-09-21T09:49:38.352 INFO:tasks.rados.rados.0.smithi017.stdout:2715: write oid 26 current snap is 344 2024-09-21T09:49:38.352 INFO:tasks.rados.rados.0.smithi017.stdout:2715: seq_num 1025 ranges {427175=721527,1930749=735360} 2024-09-21T09:49:38.357 INFO:tasks.rados.rados.0.smithi017.stdout:2715: writing smithi01785587-26 from 427175 to 1148702 tid 1 2024-09-21T09:49:38.365 INFO:tasks.rados.rados.0.smithi017.stdout:2715: writing smithi01785587-26 from 1930749 to 2666109 tid 2 2024-09-21T09:49:38.365 INFO:tasks.rados.rados.0.smithi017.stdout:2716: read oid 36 snap -1 2024-09-21T09:49:38.365 INFO:tasks.rados.rados.0.smithi017.stdout:2716: expect (ObjNum 952 snap 321 seq_num 952) 2024-09-21T09:49:38.366 INFO:tasks.rados.rados.0.smithi017.stdout:2717: snap_create 2024-09-21T09:49:38.366 INFO:tasks.rados.rados.0.smithi017.stdout:2715: finishing write tid 1 to smithi01785587-26 2024-09-21T09:49:38.369 INFO:tasks.rados.rados.0.smithi017.stdout:2716: expect (ObjNum 952 snap 321 seq_num 952) 2024-09-21T09:49:38.434 INFO:tasks.rados.rados.0.smithi017.stdout:2715: finishing write tid 2 to smithi01785587-26 2024-09-21T09:49:38.434 INFO:tasks.rados.rados.0.smithi017.stdout:2715: finishing write tid 3 to smithi01785587-26 2024-09-21T09:49:38.434 INFO:tasks.rados.rados.0.smithi017.stdout:2715: finishing write tid 4 to smithi01785587-26 2024-09-21T09:49:38.434 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1381 (ObjNum 1025 snap 344 seq_num 1025) dirty exists 2024-09-21T09:49:38.434 INFO:tasks.rados.rados.0.smithi017.stdout:2715: left oid 26 (ObjNum 1025 snap 344 seq_num 1025) 2024-09-21T09:49:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:38 smithi134 ceph-mon[90085]: osdmap e752: 8 total, 8 up, 8 in 2024-09-21T09:49:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:38 smithi134 ceph-mon[90085]: osdmap e753: 8 total, 8 up, 8 in 2024-09-21T09:49:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:38 smithi017 ceph-mon[102913]: osdmap e752: 8 total, 8 up, 8 in 2024-09-21T09:49:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:38 smithi017 ceph-mon[102913]: osdmap e753: 8 total, 8 up, 8 in 2024-09-21T09:49:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:38 smithi017 ceph-mon[105151]: osdmap e752: 8 total, 8 up, 8 in 2024-09-21T09:49:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:38 smithi017 ceph-mon[105151]: osdmap e753: 8 total, 8 up, 8 in 2024-09-21T09:49:39.225 INFO:tasks.rados.rados.0.smithi017.stdout:2714: done (3 left) 2024-09-21T09:49:39.225 INFO:tasks.rados.rados.0.smithi017.stdout:2715: done (2 left) 2024-09-21T09:49:39.225 INFO:tasks.rados.rados.0.smithi017.stdout:2716: done (1 left) 2024-09-21T09:49:39.226 INFO:tasks.rados.rados.0.smithi017.stdout:2717: done (0 left) 2024-09-21T09:49:39.226 INFO:tasks.rados.rados.0.smithi017.stdout:2718: read oid 4 snap -1 2024-09-21T09:49:39.226 INFO:tasks.rados.rados.0.smithi017.stdout:2718: expect deleted 2024-09-21T09:49:39.226 INFO:tasks.rados.rados.0.smithi017.stdout:2719: snap_remove snap 342 2024-09-21T09:49:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:39 smithi134 ceph-mon[90085]: pgmap v708: 89 pgs: 89 active+clean; 160 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-21T09:49:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:39 smithi017 ceph-mon[102913]: pgmap v708: 89 pgs: 89 active+clean; 160 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-21T09:49:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:39 smithi017 ceph-mon[105151]: pgmap v708: 89 pgs: 89 active+clean; 160 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-21T09:49:40.233 INFO:tasks.rados.rados.0.smithi017.stdout:2719: done (1 left) 2024-09-21T09:49:40.233 INFO:tasks.rados.rados.0.smithi017.stdout:2720: snap_remove snap 333 2024-09-21T09:49:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:40 smithi134 ceph-mon[90085]: osdmap e754: 8 total, 8 up, 8 in 2024-09-21T09:49:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:49:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:49:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:40 smithi017 ceph-mon[102913]: osdmap e754: 8 total, 8 up, 8 in 2024-09-21T09:49:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:40 smithi017 ceph-mon[105151]: osdmap e754: 8 total, 8 up, 8 in 2024-09-21T09:49:41.238 INFO:tasks.rados.rados.0.smithi017.stdout:2720: done (1 left) 2024-09-21T09:49:41.238 INFO:tasks.rados.rados.0.smithi017.stdout:2721: write oid 16 current snap is 345 2024-09-21T09:49:41.238 INFO:tasks.rados.rados.0.smithi017.stdout:2721: seq_num 1026 ranges {443982=796776,1989231=479018,2959167=284815} 2024-09-21T09:49:41.246 INFO:tasks.rados.rados.0.smithi017.stdout:2721: writing smithi01785587-16 from 443982 to 1240758 tid 1 2024-09-21T09:49:41.253 INFO:tasks.rados.rados.0.smithi017.stdout:2721: writing smithi01785587-16 from 1989231 to 2468249 tid 2 2024-09-21T09:49:41.257 INFO:tasks.rados.rados.0.smithi017.stdout:2721: writing smithi01785587-16 from 2959167 to 3243982 tid 3 2024-09-21T09:49:41.257 INFO:tasks.rados.rados.0.smithi017.stdout:2722: snap_create 2024-09-21T09:49:41.257 INFO:tasks.rados.rados.0.smithi017.stdout:2721: finishing write tid 1 to smithi01785587-16 2024-09-21T09:49:41.257 INFO:tasks.rados.rados.0.smithi017.stdout:2721: finishing write tid 2 to smithi01785587-16 2024-09-21T09:49:41.260 INFO:tasks.rados.rados.0.smithi017.stdout:2721: finishing write tid 3 to smithi01785587-16 2024-09-21T09:49:41.260 INFO:tasks.rados.rados.0.smithi017.stdout:2721: finishing write tid 4 to smithi01785587-16 2024-09-21T09:49:41.260 INFO:tasks.rados.rados.0.smithi017.stdout:2721: finishing write tid 5 to smithi01785587-16 2024-09-21T09:49:41.260 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1398 (ObjNum 1026 snap 345 seq_num 1026) dirty exists 2024-09-21T09:49:41.260 INFO:tasks.rados.rados.0.smithi017.stdout:2721: left oid 16 (ObjNum 1026 snap 345 seq_num 1026) 2024-09-21T09:49:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:41 smithi134 ceph-mon[90085]: pgmap v711: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 160 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 296 KiB/s wr, 3 op/s 2024-09-21T09:49:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:41 smithi134 ceph-mon[90085]: osdmap e755: 8 total, 8 up, 8 in 2024-09-21T09:49:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:41.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:41 smithi017 ceph-mon[102913]: pgmap v711: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 160 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 296 KiB/s wr, 3 op/s 2024-09-21T09:49:41.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:41 smithi017 ceph-mon[102913]: osdmap e755: 8 total, 8 up, 8 in 2024-09-21T09:49:41.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:41.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:41 smithi017 ceph-mon[105151]: pgmap v711: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 160 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 296 KiB/s wr, 3 op/s 2024-09-21T09:49:41.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:41 smithi017 ceph-mon[105151]: osdmap e755: 8 total, 8 up, 8 in 2024-09-21T09:49:41.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:42.244 INFO:tasks.rados.rados.0.smithi017.stdout:2718: done (2 left) 2024-09-21T09:49:42.244 INFO:tasks.rados.rados.0.smithi017.stdout:2721: done (1 left) 2024-09-21T09:49:42.244 INFO:tasks.rados.rados.0.smithi017.stdout:2722: done (0 left) 2024-09-21T09:49:42.244 INFO:tasks.rados.rados.0.smithi017.stdout:2723: read oid 27 snap 337 2024-09-21T09:49:42.244 INFO:tasks.rados.rados.0.smithi017.stdout:2723: expect (ObjNum 961 snap 323 seq_num 961) 2024-09-21T09:49:42.244 INFO:tasks.rados.rados.0.smithi017.stdout:2724: write oid 30 current snap is 346 2024-09-21T09:49:42.244 INFO:tasks.rados.rados.0.smithi017.stdout:2724: seq_num 1027 ranges {460789=588378,1764066=739029,3219296=41493} 2024-09-21T09:49:42.249 INFO:tasks.rados.rados.0.smithi017.stdout:2724: writing smithi01785587-30 from 460789 to 1049167 tid 1 2024-09-21T09:49:42.257 INFO:tasks.rados.rados.0.smithi017.stdout:2724: writing smithi01785587-30 from 1764066 to 2503095 tid 2 2024-09-21T09:49:42.262 INFO:tasks.rados.rados.0.smithi017.stdout:2724: writing smithi01785587-30 from 3219296 to 3260789 tid 3 2024-09-21T09:49:42.262 INFO:tasks.rados.rados.0.smithi017.stdout:2725: write oid 24 current snap is 346 2024-09-21T09:49:42.262 INFO:tasks.rados.rados.0.smithi017.stdout:2725: seq_num 1028 ranges {477596=663627,1706195=482687,2614013=567041} 2024-09-21T09:49:42.268 INFO:tasks.rados.rados.0.smithi017.stdout:2725: writing smithi01785587-24 from 477596 to 1141223 tid 1 2024-09-21T09:49:42.274 INFO:tasks.rados.rados.0.smithi017.stdout:2725: writing smithi01785587-24 from 1706195 to 2188882 tid 2 2024-09-21T09:49:42.279 INFO:tasks.rados.rados.0.smithi017.stdout:2725: writing smithi01785587-24 from 2614013 to 3181054 tid 3 2024-09-21T09:49:42.279 INFO:tasks.rados.rados.0.smithi017.stdout:2726: snap_remove snap 339 2024-09-21T09:49:42.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:42 smithi134 ceph-mon[90085]: osdmap e756: 8 total, 8 up, 8 in 2024-09-21T09:49:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:42 smithi017 ceph-mon[102913]: osdmap e756: 8 total, 8 up, 8 in 2024-09-21T09:49:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:42 smithi017 ceph-mon[105151]: osdmap e756: 8 total, 8 up, 8 in 2024-09-21T09:49:43.151 INFO:tasks.rados.rados.0.smithi017.stdout:2723: expect (ObjNum 961 snap 323 seq_num 961) 2024-09-21T09:49:43.287 INFO:tasks.rados.rados.0.smithi017.stdout:2724: finishing write tid 1 to smithi01785587-30 2024-09-21T09:49:43.287 INFO:tasks.rados.rados.0.smithi017.stdout:2724: finishing write tid 2 to smithi01785587-30 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2724: finishing write tid 3 to smithi01785587-30 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2724: finishing write tid 4 to smithi01785587-30 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2724: finishing write tid 5 to smithi01785587-30 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1154 (ObjNum 1027 snap 346 seq_num 1027) dirty exists 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2724: left oid 30 (ObjNum 1027 snap 346 seq_num 1027) 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2723: done (3 left) 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2724: done (2 left) 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2726: done (1 left) 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2727: read oid 41 snap -1 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2727: expect (ObjNum 1016 snap 343 seq_num 1016) 2024-09-21T09:49:43.288 INFO:tasks.rados.rados.0.smithi017.stdout:2728: read oid 2 snap 338 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2728: expect (ObjNum 930 snap 314 seq_num 930) 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2729: rollback oid 3 current snap is 346 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 327 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2725: finishing write tid 1 to smithi01785587-24 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2725: finishing write tid 2 to smithi01785587-24 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2725: finishing write tid 3 to smithi01785587-24 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2725: finishing write tid 4 to smithi01785587-24 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2725: finishing write tid 5 to smithi01785587-24 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 912 (ObjNum 1028 snap 346 seq_num 1028) dirty exists 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2725: left oid 24 (ObjNum 1028 snap 346 seq_num 1028) 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2725: done (3 left) 2024-09-21T09:49:43.289 INFO:tasks.rados.rados.0.smithi017.stdout:2730: snap_create 2024-09-21T09:49:43.290 INFO:tasks.rados.rados.0.smithi017.stdout:2728: expect (ObjNum 930 snap 314 seq_num 930) 2024-09-21T09:49:43.409 INFO:tasks.rados.rados.0.smithi017.stdout:2729: finishing rollback tid 0 to smithi01785587-3 2024-09-21T09:49:43.409 INFO:tasks.rados.rados.0.smithi017.stdout:2729: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:49:43.409 INFO:tasks.rados.rados.0.smithi017.stdout:2729: finishing rollback tid 2 to smithi01785587-3 2024-09-21T09:49:43.409 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1398 (ObjNum 921 snap 313 seq_num 921) dirty exists 2024-09-21T09:49:43.409 INFO:tasks.rados.rados.0.smithi017.stdout:2727: expect (ObjNum 1016 snap 343 seq_num 1016) 2024-09-21T09:49:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:43 smithi134 ceph-mon[90085]: pgmap v714: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 168 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:49:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:43 smithi134 ceph-mon[90085]: osdmap e757: 8 total, 8 up, 8 in 2024-09-21T09:49:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:43 smithi134 ceph-mon[90085]: osdmap e758: 8 total, 8 up, 8 in 2024-09-21T09:49:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:43 smithi017 ceph-mon[102913]: pgmap v714: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 168 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:49:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:43 smithi017 ceph-mon[102913]: osdmap e757: 8 total, 8 up, 8 in 2024-09-21T09:49:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:43 smithi017 ceph-mon[102913]: osdmap e758: 8 total, 8 up, 8 in 2024-09-21T09:49:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:43 smithi017 ceph-mon[105151]: pgmap v714: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 168 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:49:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:43 smithi017 ceph-mon[105151]: osdmap e757: 8 total, 8 up, 8 in 2024-09-21T09:49:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:43 smithi017 ceph-mon[105151]: osdmap e758: 8 total, 8 up, 8 in 2024-09-21T09:49:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2727: done (3 left) 2024-09-21T09:49:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2728: done (2 left) 2024-09-21T09:49:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2729: done (1 left) 2024-09-21T09:49:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2730: done (0 left) 2024-09-21T09:49:44.250 INFO:tasks.rados.rados.0.smithi017.stdout:2731: rollback oid 32 current snap is 347 2024-09-21T09:49:44.251 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 335 2024-09-21T09:49:44.251 INFO:tasks.rados.rados.0.smithi017.stdout:2732: delete oid 43 current snap is 347 2024-09-21T09:49:44.252 INFO:tasks.rados.rados.0.smithi017.stdout:2731: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:49:44.252 INFO:tasks.rados.rados.0.smithi017.stdout:2731: finishing rollback tid 2 to smithi01785587-32 2024-09-21T09:49:44.252 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1243 (ObjNum 982 snap 335 seq_num 982) dirty exists 2024-09-21T09:49:44.253 INFO:tasks.rados.rados.0.smithi017.stdout:2731: done (1 left) 2024-09-21T09:49:44.254 INFO:tasks.rados.rados.0.smithi017.stdout:2732: done (0 left) 2024-09-21T09:49:44.254 INFO:tasks.rados.rados.0.smithi017.stdout:2733: read oid 8 snap -1 2024-09-21T09:49:44.254 INFO:tasks.rados.rados.0.smithi017.stdout:2733: expect deleted 2024-09-21T09:49:44.254 INFO:tasks.rados.rados.0.smithi017.stdout:2734: snap_create 2024-09-21T09:49:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:49:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:49:44] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.51.0" 2024-09-21T09:49:45.251 INFO:tasks.rados.rados.0.smithi017.stdout:2733: done (1 left) 2024-09-21T09:49:45.251 INFO:tasks.rados.rados.0.smithi017.stdout:2734: done (0 left) 2024-09-21T09:49:45.251 INFO:tasks.rados.rados.0.smithi017.stdout:2735: read oid 7 snap 346 2024-09-21T09:49:45.251 INFO:tasks.rados.rados.0.smithi017.stdout:2735: expect (ObjNum 967 snap 325 seq_num 967) 2024-09-21T09:49:45.251 INFO:tasks.rados.rados.0.smithi017.stdout:2736: write oid 17 current snap is 348 2024-09-21T09:49:45.251 INFO:tasks.rados.rados.0.smithi017.stdout:2736: seq_num 1030 ranges {511210=414125,1423159=486356,2668859=589585} 2024-09-21T09:49:45.256 INFO:tasks.rados.rados.0.smithi017.stdout:2736: writing smithi01785587-17 from 511210 to 925335 tid 1 2024-09-21T09:49:45.262 INFO:tasks.rados.rados.0.smithi017.stdout:2736: writing smithi01785587-17 from 1423159 to 1909515 tid 2 2024-09-21T09:49:45.270 INFO:tasks.rados.rados.0.smithi017.stdout:2736: writing smithi01785587-17 from 2668859 to 3258444 tid 3 2024-09-21T09:49:45.270 INFO:tasks.rados.rados.0.smithi017.stdout:2737: read oid 13 snap -1 2024-09-21T09:49:45.270 INFO:tasks.rados.rados.0.smithi017.stdout:2737: expect (ObjNum 1013 snap 341 seq_num 1013) 2024-09-21T09:49:45.270 INFO:tasks.rados.rados.0.smithi017.stdout:2738: read oid 40 snap -1 2024-09-21T09:49:45.270 INFO:tasks.rados.rados.0.smithi017.stdout:2738: expect deleted 2024-09-21T09:49:45.270 INFO:tasks.rados.rados.0.smithi017.stdout:2739: write oid 11 current snap is 348 2024-09-21T09:49:45.270 INFO:tasks.rados.rados.0.smithi017.stdout:2739: seq_num 1031 ranges {528017=489374,1481641=630014,2696282=400857} 2024-09-21T09:49:45.275 INFO:tasks.rados.rados.0.smithi017.stdout:2739: writing smithi01785587-11 from 528017 to 1017391 tid 1 2024-09-21T09:49:45.281 INFO:tasks.rados.rados.0.smithi017.stdout:2739: writing smithi01785587-11 from 1481641 to 2111655 tid 2 2024-09-21T09:49:45.287 INFO:tasks.rados.rados.0.smithi017.stdout:2739: writing smithi01785587-11 from 2696282 to 3097139 tid 3 2024-09-21T09:49:45.287 INFO:tasks.rados.rados.0.smithi017.stdout:2740: read oid 34 snap -1 2024-09-21T09:49:45.288 INFO:tasks.rados.rados.0.smithi017.stdout:2740: expect (ObjNum 1011 snap 341 seq_num 1011) 2024-09-21T09:49:45.288 INFO:tasks.rados.rados.0.smithi017.stdout:2741: read oid 35 snap 335 2024-09-21T09:49:45.288 INFO:tasks.rados.rados.0.smithi017.stdout:2741: expect (ObjNum 978 snap 335 seq_num 978) 2024-09-21T09:49:45.288 INFO:tasks.rados.rados.0.smithi017.stdout:2742: write oid 19 current snap is 348 2024-09-21T09:49:45.288 INFO:tasks.rados.rados.0.smithi017.stdout:2742: seq_num 1032 ranges {544824=564623,1823770=490025,3007352=337472} 2024-09-21T09:49:45.294 INFO:tasks.rados.rados.0.smithi017.stdout:2742: writing smithi01785587-19 from 544824 to 1109447 tid 1 2024-09-21T09:49:45.300 INFO:tasks.rados.rados.0.smithi017.stdout:2742: writing smithi01785587-19 from 1823770 to 2313795 tid 2 2024-09-21T09:49:45.306 INFO:tasks.rados.rados.0.smithi017.stdout:2742: writing smithi01785587-19 from 3007352 to 3344824 tid 3 2024-09-21T09:49:45.306 INFO:tasks.rados.rados.0.smithi017.stdout:2743: snap_remove snap 344 2024-09-21T09:49:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:45 smithi017 ceph-mon[105151]: pgmap v717: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 168 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T09:49:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:45 smithi017 ceph-mon[105151]: osdmap e759: 8 total, 8 up, 8 in 2024-09-21T09:49:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:45 smithi017 ceph-mon[102913]: pgmap v717: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 168 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T09:49:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:45 smithi017 ceph-mon[102913]: osdmap e759: 8 total, 8 up, 8 in 2024-09-21T09:49:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:45 smithi134 ceph-mon[90085]: pgmap v717: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 168 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T09:49:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:45 smithi134 ceph-mon[90085]: osdmap e759: 8 total, 8 up, 8 in 2024-09-21T09:49:46.262 INFO:tasks.rados.rados.0.smithi017.stdout:2743: done (8 left) 2024-09-21T09:49:46.262 INFO:tasks.rados.rados.0.smithi017.stdout:2744: write oid 47 current snap is 348 2024-09-21T09:49:46.262 INFO:tasks.rados.rados.0.smithi017.stdout:2735: expect (ObjNum 967 snap 325 seq_num 967) 2024-09-21T09:49:46.396 INFO:tasks.rados.rados.0.smithi017.stdout:2736: finishing write tid 1 to smithi01785587-17 2024-09-21T09:49:46.396 INFO:tasks.rados.rados.0.smithi017.stdout:2736: finishing write tid 2 to smithi01785587-17 2024-09-21T09:49:46.396 INFO:tasks.rados.rados.0.smithi017.stdout:2736: finishing write tid 3 to smithi01785587-17 2024-09-21T09:49:46.397 INFO:tasks.rados.rados.0.smithi017.stdout:2736: finishing write tid 4 to smithi01785587-17 2024-09-21T09:49:46.397 INFO:tasks.rados.rados.0.smithi017.stdout:2736: finishing write tid 5 to smithi01785587-17 2024-09-21T09:49:46.397 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 927 (ObjNum 1030 snap 348 seq_num 1030) dirty exists 2024-09-21T09:49:46.397 INFO:tasks.rados.rados.0.smithi017.stdout:2736: left oid 17 (ObjNum 1030 snap 348 seq_num 1030) 2024-09-21T09:49:46.397 INFO:tasks.rados.rados.0.smithi017.stdout:2739: finishing write tid 1 to smithi01785587-11 2024-09-21T09:49:46.397 INFO:tasks.rados.rados.0.smithi017.stdout:2744: seq_num 1033 ranges {561631=639872,1882252=633683,3034775=326856} 2024-09-21T09:49:46.402 INFO:tasks.rados.rados.0.smithi017.stdout:2744: writing smithi01785587-47 from 561631 to 1201503 tid 1 2024-09-21T09:49:46.410 INFO:tasks.rados.rados.0.smithi017.stdout:2744: writing smithi01785587-47 from 1882252 to 2515935 tid 2 2024-09-21T09:49:46.416 INFO:tasks.rados.rados.0.smithi017.stdout:2744: writing smithi01785587-47 from 3034775 to 3361631 tid 3 2024-09-21T09:49:46.416 INFO:tasks.rados.rados.0.smithi017.stdout:2735: done (8 left) 2024-09-21T09:49:46.416 INFO:tasks.rados.rados.0.smithi017.stdout:2736: done (7 left) 2024-09-21T09:49:46.416 INFO:tasks.rados.rados.0.smithi017.stdout:2738: done (6 left) 2024-09-21T09:49:46.416 INFO:tasks.rados.rados.0.smithi017.stdout:2745: rollback oid 50 current snap is 348 2024-09-21T09:49:46.416 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 50 to 347 2024-09-21T09:49:46.416 INFO:tasks.rados.rados.0.smithi017.stdout:2737: expect (ObjNum 1013 snap 341 seq_num 1013) 2024-09-21T09:49:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:46 smithi134 ceph-mon[90085]: osdmap e760: 8 total, 8 up, 8 in 2024-09-21T09:49:46.519 INFO:tasks.rados.rados.0.smithi017.stdout:2739: finishing write tid 2 to smithi01785587-11 2024-09-21T09:49:46.519 INFO:tasks.rados.rados.0.smithi017.stdout:2741: expect (ObjNum 978 snap 335 seq_num 978) 2024-09-21T09:49:46.599 INFO:tasks.rados.rados.0.smithi017.stdout:2740: expect (ObjNum 1011 snap 341 seq_num 1011) 2024-09-21T09:49:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:46 smithi017 ceph-mon[102913]: osdmap e760: 8 total, 8 up, 8 in 2024-09-21T09:49:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:46 smithi017 ceph-mon[105151]: osdmap e760: 8 total, 8 up, 8 in 2024-09-21T09:49:46.743 INFO:tasks.rados.rados.0.smithi017.stdout:2739: finishing write tid 3 to smithi01785587-11 2024-09-21T09:49:46.743 INFO:tasks.rados.rados.0.smithi017.stdout:2739: finishing write tid 4 to smithi01785587-11 2024-09-21T09:49:46.743 INFO:tasks.rados.rados.0.smithi017.stdout:2739: finishing write tid 5 to smithi01785587-11 2024-09-21T09:49:46.743 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1247 (ObjNum 1031 snap 348 seq_num 1031) dirty exists 2024-09-21T09:49:46.743 INFO:tasks.rados.rados.0.smithi017.stdout:2739: left oid 11 (ObjNum 1031 snap 348 seq_num 1031) 2024-09-21T09:49:46.743 INFO:tasks.rados.rados.0.smithi017.stdout:2742: finishing write tid 1 to smithi01785587-19 2024-09-21T09:49:46.743 INFO:tasks.rados.rados.0.smithi017.stdout:2742: finishing write tid 2 to smithi01785587-19 2024-09-21T09:49:46.744 INFO:tasks.rados.rados.0.smithi017.stdout:2742: finishing write tid 3 to smithi01785587-19 2024-09-21T09:49:46.744 INFO:tasks.rados.rados.0.smithi017.stdout:2737: done (6 left) 2024-09-21T09:49:46.744 INFO:tasks.rados.rados.0.smithi017.stdout:2739: done (5 left) 2024-09-21T09:49:46.744 INFO:tasks.rados.rados.0.smithi017.stdout:2740: done (4 left) 2024-09-21T09:49:46.744 INFO:tasks.rados.rados.0.smithi017.stdout:2741: done (3 left) 2024-09-21T09:49:46.744 INFO:tasks.rados.rados.0.smithi017.stdout:2746: read oid 43 snap -1 2024-09-21T09:49:46.744 INFO:tasks.rados.rados.0.smithi017.stdout:2746: expect deleted 2024-09-21T09:49:46.744 INFO:tasks.rados.rados.0.smithi017.stdout:2747: snap_remove snap 323 2024-09-21T09:49:47.264 INFO:tasks.rados.rados.0.smithi017.stdout:2747: done (4 left) 2024-09-21T09:49:47.264 INFO:tasks.rados.rados.0.smithi017.stdout:2748: delete oid 20 current snap is 348 2024-09-21T09:49:47.264 INFO:tasks.rados.rados.0.smithi017.stdout:2742: finishing write tid 4 to smithi01785587-19 2024-09-21T09:49:47.264 INFO:tasks.rados.rados.0.smithi017.stdout:2742: finishing write tid 5 to smithi01785587-19 2024-09-21T09:49:47.264 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 1161 (ObjNum 1032 snap 348 seq_num 1032) dirty exists 2024-09-21T09:49:47.264 INFO:tasks.rados.rados.0.smithi017.stdout:2742: left oid 19 (ObjNum 1032 snap 348 seq_num 1032) 2024-09-21T09:49:47.264 INFO:tasks.rados.rados.0.smithi017.stdout:2744: finishing write tid 1 to smithi01785587-47 2024-09-21T09:49:47.265 INFO:tasks.rados.rados.0.smithi017.stdout:2744: finishing write tid 2 to smithi01785587-47 2024-09-21T09:49:47.265 INFO:tasks.rados.rados.0.smithi017.stdout:2745: finishing rollback tid 1 to smithi01785587-50 2024-09-21T09:49:47.265 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:47.265 INFO:tasks.rados.rados.0.smithi017.stdout:2744: finishing write tid 3 to smithi01785587-47 2024-09-21T09:49:47.265 INFO:tasks.rados.rados.0.smithi017.stdout:2744: finishing write tid 4 to smithi01785587-47 2024-09-21T09:49:47.265 INFO:tasks.rados.rados.0.smithi017.stdout:2744: finishing write tid 5 to smithi01785587-47 2024-09-21T09:49:47.265 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 930 (ObjNum 1033 snap 348 seq_num 1033) dirty exists 2024-09-21T09:49:47.265 INFO:tasks.rados.rados.0.smithi017.stdout:2744: left oid 47 (ObjNum 1033 snap 348 seq_num 1033) 2024-09-21T09:49:47.268 INFO:tasks.rados.rados.0.smithi017.stdout:2742: done (4 left) 2024-09-21T09:49:47.268 INFO:tasks.rados.rados.0.smithi017.stdout:2744: done (3 left) 2024-09-21T09:49:47.268 INFO:tasks.rados.rados.0.smithi017.stdout:2745: done (2 left) 2024-09-21T09:49:47.268 INFO:tasks.rados.rados.0.smithi017.stdout:2746: done (1 left) 2024-09-21T09:49:47.268 INFO:tasks.rados.rados.0.smithi017.stdout:2748: done (0 left) 2024-09-21T09:49:47.268 INFO:tasks.rados.rados.0.smithi017.stdout:2749: write oid 22 current snap is 348 2024-09-21T09:49:47.268 INFO:tasks.rados.rados.0.smithi017.stdout:2749: seq_num 1035 ranges {595245=506723,1715569=637352,2805974=562298} 2024-09-21T09:49:47.274 INFO:tasks.rados.rados.0.smithi017.stdout:2749: writing smithi01785587-22 from 595245 to 1101968 tid 1 2024-09-21T09:49:47.281 INFO:tasks.rados.rados.0.smithi017.stdout:2749: writing smithi01785587-22 from 1715569 to 2352921 tid 2 2024-09-21T09:49:47.287 INFO:tasks.rados.rados.0.smithi017.stdout:2749: writing smithi01785587-22 from 2805974 to 3368272 tid 3 2024-09-21T09:49:47.287 INFO:tasks.rados.rados.0.smithi017.stdout:2750: write oid 40 current snap is 348 2024-09-21T09:49:47.287 INFO:tasks.rados.rados.0.smithi017.stdout:2750: seq_num 1036 ranges {612052=581972,1657698=497363,2717044=695008} 2024-09-21T09:49:47.293 INFO:tasks.rados.rados.0.smithi017.stdout:2750: writing smithi01785587-40 from 612052 to 1194024 tid 1 2024-09-21T09:49:47.298 INFO:tasks.rados.rados.0.smithi017.stdout:2750: writing smithi01785587-40 from 1657698 to 2155061 tid 2 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2750: writing smithi01785587-40 from 2717044 to 3412052 tid 3 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2751: delete oid 17 current snap is 348 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2749: finishing write tid 1 to smithi01785587-22 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2749: finishing write tid 2 to smithi01785587-22 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2749: finishing write tid 3 to smithi01785587-22 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2749: finishing write tid 4 to smithi01785587-22 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2749: finishing write tid 5 to smithi01785587-22 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1411 (ObjNum 1035 snap 348 seq_num 1035) dirty exists 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2749: left oid 22 (ObjNum 1035 snap 348 seq_num 1035) 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2750: finishing write tid 1 to smithi01785587-40 2024-09-21T09:49:47.306 INFO:tasks.rados.rados.0.smithi017.stdout:2750: finishing write tid 2 to smithi01785587-40 2024-09-21T09:49:47.309 INFO:tasks.rados.rados.0.smithi017.stdout:2749: done (2 left) 2024-09-21T09:49:47.309 INFO:tasks.rados.rados.0.smithi017.stdout:2751: done (1 left) 2024-09-21T09:49:47.309 INFO:tasks.rados.rados.0.smithi017.stdout:2752: delete oid 19 current snap is 348 2024-09-21T09:49:47.310 INFO:tasks.rados.rados.0.smithi017.stdout:2750: finishing write tid 3 to smithi01785587-40 2024-09-21T09:49:47.310 INFO:tasks.rados.rados.0.smithi017.stdout:2750: finishing write tid 4 to smithi01785587-40 2024-09-21T09:49:47.311 INFO:tasks.rados.rados.0.smithi017.stdout:2750: finishing write tid 5 to smithi01785587-40 2024-09-21T09:49:47.311 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1432 (ObjNum 1036 snap 348 seq_num 1036) dirty exists 2024-09-21T09:49:47.311 INFO:tasks.rados.rados.0.smithi017.stdout:2750: left oid 40 (ObjNum 1036 snap 348 seq_num 1036) 2024-09-21T09:49:47.315 INFO:tasks.rados.rados.0.smithi017.stdout:2750: done (1 left) 2024-09-21T09:49:47.315 INFO:tasks.rados.rados.0.smithi017.stdout:2752: done (0 left) 2024-09-21T09:49:47.315 INFO:tasks.rados.rados.0.smithi017.stdout:2753: snap_remove snap 347 2024-09-21T09:49:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:47 smithi017 ceph-mon[102913]: pgmap v720: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 172 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 16 op/s 2024-09-21T09:49:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:47 smithi017 ceph-mon[102913]: osdmap e761: 8 total, 8 up, 8 in 2024-09-21T09:49:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:47 smithi017 ceph-mon[105151]: pgmap v720: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 172 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 16 op/s 2024-09-21T09:49:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:47 smithi017 ceph-mon[105151]: osdmap e761: 8 total, 8 up, 8 in 2024-09-21T09:49:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:47 smithi134 ceph-mon[90085]: pgmap v720: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 172 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 16 op/s 2024-09-21T09:49:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:47 smithi134 ceph-mon[90085]: osdmap e761: 8 total, 8 up, 8 in 2024-09-21T09:49:48.150 INFO:tasks.rados.rados.0.smithi017.stdout:2753: done (0 left) 2024-09-21T09:49:48.150 INFO:tasks.rados.rados.0.smithi017.stdout:2754: snap_remove snap 341 2024-09-21T09:49:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:48 smithi017 ceph-mon[102913]: osdmap e762: 8 total, 8 up, 8 in 2024-09-21T09:49:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:48 smithi017 ceph-mon[102913]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T09:49:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:48 smithi017 ceph-mon[105151]: osdmap e762: 8 total, 8 up, 8 in 2024-09-21T09:49:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:48 smithi017 ceph-mon[105151]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T09:49:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:48 smithi134 ceph-mon[90085]: osdmap e762: 8 total, 8 up, 8 in 2024-09-21T09:49:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:48 smithi134 ceph-mon[90085]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T09:49:49.154 INFO:tasks.rados.rados.0.smithi017.stdout:2754: done (0 left) 2024-09-21T09:49:49.154 INFO:tasks.rados.rados.0.smithi017.stdout:2755: snap_create 2024-09-21T09:49:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:49 smithi017 ceph-mon[102913]: pgmap v723: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 181 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-09-21T09:49:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:49 smithi017 ceph-mon[102913]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T09:49:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:49 smithi017 ceph-mon[105151]: pgmap v723: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 181 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-09-21T09:49:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:49 smithi017 ceph-mon[105151]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T09:49:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:49 smithi134 ceph-mon[90085]: pgmap v723: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 181 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-09-21T09:49:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:49 smithi134 ceph-mon[90085]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T09:49:50.154 INFO:tasks.rados.rados.0.smithi017.stdout:2755: done (0 left) 2024-09-21T09:49:50.154 INFO:tasks.rados.rados.0.smithi017.stdout:2756: read oid 24 snap -1 2024-09-21T09:49:50.154 INFO:tasks.rados.rados.0.smithi017.stdout:2756: expect (ObjNum 1028 snap 346 seq_num 1028) 2024-09-21T09:49:50.154 INFO:tasks.rados.rados.0.smithi017.stdout:2757: snap_remove snap 327 2024-09-21T09:49:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:49:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:49:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:49:51.156 INFO:tasks.rados.rados.0.smithi017.stdout:2757: done (1 left) 2024-09-21T09:49:51.156 INFO:tasks.rados.rados.0.smithi017.stdout:2758: write oid 41 current snap is 349 2024-09-21T09:49:51.157 INFO:tasks.rados.rados.0.smithi017.stdout:2758: seq_num 1039 ranges {662473=407719,1833144=644690,3199313=263160} 2024-09-21T09:49:51.162 INFO:tasks.rados.rados.0.smithi017.stdout:2758: writing smithi01785587-41 from 662473 to 1070192 tid 1 2024-09-21T09:49:51.171 INFO:tasks.rados.rados.0.smithi017.stdout:2758: writing smithi01785587-41 from 1833144 to 2477834 tid 2 2024-09-21T09:49:51.177 INFO:tasks.rados.rados.0.smithi017.stdout:2758: writing smithi01785587-41 from 3199313 to 3462473 tid 3 2024-09-21T09:49:51.177 INFO:tasks.rados.rados.0.smithi017.stdout:2759: delete oid 8 current snap is 349 2024-09-21T09:49:51.177 INFO:tasks.rados.rados.0.smithi017.stdout:2756: expect (ObjNum 1028 snap 346 seq_num 1028) 2024-09-21T09:49:51.286 INFO:tasks.rados.rados.0.smithi017.stdout:2758: finishing write tid 1 to smithi01785587-41 2024-09-21T09:49:51.286 INFO:tasks.rados.rados.0.smithi017.stdout:2758: finishing write tid 2 to smithi01785587-41 2024-09-21T09:49:51.286 INFO:tasks.rados.rados.0.smithi017.stdout:2758: finishing write tid 3 to smithi01785587-41 2024-09-21T09:49:51.286 INFO:tasks.rados.rados.0.smithi017.stdout:2758: finishing write tid 4 to smithi01785587-41 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2758: finishing write tid 5 to smithi01785587-41 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 459 (ObjNum 1039 snap 349 seq_num 1039) dirty exists 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2758: left oid 41 (ObjNum 1039 snap 349 seq_num 1039) 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2756: done (2 left) 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2758: done (1 left) 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2759: done (0 left) 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2760: read oid 41 snap -1 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2760: expect (ObjNum 1039 snap 349 seq_num 1039) 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2761: read oid 4 snap -1 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2761: expect deleted 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2762: rollback oid 45 current snap is 349 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 45 to 346 2024-09-21T09:49:51.287 INFO:tasks.rados.rados.0.smithi017.stdout:2763: snap_create 2024-09-21T09:49:51.289 INFO:tasks.rados.rados.0.smithi017.stdout:2762: finishing rollback tid 0 to smithi01785587-45 2024-09-21T09:49:51.289 INFO:tasks.rados.rados.0.smithi017.stdout:2762: finishing rollback tid 1 to smithi01785587-45 2024-09-21T09:49:51.289 INFO:tasks.rados.rados.0.smithi017.stdout:2762: finishing rollback tid 2 to smithi01785587-45 2024-09-21T09:49:51.289 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 944 (ObjNum 1008 snap 341 seq_num 1008) dirty exists 2024-09-21T09:49:51.296 INFO:tasks.rados.rados.0.smithi017.stdout:2760: expect (ObjNum 1039 snap 349 seq_num 1039) 2024-09-21T09:49:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:51 smithi017 ceph-mon[102913]: pgmap v726: 89 pgs: 89 active+clean; 184 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-21T09:49:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:51 smithi017 ceph-mon[102913]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T09:49:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:51 smithi017 ceph-mon[105151]: pgmap v726: 89 pgs: 89 active+clean; 184 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-21T09:49:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:51 smithi017 ceph-mon[105151]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T09:49:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:51 smithi134 ceph-mon[90085]: pgmap v726: 89 pgs: 89 active+clean; 184 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-21T09:49:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:51 smithi134 ceph-mon[90085]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2760: done (3 left) 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2761: done (2 left) 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2762: done (1 left) 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2763: done (0 left) 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2764: read oid 42 snap -1 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2764: expect deleted 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2765: read oid 29 snap -1 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2765: expect deleted 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2766: read oid 21 snap -1 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2766: expect deleted 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2767: read oid 45 snap -1 2024-09-21T09:49:52.170 INFO:tasks.rados.rados.0.smithi017.stdout:2767: expect (ObjNum 1008 snap 341 seq_num 1008) 2024-09-21T09:49:52.171 INFO:tasks.rados.rados.0.smithi017.stdout:2768: rollback oid 34 current snap is 350 2024-09-21T09:49:52.171 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 349 2024-09-21T09:49:52.171 INFO:tasks.rados.rados.0.smithi017.stdout:2769: snap_remove snap 345 2024-09-21T09:49:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:52 smithi017 ceph-mon[102913]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T09:49:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:52 smithi017 ceph-mon[105151]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T09:49:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:52 smithi134 ceph-mon[90085]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T09:49:53.154 INFO:tasks.rados.rados.0.smithi017.stdout:2769: done (5 left) 2024-09-21T09:49:53.154 INFO:tasks.rados.rados.0.smithi017.stdout:2770: read oid 28 snap -1 2024-09-21T09:49:53.154 INFO:tasks.rados.rados.0.smithi017.stdout:2770: expect deleted 2024-09-21T09:49:53.154 INFO:tasks.rados.rados.0.smithi017.stdout:2771: write oid 41 current snap is 350 2024-09-21T09:49:53.154 INFO:tasks.rados.rados.0.smithi017.stdout:2771: seq_num 1041 ranges {696087=558217,1833755=648359,3137806=358281} 2024-09-21T09:49:53.162 INFO:tasks.rados.rados.0.smithi017.stdout:2771: writing smithi01785587-41 from 696087 to 1254304 tid 1 2024-09-21T09:49:53.169 INFO:tasks.rados.rados.0.smithi017.stdout:2771: writing smithi01785587-41 from 1833755 to 2482114 tid 2 2024-09-21T09:49:53.175 INFO:tasks.rados.rados.0.smithi017.stdout:2771: writing smithi01785587-41 from 3137806 to 3496087 tid 3 2024-09-21T09:49:53.175 INFO:tasks.rados.rados.0.smithi017.stdout:2772: write oid 47 current snap is 350 2024-09-21T09:49:53.175 INFO:tasks.rados.rados.0.smithi017.stdout:2772: seq_num 1042 ranges {712894=749819,2008590=792017,3281582=231312} 2024-09-21T09:49:53.183 INFO:tasks.rados.rados.0.smithi017.stdout:2772: writing smithi01785587-47 from 712894 to 1462713 tid 1 2024-09-21T09:49:53.190 INFO:tasks.rados.rados.0.smithi017.stdout:2772: writing smithi01785587-47 from 2008590 to 2800607 tid 2 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:2772: writing smithi01785587-47 from 3281582 to 3512894 tid 3 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:2773: rollback oid 2 current snap is 350 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 2 to 338 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:2768: finishing rollback tid 0 to smithi01785587-34 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:2768: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:2764: done (8 left) 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:2765: done (7 left) 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:2766: done (6 left) 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:2774: rollback oid 6 current snap is 350 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 349 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:2768: finishing rollback tid 2 to smithi01785587-34 2024-09-21T09:49:53.195 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1405 (ObjNum 1011 snap 341 seq_num 1011) dirty exists 2024-09-21T09:49:53.196 INFO:tasks.rados.rados.0.smithi017.stdout:2767: expect (ObjNum 1008 snap 341 seq_num 1008) 2024-09-21T09:49:53.328 INFO:tasks.rados.rados.0.smithi017.stdout:2771: finishing write tid 1 to smithi01785587-41 2024-09-21T09:49:53.328 INFO:tasks.rados.rados.0.smithi017.stdout:2771: finishing write tid 2 to smithi01785587-41 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2771: finishing write tid 3 to smithi01785587-41 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2771: finishing write tid 4 to smithi01785587-41 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2771: finishing write tid 5 to smithi01785587-41 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 466 (ObjNum 1041 snap 350 seq_num 1041) dirty exists 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2771: left oid 41 (ObjNum 1041 snap 350 seq_num 1041) 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2772: finishing write tid 1 to smithi01785587-47 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2772: finishing write tid 2 to smithi01785587-47 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2767: done (6 left) 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2768: done (5 left) 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2770: done (4 left) 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2771: done (3 left) 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2775: read oid 34 snap -1 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2775: expect (ObjNum 1011 snap 341 seq_num 1011) 2024-09-21T09:49:53.329 INFO:tasks.rados.rados.0.smithi017.stdout:2776: write oid 42 current snap is 350 2024-09-21T09:49:53.330 INFO:tasks.rados.rados.0.smithi017.stdout:2773: finishing rollback tid 0 to smithi01785587-2 2024-09-21T09:49:53.330 INFO:tasks.rados.rados.0.smithi017.stdout:2772: finishing write tid 3 to smithi01785587-47 2024-09-21T09:49:53.330 INFO:tasks.rados.rados.0.smithi017.stdout:2774: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:49:53.330 INFO:tasks.rados.rados.0.smithi017.stdout:2773: finishing rollback tid 1 to smithi01785587-2 2024-09-21T09:49:53.330 INFO:tasks.rados.rados.0.smithi017.stdout:2772: finishing write tid 4 to smithi01785587-47 2024-09-21T09:49:53.330 INFO:tasks.rados.rados.0.smithi017.stdout:2776: seq_num 1043 ranges {729701=425068,1667072=652028,2909005=485180} 2024-09-21T09:49:53.334 INFO:tasks.rados.rados.0.smithi017.stdout:2776: writing smithi01785587-42 from 729701 to 1154769 tid 1 2024-09-21T09:49:53.341 INFO:tasks.rados.rados.0.smithi017.stdout:2776: writing smithi01785587-42 from 1667072 to 2319100 tid 2 2024-09-21T09:49:53.347 INFO:tasks.rados.rados.0.smithi017.stdout:2776: writing smithi01785587-42 from 2909005 to 3394185 tid 3 2024-09-21T09:49:53.347 INFO:tasks.rados.rados.0.smithi017.stdout:2777: read oid 35 snap -1 2024-09-21T09:49:53.347 INFO:tasks.rados.rados.0.smithi017.stdout:2777: expect (ObjNum 945 snap 321 seq_num 945) 2024-09-21T09:49:53.347 INFO:tasks.rados.rados.0.smithi017.stdout:2778: write oid 33 current snap is 350 2024-09-21T09:49:53.347 INFO:tasks.rados.rados.0.smithi017.stdout:2774: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:49:53.348 INFO:tasks.rados.rados.0.smithi017.stdout:2773: finishing rollback tid 2 to smithi01785587-2 2024-09-21T09:49:53.348 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 562 (ObjNum 930 snap 314 seq_num 930) dirty exists 2024-09-21T09:49:53.348 INFO:tasks.rados.rados.0.smithi017.stdout:2774: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:49:53.348 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 852 (ObjNum 1010 snap 341 seq_num 1010) dirty exists 2024-09-21T09:49:53.348 INFO:tasks.rados.rados.0.smithi017.stdout:2772: finishing write tid 5 to smithi01785587-47 2024-09-21T09:49:53.348 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 945 (ObjNum 1042 snap 350 seq_num 1042) dirty exists 2024-09-21T09:49:53.348 INFO:tasks.rados.rados.0.smithi017.stdout:2772: left oid 47 (ObjNum 1042 snap 350 seq_num 1042) 2024-09-21T09:49:53.348 INFO:tasks.rados.rados.0.smithi017.stdout:2778: seq_num 1044 ranges {746508=500317,2009201=795686,3220075=326433} 2024-09-21T09:49:53.354 INFO:tasks.rados.rados.0.smithi017.stdout:2778: writing smithi01785587-33 from 746508 to 1246825 tid 1 2024-09-21T09:49:53.362 INFO:tasks.rados.rados.0.smithi017.stdout:2778: writing smithi01785587-33 from 2009201 to 2804887 tid 2 2024-09-21T09:49:53.367 INFO:tasks.rados.rados.0.smithi017.stdout:2778: writing smithi01785587-33 from 3220075 to 3546508 tid 3 2024-09-21T09:49:53.367 INFO:tasks.rados.rados.0.smithi017.stdout:2772: done (6 left) 2024-09-21T09:49:53.367 INFO:tasks.rados.rados.0.smithi017.stdout:2773: done (5 left) 2024-09-21T09:49:53.367 INFO:tasks.rados.rados.0.smithi017.stdout:2774: done (4 left) 2024-09-21T09:49:53.367 INFO:tasks.rados.rados.0.smithi017.stdout:2779: write oid 40 current snap is 350 2024-09-21T09:49:53.367 INFO:tasks.rados.rados.0.smithi017.stdout:2779: seq_num 1045 ranges {763315=575566,2067683=655697,3247498=315817} 2024-09-21T09:49:53.374 INFO:tasks.rados.rados.0.smithi017.stdout:2779: writing smithi01785587-40 from 763315 to 1338881 tid 1 2024-09-21T09:49:53.381 INFO:tasks.rados.rados.0.smithi017.stdout:2779: writing smithi01785587-40 from 2067683 to 2723380 tid 2 2024-09-21T09:49:53.386 INFO:tasks.rados.rados.0.smithi017.stdout:2779: writing smithi01785587-40 from 3247498 to 3563315 tid 3 2024-09-21T09:49:53.387 INFO:tasks.rados.rados.0.smithi017.stdout:2780: read oid 14 snap -1 2024-09-21T09:49:53.387 INFO:tasks.rados.rados.0.smithi017.stdout:2780: expect (ObjNum 1021 snap 344 seq_num 1021) 2024-09-21T09:49:53.387 INFO:tasks.rados.rados.0.smithi017.stdout:2781: rollback oid 30 current snap is 350 2024-09-21T09:49:53.387 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 349 2024-09-21T09:49:53.387 INFO:tasks.rados.rados.0.smithi017.stdout:2775: expect (ObjNum 1011 snap 341 seq_num 1011) 2024-09-21T09:49:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:53 smithi017 ceph-mon[102913]: pgmap v729: 89 pgs: 89 active+clean; 169 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-21T09:49:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:53 smithi017 ceph-mon[102913]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T09:49:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:53 smithi017 ceph-mon[102913]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T09:49:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:53 smithi017 ceph-mon[105151]: pgmap v729: 89 pgs: 89 active+clean; 169 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-21T09:49:53.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:53 smithi017 ceph-mon[105151]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T09:49:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:53 smithi017 ceph-mon[105151]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T09:49:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:53 smithi134 ceph-mon[90085]: pgmap v729: 89 pgs: 89 active+clean; 169 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-21T09:49:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:53 smithi134 ceph-mon[90085]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T09:49:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:53 smithi134 ceph-mon[90085]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T09:49:53.531 INFO:tasks.rados.rados.0.smithi017.stdout:2776: finishing write tid 1 to smithi01785587-42 2024-09-21T09:49:53.531 INFO:tasks.rados.rados.0.smithi017.stdout:2776: finishing write tid 2 to smithi01785587-42 2024-09-21T09:49:53.531 INFO:tasks.rados.rados.0.smithi017.stdout:2776: finishing write tid 3 to smithi01785587-42 2024-09-21T09:49:53.531 INFO:tasks.rados.rados.0.smithi017.stdout:2776: finishing write tid 4 to smithi01785587-42 2024-09-21T09:49:53.531 INFO:tasks.rados.rados.0.smithi017.stdout:2776: finishing write tid 5 to smithi01785587-42 2024-09-21T09:49:53.531 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 949 (ObjNum 1043 snap 350 seq_num 1043) dirty exists 2024-09-21T09:49:53.531 INFO:tasks.rados.rados.0.smithi017.stdout:2776: left oid 42 (ObjNum 1043 snap 350 seq_num 1043) 2024-09-21T09:49:53.531 INFO:tasks.rados.rados.0.smithi017.stdout:2778: finishing write tid 1 to smithi01785587-33 2024-09-21T09:49:53.531 INFO:tasks.rados.rados.0.smithi017.stdout:2777: expect (ObjNum 945 snap 321 seq_num 945) 2024-09-21T09:49:53.661 INFO:tasks.rados.rados.0.smithi017.stdout:2778: finishing write tid 2 to smithi01785587-33 2024-09-21T09:49:53.661 INFO:tasks.rados.rados.0.smithi017.stdout:2778: finishing write tid 3 to smithi01785587-33 2024-09-21T09:49:53.661 INFO:tasks.rados.rados.0.smithi017.stdout:2778: finishing write tid 4 to smithi01785587-33 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2778: finishing write tid 5 to smithi01785587-33 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1434 (ObjNum 1044 snap 350 seq_num 1044) dirty exists 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2778: left oid 33 (ObjNum 1044 snap 350 seq_num 1044) 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2779: finishing write tid 1 to smithi01785587-40 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2779: finishing write tid 2 to smithi01785587-40 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2775: done (6 left) 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2776: done (5 left) 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2777: done (4 left) 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2778: done (3 left) 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2782: read oid 5 snap -1 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2782: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2783: snap_create 2024-09-21T09:49:53.662 INFO:tasks.rados.rados.0.smithi017.stdout:2779: finishing write tid 3 to smithi01785587-40 2024-09-21T09:49:53.663 INFO:tasks.rados.rados.0.smithi017.stdout:2779: finishing write tid 4 to smithi01785587-40 2024-09-21T09:49:53.663 INFO:tasks.rados.rados.0.smithi017.stdout:2779: finishing write tid 5 to smithi01785587-40 2024-09-21T09:49:53.663 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1441 (ObjNum 1045 snap 350 seq_num 1045) dirty exists 2024-09-21T09:49:53.663 INFO:tasks.rados.rados.0.smithi017.stdout:2779: left oid 40 (ObjNum 1045 snap 350 seq_num 1045) 2024-09-21T09:49:53.663 INFO:tasks.rados.rados.0.smithi017.stdout:2781: finishing rollback tid 0 to smithi01785587-30 2024-09-21T09:49:53.663 INFO:tasks.rados.rados.0.smithi017.stdout:2781: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:49:53.663 INFO:tasks.rados.rados.0.smithi017.stdout:2781: finishing rollback tid 2 to smithi01785587-30 2024-09-21T09:49:53.663 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1186 (ObjNum 1027 snap 346 seq_num 1027) dirty exists 2024-09-21T09:49:53.663 INFO:tasks.rados.rados.0.smithi017.stdout:2780: expect (ObjNum 1021 snap 344 seq_num 1021) 2024-09-21T09:49:53.770 INFO:tasks.rados.rados.0.smithi017.stdout:2782: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:49:54.174 INFO:tasks.rados.rados.0.smithi017.stdout:2779: done (4 left) 2024-09-21T09:49:54.174 INFO:tasks.rados.rados.0.smithi017.stdout:2780: done (3 left) 2024-09-21T09:49:54.174 INFO:tasks.rados.rados.0.smithi017.stdout:2781: done (2 left) 2024-09-21T09:49:54.175 INFO:tasks.rados.rados.0.smithi017.stdout:2782: done (1 left) 2024-09-21T09:49:54.175 INFO:tasks.rados.rados.0.smithi017.stdout:2783: done (0 left) 2024-09-21T09:49:54.175 INFO:tasks.rados.rados.0.smithi017.stdout:2784: delete oid 30 current snap is 351 2024-09-21T09:49:54.179 INFO:tasks.rados.rados.0.smithi017.stdout:2784: done (0 left) 2024-09-21T09:49:54.179 INFO:tasks.rados.rados.0.smithi017.stdout:2785: rollback oid 50 current snap is 351 2024-09-21T09:49:54.179 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 50 to 338 2024-09-21T09:49:54.179 INFO:tasks.rados.rados.0.smithi017.stdout:2786: read oid 9 snap -1 2024-09-21T09:49:54.180 INFO:tasks.rados.rados.0.smithi017.stdout:2786: expect deleted 2024-09-21T09:49:54.180 INFO:tasks.rados.rados.0.smithi017.stdout:2787: write oid 23 current snap is 351 2024-09-21T09:49:54.180 INFO:tasks.rados.rados.0.smithi017.stdout:2787: seq_num 1047 ranges {796929=726064,2184647=659366,3302344=294585} 2024-09-21T09:49:54.187 INFO:tasks.rados.rados.0.smithi017.stdout:2787: writing smithi01785587-23 from 796929 to 1522993 tid 1 2024-09-21T09:49:54.195 INFO:tasks.rados.rados.0.smithi017.stdout:2787: writing smithi01785587-23 from 2184647 to 2844013 tid 2 2024-09-21T09:49:54.199 INFO:tasks.rados.rados.0.smithi017.stdout:2787: writing smithi01785587-23 from 3302344 to 3596929 tid 3 2024-09-21T09:49:54.199 INFO:tasks.rados.rados.0.smithi017.stdout:2788: write oid 13 current snap is 351 2024-09-21T09:49:54.199 INFO:tasks.rados.rados.0.smithi017.stdout:2788: seq_num 1048 ranges {413736=401313,1326776=403024,2413414=457893,3438716=175020} 2024-09-21T09:49:54.203 INFO:tasks.rados.rados.0.smithi017.stdout:2788: writing smithi01785587-13 from 413736 to 815049 tid 1 2024-09-21T09:49:54.208 INFO:tasks.rados.rados.0.smithi017.stdout:2788: writing smithi01785587-13 from 1326776 to 1729800 tid 2 2024-09-21T09:49:54.214 INFO:tasks.rados.rados.0.smithi017.stdout:2788: writing smithi01785587-13 from 2413414 to 2871307 tid 3 2024-09-21T09:49:54.219 INFO:tasks.rados.rados.0.smithi017.stdout:2788: writing smithi01785587-13 from 3438716 to 3613736 tid 4 2024-09-21T09:49:54.219 INFO:tasks.rados.rados.0.smithi017.stdout:2789: rollback oid 20 current snap is 351 2024-09-21T09:49:54.219 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 350 2024-09-21T09:49:54.219 INFO:tasks.rados.rados.0.smithi017.stdout:2785: finishing rollback tid 1 to smithi01785587-50 2024-09-21T09:49:54.219 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2787: finishing write tid 1 to smithi01785587-23 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2787: finishing write tid 2 to smithi01785587-23 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2787: finishing write tid 3 to smithi01785587-23 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2787: finishing write tid 4 to smithi01785587-23 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2787: finishing write tid 5 to smithi01785587-23 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1252 (ObjNum 1047 snap 351 seq_num 1047) dirty exists 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2787: left oid 23 (ObjNum 1047 snap 351 seq_num 1047) 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2788: finishing write tid 1 to smithi01785587-13 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2785: done (4 left) 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2786: done (3 left) 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2787: done (2 left) 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2790: write oid 42 current snap is 351 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2788: finishing write tid 2 to smithi01785587-13 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2788: finishing write tid 3 to smithi01785587-13 2024-09-21T09:49:54.220 INFO:tasks.rados.rados.0.smithi017.stdout:2790: seq_num 1049 ranges {430543=476562,1385258=663035,2840837=669165} 2024-09-21T09:49:54.224 INFO:tasks.rados.rados.0.smithi017.stdout:2790: writing smithi01785587-42 from 430543 to 907105 tid 1 2024-09-21T09:49:54.230 INFO:tasks.rados.rados.0.smithi017.stdout:2790: writing smithi01785587-42 from 1385258 to 2048293 tid 2 2024-09-21T09:49:54.238 INFO:tasks.rados.rados.0.smithi017.stdout:2790: writing smithi01785587-42 from 2840837 to 3510002 tid 3 2024-09-21T09:49:54.238 INFO:tasks.rados.rados.0.smithi017.stdout:2791: snap_remove snap 338 2024-09-21T09:49:55.173 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:49:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:49:54] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.51.0" 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:2788: finishing write tid 4 to smithi01785587-13 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:2788: finishing write tid 5 to smithi01785587-13 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:2788: finishing write tid 6 to smithi01785587-13 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1194 (ObjNum 1048 snap 351 seq_num 1048) dirty exists 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:2788: left oid 13 (ObjNum 1048 snap 351 seq_num 1048) 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:2789: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 757 (ObjNum 33 snap 0 seq_num 2399942592) dirty dne 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:2790: finishing write tid 1 to smithi01785587-42 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:2790: finishing write tid 2 to smithi01785587-42 2024-09-21T09:49:55.183 INFO:tasks.rados.rados.0.smithi017.stdout:2790: finishing write tid 3 to smithi01785587-42 2024-09-21T09:49:55.184 INFO:tasks.rados.rados.0.smithi017.stdout:2790: finishing write tid 4 to smithi01785587-42 2024-09-21T09:49:55.184 INFO:tasks.rados.rados.0.smithi017.stdout:2790: finishing write tid 5 to smithi01785587-42 2024-09-21T09:49:55.184 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 954 (ObjNum 1049 snap 351 seq_num 1049) dirty exists 2024-09-21T09:49:55.184 INFO:tasks.rados.rados.0.smithi017.stdout:2790: left oid 42 (ObjNum 1049 snap 351 seq_num 1049) 2024-09-21T09:49:55.184 INFO:tasks.rados.rados.0.smithi017.stdout:2788: done (3 left) 2024-09-21T09:49:55.184 INFO:tasks.rados.rados.0.smithi017.stdout:2789: done (2 left) 2024-09-21T09:49:55.184 INFO:tasks.rados.rados.0.smithi017.stdout:2790: done (1 left) 2024-09-21T09:49:55.184 INFO:tasks.rados.rados.0.smithi017.stdout:2791: done (0 left) 2024-09-21T09:49:55.184 INFO:tasks.rados.rados.0.smithi017.stdout:2792: snap_remove snap 337 2024-09-21T09:49:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:55 smithi017 ceph-mon[102913]: pgmap v732: 89 pgs: 89 active+clean; 167 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 808 KiB/s wr, 8 op/s 2024-09-21T09:49:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:55 smithi017 ceph-mon[102913]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T09:49:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:55 smithi017 ceph-mon[105151]: pgmap v732: 89 pgs: 89 active+clean; 167 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 808 KiB/s wr, 8 op/s 2024-09-21T09:49:55.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:55 smithi017 ceph-mon[105151]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T09:49:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:55 smithi134 ceph-mon[90085]: pgmap v732: 89 pgs: 89 active+clean; 167 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 808 KiB/s wr, 8 op/s 2024-09-21T09:49:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:55 smithi134 ceph-mon[90085]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T09:49:56.189 INFO:tasks.rados.rados.0.smithi017.stdout:2792: done (0 left) 2024-09-21T09:49:56.189 INFO:tasks.rados.rados.0.smithi017.stdout:2793: write oid 28 current snap is 351 2024-09-21T09:49:56.189 INFO:tasks.rados.rados.0.smithi017.stdout:2793: seq_num 1050 ranges {447350=668164,1560093=406693,2584613=480437} 2024-09-21T09:49:56.196 INFO:tasks.rados.rados.0.smithi017.stdout:2793: writing smithi01785587-28 from 447350 to 1115514 tid 1 2024-09-21T09:49:56.202 INFO:tasks.rados.rados.0.smithi017.stdout:2793: writing smithi01785587-28 from 1560093 to 1966786 tid 2 2024-09-21T09:49:56.208 INFO:tasks.rados.rados.0.smithi017.stdout:2793: writing smithi01785587-28 from 2584613 to 3065050 tid 3 2024-09-21T09:49:56.208 INFO:tasks.rados.rados.0.smithi017.stdout:2794: snap_create 2024-09-21T09:49:56.208 INFO:tasks.rados.rados.0.smithi017.stdout:2793: finishing write tid 1 to smithi01785587-28 2024-09-21T09:49:56.208 INFO:tasks.rados.rados.0.smithi017.stdout:2793: finishing write tid 2 to smithi01785587-28 2024-09-21T09:49:56.211 INFO:tasks.rados.rados.0.smithi017.stdout:2793: finishing write tid 3 to smithi01785587-28 2024-09-21T09:49:56.211 INFO:tasks.rados.rados.0.smithi017.stdout:2793: finishing write tid 4 to smithi01785587-28 2024-09-21T09:49:56.211 INFO:tasks.rados.rados.0.smithi017.stdout:2793: finishing write tid 5 to smithi01785587-28 2024-09-21T09:49:56.211 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 1454 (ObjNum 1050 snap 351 seq_num 1050) dirty exists 2024-09-21T09:49:56.211 INFO:tasks.rados.rados.0.smithi017.stdout:2793: left oid 28 (ObjNum 1050 snap 351 seq_num 1050) 2024-09-21T09:49:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:56 smithi017 ceph-mon[102913]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T09:49:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:56 smithi017 ceph-mon[105151]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T09:49:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:56 smithi134 ceph-mon[90085]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T09:49:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:57.191 INFO:tasks.rados.rados.0.smithi017.stdout:2793: done (1 left) 2024-09-21T09:49:57.191 INFO:tasks.rados.rados.0.smithi017.stdout:2794: done (0 left) 2024-09-21T09:49:57.192 INFO:tasks.rados.rados.0.smithi017.stdout:2795: delete oid 8 current snap is 352 2024-09-21T09:49:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:2795: done (0 left) 2024-09-21T09:49:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:2796: read oid 29 snap -1 2024-09-21T09:49:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:2796: expect deleted 2024-09-21T09:49:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:2797: read oid 43 snap -1 2024-09-21T09:49:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:2797: expect deleted 2024-09-21T09:49:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:2798: rollback oid 3 current snap is 352 2024-09-21T09:49:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 343 2024-09-21T09:49:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:2799: write oid 44 current snap is 352 2024-09-21T09:49:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:2799: seq_num 1052 ranges {480964=418662,1560704=410362,2523106=619334} 2024-09-21T09:49:57.199 INFO:tasks.rados.rados.0.smithi017.stdout:2799: writing smithi01785587-44 from 480964 to 899626 tid 1 2024-09-21T09:49:57.205 INFO:tasks.rados.rados.0.smithi017.stdout:2799: writing smithi01785587-44 from 1560704 to 1971066 tid 2 2024-09-21T09:49:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:2799: writing smithi01785587-44 from 2523106 to 3142440 tid 3 2024-09-21T09:49:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:2800: delete oid 37 current snap is 352 2024-09-21T09:49:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:2798: finishing rollback tid 0 to smithi01785587-3 2024-09-21T09:49:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:2798: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:49:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:2798: finishing rollback tid 2 to smithi01785587-3 2024-09-21T09:49:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1438 (ObjNum 1002 snap 340 seq_num 1002) dirty exists 2024-09-21T09:49:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:2799: finishing write tid 1 to smithi01785587-44 2024-09-21T09:49:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:2799: finishing write tid 2 to smithi01785587-44 2024-09-21T09:49:57.215 INFO:tasks.rados.rados.0.smithi017.stdout:2799: finishing write tid 3 to smithi01785587-44 2024-09-21T09:49:57.215 INFO:tasks.rados.rados.0.smithi017.stdout:2799: finishing write tid 4 to smithi01785587-44 2024-09-21T09:49:57.216 INFO:tasks.rados.rados.0.smithi017.stdout:2799: finishing write tid 5 to smithi01785587-44 2024-09-21T09:49:57.216 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1443 (ObjNum 1052 snap 352 seq_num 1052) dirty exists 2024-09-21T09:49:57.216 INFO:tasks.rados.rados.0.smithi017.stdout:2799: left oid 44 (ObjNum 1052 snap 352 seq_num 1052) 2024-09-21T09:49:57.217 INFO:tasks.rados.rados.0.smithi017.stdout:2796: done (4 left) 2024-09-21T09:49:57.217 INFO:tasks.rados.rados.0.smithi017.stdout:2797: done (3 left) 2024-09-21T09:49:57.217 INFO:tasks.rados.rados.0.smithi017.stdout:2798: done (2 left) 2024-09-21T09:49:57.217 INFO:tasks.rados.rados.0.smithi017.stdout:2799: done (1 left) 2024-09-21T09:49:57.217 INFO:tasks.rados.rados.0.smithi017.stdout:2800: done (0 left) 2024-09-21T09:49:57.217 INFO:tasks.rados.rados.0.smithi017.stdout:2801: write oid 44 current snap is 352 2024-09-21T09:49:57.217 INFO:tasks.rados.rados.0.smithi017.stdout:2801: seq_num 1054 ranges {514578=569160,1677668=414031,2577952=641878} 2024-09-21T09:49:57.222 INFO:tasks.rados.rados.0.smithi017.stdout:2801: writing smithi01785587-44 from 514578 to 1083738 tid 1 2024-09-21T09:49:57.228 INFO:tasks.rados.rados.0.smithi017.stdout:2801: writing smithi01785587-44 from 1677668 to 2091699 tid 2 2024-09-21T09:49:57.234 INFO:tasks.rados.rados.0.smithi017.stdout:2801: writing smithi01785587-44 from 2577952 to 3219830 tid 3 2024-09-21T09:49:57.234 INFO:tasks.rados.rados.0.smithi017.stdout:2802: write oid 13 current snap is 352 2024-09-21T09:49:57.234 INFO:tasks.rados.rados.0.smithi017.stdout:2801: finishing write tid 1 to smithi01785587-44 2024-09-21T09:49:57.234 INFO:tasks.rados.rados.0.smithi017.stdout:2801: finishing write tid 2 to smithi01785587-44 2024-09-21T09:49:57.235 INFO:tasks.rados.rados.0.smithi017.stdout:2802: seq_num 1055 ranges {531385=644409,1736150=557689,3005375=569503} 2024-09-21T09:49:57.241 INFO:tasks.rados.rados.0.smithi017.stdout:2802: writing smithi01785587-13 from 531385 to 1175794 tid 1 2024-09-21T09:49:57.247 INFO:tasks.rados.rados.0.smithi017.stdout:2802: writing smithi01785587-13 from 1736150 to 2293839 tid 2 2024-09-21T09:49:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:2802: writing smithi01785587-13 from 3005375 to 3574878 tid 3 2024-09-21T09:49:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:2803: delete oid 49 current snap is 352 2024-09-21T09:49:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:2801: finishing write tid 3 to smithi01785587-44 2024-09-21T09:49:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:2801: finishing write tid 4 to smithi01785587-44 2024-09-21T09:49:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:2801: finishing write tid 5 to smithi01785587-44 2024-09-21T09:49:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1447 (ObjNum 1054 snap 352 seq_num 1054) dirty exists 2024-09-21T09:49:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:2801: left oid 44 (ObjNum 1054 snap 352 seq_num 1054) 2024-09-21T09:49:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:2802: finishing write tid 1 to smithi01785587-13 2024-09-21T09:49:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:2802: finishing write tid 2 to smithi01785587-13 2024-09-21T09:49:57.256 INFO:tasks.rados.rados.0.smithi017.stdout:2801: done (2 left) 2024-09-21T09:49:57.256 INFO:tasks.rados.rados.0.smithi017.stdout:2803: done (1 left) 2024-09-21T09:49:57.256 INFO:tasks.rados.rados.0.smithi017.stdout:2804: read oid 48 snap -1 2024-09-21T09:49:57.256 INFO:tasks.rados.rados.0.smithi017.stdout:2804: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:49:57.256 INFO:tasks.rados.rados.0.smithi017.stdout:2805: delete oid 35 current snap is 352 2024-09-21T09:49:57.258 INFO:tasks.rados.rados.0.smithi017.stdout:2805: done (2 left) 2024-09-21T09:49:57.258 INFO:tasks.rados.rados.0.smithi017.stdout:2806: snap_create 2024-09-21T09:49:57.267 INFO:tasks.rados.rados.0.smithi017.stdout:2802: finishing write tid 3 to smithi01785587-13 2024-09-21T09:49:57.267 INFO:tasks.rados.rados.0.smithi017.stdout:2802: finishing write tid 4 to smithi01785587-13 2024-09-21T09:49:57.267 INFO:tasks.rados.rados.0.smithi017.stdout:2802: finishing write tid 5 to smithi01785587-13 2024-09-21T09:49:57.268 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1203 (ObjNum 1055 snap 352 seq_num 1055) dirty exists 2024-09-21T09:49:57.268 INFO:tasks.rados.rados.0.smithi017.stdout:2802: left oid 13 (ObjNum 1055 snap 352 seq_num 1055) 2024-09-21T09:49:57.269 INFO:tasks.rados.rados.0.smithi017.stdout:2804: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:49:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:57 smithi017 ceph-mon[102913]: pgmap v735: 89 pgs: 89 active+clean; 165 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:49:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:57 smithi017 ceph-mon[102913]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T09:49:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:57 smithi017 ceph-mon[105151]: pgmap v735: 89 pgs: 89 active+clean; 165 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:49:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:57 smithi017 ceph-mon[105151]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T09:49:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:57 smithi134 ceph-mon[90085]: pgmap v735: 89 pgs: 89 active+clean; 165 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:49:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:57 smithi134 ceph-mon[90085]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T09:49:58.155 INFO:tasks.rados.rados.0.smithi017.stdout:2802: done (2 left) 2024-09-21T09:49:58.155 INFO:tasks.rados.rados.0.smithi017.stdout:2804: done (1 left) 2024-09-21T09:49:58.155 INFO:tasks.rados.rados.0.smithi017.stdout:2806: done (0 left) 2024-09-21T09:49:58.155 INFO:tasks.rados.rados.0.smithi017.stdout:2807: snap_remove snap 348 2024-09-21T09:49:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:58 smithi017 ceph-mon[102913]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T09:49:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:58 smithi017 ceph-mon[102913]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T09:49:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:58 smithi017 ceph-mon[105151]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T09:49:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:58 smithi017 ceph-mon[105151]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T09:49:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:58 smithi134 ceph-mon[90085]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T09:49:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:58 smithi134 ceph-mon[90085]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T09:49:59.157 INFO:tasks.rados.rados.0.smithi017.stdout:2807: done (0 left) 2024-09-21T09:49:59.158 INFO:tasks.rados.rados.0.smithi017.stdout:2808: write oid 4 current snap is 353 2024-09-21T09:49:59.158 INFO:tasks.rados.rados.0.smithi017.stdout:2808: seq_num 1058 ranges {581806=586509,1911596=421369,3087644=403319} 2024-09-21T09:49:59.164 INFO:tasks.rados.rados.0.smithi017.stdout:2808: writing smithi01785587-4 from 581806 to 1168315 tid 1 2024-09-21T09:49:59.172 INFO:tasks.rados.rados.0.smithi017.stdout:2808: writing smithi01785587-4 from 1911596 to 2332965 tid 2 2024-09-21T09:49:59.178 INFO:tasks.rados.rados.0.smithi017.stdout:2808: writing smithi01785587-4 from 3087644 to 3490963 tid 3 2024-09-21T09:49:59.178 INFO:tasks.rados.rados.0.smithi017.stdout:2809: write oid 11 current snap is 353 2024-09-21T09:49:59.178 INFO:tasks.rados.rados.0.smithi017.stdout:2809: seq_num 1059 ranges {598613=661758,1970078=565027,3115067=614591} 2024-09-21T09:49:59.185 INFO:tasks.rados.rados.0.smithi017.stdout:2809: writing smithi01785587-11 from 598613 to 1260371 tid 1 2024-09-21T09:49:59.192 INFO:tasks.rados.rados.0.smithi017.stdout:2809: writing smithi01785587-11 from 1970078 to 2535105 tid 2 2024-09-21T09:49:59.199 INFO:tasks.rados.rados.0.smithi017.stdout:2809: writing smithi01785587-11 from 3115067 to 3729658 tid 3 2024-09-21T09:49:59.199 INFO:tasks.rados.rados.0.smithi017.stdout:2810: write oid 39 current snap is 353 2024-09-21T09:49:59.199 INFO:tasks.rados.rados.0.smithi017.stdout:2808: finishing write tid 1 to smithi01785587-4 2024-09-21T09:49:59.199 INFO:tasks.rados.rados.0.smithi017.stdout:2810: seq_num 1060 ranges {615420=737007,1912207=425038,3026137=542216} 2024-09-21T09:49:59.206 INFO:tasks.rados.rados.0.smithi017.stdout:2810: writing smithi01785587-39 from 615420 to 1352427 tid 1 2024-09-21T09:49:59.212 INFO:tasks.rados.rados.0.smithi017.stdout:2810: writing smithi01785587-39 from 1912207 to 2337245 tid 2 2024-09-21T09:49:59.219 INFO:tasks.rados.rados.0.smithi017.stdout:2810: writing smithi01785587-39 from 3026137 to 3568353 tid 3 2024-09-21T09:49:59.219 INFO:tasks.rados.rados.0.smithi017.stdout:2811: delete oid 36 current snap is 353 2024-09-21T09:49:59.219 INFO:tasks.rados.rados.0.smithi017.stdout:2808: finishing write tid 2 to smithi01785587-4 2024-09-21T09:49:59.219 INFO:tasks.rados.rados.0.smithi017.stdout:2808: finishing write tid 3 to smithi01785587-4 2024-09-21T09:49:59.219 INFO:tasks.rados.rados.0.smithi017.stdout:2808: finishing write tid 4 to smithi01785587-4 2024-09-21T09:49:59.219 INFO:tasks.rados.rados.0.smithi017.stdout:2808: finishing write tid 5 to smithi01785587-4 2024-09-21T09:49:59.219 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1460 (ObjNum 1058 snap 353 seq_num 1058) dirty exists 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:2808: left oid 4 (ObjNum 1058 snap 353 seq_num 1058) 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:2809: finishing write tid 1 to smithi01785587-11 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:2809: finishing write tid 2 to smithi01785587-11 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:2809: finishing write tid 3 to smithi01785587-11 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:2809: finishing write tid 4 to smithi01785587-11 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:2809: finishing write tid 5 to smithi01785587-11 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1258 (ObjNum 1059 snap 353 seq_num 1059) dirty exists 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:2809: left oid 11 (ObjNum 1059 snap 353 seq_num 1059) 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:2810: finishing write tid 1 to smithi01785587-39 2024-09-21T09:49:59.220 INFO:tasks.rados.rados.0.smithi017.stdout:2810: finishing write tid 2 to smithi01785587-39 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:2810: finishing write tid 3 to smithi01785587-39 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:2810: finishing write tid 4 to smithi01785587-39 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:2810: finishing write tid 5 to smithi01785587-39 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1455 (ObjNum 1060 snap 353 seq_num 1060) dirty exists 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:2810: left oid 39 (ObjNum 1060 snap 353 seq_num 1060) 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:2808: done (3 left) 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:2809: done (2 left) 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:2810: done (1 left) 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:2811: done (0 left) 2024-09-21T09:49:59.223 INFO:tasks.rados.rados.0.smithi017.stdout:2812: read oid 26 snap -1 2024-09-21T09:49:59.224 INFO:tasks.rados.rados.0.smithi017.stdout:2812: expect (ObjNum 1025 snap 344 seq_num 1025) 2024-09-21T09:49:59.224 INFO:tasks.rados.rados.0.smithi017.stdout:2813: write oid 8 current snap is 353 2024-09-21T09:49:59.224 INFO:tasks.rados.rados.0.smithi017.stdout:2813: seq_num 1062 ranges {649034=487505,1629171=428707,2680983=564760,3798768=50266} 2024-09-21T09:49:59.229 INFO:tasks.rados.rados.0.smithi017.stdout:2813: writing smithi01785587-8 from 649034 to 1136539 tid 1 2024-09-21T09:49:59.234 INFO:tasks.rados.rados.0.smithi017.stdout:2813: writing smithi01785587-8 from 1629171 to 2057878 tid 2 2024-09-21T09:49:59.241 INFO:tasks.rados.rados.0.smithi017.stdout:2813: writing smithi01785587-8 from 2680983 to 3245743 tid 3 2024-09-21T09:49:59.245 INFO:tasks.rados.rados.0.smithi017.stdout:2813: writing smithi01785587-8 from 3798768 to 3849034 tid 4 2024-09-21T09:49:59.245 INFO:tasks.rados.rados.0.smithi017.stdout:2814: write oid 23 current snap is 353 2024-09-21T09:49:59.245 INFO:tasks.rados.rados.0.smithi017.stdout:2814: seq_num 1063 ranges {665841=562754,1687653=572365,2708406=776032} 2024-09-21T09:49:59.251 INFO:tasks.rados.rados.0.smithi017.stdout:2814: writing smithi01785587-23 from 665841 to 1228595 tid 1 2024-09-21T09:49:59.257 INFO:tasks.rados.rados.0.smithi017.stdout:2814: writing smithi01785587-23 from 1687653 to 2260018 tid 2 2024-09-21T09:49:59.264 INFO:tasks.rados.rados.0.smithi017.stdout:2814: writing smithi01785587-23 from 2708406 to 3484438 tid 3 2024-09-21T09:49:59.264 INFO:tasks.rados.rados.0.smithi017.stdout:2815: read oid 33 snap -1 2024-09-21T09:49:59.264 INFO:tasks.rados.rados.0.smithi017.stdout:2815: expect (ObjNum 1044 snap 350 seq_num 1044) 2024-09-21T09:49:59.264 INFO:tasks.rados.rados.0.smithi017.stdout:2813: finishing write tid 1 to smithi01785587-8 2024-09-21T09:49:59.264 INFO:tasks.rados.rados.0.smithi017.stdout:2812: expect (ObjNum 1025 snap 344 seq_num 1025) 2024-09-21T09:49:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:59 smithi017 ceph-mon[105151]: pgmap v738: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 3.0 MiB/s wr, 24 op/s 2024-09-21T09:49:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:49:59 smithi017 ceph-mon[105151]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T09:49:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:59 smithi017 ceph-mon[102913]: pgmap v738: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 3.0 MiB/s wr, 24 op/s 2024-09-21T09:49:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:49:59 smithi017 ceph-mon[102913]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T09:49:59.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:59 smithi134 ceph-mon[90085]: pgmap v738: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 3.0 MiB/s wr, 24 op/s 2024-09-21T09:49:59.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:49:59 smithi134 ceph-mon[90085]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T09:49:59.559 INFO:tasks.rados.rados.0.smithi017.stdout:2813: finishing write tid 2 to smithi01785587-8 2024-09-21T09:49:59.559 INFO:tasks.rados.rados.0.smithi017.stdout:2813: finishing write tid 3 to smithi01785587-8 2024-09-21T09:49:59.559 INFO:tasks.rados.rados.0.smithi017.stdout:2812: done (3 left) 2024-09-21T09:49:59.559 INFO:tasks.rados.rados.0.smithi017.stdout:2816: rollback oid 21 current snap is 353 2024-09-21T09:49:59.559 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 335 2024-09-21T09:49:59.559 INFO:tasks.rados.rados.0.smithi017.stdout:2813: finishing write tid 4 to smithi01785587-8 2024-09-21T09:49:59.559 INFO:tasks.rados.rados.0.smithi017.stdout:2813: finishing write tid 5 to smithi01785587-8 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2813: finishing write tid 6 to smithi01785587-8 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1263 (ObjNum 1062 snap 353 seq_num 1062) dirty exists 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2813: left oid 8 (ObjNum 1062 snap 353 seq_num 1062) 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2814: finishing write tid 1 to smithi01785587-23 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2814: finishing write tid 2 to smithi01785587-23 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2814: finishing write tid 3 to smithi01785587-23 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2814: finishing write tid 4 to smithi01785587-23 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2814: finishing write tid 5 to smithi01785587-23 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1268 (ObjNum 1063 snap 353 seq_num 1063) dirty exists 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2814: left oid 23 (ObjNum 1063 snap 353 seq_num 1063) 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2813: done (3 left) 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2814: done (2 left) 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2817: snap_create 2024-09-21T09:49:59.560 INFO:tasks.rados.rados.0.smithi017.stdout:2816: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:49:59.561 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:59.570 INFO:tasks.rados.rados.0.smithi017.stdout:2815: expect (ObjNum 1044 snap 350 seq_num 1044) 2024-09-21T09:50:00.202 INFO:tasks.rados.rados.0.smithi017.stdout:2815: done (2 left) 2024-09-21T09:50:00.202 INFO:tasks.rados.rados.0.smithi017.stdout:2816: done (1 left) 2024-09-21T09:50:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:2817: done (0 left) 2024-09-21T09:50:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:2818: snap_remove snap 351 2024-09-21T09:50:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:00 smithi017 ceph-mon[102913]: overall HEALTH_OK 2024-09-21T09:50:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:00 smithi017 ceph-mon[105151]: overall HEALTH_OK 2024-09-21T09:50:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:00 smithi134 ceph-mon[90085]: overall HEALTH_OK 2024-09-21T09:50:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:50:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:50:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:50:01.206 INFO:tasks.rados.rados.0.smithi017.stdout:2818: done (0 left) 2024-09-21T09:50:01.206 INFO:tasks.rados.rados.0.smithi017.stdout:2819: rollback oid 26 current snap is 354 2024-09-21T09:50:01.206 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 26 to 343 2024-09-21T09:50:01.206 INFO:tasks.rados.rados.0.smithi017.stdout:2820: delete oid 4 current snap is 354 2024-09-21T09:50:01.210 INFO:tasks.rados.rados.0.smithi017.stdout:2819: finishing rollback tid 0 to smithi01785587-26 2024-09-21T09:50:01.211 INFO:tasks.rados.rados.0.smithi017.stdout:2819: finishing rollback tid 1 to smithi01785587-26 2024-09-21T09:50:01.212 INFO:tasks.rados.rados.0.smithi017.stdout:2819: finishing rollback tid 2 to smithi01785587-26 2024-09-21T09:50:01.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1464 (ObjNum 1015 snap 343 seq_num 1015) dirty exists 2024-09-21T09:50:01.212 INFO:tasks.rados.rados.0.smithi017.stdout:2819: done (1 left) 2024-09-21T09:50:01.212 INFO:tasks.rados.rados.0.smithi017.stdout:2820: done (0 left) 2024-09-21T09:50:01.212 INFO:tasks.rados.rados.0.smithi017.stdout:2821: read oid 28 snap 346 2024-09-21T09:50:01.212 INFO:tasks.rados.rados.0.smithi017.stdout:2821: expect deleted 2024-09-21T09:50:01.212 INFO:tasks.rados.rados.0.smithi017.stdout:2822: write oid 44 current snap is 354 2024-09-21T09:50:01.213 INFO:tasks.rados.rados.0.smithi017.stdout:2822: seq_num 1065 ranges {699455=429605,1804617=576034,3163252=514929} 2024-09-21T09:50:01.219 INFO:tasks.rados.rados.0.smithi017.stdout:2822: writing smithi01785587-44 from 699455 to 1129060 tid 1 2024-09-21T09:50:01.226 INFO:tasks.rados.rados.0.smithi017.stdout:2822: writing smithi01785587-44 from 1804617 to 2380651 tid 2 2024-09-21T09:50:01.233 INFO:tasks.rados.rados.0.smithi017.stdout:2822: writing smithi01785587-44 from 3163252 to 3678181 tid 3 2024-09-21T09:50:01.233 INFO:tasks.rados.rados.0.smithi017.stdout:2823: snap_create 2024-09-21T09:50:01.233 INFO:tasks.rados.rados.0.smithi017.stdout:2822: finishing write tid 1 to smithi01785587-44 2024-09-21T09:50:01.233 INFO:tasks.rados.rados.0.smithi017.stdout:2822: finishing write tid 2 to smithi01785587-44 2024-09-21T09:50:01.236 INFO:tasks.rados.rados.0.smithi017.stdout:2822: finishing write tid 3 to smithi01785587-44 2024-09-21T09:50:01.236 INFO:tasks.rados.rados.0.smithi017.stdout:2822: finishing write tid 4 to smithi01785587-44 2024-09-21T09:50:01.236 INFO:tasks.rados.rados.0.smithi017.stdout:2822: finishing write tid 5 to smithi01785587-44 2024-09-21T09:50:01.236 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1460 (ObjNum 1065 snap 354 seq_num 1065) dirty exists 2024-09-21T09:50:01.236 INFO:tasks.rados.rados.0.smithi017.stdout:2822: left oid 44 (ObjNum 1065 snap 354 seq_num 1065) 2024-09-21T09:50:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:01 smithi134 ceph-mon[90085]: pgmap v741: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-21T09:50:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:01 smithi134 ceph-mon[90085]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T09:50:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:01 smithi017 ceph-mon[105151]: pgmap v741: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-21T09:50:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:01 smithi017 ceph-mon[105151]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T09:50:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:01 smithi017 ceph-mon[102913]: pgmap v741: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-09-21T09:50:01.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:01 smithi017 ceph-mon[102913]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T09:50:02.217 INFO:tasks.rados.rados.0.smithi017.stdout:2821: done (2 left) 2024-09-21T09:50:02.217 INFO:tasks.rados.rados.0.smithi017.stdout:2822: done (1 left) 2024-09-21T09:50:02.217 INFO:tasks.rados.rados.0.smithi017.stdout:2823: done (0 left) 2024-09-21T09:50:02.217 INFO:tasks.rados.rados.0.smithi017.stdout:2824: delete oid 25 current snap is 355 2024-09-21T09:50:02.223 INFO:tasks.rados.rados.0.smithi017.stdout:2824: done (0 left) 2024-09-21T09:50:02.223 INFO:tasks.rados.rados.0.smithi017.stdout:2825: snap_create 2024-09-21T09:50:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:02 smithi134 ceph-mon[90085]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T09:50:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:02 smithi017 ceph-mon[102913]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T09:50:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:02 smithi017 ceph-mon[105151]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T09:50:03.157 INFO:tasks.rados.rados.0.smithi017.stdout:2825: done (0 left) 2024-09-21T09:50:03.157 INFO:tasks.rados.rados.0.smithi017.stdout:2826: rollback oid 38 current snap is 356 2024-09-21T09:50:03.158 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 354 2024-09-21T09:50:03.158 INFO:tasks.rados.rados.0.smithi017.stdout:2827: read oid 16 snap 349 2024-09-21T09:50:03.158 INFO:tasks.rados.rados.0.smithi017.stdout:2827: expect (ObjNum 1026 snap 345 seq_num 1026) 2024-09-21T09:50:03.158 INFO:tasks.rados.rados.0.smithi017.stdout:2828: read oid 33 snap -1 2024-09-21T09:50:03.158 INFO:tasks.rados.rados.0.smithi017.stdout:2828: expect (ObjNum 1044 snap 350 seq_num 1044) 2024-09-21T09:50:03.158 INFO:tasks.rados.rados.0.smithi017.stdout:2829: rollback oid 34 current snap is 356 2024-09-21T09:50:03.158 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 355 2024-09-21T09:50:03.158 INFO:tasks.rados.rados.0.smithi017.stdout:2830: write oid 3 current snap is 356 2024-09-21T09:50:03.158 INFO:tasks.rados.rados.0.smithi017.stdout:2830: seq_num 1067 ranges {733069=580103,1921581=579703,3218098=537473} 2024-09-21T09:50:03.164 INFO:tasks.rados.rados.0.smithi017.stdout:2830: writing smithi01785587-3 from 733069 to 1313172 tid 1 2024-09-21T09:50:03.172 INFO:tasks.rados.rados.0.smithi017.stdout:2830: writing smithi01785587-3 from 1921581 to 2501284 tid 2 2024-09-21T09:50:03.179 INFO:tasks.rados.rados.0.smithi017.stdout:2830: writing smithi01785587-3 from 3218098 to 3755571 tid 3 2024-09-21T09:50:03.179 INFO:tasks.rados.rados.0.smithi017.stdout:2831: write oid 37 current snap is 356 2024-09-21T09:50:03.179 INFO:tasks.rados.rados.0.smithi017.stdout:2831: seq_num 1068 ranges {749876=655352,1863710=723361,3012815=748745} 2024-09-21T09:50:03.186 INFO:tasks.rados.rados.0.smithi017.stdout:2831: writing smithi01785587-37 from 749876 to 1405228 tid 1 2024-09-21T09:50:03.193 INFO:tasks.rados.rados.0.smithi017.stdout:2831: writing smithi01785587-37 from 1863710 to 2587071 tid 2 2024-09-21T09:50:03.200 INFO:tasks.rados.rados.0.smithi017.stdout:2831: writing smithi01785587-37 from 3012815 to 3761560 tid 3 2024-09-21T09:50:03.200 INFO:tasks.rados.rados.0.smithi017.stdout:2832: write oid 35 current snap is 356 2024-09-21T09:50:03.200 INFO:tasks.rados.rados.0.smithi017.stdout:2832: seq_num 1069 ranges {766683=730601,1922192=583372,3156591=676370} 2024-09-21T09:50:03.208 INFO:tasks.rados.rados.0.smithi017.stdout:2832: writing smithi01785587-35 from 766683 to 1497284 tid 1 2024-09-21T09:50:03.214 INFO:tasks.rados.rados.0.smithi017.stdout:2832: writing smithi01785587-35 from 1922192 to 2505564 tid 2 2024-09-21T09:50:03.222 INFO:tasks.rados.rados.0.smithi017.stdout:2832: writing smithi01785587-35 from 3156591 to 3832961 tid 3 2024-09-21T09:50:03.222 INFO:tasks.rados.rados.0.smithi017.stdout:2833: snap_create 2024-09-21T09:50:03.222 INFO:tasks.rados.rados.0.smithi017.stdout:2826: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:50:03.222 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 888 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:50:03.222 INFO:tasks.rados.rados.0.smithi017.stdout:2829: finishing rollback tid 0 to smithi01785587-34 2024-09-21T09:50:03.222 INFO:tasks.rados.rados.0.smithi017.stdout:2829: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:50:03.222 INFO:tasks.rados.rados.0.smithi017.stdout:2829: finishing rollback tid 2 to smithi01785587-34 2024-09-21T09:50:03.222 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1418 (ObjNum 1011 snap 341 seq_num 1011) dirty exists 2024-09-21T09:50:03.223 INFO:tasks.rados.rados.0.smithi017.stdout:2830: finishing write tid 1 to smithi01785587-3 2024-09-21T09:50:03.223 INFO:tasks.rados.rados.0.smithi017.stdout:2830: finishing write tid 2 to smithi01785587-3 2024-09-21T09:50:03.223 INFO:tasks.rados.rados.0.smithi017.stdout:2827: expect (ObjNum 1026 snap 345 seq_num 1026) 2024-09-21T09:50:03.369 INFO:tasks.rados.rados.0.smithi017.stdout:2830: finishing write tid 3 to smithi01785587-3 2024-09-21T09:50:03.369 INFO:tasks.rados.rados.0.smithi017.stdout:2830: finishing write tid 4 to smithi01785587-3 2024-09-21T09:50:03.369 INFO:tasks.rados.rados.0.smithi017.stdout:2830: finishing write tid 5 to smithi01785587-3 2024-09-21T09:50:03.369 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1465 (ObjNum 1067 snap 356 seq_num 1067) dirty exists 2024-09-21T09:50:03.369 INFO:tasks.rados.rados.0.smithi017.stdout:2830: left oid 3 (ObjNum 1067 snap 356 seq_num 1067) 2024-09-21T09:50:03.369 INFO:tasks.rados.rados.0.smithi017.stdout:2828: expect (ObjNum 1044 snap 350 seq_num 1044) 2024-09-21T09:50:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:03 smithi134 ceph-mon[90085]: pgmap v744: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 21 op/s 2024-09-21T09:50:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:03 smithi134 ceph-mon[90085]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T09:50:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:03 smithi134 ceph-mon[90085]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T09:50:03.524 INFO:tasks.rados.rados.0.smithi017.stdout:2831: finishing write tid 1 to smithi01785587-37 2024-09-21T09:50:03.524 INFO:tasks.rados.rados.0.smithi017.stdout:2831: finishing write tid 2 to smithi01785587-37 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2831: finishing write tid 3 to smithi01785587-37 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2831: finishing write tid 4 to smithi01785587-37 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2831: finishing write tid 5 to smithi01785587-37 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1465 (ObjNum 1068 snap 356 seq_num 1068) dirty exists 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2831: left oid 37 (ObjNum 1068 snap 356 seq_num 1068) 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2832: finishing write tid 1 to smithi01785587-35 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2832: finishing write tid 2 to smithi01785587-35 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2832: finishing write tid 3 to smithi01785587-35 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2832: finishing write tid 4 to smithi01785587-35 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2832: finishing write tid 5 to smithi01785587-35 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 317 (ObjNum 1069 snap 356 seq_num 1069) dirty exists 2024-09-21T09:50:03.525 INFO:tasks.rados.rados.0.smithi017.stdout:2832: left oid 35 (ObjNum 1069 snap 356 seq_num 1069) 2024-09-21T09:50:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:03 smithi017 ceph-mon[102913]: pgmap v744: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 21 op/s 2024-09-21T09:50:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:03 smithi017 ceph-mon[102913]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T09:50:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:03 smithi017 ceph-mon[102913]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T09:50:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:03 smithi017 ceph-mon[105151]: pgmap v744: 89 pgs: 89 active+clean; 194 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 21 op/s 2024-09-21T09:50:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:03 smithi017 ceph-mon[105151]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T09:50:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:03 smithi017 ceph-mon[105151]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2826: done (7 left) 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2827: done (6 left) 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2828: done (5 left) 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2829: done (4 left) 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2830: done (3 left) 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2831: done (2 left) 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2832: done (1 left) 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2833: done (0 left) 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2834: read oid 45 snap -1 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2834: expect (ObjNum 1008 snap 341 seq_num 1008) 2024-09-21T09:50:04.223 INFO:tasks.rados.rados.0.smithi017.stdout:2835: write oid 8 current snap is 357 2024-09-21T09:50:04.224 INFO:tasks.rados.rados.0.smithi017.stdout:2835: seq_num 1070 ranges {783490=405850,1980674=727030,3184014=487642} 2024-09-21T09:50:04.229 INFO:tasks.rados.rados.0.smithi017.stdout:2835: writing smithi01785587-8 from 783490 to 1189340 tid 1 2024-09-21T09:50:04.237 INFO:tasks.rados.rados.0.smithi017.stdout:2835: writing smithi01785587-8 from 1980674 to 2707704 tid 2 2024-09-21T09:50:04.243 INFO:tasks.rados.rados.0.smithi017.stdout:2835: writing smithi01785587-8 from 3184014 to 3671656 tid 3 2024-09-21T09:50:04.243 INFO:tasks.rados.rados.0.smithi017.stdout:2836: read oid 39 snap 354 2024-09-21T09:50:04.243 INFO:tasks.rados.rados.0.smithi017.stdout:2836: expect (ObjNum 1060 snap 353 seq_num 1060) 2024-09-21T09:50:04.243 INFO:tasks.rados.rados.0.smithi017.stdout:2837: rollback oid 40 current snap is 357 2024-09-21T09:50:04.244 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 350 2024-09-21T09:50:04.244 INFO:tasks.rados.rados.0.smithi017.stdout:2834: expect (ObjNum 1008 snap 341 seq_num 1008) 2024-09-21T09:50:04.377 INFO:tasks.rados.rados.0.smithi017.stdout:2835: finishing write tid 1 to smithi01785587-8 2024-09-21T09:50:04.377 INFO:tasks.rados.rados.0.smithi017.stdout:2835: finishing write tid 2 to smithi01785587-8 2024-09-21T09:50:04.377 INFO:tasks.rados.rados.0.smithi017.stdout:2835: finishing write tid 3 to smithi01785587-8 2024-09-21T09:50:04.377 INFO:tasks.rados.rados.0.smithi017.stdout:2835: finishing write tid 4 to smithi01785587-8 2024-09-21T09:50:04.377 INFO:tasks.rados.rados.0.smithi017.stdout:2835: finishing write tid 5 to smithi01785587-8 2024-09-21T09:50:04.377 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1277 (ObjNum 1070 snap 357 seq_num 1070) dirty exists 2024-09-21T09:50:04.377 INFO:tasks.rados.rados.0.smithi017.stdout:2835: left oid 8 (ObjNum 1070 snap 357 seq_num 1070) 2024-09-21T09:50:04.378 INFO:tasks.rados.rados.0.smithi017.stdout:2836: expect (ObjNum 1060 snap 353 seq_num 1060) 2024-09-21T09:50:04.607 INFO:tasks.rados.rados.0.smithi017.stdout:2837: finishing rollback tid 0 to smithi01785587-40 2024-09-21T09:50:04.607 INFO:tasks.rados.rados.0.smithi017.stdout:2837: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:50:04.607 INFO:tasks.rados.rados.0.smithi017.stdout:2837: finishing rollback tid 2 to smithi01785587-40 2024-09-21T09:50:04.607 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1469 (ObjNum 1045 snap 350 seq_num 1045) dirty exists 2024-09-21T09:50:04.607 INFO:tasks.rados.rados.0.smithi017.stdout:2834: done (3 left) 2024-09-21T09:50:04.607 INFO:tasks.rados.rados.0.smithi017.stdout:2835: done (2 left) 2024-09-21T09:50:04.607 INFO:tasks.rados.rados.0.smithi017.stdout:2836: done (1 left) 2024-09-21T09:50:04.607 INFO:tasks.rados.rados.0.smithi017.stdout:2837: done (0 left) 2024-09-21T09:50:04.607 INFO:tasks.rados.rados.0.smithi017.stdout:2838: read oid 14 snap -1 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:2838: expect (ObjNum 1021 snap 344 seq_num 1021) 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:2839: read oid 44 snap -1 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:2839: expect (ObjNum 1065 snap 354 seq_num 1065) 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:2840: rollback oid 34 current snap is 357 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 356 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:2841: read oid 43 snap 335 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:2841: expect deleted 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:2842: read oid 11 snap -1 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:2842: expect (ObjNum 1059 snap 353 seq_num 1059) 2024-09-21T09:50:04.608 INFO:tasks.rados.rados.0.smithi017.stdout:2843: snap_remove snap 346 2024-09-21T09:50:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:50:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:50:04] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.51.0" 2024-09-21T09:50:05.229 INFO:tasks.rados.rados.0.smithi017.stdout:2840: finishing rollback tid 0 to smithi01785587-34 2024-09-21T09:50:05.229 INFO:tasks.rados.rados.0.smithi017.stdout:2840: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:50:05.229 INFO:tasks.rados.rados.0.smithi017.stdout:2840: finishing rollback tid 2 to smithi01785587-34 2024-09-21T09:50:05.229 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1422 (ObjNum 1011 snap 341 seq_num 1011) dirty exists 2024-09-21T09:50:05.230 INFO:tasks.rados.rados.0.smithi017.stdout:2838: expect (ObjNum 1021 snap 344 seq_num 1021) 2024-09-21T09:50:05.341 INFO:tasks.rados.rados.0.smithi017.stdout:2842: expect (ObjNum 1059 snap 353 seq_num 1059) 2024-09-21T09:50:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:05 smithi134 ceph-mon[90085]: pgmap v747: 89 pgs: 89 active+clean; 196 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 21 op/s 2024-09-21T09:50:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:05 smithi134 ceph-mon[90085]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T09:50:05.515 INFO:tasks.rados.rados.0.smithi017.stdout:2839: expect (ObjNum 1065 snap 354 seq_num 1065) 2024-09-21T09:50:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:05 smithi017 ceph-mon[102913]: pgmap v747: 89 pgs: 89 active+clean; 196 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 21 op/s 2024-09-21T09:50:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:05 smithi017 ceph-mon[102913]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T09:50:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:05 smithi017 ceph-mon[105151]: pgmap v747: 89 pgs: 89 active+clean; 196 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 21 op/s 2024-09-21T09:50:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:05 smithi017 ceph-mon[105151]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T09:50:05.808 INFO:tasks.rados.rados.0.smithi017.stdout:2838: done (5 left) 2024-09-21T09:50:05.808 INFO:tasks.rados.rados.0.smithi017.stdout:2839: done (4 left) 2024-09-21T09:50:05.808 INFO:tasks.rados.rados.0.smithi017.stdout:2840: done (3 left) 2024-09-21T09:50:05.809 INFO:tasks.rados.rados.0.smithi017.stdout:2841: done (2 left) 2024-09-21T09:50:05.809 INFO:tasks.rados.rados.0.smithi017.stdout:2842: done (1 left) 2024-09-21T09:50:05.809 INFO:tasks.rados.rados.0.smithi017.stdout:2843: done (0 left) 2024-09-21T09:50:05.809 INFO:tasks.rados.rados.0.smithi017.stdout:2844: write oid 11 current snap is 357 2024-09-21T09:50:05.809 INFO:tasks.rados.rados.0.smithi017.stdout:2844: seq_num 1071 ranges {400297=481099,1639156=361141} 2024-09-21T09:50:05.813 INFO:tasks.rados.rados.0.smithi017.stdout:2844: writing smithi01785587-11 from 400297 to 881396 tid 1 2024-09-21T09:50:05.819 INFO:tasks.rados.rados.0.smithi017.stdout:2844: writing smithi01785587-11 from 1639156 to 2000297 tid 2 2024-09-21T09:50:05.819 INFO:tasks.rados.rados.0.smithi017.stdout:2845: read oid 16 snap -1 2024-09-21T09:50:05.819 INFO:tasks.rados.rados.0.smithi017.stdout:2845: expect (ObjNum 1026 snap 345 seq_num 1026) 2024-09-21T09:50:05.820 INFO:tasks.rados.rados.0.smithi017.stdout:2846: snap_create 2024-09-21T09:50:05.820 INFO:tasks.rados.rados.0.smithi017.stdout:2844: finishing write tid 1 to smithi01785587-11 2024-09-21T09:50:05.822 INFO:tasks.rados.rados.0.smithi017.stdout:2844: finishing write tid 2 to smithi01785587-11 2024-09-21T09:50:05.822 INFO:tasks.rados.rados.0.smithi017.stdout:2844: finishing write tid 3 to smithi01785587-11 2024-09-21T09:50:05.823 INFO:tasks.rados.rados.0.smithi017.stdout:2844: finishing write tid 4 to smithi01785587-11 2024-09-21T09:50:05.823 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1281 (ObjNum 1071 snap 357 seq_num 1071) dirty exists 2024-09-21T09:50:05.823 INFO:tasks.rados.rados.0.smithi017.stdout:2844: left oid 11 (ObjNum 1071 snap 357 seq_num 1071) 2024-09-21T09:50:05.828 INFO:tasks.rados.rados.0.smithi017.stdout:2845: expect (ObjNum 1026 snap 345 seq_num 1026) 2024-09-21T09:50:06.233 INFO:tasks.rados.rados.0.smithi017.stdout:2844: done (2 left) 2024-09-21T09:50:06.233 INFO:tasks.rados.rados.0.smithi017.stdout:2845: done (1 left) 2024-09-21T09:50:06.233 INFO:tasks.rados.rados.0.smithi017.stdout:2846: done (0 left) 2024-09-21T09:50:06.233 INFO:tasks.rados.rados.0.smithi017.stdout:2847: write oid 10 current snap is 358 2024-09-21T09:50:06.233 INFO:tasks.rados.rados.0.smithi017.stdout:2847: seq_num 1072 ranges {417104=672701,1813991=203113} 2024-09-21T09:50:06.238 INFO:tasks.rados.rados.0.smithi017.stdout:2847: writing smithi01785587-10 from 417104 to 1089805 tid 1 2024-09-21T09:50:06.244 INFO:tasks.rados.rados.0.smithi017.stdout:2847: writing smithi01785587-10 from 1813991 to 2017104 tid 2 2024-09-21T09:50:06.244 INFO:tasks.rados.rados.0.smithi017.stdout:2848: snap_create 2024-09-21T09:50:06.244 INFO:tasks.rados.rados.0.smithi017.stdout:2847: finishing write tid 1 to smithi01785587-10 2024-09-21T09:50:06.245 INFO:tasks.rados.rados.0.smithi017.stdout:2847: finishing write tid 2 to smithi01785587-10 2024-09-21T09:50:06.245 INFO:tasks.rados.rados.0.smithi017.stdout:2847: finishing write tid 3 to smithi01785587-10 2024-09-21T09:50:06.246 INFO:tasks.rados.rados.0.smithi017.stdout:2847: finishing write tid 4 to smithi01785587-10 2024-09-21T09:50:06.246 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 240 (ObjNum 1072 snap 358 seq_num 1072) dirty exists 2024-09-21T09:50:06.246 INFO:tasks.rados.rados.0.smithi017.stdout:2847: left oid 10 (ObjNum 1072 snap 358 seq_num 1072) 2024-09-21T09:50:06.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:06 smithi134 ceph-mon[90085]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T09:50:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:06 smithi017 ceph-mon[102913]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T09:50:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:06 smithi017 ceph-mon[105151]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T09:50:07.239 INFO:tasks.rados.rados.0.smithi017.stdout:2847: done (1 left) 2024-09-21T09:50:07.239 INFO:tasks.rados.rados.0.smithi017.stdout:2848: done (0 left) 2024-09-21T09:50:07.239 INFO:tasks.rados.rados.0.smithi017.stdout:2849: write oid 12 current snap is 359 2024-09-21T09:50:07.239 INFO:tasks.rados.rados.0.smithi017.stdout:2849: seq_num 1073 ranges {433911=747950,1756120=277791} 2024-09-21T09:50:07.245 INFO:tasks.rados.rados.0.smithi017.stdout:2849: writing smithi01785587-12 from 433911 to 1181861 tid 1 2024-09-21T09:50:07.250 INFO:tasks.rados.rados.0.smithi017.stdout:2849: writing smithi01785587-12 from 1756120 to 2033911 tid 2 2024-09-21T09:50:07.250 INFO:tasks.rados.rados.0.smithi017.stdout:2850: rollback oid 13 current snap is 359 2024-09-21T09:50:07.251 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 13 to 357 2024-09-21T09:50:07.251 INFO:tasks.rados.rados.0.smithi017.stdout:2851: delete oid 41 current snap is 359 2024-09-21T09:50:07.252 INFO:tasks.rados.rados.0.smithi017.stdout:2849: finishing write tid 1 to smithi01785587-12 2024-09-21T09:50:07.253 INFO:tasks.rados.rados.0.smithi017.stdout:2851: done (2 left) 2024-09-21T09:50:07.253 INFO:tasks.rados.rados.0.smithi017.stdout:2852: snap_create 2024-09-21T09:50:07.255 INFO:tasks.rados.rados.0.smithi017.stdout:2849: finishing write tid 2 to smithi01785587-12 2024-09-21T09:50:07.255 INFO:tasks.rados.rados.0.smithi017.stdout:2849: finishing write tid 3 to smithi01785587-12 2024-09-21T09:50:07.255 INFO:tasks.rados.rados.0.smithi017.stdout:2849: finishing write tid 4 to smithi01785587-12 2024-09-21T09:50:07.255 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 965 (ObjNum 1073 snap 359 seq_num 1073) dirty exists 2024-09-21T09:50:07.255 INFO:tasks.rados.rados.0.smithi017.stdout:2849: left oid 12 (ObjNum 1073 snap 359 seq_num 1073) 2024-09-21T09:50:07.256 INFO:tasks.rados.rados.0.smithi017.stdout:2850: finishing rollback tid 0 to smithi01785587-13 2024-09-21T09:50:07.256 INFO:tasks.rados.rados.0.smithi017.stdout:2850: finishing rollback tid 1 to smithi01785587-13 2024-09-21T09:50:07.256 INFO:tasks.rados.rados.0.smithi017.stdout:2850: finishing rollback tid 2 to smithi01785587-13 2024-09-21T09:50:07.256 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1219 (ObjNum 1055 snap 352 seq_num 1055) dirty exists 2024-09-21T09:50:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:07 smithi134 ceph-mon[90085]: pgmap v750: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-21T09:50:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:07 smithi134 ceph-mon[90085]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T09:50:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:07 smithi017 ceph-mon[102913]: pgmap v750: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-21T09:50:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:07 smithi017 ceph-mon[102913]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T09:50:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:07 smithi017 ceph-mon[105151]: pgmap v750: 89 pgs: 89 active+clean; 204 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-21T09:50:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:07 smithi017 ceph-mon[105151]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T09:50:08.158 INFO:tasks.rados.rados.0.smithi017.stdout:2849: done (2 left) 2024-09-21T09:50:08.158 INFO:tasks.rados.rados.0.smithi017.stdout:2850: done (1 left) 2024-09-21T09:50:08.158 INFO:tasks.rados.rados.0.smithi017.stdout:2852: done (0 left) 2024-09-21T09:50:08.158 INFO:tasks.rados.rados.0.smithi017.stdout:2853: read oid 48 snap -1 2024-09-21T09:50:08.158 INFO:tasks.rados.rados.0.smithi017.stdout:2853: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:50:08.158 INFO:tasks.rados.rados.0.smithi017.stdout:2854: delete oid 23 current snap is 360 2024-09-21T09:50:08.162 INFO:tasks.rados.rados.0.smithi017.stdout:2854: done (1 left) 2024-09-21T09:50:08.162 INFO:tasks.rados.rados.0.smithi017.stdout:2855: read oid 5 snap -1 2024-09-21T09:50:08.162 INFO:tasks.rados.rados.0.smithi017.stdout:2855: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:50:08.162 INFO:tasks.rados.rados.0.smithi017.stdout:2856: read oid 23 snap 350 2024-09-21T09:50:08.162 INFO:tasks.rados.rados.0.smithi017.stdout:2856: expect deleted 2024-09-21T09:50:08.162 INFO:tasks.rados.rados.0.smithi017.stdout:2857: snap_remove snap 335 2024-09-21T09:50:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:08 smithi134 ceph-mon[90085]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T09:50:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:08 smithi134 ceph-mon[90085]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T09:50:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:08 smithi017 ceph-mon[102913]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T09:50:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:08 smithi017 ceph-mon[102913]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T09:50:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:08 smithi017 ceph-mon[105151]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T09:50:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:08 smithi017 ceph-mon[105151]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T09:50:09.160 INFO:tasks.rados.rados.0.smithi017.stdout:2853: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:50:09.278 INFO:tasks.rados.rados.0.smithi017.stdout:2855: expect (ObjNum 949 snap 321 seq_num 949) 2024-09-21T09:50:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:09 smithi134 ceph-mon[90085]: pgmap v753: 89 pgs: 89 active+clean; 206 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.0 MiB/s wr, 23 op/s 2024-09-21T09:50:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:09 smithi134 ceph-mon[90085]: osdmap e784: 8 total, 8 up, 8 in 2024-09-21T09:50:09.533 INFO:tasks.rados.rados.0.smithi017.stdout:2853: done (3 left) 2024-09-21T09:50:09.533 INFO:tasks.rados.rados.0.smithi017.stdout:2855: done (2 left) 2024-09-21T09:50:09.533 INFO:tasks.rados.rados.0.smithi017.stdout:2856: done (1 left) 2024-09-21T09:50:09.533 INFO:tasks.rados.rados.0.smithi017.stdout:2857: done (0 left) 2024-09-21T09:50:09.533 INFO:tasks.rados.rados.0.smithi017.stdout:2858: write oid 2 current snap is 360 2024-09-21T09:50:09.534 INFO:tasks.rados.rados.0.smithi017.stdout:2858: seq_num 1076 ranges {484332=573697,1531566=552766} 2024-09-21T09:50:09.538 INFO:tasks.rados.rados.0.smithi017.stdout:2858: writing smithi01785587-2 from 484332 to 1058029 tid 1 2024-09-21T09:50:09.544 INFO:tasks.rados.rados.0.smithi017.stdout:2858: writing smithi01785587-2 from 1531566 to 2084332 tid 2 2024-09-21T09:50:09.544 INFO:tasks.rados.rados.0.smithi017.stdout:2859: snap_create 2024-09-21T09:50:09.544 INFO:tasks.rados.rados.0.smithi017.stdout:2858: finishing write tid 1 to smithi01785587-2 2024-09-21T09:50:09.547 INFO:tasks.rados.rados.0.smithi017.stdout:2858: finishing write tid 2 to smithi01785587-2 2024-09-21T09:50:09.547 INFO:tasks.rados.rados.0.smithi017.stdout:2858: finishing write tid 3 to smithi01785587-2 2024-09-21T09:50:09.547 INFO:tasks.rados.rados.0.smithi017.stdout:2858: finishing write tid 4 to smithi01785587-2 2024-09-21T09:50:09.547 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 580 (ObjNum 1076 snap 360 seq_num 1076) dirty exists 2024-09-21T09:50:09.547 INFO:tasks.rados.rados.0.smithi017.stdout:2858: left oid 2 (ObjNum 1076 snap 360 seq_num 1076) 2024-09-21T09:50:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:09 smithi017 ceph-mon[102913]: pgmap v753: 89 pgs: 89 active+clean; 206 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.0 MiB/s wr, 23 op/s 2024-09-21T09:50:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:09 smithi017 ceph-mon[102913]: osdmap e784: 8 total, 8 up, 8 in 2024-09-21T09:50:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:09 smithi017 ceph-mon[105151]: pgmap v753: 89 pgs: 89 active+clean; 206 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.0 MiB/s wr, 23 op/s 2024-09-21T09:50:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:09 smithi017 ceph-mon[105151]: osdmap e784: 8 total, 8 up, 8 in 2024-09-21T09:50:10.250 INFO:tasks.rados.rados.0.smithi017.stdout:2858: done (1 left) 2024-09-21T09:50:10.250 INFO:tasks.rados.rados.0.smithi017.stdout:2859: done (0 left) 2024-09-21T09:50:10.250 INFO:tasks.rados.rados.0.smithi017.stdout:2860: snap_create 2024-09-21T09:50:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:50:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:50:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:50:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:50:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=sqlstore.transactions t=2024-09-21T09:50:10.296424929Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:50:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:50:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=cleanup t=2024-09-21T09:50:10.298750888Z level=info msg="Completed cleanup jobs" duration=12.802286ms 2024-09-21T09:50:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:10.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:11.252 INFO:tasks.rados.rados.0.smithi017.stdout:2860: done (0 left) 2024-09-21T09:50:11.252 INFO:tasks.rados.rados.0.smithi017.stdout:2861: rollback oid 30 current snap is 362 2024-09-21T09:50:11.252 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 343 2024-09-21T09:50:11.253 INFO:tasks.rados.rados.0.smithi017.stdout:2862: read oid 44 snap -1 2024-09-21T09:50:11.253 INFO:tasks.rados.rados.0.smithi017.stdout:2862: expect (ObjNum 1065 snap 354 seq_num 1065) 2024-09-21T09:50:11.253 INFO:tasks.rados.rados.0.smithi017.stdout:2863: snap_remove snap 349 2024-09-21T09:50:11.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:11 smithi017 ceph-mon[102913]: pgmap v756: 89 pgs: 89 active+clean; 210 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:50:11.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:11 smithi017 ceph-mon[102913]: osdmap e785: 8 total, 8 up, 8 in 2024-09-21T09:50:11.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:11 smithi017 ceph-mon[105151]: pgmap v756: 89 pgs: 89 active+clean; 210 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:50:11.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:11 smithi017 ceph-mon[105151]: osdmap e785: 8 total, 8 up, 8 in 2024-09-21T09:50:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:11 smithi134 ceph-mon[90085]: pgmap v756: 89 pgs: 89 active+clean; 210 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-21T09:50:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:11 smithi134 ceph-mon[90085]: osdmap e785: 8 total, 8 up, 8 in 2024-09-21T09:50:12.264 INFO:tasks.rados.rados.0.smithi017.stdout:2863: done (2 left) 2024-09-21T09:50:12.264 INFO:tasks.rados.rados.0.smithi017.stdout:2864: read oid 7 snap -1 2024-09-21T09:50:12.265 INFO:tasks.rados.rados.0.smithi017.stdout:2864: expect (ObjNum 967 snap 325 seq_num 967) 2024-09-21T09:50:12.265 INFO:tasks.rados.rados.0.smithi017.stdout:2865: read oid 48 snap -1 2024-09-21T09:50:12.265 INFO:tasks.rados.rados.0.smithi017.stdout:2865: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:50:12.265 INFO:tasks.rados.rados.0.smithi017.stdout:2866: rollback oid 50 current snap is 362 2024-09-21T09:50:12.265 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 50 to 343 2024-09-21T09:50:12.265 INFO:tasks.rados.rados.0.smithi017.stdout:2867: read oid 34 snap -1 2024-09-21T09:50:12.265 INFO:tasks.rados.rados.0.smithi017.stdout:2867: expect (ObjNum 1011 snap 341 seq_num 1011) 2024-09-21T09:50:12.265 INFO:tasks.rados.rados.0.smithi017.stdout:2868: snap_remove snap 361 2024-09-21T09:50:12.512 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:12 smithi017 ceph-mon[102913]: osdmap e786: 8 total, 8 up, 8 in 2024-09-21T09:50:12.513 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:12 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:12.513 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:12 smithi017 ceph-mon[105151]: osdmap e786: 8 total, 8 up, 8 in 2024-09-21T09:50:12.513 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:12 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:12 smithi134 ceph-mon[90085]: osdmap e786: 8 total, 8 up, 8 in 2024-09-21T09:50:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:12 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:13.159 INFO:tasks.rados.rados.0.smithi017.stdout:2868: done (6 left) 2024-09-21T09:50:13.159 INFO:tasks.rados.rados.0.smithi017.stdout:2869: read oid 18 snap -1 2024-09-21T09:50:13.159 INFO:tasks.rados.rados.0.smithi017.stdout:2869: expect deleted 2024-09-21T09:50:13.159 INFO:tasks.rados.rados.0.smithi017.stdout:2870: write oid 49 current snap is 362 2024-09-21T09:50:13.159 INFO:tasks.rados.rados.0.smithi017.stdout:2861: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:50:13.159 INFO:tasks.rados.rados.0.smithi017.stdout:2861: finishing rollback tid 2 to smithi01785587-30 2024-09-21T09:50:13.159 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1225 (ObjNum 996 snap 338 seq_num 996) dirty exists 2024-09-21T09:50:13.159 INFO:tasks.rados.rados.0.smithi017.stdout:2862: expect (ObjNum 1065 snap 354 seq_num 1065) 2024-09-21T09:50:13.459 INFO:tasks.rados.rados.0.smithi017.stdout:2864: expect (ObjNum 967 snap 325 seq_num 967) 2024-09-21T09:50:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:13 smithi134 ceph-mon[90085]: pgmap v759: 89 pgs: 89 active+clean; 176 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 770 KiB/s wr, 13 op/s 2024-09-21T09:50:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:13 smithi134 ceph-mon[90085]: osdmap e787: 8 total, 8 up, 8 in 2024-09-21T09:50:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:13 smithi134 ceph-mon[90085]: osdmap e788: 8 total, 8 up, 8 in 2024-09-21T09:50:13.604 INFO:tasks.rados.rados.0.smithi017.stdout:2866: finishing rollback tid 1 to smithi01785587-50 2024-09-21T09:50:13.604 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:50:13.604 INFO:tasks.rados.rados.0.smithi017.stdout:2867: expect (ObjNum 1011 snap 341 seq_num 1011) 2024-09-21T09:50:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:13 smithi017 ceph-mon[102913]: pgmap v759: 89 pgs: 89 active+clean; 176 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 770 KiB/s wr, 13 op/s 2024-09-21T09:50:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:13 smithi017 ceph-mon[102913]: osdmap e787: 8 total, 8 up, 8 in 2024-09-21T09:50:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:13 smithi017 ceph-mon[102913]: osdmap e788: 8 total, 8 up, 8 in 2024-09-21T09:50:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:13 smithi017 ceph-mon[105151]: pgmap v759: 89 pgs: 89 active+clean; 176 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 770 KiB/s wr, 13 op/s 2024-09-21T09:50:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:13 smithi017 ceph-mon[105151]: osdmap e787: 8 total, 8 up, 8 in 2024-09-21T09:50:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:13 smithi017 ceph-mon[105151]: osdmap e788: 8 total, 8 up, 8 in 2024-09-21T09:50:13.753 INFO:tasks.rados.rados.0.smithi017.stdout:2865: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:50:13.865 INFO:tasks.rados.rados.0.smithi017.stdout:2870: seq_num 1077 ranges {501139=648946,1873695=227444} 2024-09-21T09:50:13.872 INFO:tasks.rados.rados.0.smithi017.stdout:2870: writing smithi01785587-49 from 501139 to 1150085 tid 1 2024-09-21T09:50:13.879 INFO:tasks.rados.rados.0.smithi017.stdout:2870: writing smithi01785587-49 from 1873695 to 2101139 tid 2 2024-09-21T09:50:13.879 INFO:tasks.rados.rados.0.smithi017.stdout:2861: done (7 left) 2024-09-21T09:50:13.879 INFO:tasks.rados.rados.0.smithi017.stdout:2862: done (6 left) 2024-09-21T09:50:13.879 INFO:tasks.rados.rados.0.smithi017.stdout:2864: done (5 left) 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:2865: done (4 left) 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:2866: done (3 left) 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:2867: done (2 left) 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:2869: done (1 left) 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:2871: read oid 32 snap 355 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:2871: expect (ObjNum 982 snap 335 seq_num 982) 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:2872: read oid 50 snap -1 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:2872: expect deleted 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:2873: rollback oid 38 current snap is 362 2024-09-21T09:50:13.880 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 357 2024-09-21T09:50:13.881 INFO:tasks.rados.rados.0.smithi017.stdout:2870: finishing write tid 1 to smithi01785587-49 2024-09-21T09:50:13.881 INFO:tasks.rados.rados.0.smithi017.stdout:2874: write oid 18 current snap is 362 2024-09-21T09:50:13.881 INFO:tasks.rados.rados.0.smithi017.stdout:2874: seq_num 1078 ranges {517946=724195,1932177=185769} 2024-09-21T09:50:13.885 INFO:tasks.rados.rados.0.smithi017.stdout:2874: writing smithi01785587-18 from 517946 to 1242141 tid 1 2024-09-21T09:50:13.890 INFO:tasks.rados.rados.0.smithi017.stdout:2874: writing smithi01785587-18 from 1932177 to 2117946 tid 2 2024-09-21T09:50:13.890 INFO:tasks.rados.rados.0.smithi017.stdout:2875: read oid 23 snap -1 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:2875: expect deleted 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:2876: delete oid 41 current snap is 362 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:2873: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 0 (ObjNum 209 snap 0 seq_num 4181695904) dirty dne 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:2870: finishing write tid 2 to smithi01785587-49 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:2870: finishing write tid 3 to smithi01785587-49 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:2870: finishing write tid 4 to smithi01785587-49 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1431 (ObjNum 1077 snap 362 seq_num 1077) dirty exists 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:2870: left oid 49 (ObjNum 1077 snap 362 seq_num 1077) 2024-09-21T09:50:13.891 INFO:tasks.rados.rados.0.smithi017.stdout:2871: expect (ObjNum 982 snap 335 seq_num 982) 2024-09-21T09:50:13.975 INFO:tasks.rados.rados.0.smithi017.stdout:2874: finishing write tid 1 to smithi01785587-18 2024-09-21T09:50:13.975 INFO:tasks.rados.rados.0.smithi017.stdout:2874: finishing write tid 2 to smithi01785587-18 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:2870: done (6 left) 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:2871: done (5 left) 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:2872: done (4 left) 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:2873: done (3 left) 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:2875: done (2 left) 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:2876: done (1 left) 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:2877: rollback oid 21 current snap is 362 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 355 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:2874: finishing write tid 3 to smithi01785587-18 2024-09-21T09:50:13.976 INFO:tasks.rados.rados.0.smithi017.stdout:2874: finishing write tid 4 to smithi01785587-18 2024-09-21T09:50:13.977 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 261 (ObjNum 1078 snap 362 seq_num 1078) dirty exists 2024-09-21T09:50:13.977 INFO:tasks.rados.rados.0.smithi017.stdout:2874: left oid 18 (ObjNum 1078 snap 362 seq_num 1078) 2024-09-21T09:50:13.977 INFO:tasks.rados.rados.0.smithi017.stdout:2874: done (1 left) 2024-09-21T09:50:13.977 INFO:tasks.rados.rados.0.smithi017.stdout:2878: write oid 1 current snap is 362 2024-09-21T09:50:13.977 INFO:tasks.rados.rados.0.smithi017.stdout:2878: seq_num 1080 ranges {551560=591046,1765494=386066} 2024-09-21T09:50:13.981 INFO:tasks.rados.rados.0.smithi017.stdout:2878: writing smithi01785587-1 from 551560 to 1142606 tid 1 2024-09-21T09:50:13.986 INFO:tasks.rados.rados.0.smithi017.stdout:2878: writing smithi01785587-1 from 1765494 to 2151560 tid 2 2024-09-21T09:50:13.986 INFO:tasks.rados.rados.0.smithi017.stdout:2879: snap_remove snap 350 2024-09-21T09:50:14.267 INFO:tasks.rados.rados.0.smithi017.stdout:2877: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:50:14.267 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 0 (ObjNum 49 snap 0 seq_num 2399795072) dirty dne 2024-09-21T09:50:14.267 INFO:tasks.rados.rados.0.smithi017.stdout:2878: finishing write tid 1 to smithi01785587-1 2024-09-21T09:50:14.267 INFO:tasks.rados.rados.0.smithi017.stdout:2878: finishing write tid 2 to smithi01785587-1 2024-09-21T09:50:14.267 INFO:tasks.rados.rados.0.smithi017.stdout:2878: finishing write tid 3 to smithi01785587-1 2024-09-21T09:50:14.267 INFO:tasks.rados.rados.0.smithi017.stdout:2878: finishing write tid 4 to smithi01785587-1 2024-09-21T09:50:14.268 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 867 (ObjNum 1080 snap 362 seq_num 1080) dirty exists 2024-09-21T09:50:14.268 INFO:tasks.rados.rados.0.smithi017.stdout:2878: left oid 1 (ObjNum 1080 snap 362 seq_num 1080) 2024-09-21T09:50:14.268 INFO:tasks.rados.rados.0.smithi017.stdout:2877: done (2 left) 2024-09-21T09:50:14.268 INFO:tasks.rados.rados.0.smithi017.stdout:2878: done (1 left) 2024-09-21T09:50:14.268 INFO:tasks.rados.rados.0.smithi017.stdout:2879: done (0 left) 2024-09-21T09:50:14.268 INFO:tasks.rados.rados.0.smithi017.stdout:2880: delete oid 20 current snap is 362 2024-09-21T09:50:14.276 INFO:tasks.rados.rados.0.smithi017.stdout:2880: done (0 left) 2024-09-21T09:50:14.276 INFO:tasks.rados.rados.0.smithi017.stdout:2881: snap_remove snap 358 2024-09-21T09:50:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:50:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:50:14] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:50:15.271 INFO:tasks.rados.rados.0.smithi017.stdout:2881: done (0 left) 2024-09-21T09:50:15.271 INFO:tasks.rados.rados.0.smithi017.stdout:2882: delete oid 43 current snap is 362 2024-09-21T09:50:15.276 INFO:tasks.rados.rados.0.smithi017.stdout:2882: done (0 left) 2024-09-21T09:50:15.276 INFO:tasks.rados.rados.0.smithi017.stdout:2883: write oid 30 current snap is 362 2024-09-21T09:50:15.276 INFO:tasks.rados.rados.0.smithi017.stdout:2883: seq_num 1083 ranges {601981=416793,1424587=492702} 2024-09-21T09:50:15.281 INFO:tasks.rados.rados.0.smithi017.stdout:2883: writing smithi01785587-30 from 601981 to 1018774 tid 1 2024-09-21T09:50:15.286 INFO:tasks.rados.rados.0.smithi017.stdout:2883: writing smithi01785587-30 from 1424587 to 1917289 tid 2 2024-09-21T09:50:15.287 INFO:tasks.rados.rados.0.smithi017.stdout:2884: write oid 26 current snap is 362 2024-09-21T09:50:15.287 INFO:tasks.rados.rados.0.smithi017.stdout:2884: seq_num 1084 ranges {618788=492042,1883069=335719} 2024-09-21T09:50:15.294 INFO:tasks.rados.rados.0.smithi017.stdout:2884: writing smithi01785587-26 from 618788 to 1110830 tid 1 2024-09-21T09:50:15.302 INFO:tasks.rados.rados.0.smithi017.stdout:2884: writing smithi01785587-26 from 1883069 to 2218788 tid 2 2024-09-21T09:50:15.302 INFO:tasks.rados.rados.0.smithi017.stdout:2885: snap_create 2024-09-21T09:50:15.303 INFO:tasks.rados.rados.0.smithi017.stdout:2883: finishing write tid 1 to smithi01785587-30 2024-09-21T09:50:15.303 INFO:tasks.rados.rados.0.smithi017.stdout:2883: finishing write tid 2 to smithi01785587-30 2024-09-21T09:50:15.303 INFO:tasks.rados.rados.0.smithi017.stdout:2883: finishing write tid 3 to smithi01785587-30 2024-09-21T09:50:15.303 INFO:tasks.rados.rados.0.smithi017.stdout:2883: finishing write tid 4 to smithi01785587-30 2024-09-21T09:50:15.303 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1238 (ObjNum 1083 snap 362 seq_num 1083) dirty exists 2024-09-21T09:50:15.303 INFO:tasks.rados.rados.0.smithi017.stdout:2883: left oid 30 (ObjNum 1083 snap 362 seq_num 1083) 2024-09-21T09:50:15.303 INFO:tasks.rados.rados.0.smithi017.stdout:2884: finishing write tid 1 to smithi01785587-26 2024-09-21T09:50:15.305 INFO:tasks.rados.rados.0.smithi017.stdout:2884: finishing write tid 2 to smithi01785587-26 2024-09-21T09:50:15.305 INFO:tasks.rados.rados.0.smithi017.stdout:2884: finishing write tid 3 to smithi01785587-26 2024-09-21T09:50:15.305 INFO:tasks.rados.rados.0.smithi017.stdout:2884: finishing write tid 4 to smithi01785587-26 2024-09-21T09:50:15.305 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1492 (ObjNum 1084 snap 362 seq_num 1084) dirty exists 2024-09-21T09:50:15.305 INFO:tasks.rados.rados.0.smithi017.stdout:2884: left oid 26 (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:50:15.571 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:15 smithi017 ceph-mon[102913]: pgmap v762: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 526 KiB/s wr, 7 op/s 2024-09-21T09:50:15.571 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:15 smithi017 ceph-mon[102913]: osdmap e789: 8 total, 8 up, 8 in 2024-09-21T09:50:15.571 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:15 smithi017 ceph-mon[105151]: pgmap v762: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 526 KiB/s wr, 7 op/s 2024-09-21T09:50:15.572 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:15 smithi017 ceph-mon[105151]: osdmap e789: 8 total, 8 up, 8 in 2024-09-21T09:50:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:15 smithi134 ceph-mon[90085]: pgmap v762: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 526 KiB/s wr, 7 op/s 2024-09-21T09:50:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:15 smithi134 ceph-mon[90085]: osdmap e789: 8 total, 8 up, 8 in 2024-09-21T09:50:16.272 INFO:tasks.rados.rados.0.smithi017.stdout:2883: done (2 left) 2024-09-21T09:50:16.272 INFO:tasks.rados.rados.0.smithi017.stdout:2884: done (1 left) 2024-09-21T09:50:16.272 INFO:tasks.rados.rados.0.smithi017.stdout:2885: done (0 left) 2024-09-21T09:50:16.272 INFO:tasks.rados.rados.0.smithi017.stdout:2886: delete oid 34 current snap is 363 2024-09-21T09:50:16.275 INFO:tasks.rados.rados.0.smithi017.stdout:2886: done (0 left) 2024-09-21T09:50:16.275 INFO:tasks.rados.rados.0.smithi017.stdout:2887: read oid 43 snap -1 2024-09-21T09:50:16.275 INFO:tasks.rados.rados.0.smithi017.stdout:2887: expect deleted 2024-09-21T09:50:16.276 INFO:tasks.rados.rados.0.smithi017.stdout:2888: delete oid 36 current snap is 363 2024-09-21T09:50:16.278 INFO:tasks.rados.rados.0.smithi017.stdout:2887: done (1 left) 2024-09-21T09:50:16.278 INFO:tasks.rados.rados.0.smithi017.stdout:2888: done (0 left) 2024-09-21T09:50:16.278 INFO:tasks.rados.rados.0.smithi017.stdout:2889: rollback oid 17 current snap is 363 2024-09-21T09:50:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 355 2024-09-21T09:50:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:2890: read oid 28 snap 354 2024-09-21T09:50:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:2890: expect (ObjNum 1050 snap 351 seq_num 1050) 2024-09-21T09:50:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:2891: rollback oid 31 current snap is 363 2024-09-21T09:50:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 31 to 362 2024-09-21T09:50:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:2892: rollback oid 30 current snap is 363 2024-09-21T09:50:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 353 2024-09-21T09:50:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:2893: write oid 42 current snap is 363 2024-09-21T09:50:16.279 INFO:tasks.rados.rados.0.smithi017.stdout:2893: seq_num 1087 ranges {669209=717789,1942162=327047} 2024-09-21T09:50:16.286 INFO:tasks.rados.rados.0.smithi017.stdout:2893: writing smithi01785587-42 from 669209 to 1386998 tid 1 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:2893: writing smithi01785587-42 from 1942162 to 2269209 tid 2 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:2894: read oid 18 snap -1 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:2894: expect (ObjNum 1078 snap 362 seq_num 1078) 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:2895: write oid 38 current snap is 363 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:2889: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 0 (ObjNum 129 snap 0 seq_num 2399463368) dirty dne 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:2892: finishing rollback tid 0 to smithi01785587-30 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:2892: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1241 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:2891: finishing rollback tid 0 to smithi01785587-31 2024-09-21T09:50:16.292 INFO:tasks.rados.rados.0.smithi017.stdout:2891: finishing rollback tid 1 to smithi01785587-31 2024-09-21T09:50:16.293 INFO:tasks.rados.rados.0.smithi017.stdout:2891: finishing rollback tid 2 to smithi01785587-31 2024-09-21T09:50:16.293 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1487 (ObjNum 1023 snap 344 seq_num 1023) dirty exists 2024-09-21T09:50:16.293 INFO:tasks.rados.rados.0.smithi017.stdout:2890: expect (ObjNum 1050 snap 351 seq_num 1050) 2024-09-21T09:50:16.415 INFO:tasks.rados.rados.0.smithi017.stdout:2893: finishing write tid 1 to smithi01785587-42 2024-09-21T09:50:16.415 INFO:tasks.rados.rados.0.smithi017.stdout:2895: seq_num 1088 ranges {686016=509391,1716997=569019} 2024-09-21T09:50:16.421 INFO:tasks.rados.rados.0.smithi017.stdout:2895: writing smithi01785587-38 from 686016 to 1195407 tid 1 2024-09-21T09:50:16.427 INFO:tasks.rados.rados.0.smithi017.stdout:2895: writing smithi01785587-38 from 1716997 to 2286016 tid 2 2024-09-21T09:50:16.427 INFO:tasks.rados.rados.0.smithi017.stdout:2889: done (6 left) 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:2890: done (5 left) 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:2891: done (4 left) 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:2892: done (3 left) 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:2896: delete oid 39 current snap is 363 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:2893: finishing write tid 2 to smithi01785587-42 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:2893: finishing write tid 3 to smithi01785587-42 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:2893: finishing write tid 4 to smithi01785587-42 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 975 (ObjNum 1087 snap 363 seq_num 1087) dirty exists 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:2893: left oid 42 (ObjNum 1087 snap 363 seq_num 1087) 2024-09-21T09:50:16.428 INFO:tasks.rados.rados.0.smithi017.stdout:2894: expect (ObjNum 1078 snap 362 seq_num 1078) 2024-09-21T09:50:16.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:16 smithi134 ceph-mon[90085]: osdmap e790: 8 total, 8 up, 8 in 2024-09-21T09:50:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:16 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:16 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:16.499 INFO:tasks.rados.rados.0.smithi017.stdout:2895: finishing write tid 1 to smithi01785587-38 2024-09-21T09:50:16.499 INFO:tasks.rados.rados.0.smithi017.stdout:2895: finishing write tid 2 to smithi01785587-38 2024-09-21T09:50:16.499 INFO:tasks.rados.rados.0.smithi017.stdout:2895: finishing write tid 3 to smithi01785587-38 2024-09-21T09:50:16.499 INFO:tasks.rados.rados.0.smithi017.stdout:2895: finishing write tid 4 to smithi01785587-38 2024-09-21T09:50:16.500 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 924 (ObjNum 1088 snap 363 seq_num 1088) dirty exists 2024-09-21T09:50:16.500 INFO:tasks.rados.rados.0.smithi017.stdout:2895: left oid 38 (ObjNum 1088 snap 363 seq_num 1088) 2024-09-21T09:50:16.500 INFO:tasks.rados.rados.0.smithi017.stdout:2893: done (3 left) 2024-09-21T09:50:16.500 INFO:tasks.rados.rados.0.smithi017.stdout:2894: done (2 left) 2024-09-21T09:50:16.500 INFO:tasks.rados.rados.0.smithi017.stdout:2895: done (1 left) 2024-09-21T09:50:16.500 INFO:tasks.rados.rados.0.smithi017.stdout:2896: done (0 left) 2024-09-21T09:50:16.500 INFO:tasks.rados.rados.0.smithi017.stdout:2897: write oid 21 current snap is 363 2024-09-21T09:50:16.500 INFO:tasks.rados.rados.0.smithi017.stdout:2897: seq_num 1090 ranges {719630=659889,2117608=202022} 2024-09-21T09:50:16.506 INFO:tasks.rados.rados.0.smithi017.stdout:2897: writing smithi01785587-21 from 719630 to 1379519 tid 1 2024-09-21T09:50:16.512 INFO:tasks.rados.rados.0.smithi017.stdout:2897: writing smithi01785587-21 from 2117608 to 2319630 tid 2 2024-09-21T09:50:16.512 INFO:tasks.rados.rados.0.smithi017.stdout:2898: read oid 35 snap -1 2024-09-21T09:50:16.513 INFO:tasks.rados.rados.0.smithi017.stdout:2898: expect (ObjNum 1069 snap 356 seq_num 1069) 2024-09-21T09:50:16.513 INFO:tasks.rados.rados.0.smithi017.stdout:2899: write oid 38 current snap is 363 2024-09-21T09:50:16.513 INFO:tasks.rados.rados.0.smithi017.stdout:2897: finishing write tid 1 to smithi01785587-21 2024-09-21T09:50:16.513 INFO:tasks.rados.rados.0.smithi017.stdout:2899: seq_num 1091 ranges {736437=735138,2176090=160347} 2024-09-21T09:50:16.520 INFO:tasks.rados.rados.0.smithi017.stdout:2899: writing smithi01785587-38 from 736437 to 1471575 tid 1 2024-09-21T09:50:16.525 INFO:tasks.rados.rados.0.smithi017.stdout:2899: writing smithi01785587-38 from 2176090 to 2336437 tid 2 2024-09-21T09:50:16.525 INFO:tasks.rados.rados.0.smithi017.stdout:2900: snap_remove snap 362 2024-09-21T09:50:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:16 smithi017 ceph-mon[102913]: osdmap e790: 8 total, 8 up, 8 in 2024-09-21T09:50:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:16 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:16 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:16 smithi017 ceph-mon[105151]: osdmap e790: 8 total, 8 up, 8 in 2024-09-21T09:50:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:16 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:16.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:16 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:17.282 INFO:tasks.rados.rados.0.smithi017.stdout:2900: done (3 left) 2024-09-21T09:50:17.282 INFO:tasks.rados.rados.0.smithi017.stdout:2901: snap_remove snap 357 2024-09-21T09:50:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[102913]: pgmap v765: 89 pgs: 89 active+clean; 167 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:50:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[102913]: osdmap e791: 8 total, 8 up, 8 in 2024-09-21T09:50:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:17.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:17.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[105151]: pgmap v765: 89 pgs: 89 active+clean; 167 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:50:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[105151]: osdmap e791: 8 total, 8 up, 8 in 2024-09-21T09:50:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:17 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:17 smithi134 ceph-mon[90085]: pgmap v765: 89 pgs: 89 active+clean; 167 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:50:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:17 smithi134 ceph-mon[90085]: osdmap e791: 8 total, 8 up, 8 in 2024-09-21T09:50:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:17 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:17 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:17 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:17 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:17 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:50:18.162 INFO:tasks.rados.rados.0.smithi017.stdout:2897: finishing write tid 2 to smithi01785587-21 2024-09-21T09:50:18.162 INFO:tasks.rados.rados.0.smithi017.stdout:2897: finishing write tid 3 to smithi01785587-21 2024-09-21T09:50:18.162 INFO:tasks.rados.rados.0.smithi017.stdout:2897: finishing write tid 4 to smithi01785587-21 2024-09-21T09:50:18.162 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1439 (ObjNum 1090 snap 363 seq_num 1090) dirty exists 2024-09-21T09:50:18.162 INFO:tasks.rados.rados.0.smithi017.stdout:2897: left oid 21 (ObjNum 1090 snap 363 seq_num 1090) 2024-09-21T09:50:18.162 INFO:tasks.rados.rados.0.smithi017.stdout:2898: expect (ObjNum 1069 snap 356 seq_num 1069) 2024-09-21T09:50:18.295 INFO:tasks.rados.rados.0.smithi017.stdout:2899: finishing write tid 1 to smithi01785587-38 2024-09-21T09:50:18.295 INFO:tasks.rados.rados.0.smithi017.stdout:2899: finishing write tid 2 to smithi01785587-38 2024-09-21T09:50:18.295 INFO:tasks.rados.rados.0.smithi017.stdout:2899: finishing write tid 3 to smithi01785587-38 2024-09-21T09:50:18.295 INFO:tasks.rados.rados.0.smithi017.stdout:2899: finishing write tid 4 to smithi01785587-38 2024-09-21T09:50:18.295 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 927 (ObjNum 1091 snap 363 seq_num 1091) dirty exists 2024-09-21T09:50:18.295 INFO:tasks.rados.rados.0.smithi017.stdout:2899: left oid 38 (ObjNum 1091 snap 363 seq_num 1091) 2024-09-21T09:50:18.295 INFO:tasks.rados.rados.0.smithi017.stdout:2897: done (3 left) 2024-09-21T09:50:18.295 INFO:tasks.rados.rados.0.smithi017.stdout:2898: done (2 left) 2024-09-21T09:50:18.295 INFO:tasks.rados.rados.0.smithi017.stdout:2899: done (1 left) 2024-09-21T09:50:18.296 INFO:tasks.rados.rados.0.smithi017.stdout:2901: done (0 left) 2024-09-21T09:50:18.296 INFO:tasks.rados.rados.0.smithi017.stdout:2902: delete oid 30 current snap is 363 2024-09-21T09:50:18.297 INFO:tasks.rados.rados.0.smithi017.stdout:2902: done (0 left) 2024-09-21T09:50:18.297 INFO:tasks.rados.rados.0.smithi017.stdout:2903: read oid 32 snap -1 2024-09-21T09:50:18.297 INFO:tasks.rados.rados.0.smithi017.stdout:2903: expect (ObjNum 982 snap 335 seq_num 982) 2024-09-21T09:50:18.297 INFO:tasks.rados.rados.0.smithi017.stdout:2904: read oid 12 snap -1 2024-09-21T09:50:18.297 INFO:tasks.rados.rados.0.smithi017.stdout:2904: expect (ObjNum 1073 snap 359 seq_num 1073) 2024-09-21T09:50:18.297 INFO:tasks.rados.rados.0.smithi017.stdout:2905: read oid 39 snap -1 2024-09-21T09:50:18.297 INFO:tasks.rados.rados.0.smithi017.stdout:2905: expect deleted 2024-09-21T09:50:18.297 INFO:tasks.rados.rados.0.smithi017.stdout:2906: write oid 50 current snap is 363 2024-09-21T09:50:18.298 INFO:tasks.rados.rados.0.smithi017.stdout:2906: seq_num 1093 ranges {770051=485636,1776701=511047} 2024-09-21T09:50:18.304 INFO:tasks.rados.rados.0.smithi017.stdout:2906: writing smithi01785587-50 from 770051 to 1255687 tid 1 2024-09-21T09:50:18.310 INFO:tasks.rados.rados.0.smithi017.stdout:2906: writing smithi01785587-50 from 1776701 to 2287748 tid 2 2024-09-21T09:50:18.310 INFO:tasks.rados.rados.0.smithi017.stdout:2907: delete oid 11 current snap is 363 2024-09-21T09:50:18.310 INFO:tasks.rados.rados.0.smithi017.stdout:2904: expect (ObjNum 1073 snap 359 seq_num 1073) 2024-09-21T09:50:18.377 INFO:tasks.rados.rados.0.smithi017.stdout:2903: expect (ObjNum 982 snap 335 seq_num 982) 2024-09-21T09:50:18.460 INFO:tasks.rados.rados.0.smithi017.stdout:2906: finishing write tid 1 to smithi01785587-50 2024-09-21T09:50:18.460 INFO:tasks.rados.rados.0.smithi017.stdout:2906: finishing write tid 2 to smithi01785587-50 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2906: finishing write tid 3 to smithi01785587-50 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2906: finishing write tid 4 to smithi01785587-50 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 930 (ObjNum 1093 snap 363 seq_num 1093) dirty exists 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2906: left oid 50 (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2903: done (4 left) 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2904: done (3 left) 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2905: done (2 left) 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2906: done (1 left) 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2907: done (0 left) 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2908: write oid 13 current snap is 363 2024-09-21T09:50:18.461 INFO:tasks.rados.rados.0.smithi017.stdout:2908: seq_num 1095 ranges {403665=752487,1610018=514716} 2024-09-21T09:50:18.466 INFO:tasks.rados.rados.0.smithi017.stdout:2908: writing smithi01785587-13 from 403665 to 1156152 tid 1 2024-09-21T09:50:18.472 INFO:tasks.rados.rados.0.smithi017.stdout:2908: writing smithi01785587-13 from 1610018 to 2124734 tid 2 2024-09-21T09:50:18.472 INFO:tasks.rados.rados.0.smithi017.stdout:2909: write oid 41 current snap is 363 2024-09-21T09:50:18.472 INFO:tasks.rados.rados.0.smithi017.stdout:2909: seq_num 1096 ranges {420472=427736,1268500=658374,2347953=72519} 2024-09-21T09:50:18.476 INFO:tasks.rados.rados.0.smithi017.stdout:2909: writing smithi01785587-41 from 420472 to 848208 tid 1 2024-09-21T09:50:18.482 INFO:tasks.rados.rados.0.smithi017.stdout:2909: writing smithi01785587-41 from 1268500 to 1926874 tid 2 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2909: writing smithi01785587-41 from 2347953 to 2420472 tid 3 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2910: rollback oid 12 current snap is 363 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 12 to 353 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2908: finishing write tid 1 to smithi01785587-13 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2911: read oid 11 snap -1 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2911: expect deleted 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2912: rollback oid 16 current snap is 363 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 16 to 353 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2908: finishing write tid 2 to smithi01785587-13 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2908: finishing write tid 3 to smithi01785587-13 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2908: finishing write tid 4 to smithi01785587-13 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1250 (ObjNum 1095 snap 363 seq_num 1095) dirty exists 2024-09-21T09:50:18.486 INFO:tasks.rados.rados.0.smithi017.stdout:2908: left oid 13 (ObjNum 1095 snap 363 seq_num 1095) 2024-09-21T09:50:18.487 INFO:tasks.rados.rados.0.smithi017.stdout:2909: finishing write tid 1 to smithi01785587-41 2024-09-21T09:50:18.487 INFO:tasks.rados.rados.0.smithi017.stdout:2908: done (4 left) 2024-09-21T09:50:18.487 INFO:tasks.rados.rados.0.smithi017.stdout:2913: write oid 39 current snap is 363 2024-09-21T09:50:18.487 INFO:tasks.rados.rados.0.smithi017.stdout:2913: seq_num 1097 ranges {437279=502985,1610629=518385} 2024-09-21T09:50:18.491 INFO:tasks.rados.rados.0.smithi017.stdout:2913: writing smithi01785587-39 from 437279 to 940264 tid 1 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:2913: writing smithi01785587-39 from 1610629 to 2129014 tid 2 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:2914: rollback oid 25 current snap is 363 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 25 to 355 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:2909: finishing write tid 2 to smithi01785587-41 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:2909: finishing write tid 3 to smithi01785587-41 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:2909: finishing write tid 4 to smithi01785587-41 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:2911: done (5 left) 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:2915: write oid 43 current snap is 363 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:2909: finishing write tid 5 to smithi01785587-41 2024-09-21T09:50:18.498 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 491 (ObjNum 1096 snap 363 seq_num 1096) dirty exists 2024-09-21T09:50:18.499 INFO:tasks.rados.rados.0.smithi017.stdout:2909: left oid 41 (ObjNum 1096 snap 363 seq_num 1096) 2024-09-21T09:50:18.499 INFO:tasks.rados.rados.0.smithi017.stdout:2910: finishing rollback tid 0 to smithi01785587-12 2024-09-21T09:50:18.499 INFO:tasks.rados.rados.0.smithi017.stdout:2915: seq_num 1098 ranges {454086=578234,1669111=662043} 2024-09-21T09:50:18.503 INFO:tasks.rados.rados.0.smithi017.stdout:2915: writing smithi01785587-43 from 454086 to 1032320 tid 1 2024-09-21T09:50:18.511 INFO:tasks.rados.rados.0.smithi017.stdout:2915: writing smithi01785587-43 from 1669111 to 2331154 tid 2 2024-09-21T09:50:18.511 INFO:tasks.rados.rados.0.smithi017.stdout:2909: done (5 left) 2024-09-21T09:50:18.511 INFO:tasks.rados.rados.0.smithi017.stdout:2916: write oid 24 current snap is 363 2024-09-21T09:50:18.511 INFO:tasks.rados.rados.0.smithi017.stdout:2910: finishing rollback tid 1 to smithi01785587-12 2024-09-21T09:50:18.511 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 982 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:50:18.511 INFO:tasks.rados.rados.0.smithi017.stdout:2912: finishing rollback tid 0 to smithi01785587-16 2024-09-21T09:50:18.511 INFO:tasks.rados.rados.0.smithi017.stdout:2912: finishing rollback tid 1 to smithi01785587-16 2024-09-21T09:50:18.511 INFO:tasks.rados.rados.0.smithi017.stdout:2912: finishing rollback tid 2 to smithi01785587-16 2024-09-21T09:50:18.511 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1498 (ObjNum 1026 snap 345 seq_num 1026) dirty exists 2024-09-21T09:50:18.512 INFO:tasks.rados.rados.0.smithi017.stdout:2913: finishing write tid 1 to smithi01785587-39 2024-09-21T09:50:18.512 INFO:tasks.rados.rados.0.smithi017.stdout:2916: seq_num 1099 ranges {470893=653483,1727593=522054} 2024-09-21T09:50:18.516 INFO:tasks.rados.rados.0.smithi017.stdout:2916: writing smithi01785587-24 from 470893 to 1124376 tid 1 2024-09-21T09:50:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:2916: writing smithi01785587-24 from 1727593 to 2249647 tid 2 2024-09-21T09:50:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:2910: done (5 left) 2024-09-21T09:50:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:2912: done (4 left) 2024-09-21T09:50:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:2917: rollback oid 20 current snap is 363 2024-09-21T09:50:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 356 2024-09-21T09:50:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:2914: finishing rollback tid 1 to smithi01785587-25 2024-09-21T09:50:18.523 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 778 (ObjNum 129 snap 0 seq_num 2399605904) dirty dne 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2913: finishing write tid 2 to smithi01785587-39 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2913: finishing write tid 3 to smithi01785587-39 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2913: finishing write tid 4 to smithi01785587-39 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1501 (ObjNum 1097 snap 363 seq_num 1097) dirty exists 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2913: left oid 39 (ObjNum 1097 snap 363 seq_num 1097) 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2913: done (4 left) 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2914: done (3 left) 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2918: delete oid 42 current snap is 363 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2915: finishing write tid 1 to smithi01785587-43 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2915: finishing write tid 2 to smithi01785587-43 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2915: finishing write tid 3 to smithi01785587-43 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:2915: finishing write tid 4 to smithi01785587-43 2024-09-21T09:50:18.524 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 871 (ObjNum 1098 snap 363 seq_num 1098) dirty exists 2024-09-21T09:50:18.525 INFO:tasks.rados.rados.0.smithi017.stdout:2915: left oid 43 (ObjNum 1098 snap 363 seq_num 1098) 2024-09-21T09:50:18.525 INFO:tasks.rados.rados.0.smithi017.stdout:2916: finishing write tid 1 to smithi01785587-24 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:2917: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 757 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:2915: done (3 left) 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:2917: done (2 left) 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:2918: done (1 left) 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:2919: rollback oid 31 current snap is 363 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 31 to 353 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:2920: read oid 27 snap -1 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:2920: expect (ObjNum 1009 snap 341 seq_num 1009) 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:2921: delete oid 38 current snap is 363 2024-09-21T09:50:18.526 INFO:tasks.rados.rados.0.smithi017.stdout:2916: finishing write tid 2 to smithi01785587-24 2024-09-21T09:50:18.527 INFO:tasks.rados.rados.0.smithi017.stdout:2916: finishing write tid 3 to smithi01785587-24 2024-09-21T09:50:18.527 INFO:tasks.rados.rados.0.smithi017.stdout:2916: finishing write tid 4 to smithi01785587-24 2024-09-21T09:50:18.527 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 934 (ObjNum 1099 snap 363 seq_num 1099) dirty exists 2024-09-21T09:50:18.527 INFO:tasks.rados.rados.0.smithi017.stdout:2916: left oid 24 (ObjNum 1099 snap 363 seq_num 1099) 2024-09-21T09:50:18.527 INFO:tasks.rados.rados.0.smithi017.stdout:2916: done (3 left) 2024-09-21T09:50:18.527 INFO:tasks.rados.rados.0.smithi017.stdout:2921: done (2 left) 2024-09-21T09:50:18.528 INFO:tasks.rados.rados.0.smithi017.stdout:2922: snap_create 2024-09-21T09:50:18.529 INFO:tasks.rados.rados.0.smithi017.stdout:2919: finishing rollback tid 0 to smithi01785587-31 2024-09-21T09:50:18.529 INFO:tasks.rados.rados.0.smithi017.stdout:2919: finishing rollback tid 1 to smithi01785587-31 2024-09-21T09:50:18.530 INFO:tasks.rados.rados.0.smithi017.stdout:2919: finishing rollback tid 2 to smithi01785587-31 2024-09-21T09:50:18.530 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1494 (ObjNum 1023 snap 344 seq_num 1023) dirty exists 2024-09-21T09:50:18.530 INFO:tasks.rados.rados.0.smithi017.stdout:2920: expect (ObjNum 1009 snap 341 seq_num 1009) 2024-09-21T09:50:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:18 smithi017 ceph-mon[102913]: osdmap e792: 8 total, 8 up, 8 in 2024-09-21T09:50:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:18 smithi017 ceph-mon[102913]: osdmap e793: 8 total, 8 up, 8 in 2024-09-21T09:50:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:18 smithi017 ceph-mon[105151]: osdmap e792: 8 total, 8 up, 8 in 2024-09-21T09:50:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:18 smithi017 ceph-mon[105151]: osdmap e793: 8 total, 8 up, 8 in 2024-09-21T09:50:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:18 smithi134 ceph-mon[90085]: osdmap e792: 8 total, 8 up, 8 in 2024-09-21T09:50:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:18 smithi134 ceph-mon[90085]: osdmap e793: 8 total, 8 up, 8 in 2024-09-21T09:50:19.290 INFO:tasks.rados.rados.0.smithi017.stdout:2919: done (2 left) 2024-09-21T09:50:19.290 INFO:tasks.rados.rados.0.smithi017.stdout:2920: done (1 left) 2024-09-21T09:50:19.290 INFO:tasks.rados.rados.0.smithi017.stdout:2922: done (0 left) 2024-09-21T09:50:19.290 INFO:tasks.rados.rados.0.smithi017.stdout:2923: write oid 3 current snap is 364 2024-09-21T09:50:19.290 INFO:tasks.rados.rados.0.smithi017.stdout:2923: seq_num 1102 ranges {521314=479230,1786686=669381} 2024-09-21T09:50:19.296 INFO:tasks.rados.rados.0.smithi017.stdout:2923: writing smithi01785587-3 from 521314 to 1000544 tid 1 2024-09-21T09:50:19.304 INFO:tasks.rados.rados.0.smithi017.stdout:2923: writing smithi01785587-3 from 1786686 to 2456067 tid 2 2024-09-21T09:50:19.304 INFO:tasks.rados.rados.0.smithi017.stdout:2924: snap_create 2024-09-21T09:50:19.304 INFO:tasks.rados.rados.0.smithi017.stdout:2923: finishing write tid 1 to smithi01785587-3 2024-09-21T09:50:19.308 INFO:tasks.rados.rados.0.smithi017.stdout:2923: finishing write tid 2 to smithi01785587-3 2024-09-21T09:50:19.308 INFO:tasks.rados.rados.0.smithi017.stdout:2923: finishing write tid 3 to smithi01785587-3 2024-09-21T09:50:19.309 INFO:tasks.rados.rados.0.smithi017.stdout:2923: finishing write tid 4 to smithi01785587-3 2024-09-21T09:50:19.309 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1498 (ObjNum 1102 snap 364 seq_num 1102) dirty exists 2024-09-21T09:50:19.309 INFO:tasks.rados.rados.0.smithi017.stdout:2923: left oid 3 (ObjNum 1102 snap 364 seq_num 1102) 2024-09-21T09:50:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:19 smithi017 ceph-mon[105151]: pgmap v768: 89 pgs: 89 active+clean; 165 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 17 op/s 2024-09-21T09:50:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:19 smithi017 ceph-mon[102913]: pgmap v768: 89 pgs: 89 active+clean; 165 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 17 op/s 2024-09-21T09:50:19.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:19 smithi134 ceph-mon[90085]: pgmap v768: 89 pgs: 89 active+clean; 165 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 17 op/s 2024-09-21T09:50:20.303 INFO:tasks.rados.rados.0.smithi017.stdout:2923: done (1 left) 2024-09-21T09:50:20.303 INFO:tasks.rados.rados.0.smithi017.stdout:2924: done (0 left) 2024-09-21T09:50:20.303 INFO:tasks.rados.rados.0.smithi017.stdout:2925: delete oid 47 current snap is 365 2024-09-21T09:50:20.306 INFO:tasks.rados.rados.0.smithi017.stdout:2925: done (0 left) 2024-09-21T09:50:20.306 INFO:tasks.rados.rados.0.smithi017.stdout:2926: rollback oid 10 current snap is 365 2024-09-21T09:50:20.307 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 353 2024-09-21T09:50:20.307 INFO:tasks.rados.rados.0.smithi017.stdout:2927: snap_create 2024-09-21T09:50:20.309 INFO:tasks.rados.rados.0.smithi017.stdout:2926: finishing rollback tid 0 to smithi01785587-10 2024-09-21T09:50:20.309 INFO:tasks.rados.rados.0.smithi017.stdout:2926: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:50:20.309 INFO:tasks.rados.rados.0.smithi017.stdout:2926: finishing rollback tid 2 to smithi01785587-10 2024-09-21T09:50:20.310 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 252 (ObjNum 985 snap 336 seq_num 985) dirty exists 2024-09-21T09:50:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:20 smithi134 ceph-mon[90085]: osdmap e794: 8 total, 8 up, 8 in 2024-09-21T09:50:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:50:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:50:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:50:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:20 smithi017 ceph-mon[102913]: osdmap e794: 8 total, 8 up, 8 in 2024-09-21T09:50:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:20 smithi017 ceph-mon[105151]: osdmap e794: 8 total, 8 up, 8 in 2024-09-21T09:50:21.304 INFO:tasks.rados.rados.0.smithi017.stdout:2926: done (1 left) 2024-09-21T09:50:21.304 INFO:tasks.rados.rados.0.smithi017.stdout:2927: done (0 left) 2024-09-21T09:50:21.304 INFO:tasks.rados.rados.0.smithi017.stdout:2928: read oid 35 snap -1 2024-09-21T09:50:21.304 INFO:tasks.rados.rados.0.smithi017.stdout:2928: expect (ObjNum 1069 snap 356 seq_num 1069) 2024-09-21T09:50:21.304 INFO:tasks.rados.rados.0.smithi017.stdout:2929: read oid 48 snap 355 2024-09-21T09:50:21.304 INFO:tasks.rados.rados.0.smithi017.stdout:2929: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:50:21.305 INFO:tasks.rados.rados.0.smithi017.stdout:2930: write oid 46 current snap is 366 2024-09-21T09:50:21.305 INFO:tasks.rados.rados.0.smithi017.stdout:2930: seq_num 1104 ranges {554928=746081,2020003=534925} 2024-09-21T09:50:21.311 INFO:tasks.rados.rados.0.smithi017.stdout:2930: writing smithi01785587-46 from 554928 to 1301009 tid 1 2024-09-21T09:50:21.319 INFO:tasks.rados.rados.0.smithi017.stdout:2930: writing smithi01785587-46 from 2020003 to 2554928 tid 2 2024-09-21T09:50:21.319 INFO:tasks.rados.rados.0.smithi017.stdout:2931: read oid 4 snap 356 2024-09-21T09:50:21.319 INFO:tasks.rados.rados.0.smithi017.stdout:2931: expect deleted 2024-09-21T09:50:21.319 INFO:tasks.rados.rados.0.smithi017.stdout:2932: write oid 22 current snap is 366 2024-09-21T09:50:21.319 INFO:tasks.rados.rados.0.smithi017.stdout:2932: seq_num 1105 ranges {571735=421330,1562132=416708} 2024-09-21T09:50:21.324 INFO:tasks.rados.rados.0.smithi017.stdout:2932: writing smithi01785587-22 from 571735 to 993065 tid 1 2024-09-21T09:50:21.330 INFO:tasks.rados.rados.0.smithi017.stdout:2932: writing smithi01785587-22 from 1562132 to 1978840 tid 2 2024-09-21T09:50:21.330 INFO:tasks.rados.rados.0.smithi017.stdout:2933: write oid 49 current snap is 366 2024-09-21T09:50:21.330 INFO:tasks.rados.rados.0.smithi017.stdout:2933: seq_num 1106 ranges {588542=496579,1620614=676719} 2024-09-21T09:50:21.336 INFO:tasks.rados.rados.0.smithi017.stdout:2933: writing smithi01785587-49 from 588542 to 1085121 tid 1 2024-09-21T09:50:21.343 INFO:tasks.rados.rados.0.smithi017.stdout:2933: writing smithi01785587-49 from 1620614 to 2297333 tid 2 2024-09-21T09:50:21.343 INFO:tasks.rados.rados.0.smithi017.stdout:2934: read oid 44 snap -1 2024-09-21T09:50:21.343 INFO:tasks.rados.rados.0.smithi017.stdout:2934: expect (ObjNum 1065 snap 354 seq_num 1065) 2024-09-21T09:50:21.343 INFO:tasks.rados.rados.0.smithi017.stdout:2935: snap_remove snap 353 2024-09-21T09:50:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:21 smithi017 ceph-mon[102913]: pgmap v771: 89 pgs: 89 active+clean; 165 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 214 KiB/s wr, 7 op/s 2024-09-21T09:50:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:21 smithi017 ceph-mon[102913]: osdmap e795: 8 total, 8 up, 8 in 2024-09-21T09:50:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:21 smithi017 ceph-mon[105151]: pgmap v771: 89 pgs: 89 active+clean; 165 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 214 KiB/s wr, 7 op/s 2024-09-21T09:50:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:21 smithi017 ceph-mon[105151]: osdmap e795: 8 total, 8 up, 8 in 2024-09-21T09:50:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:21 smithi134 ceph-mon[90085]: pgmap v771: 89 pgs: 89 active+clean; 165 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 214 KiB/s wr, 7 op/s 2024-09-21T09:50:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:21 smithi134 ceph-mon[90085]: osdmap e795: 8 total, 8 up, 8 in 2024-09-21T09:50:22.315 INFO:tasks.rados.rados.0.smithi017.stdout:2935: done (7 left) 2024-09-21T09:50:22.316 INFO:tasks.rados.rados.0.smithi017.stdout:2936: rollback oid 31 current snap is 366 2024-09-21T09:50:22.316 INFO:tasks.rados.rados.0.smithi017.stdout:2929: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:50:22.431 INFO:tasks.rados.rados.0.smithi017.stdout:2930: finishing write tid 1 to smithi01785587-46 2024-09-21T09:50:22.431 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 31 to 352 2024-09-21T09:50:22.431 INFO:tasks.rados.rados.0.smithi017.stdout:2930: finishing write tid 2 to smithi01785587-46 2024-09-21T09:50:22.431 INFO:tasks.rados.rados.0.smithi017.stdout:2930: finishing write tid 3 to smithi01785587-46 2024-09-21T09:50:22.431 INFO:tasks.rados.rados.0.smithi017.stdout:2930: finishing write tid 4 to smithi01785587-46 2024-09-21T09:50:22.431 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1299 (ObjNum 1104 snap 366 seq_num 1104) dirty exists 2024-09-21T09:50:22.431 INFO:tasks.rados.rados.0.smithi017.stdout:2930: left oid 46 (ObjNum 1104 snap 366 seq_num 1104) 2024-09-21T09:50:22.432 INFO:tasks.rados.rados.0.smithi017.stdout:2928: expect (ObjNum 1069 snap 356 seq_num 1069) 2024-09-21T09:50:22.560 INFO:tasks.rados.rados.0.smithi017.stdout:2932: finishing write tid 1 to smithi01785587-22 2024-09-21T09:50:22.561 INFO:tasks.rados.rados.0.smithi017.stdout:2932: finishing write tid 2 to smithi01785587-22 2024-09-21T09:50:22.561 INFO:tasks.rados.rados.0.smithi017.stdout:2928: done (7 left) 2024-09-21T09:50:22.561 INFO:tasks.rados.rados.0.smithi017.stdout:2929: done (6 left) 2024-09-21T09:50:22.561 INFO:tasks.rados.rados.0.smithi017.stdout:2930: done (5 left) 2024-09-21T09:50:22.561 INFO:tasks.rados.rados.0.smithi017.stdout:2931: done (4 left) 2024-09-21T09:50:22.561 INFO:tasks.rados.rados.0.smithi017.stdout:2937: snap_remove snap 355 2024-09-21T09:50:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:22 smithi017 ceph-mon[102913]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T09:50:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:22 smithi017 ceph-mon[105151]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T09:50:22.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:22 smithi134 ceph-mon[90085]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T09:50:23.164 INFO:tasks.rados.rados.0.smithi017.stdout:2937: done (4 left) 2024-09-21T09:50:23.164 INFO:tasks.rados.rados.0.smithi017.stdout:2938: read oid 37 snap -1 2024-09-21T09:50:23.164 INFO:tasks.rados.rados.0.smithi017.stdout:2938: expect (ObjNum 1068 snap 356 seq_num 1068) 2024-09-21T09:50:23.164 INFO:tasks.rados.rados.0.smithi017.stdout:2939: delete oid 47 current snap is 366 2024-09-21T09:50:23.164 INFO:tasks.rados.rados.0.smithi017.stdout:2932: finishing write tid 3 to smithi01785587-22 2024-09-21T09:50:23.164 INFO:tasks.rados.rados.0.smithi017.stdout:2932: finishing write tid 4 to smithi01785587-22 2024-09-21T09:50:23.164 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1502 (ObjNum 1105 snap 366 seq_num 1105) dirty exists 2024-09-21T09:50:23.165 INFO:tasks.rados.rados.0.smithi017.stdout:2932: left oid 22 (ObjNum 1105 snap 366 seq_num 1105) 2024-09-21T09:50:23.165 INFO:tasks.rados.rados.0.smithi017.stdout:2933: finishing write tid 1 to smithi01785587-49 2024-09-21T09:50:23.165 INFO:tasks.rados.rados.0.smithi017.stdout:2933: finishing write tid 2 to smithi01785587-49 2024-09-21T09:50:23.165 INFO:tasks.rados.rados.0.smithi017.stdout:2933: finishing write tid 3 to smithi01785587-49 2024-09-21T09:50:23.165 INFO:tasks.rados.rados.0.smithi017.stdout:2933: finishing write tid 4 to smithi01785587-49 2024-09-21T09:50:23.165 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1447 (ObjNum 1106 snap 366 seq_num 1106) dirty exists 2024-09-21T09:50:23.165 INFO:tasks.rados.rados.0.smithi017.stdout:2933: left oid 49 (ObjNum 1106 snap 366 seq_num 1106) 2024-09-21T09:50:23.165 INFO:tasks.rados.rados.0.smithi017.stdout:2934: expect (ObjNum 1065 snap 354 seq_num 1065) 2024-09-21T09:50:23.462 INFO:tasks.rados.rados.0.smithi017.stdout:2936: finishing rollback tid 0 to smithi01785587-31 2024-09-21T09:50:23.462 INFO:tasks.rados.rados.0.smithi017.stdout:2936: finishing rollback tid 1 to smithi01785587-31 2024-09-21T09:50:23.462 INFO:tasks.rados.rados.0.smithi017.stdout:2936: finishing rollback tid 2 to smithi01785587-31 2024-09-21T09:50:23.462 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1508 (ObjNum 1023 snap 344 seq_num 1023) dirty exists 2024-09-21T09:50:23.462 INFO:tasks.rados.rados.0.smithi017.stdout:2932: done (5 left) 2024-09-21T09:50:23.462 INFO:tasks.rados.rados.0.smithi017.stdout:2933: done (4 left) 2024-09-21T09:50:23.462 INFO:tasks.rados.rados.0.smithi017.stdout:2934: done (3 left) 2024-09-21T09:50:23.462 INFO:tasks.rados.rados.0.smithi017.stdout:2936: done (2 left) 2024-09-21T09:50:23.463 INFO:tasks.rados.rados.0.smithi017.stdout:2939: done (1 left) 2024-09-21T09:50:23.463 INFO:tasks.rados.rados.0.smithi017.stdout:2940: read oid 7 snap -1 2024-09-21T09:50:23.463 INFO:tasks.rados.rados.0.smithi017.stdout:2940: expect (ObjNum 967 snap 325 seq_num 967) 2024-09-21T09:50:23.463 INFO:tasks.rados.rados.0.smithi017.stdout:2941: write oid 48 current snap is 366 2024-09-21T09:50:23.463 INFO:tasks.rados.rados.0.smithi017.stdout:2938: expect (ObjNum 1068 snap 356 seq_num 1068) 2024-09-21T09:50:23.589 INFO:tasks.rados.rados.0.smithi017.stdout:2940: expect (ObjNum 967 snap 325 seq_num 967) 2024-09-21T09:50:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:23 smithi017 ceph-mon[102913]: pgmap v774: 89 pgs: 89 active+clean; 177 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 28 op/s 2024-09-21T09:50:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:23 smithi017 ceph-mon[102913]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T09:50:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:23 smithi017 ceph-mon[102913]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T09:50:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:23 smithi017 ceph-mon[105151]: pgmap v774: 89 pgs: 89 active+clean; 177 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 28 op/s 2024-09-21T09:50:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:23 smithi017 ceph-mon[105151]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T09:50:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:23 smithi017 ceph-mon[105151]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T09:50:23.723 INFO:tasks.rados.rados.0.smithi017.stdout:2941: seq_num 1108 ranges {622156=647077,1737578=680388} 2024-09-21T09:50:23.730 INFO:tasks.rados.rados.0.smithi017.stdout:2941: writing smithi01785587-48 from 622156 to 1269233 tid 1 2024-09-21T09:50:23.736 INFO:tasks.rados.rados.0.smithi017.stdout:2941: writing smithi01785587-48 from 1737578 to 2417966 tid 2 2024-09-21T09:50:23.736 INFO:tasks.rados.rados.0.smithi017.stdout:2938: done (2 left) 2024-09-21T09:50:23.737 INFO:tasks.rados.rados.0.smithi017.stdout:2940: done (1 left) 2024-09-21T09:50:23.737 INFO:tasks.rados.rados.0.smithi017.stdout:2942: write oid 25 current snap is 366 2024-09-21T09:50:23.737 INFO:tasks.rados.rados.0.smithi017.stdout:2941: finishing write tid 1 to smithi01785587-48 2024-09-21T09:50:23.737 INFO:tasks.rados.rados.0.smithi017.stdout:2942: seq_num 1109 ranges {638963=722326,2079707=424046} 2024-09-21T09:50:23.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:23 smithi134 ceph-mon[90085]: pgmap v774: 89 pgs: 89 active+clean; 177 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 28 op/s 2024-09-21T09:50:23.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:23 smithi134 ceph-mon[90085]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T09:50:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:23 smithi134 ceph-mon[90085]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T09:50:23.744 INFO:tasks.rados.rados.0.smithi017.stdout:2942: writing smithi01785587-25 from 638963 to 1361289 tid 1 2024-09-21T09:50:23.750 INFO:tasks.rados.rados.0.smithi017.stdout:2942: writing smithi01785587-25 from 2079707 to 2503753 tid 2 2024-09-21T09:50:23.750 INFO:tasks.rados.rados.0.smithi017.stdout:2943: delete oid 31 current snap is 366 2024-09-21T09:50:23.750 INFO:tasks.rados.rados.0.smithi017.stdout:2941: finishing write tid 2 to smithi01785587-48 2024-09-21T09:50:23.750 INFO:tasks.rados.rados.0.smithi017.stdout:2941: finishing write tid 3 to smithi01785587-48 2024-09-21T09:50:23.750 INFO:tasks.rados.rados.0.smithi017.stdout:2941: finishing write tid 4 to smithi01785587-48 2024-09-21T09:50:23.750 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 1258 (ObjNum 1108 snap 366 seq_num 1108) dirty exists 2024-09-21T09:50:23.750 INFO:tasks.rados.rados.0.smithi017.stdout:2941: left oid 48 (ObjNum 1108 snap 366 seq_num 1108) 2024-09-21T09:50:23.751 INFO:tasks.rados.rados.0.smithi017.stdout:2942: finishing write tid 1 to smithi01785587-25 2024-09-21T09:50:23.752 INFO:tasks.rados.rados.0.smithi017.stdout:2941: done (2 left) 2024-09-21T09:50:23.752 INFO:tasks.rados.rados.0.smithi017.stdout:2943: done (1 left) 2024-09-21T09:50:23.752 INFO:tasks.rados.rados.0.smithi017.stdout:2944: write oid 4 current snap is 366 2024-09-21T09:50:23.752 INFO:tasks.rados.rados.0.smithi017.stdout:2944: seq_num 1111 ranges {672577=589177,1913024=427715} 2024-09-21T09:50:23.759 INFO:tasks.rados.rados.0.smithi017.stdout:2944: writing smithi01785587-4 from 672577 to 1261754 tid 1 2024-09-21T09:50:23.765 INFO:tasks.rados.rados.0.smithi017.stdout:2944: writing smithi01785587-4 from 1913024 to 2340739 tid 2 2024-09-21T09:50:23.765 INFO:tasks.rados.rados.0.smithi017.stdout:2945: snap_create 2024-09-21T09:50:23.765 INFO:tasks.rados.rados.0.smithi017.stdout:2942: finishing write tid 2 to smithi01785587-25 2024-09-21T09:50:23.765 INFO:tasks.rados.rados.0.smithi017.stdout:2942: finishing write tid 3 to smithi01785587-25 2024-09-21T09:50:23.765 INFO:tasks.rados.rados.0.smithi017.stdout:2942: finishing write tid 4 to smithi01785587-25 2024-09-21T09:50:23.766 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 796 (ObjNum 1109 snap 366 seq_num 1109) dirty exists 2024-09-21T09:50:23.766 INFO:tasks.rados.rados.0.smithi017.stdout:2942: left oid 25 (ObjNum 1109 snap 366 seq_num 1109) 2024-09-21T09:50:23.766 INFO:tasks.rados.rados.0.smithi017.stdout:2944: finishing write tid 1 to smithi01785587-4 2024-09-21T09:50:23.768 INFO:tasks.rados.rados.0.smithi017.stdout:2944: finishing write tid 2 to smithi01785587-4 2024-09-21T09:50:23.768 INFO:tasks.rados.rados.0.smithi017.stdout:2944: finishing write tid 3 to smithi01785587-4 2024-09-21T09:50:23.768 INFO:tasks.rados.rados.0.smithi017.stdout:2944: finishing write tid 4 to smithi01785587-4 2024-09-21T09:50:23.768 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1519 (ObjNum 1111 snap 366 seq_num 1111) dirty exists 2024-09-21T09:50:23.768 INFO:tasks.rados.rados.0.smithi017.stdout:2944: left oid 4 (ObjNum 1111 snap 366 seq_num 1111) 2024-09-21T09:50:24.320 INFO:tasks.rados.rados.0.smithi017.stdout:2942: done (2 left) 2024-09-21T09:50:24.320 INFO:tasks.rados.rados.0.smithi017.stdout:2944: done (1 left) 2024-09-21T09:50:24.320 INFO:tasks.rados.rados.0.smithi017.stdout:2945: done (0 left) 2024-09-21T09:50:24.320 INFO:tasks.rados.rados.0.smithi017.stdout:2946: write oid 23 current snap is 367 2024-09-21T09:50:24.320 INFO:tasks.rados.rados.0.smithi017.stdout:2946: seq_num 1112 ranges {689384=664426,1971506=687726} 2024-09-21T09:50:24.327 INFO:tasks.rados.rados.0.smithi017.stdout:2946: writing smithi01785587-23 from 689384 to 1353810 tid 1 2024-09-21T09:50:24.334 INFO:tasks.rados.rados.0.smithi017.stdout:2946: writing smithi01785587-23 from 1971506 to 2659232 tid 2 2024-09-21T09:50:24.335 INFO:tasks.rados.rados.0.smithi017.stdout:2947: snap_remove snap 343 2024-09-21T09:50:25.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:50:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:50:24] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:50:25.322 INFO:tasks.rados.rados.0.smithi017.stdout:2947: done (1 left) 2024-09-21T09:50:25.323 INFO:tasks.rados.rados.0.smithi017.stdout:2948: snap_create 2024-09-21T09:50:25.323 INFO:tasks.rados.rados.0.smithi017.stdout:2946: finishing write tid 1 to smithi01785587-23 2024-09-21T09:50:25.323 INFO:tasks.rados.rados.0.smithi017.stdout:2946: finishing write tid 2 to smithi01785587-23 2024-09-21T09:50:25.323 INFO:tasks.rados.rados.0.smithi017.stdout:2946: finishing write tid 3 to smithi01785587-23 2024-09-21T09:50:25.323 INFO:tasks.rados.rados.0.smithi017.stdout:2946: finishing write tid 4 to smithi01785587-23 2024-09-21T09:50:25.323 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1318 (ObjNum 1112 snap 367 seq_num 1112) dirty exists 2024-09-21T09:50:25.323 INFO:tasks.rados.rados.0.smithi017.stdout:2946: left oid 23 (ObjNum 1112 snap 367 seq_num 1112) 2024-09-21T09:50:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:25 smithi017 ceph-mon[102913]: pgmap v777: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 180 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2024-09-21T09:50:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:25 smithi017 ceph-mon[102913]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T09:50:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:25 smithi017 ceph-mon[105151]: pgmap v777: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 180 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2024-09-21T09:50:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:25 smithi017 ceph-mon[105151]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T09:50:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:25 smithi134 ceph-mon[90085]: pgmap v777: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 180 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2024-09-21T09:50:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:25 smithi134 ceph-mon[90085]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T09:50:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:26.335 INFO:tasks.rados.rados.0.smithi017.stdout:2946: done (1 left) 2024-09-21T09:50:26.335 INFO:tasks.rados.rados.0.smithi017.stdout:2948: done (0 left) 2024-09-21T09:50:26.335 INFO:tasks.rados.rados.0.smithi017.stdout:2949: rollback oid 14 current snap is 368 2024-09-21T09:50:26.335 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 360 2024-09-21T09:50:26.335 INFO:tasks.rados.rados.0.smithi017.stdout:2950: read oid 17 snap -1 2024-09-21T09:50:26.335 INFO:tasks.rados.rados.0.smithi017.stdout:2950: expect deleted 2024-09-21T09:50:26.335 INFO:tasks.rados.rados.0.smithi017.stdout:2951: snap_remove snap 363 2024-09-21T09:50:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:26 smithi017 ceph-mon[102913]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T09:50:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:26 smithi017 ceph-mon[105151]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T09:50:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:26 smithi134 ceph-mon[90085]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T09:50:27.337 INFO:tasks.rados.rados.0.smithi017.stdout:2951: done (2 left) 2024-09-21T09:50:27.338 INFO:tasks.rados.rados.0.smithi017.stdout:2952: delete oid 46 current snap is 368 2024-09-21T09:50:27.338 INFO:tasks.rados.rados.0.smithi017.stdout:2949: finishing rollback tid 0 to smithi01785587-14 2024-09-21T09:50:27.338 INFO:tasks.rados.rados.0.smithi017.stdout:2949: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:50:27.338 INFO:tasks.rados.rados.0.smithi017.stdout:2949: finishing rollback tid 2 to smithi01785587-14 2024-09-21T09:50:27.338 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1521 (ObjNum 1021 snap 344 seq_num 1021) dirty exists 2024-09-21T09:50:27.343 INFO:tasks.rados.rados.0.smithi017.stdout:2949: done (2 left) 2024-09-21T09:50:27.343 INFO:tasks.rados.rados.0.smithi017.stdout:2950: done (1 left) 2024-09-21T09:50:27.343 INFO:tasks.rados.rados.0.smithi017.stdout:2952: done (0 left) 2024-09-21T09:50:27.343 INFO:tasks.rados.rados.0.smithi017.stdout:2953: delete oid 24 current snap is 368 2024-09-21T09:50:27.347 INFO:tasks.rados.rados.0.smithi017.stdout:2953: done (0 left) 2024-09-21T09:50:27.347 INFO:tasks.rados.rados.0.smithi017.stdout:2954: delete oid 15 current snap is 368 2024-09-21T09:50:27.348 INFO:tasks.rados.rados.0.smithi017.stdout:2954: done (0 left) 2024-09-21T09:50:27.349 INFO:tasks.rados.rados.0.smithi017.stdout:2955: rollback oid 32 current snap is 368 2024-09-21T09:50:27.349 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 360 2024-09-21T09:50:27.349 INFO:tasks.rados.rados.0.smithi017.stdout:2956: write oid 48 current snap is 368 2024-09-21T09:50:27.349 INFO:tasks.rados.rados.0.smithi017.stdout:2956: seq_num 1116 ranges {756612=565422,2089081=578711} 2024-09-21T09:50:27.356 INFO:tasks.rados.rados.0.smithi017.stdout:2956: writing smithi01785587-48 from 756612 to 1322034 tid 1 2024-09-21T09:50:27.363 INFO:tasks.rados.rados.0.smithi017.stdout:2956: writing smithi01785587-48 from 2089081 to 2667792 tid 2 2024-09-21T09:50:27.363 INFO:tasks.rados.rados.0.smithi017.stdout:2957: write oid 28 current snap is 368 2024-09-21T09:50:27.364 INFO:tasks.rados.rados.0.smithi017.stdout:2955: finishing rollback tid 0 to smithi01785587-32 2024-09-21T09:50:27.364 INFO:tasks.rados.rados.0.smithi017.stdout:2955: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:50:27.364 INFO:tasks.rados.rados.0.smithi017.stdout:2955: finishing rollback tid 2 to smithi01785587-32 2024-09-21T09:50:27.364 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1328 (ObjNum 982 snap 335 seq_num 982) dirty exists 2024-09-21T09:50:27.364 INFO:tasks.rados.rados.0.smithi017.stdout:2956: finishing write tid 1 to smithi01785587-48 2024-09-21T09:50:27.364 INFO:tasks.rados.rados.0.smithi017.stdout:2957: seq_num 1117 ranges {773419=640671,2147563=438722} 2024-09-21T09:50:27.371 INFO:tasks.rados.rados.0.smithi017.stdout:2957: writing smithi01785587-28 from 773419 to 1414090 tid 1 2024-09-21T09:50:27.378 INFO:tasks.rados.rados.0.smithi017.stdout:2957: writing smithi01785587-28 from 2147563 to 2586285 tid 2 2024-09-21T09:50:27.378 INFO:tasks.rados.rados.0.smithi017.stdout:2955: done (2 left) 2024-09-21T09:50:27.378 INFO:tasks.rados.rados.0.smithi017.stdout:2958: read oid 47 snap -1 2024-09-21T09:50:27.378 INFO:tasks.rados.rados.0.smithi017.stdout:2958: expect deleted 2024-09-21T09:50:27.378 INFO:tasks.rados.rados.0.smithi017.stdout:2959: snap_create 2024-09-21T09:50:27.378 INFO:tasks.rados.rados.0.smithi017.stdout:2956: finishing write tid 2 to smithi01785587-48 2024-09-21T09:50:27.378 INFO:tasks.rados.rados.0.smithi017.stdout:2956: finishing write tid 3 to smithi01785587-48 2024-09-21T09:50:27.379 INFO:tasks.rados.rados.0.smithi017.stdout:2956: finishing write tid 4 to smithi01785587-48 2024-09-21T09:50:27.379 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 1272 (ObjNum 1116 snap 368 seq_num 1116) dirty exists 2024-09-21T09:50:27.379 INFO:tasks.rados.rados.0.smithi017.stdout:2956: left oid 48 (ObjNum 1116 snap 368 seq_num 1116) 2024-09-21T09:50:27.379 INFO:tasks.rados.rados.0.smithi017.stdout:2957: finishing write tid 1 to smithi01785587-28 2024-09-21T09:50:27.381 INFO:tasks.rados.rados.0.smithi017.stdout:2957: finishing write tid 2 to smithi01785587-28 2024-09-21T09:50:27.382 INFO:tasks.rados.rados.0.smithi017.stdout:2957: finishing write tid 3 to smithi01785587-28 2024-09-21T09:50:27.382 INFO:tasks.rados.rados.0.smithi017.stdout:2957: finishing write tid 4 to smithi01785587-28 2024-09-21T09:50:27.382 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 1533 (ObjNum 1117 snap 368 seq_num 1117) dirty exists 2024-09-21T09:50:27.382 INFO:tasks.rados.rados.0.smithi017.stdout:2957: left oid 28 (ObjNum 1117 snap 368 seq_num 1117) 2024-09-21T09:50:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:27 smithi017 ceph-mon[102913]: pgmap v780: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:50:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:27 smithi017 ceph-mon[102913]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T09:50:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:27 smithi017 ceph-mon[105151]: pgmap v780: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:50:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:27 smithi017 ceph-mon[105151]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T09:50:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:27 smithi134 ceph-mon[90085]: pgmap v780: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:50:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:27 smithi134 ceph-mon[90085]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T09:50:28.165 INFO:tasks.rados.rados.0.smithi017.stdout:2956: done (3 left) 2024-09-21T09:50:28.166 INFO:tasks.rados.rados.0.smithi017.stdout:2957: done (2 left) 2024-09-21T09:50:28.166 INFO:tasks.rados.rados.0.smithi017.stdout:2958: done (1 left) 2024-09-21T09:50:28.166 INFO:tasks.rados.rados.0.smithi017.stdout:2959: done (0 left) 2024-09-21T09:50:28.166 INFO:tasks.rados.rados.0.smithi017.stdout:2960: write oid 45 current snap is 369 2024-09-21T09:50:28.166 INFO:tasks.rados.rados.0.smithi017.stdout:2960: seq_num 1118 ranges {790226=432273,1806045=582380} 2024-09-21T09:50:28.172 INFO:tasks.rados.rados.0.smithi017.stdout:2960: writing smithi01785587-45 from 790226 to 1222499 tid 1 2024-09-21T09:50:28.179 INFO:tasks.rados.rados.0.smithi017.stdout:2960: writing smithi01785587-45 from 1806045 to 2388425 tid 2 2024-09-21T09:50:28.179 INFO:tasks.rados.rados.0.smithi017.stdout:2961: write oid 4 current snap is 369 2024-09-21T09:50:28.180 INFO:tasks.rados.rados.0.smithi017.stdout:2961: seq_num 1119 ranges {407033=507522,1464527=442391,2345976=461057} 2024-09-21T09:50:28.185 INFO:tasks.rados.rados.0.smithi017.stdout:2961: writing smithi01785587-4 from 407033 to 914555 tid 1 2024-09-21T09:50:28.190 INFO:tasks.rados.rados.0.smithi017.stdout:2961: writing smithi01785587-4 from 1464527 to 1906918 tid 2 2024-09-21T09:50:28.196 INFO:tasks.rados.rados.0.smithi017.stdout:2961: writing smithi01785587-4 from 2345976 to 2807033 tid 3 2024-09-21T09:50:28.196 INFO:tasks.rados.rados.0.smithi017.stdout:2962: read oid 26 snap -1 2024-09-21T09:50:28.196 INFO:tasks.rados.rados.0.smithi017.stdout:2962: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:50:28.196 INFO:tasks.rados.rados.0.smithi017.stdout:2963: delete oid 13 current snap is 369 2024-09-21T09:50:28.196 INFO:tasks.rados.rados.0.smithi017.stdout:2960: finishing write tid 1 to smithi01785587-45 2024-09-21T09:50:28.196 INFO:tasks.rados.rados.0.smithi017.stdout:2960: finishing write tid 2 to smithi01785587-45 2024-09-21T09:50:28.196 INFO:tasks.rados.rados.0.smithi017.stdout:2960: finishing write tid 3 to smithi01785587-45 2024-09-21T09:50:28.197 INFO:tasks.rados.rados.0.smithi017.stdout:2960: finishing write tid 4 to smithi01785587-45 2024-09-21T09:50:28.197 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 976 (ObjNum 1118 snap 369 seq_num 1118) dirty exists 2024-09-21T09:50:28.197 INFO:tasks.rados.rados.0.smithi017.stdout:2960: left oid 45 (ObjNum 1118 snap 369 seq_num 1118) 2024-09-21T09:50:28.197 INFO:tasks.rados.rados.0.smithi017.stdout:2961: finishing write tid 1 to smithi01785587-4 2024-09-21T09:50:28.197 INFO:tasks.rados.rados.0.smithi017.stdout:2961: finishing write tid 2 to smithi01785587-4 2024-09-21T09:50:28.210 INFO:tasks.rados.rados.0.smithi017.stdout:2962: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:50:28.371 INFO:tasks.rados.rados.0.smithi017.stdout:2961: finishing write tid 3 to smithi01785587-4 2024-09-21T09:50:28.371 INFO:tasks.rados.rados.0.smithi017.stdout:2961: finishing write tid 4 to smithi01785587-4 2024-09-21T09:50:28.371 INFO:tasks.rados.rados.0.smithi017.stdout:2961: finishing write tid 5 to smithi01785587-4 2024-09-21T09:50:28.372 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1538 (ObjNum 1119 snap 369 seq_num 1119) dirty exists 2024-09-21T09:50:28.372 INFO:tasks.rados.rados.0.smithi017.stdout:2961: left oid 4 (ObjNum 1119 snap 369 seq_num 1119) 2024-09-21T09:50:28.372 INFO:tasks.rados.rados.0.smithi017.stdout:2960: done (3 left) 2024-09-21T09:50:28.372 INFO:tasks.rados.rados.0.smithi017.stdout:2961: done (2 left) 2024-09-21T09:50:28.372 INFO:tasks.rados.rados.0.smithi017.stdout:2962: done (1 left) 2024-09-21T09:50:28.372 INFO:tasks.rados.rados.0.smithi017.stdout:2963: done (0 left) 2024-09-21T09:50:28.372 INFO:tasks.rados.rados.0.smithi017.stdout:2964: rollback oid 46 current snap is 369 2024-09-21T09:50:28.372 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 352 2024-09-21T09:50:28.372 INFO:tasks.rados.rados.0.smithi017.stdout:2965: delete oid 7 current snap is 369 2024-09-21T09:50:28.373 INFO:tasks.rados.rados.0.smithi017.stdout:2965: done (1 left) 2024-09-21T09:50:28.373 INFO:tasks.rados.rados.0.smithi017.stdout:2966: delete oid 1 current snap is 369 2024-09-21T09:50:28.374 INFO:tasks.rados.rados.0.smithi017.stdout:2964: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:50:28.374 INFO:tasks.rados.rados.0.smithi017.stdout:2964: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:50:28.374 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1330 (ObjNum 791 snap 271 seq_num 791) dirty exists 2024-09-21T09:50:28.375 INFO:tasks.rados.rados.0.smithi017.stdout:2964: done (1 left) 2024-09-21T09:50:28.375 INFO:tasks.rados.rados.0.smithi017.stdout:2966: done (0 left) 2024-09-21T09:50:28.375 INFO:tasks.rados.rados.0.smithi017.stdout:2967: read oid 48 snap -1 2024-09-21T09:50:28.376 INFO:tasks.rados.rados.0.smithi017.stdout:2967: expect (ObjNum 1116 snap 368 seq_num 1116) 2024-09-21T09:50:28.376 INFO:tasks.rados.rados.0.smithi017.stdout:2968: write oid 22 current snap is 369 2024-09-21T09:50:28.376 INFO:tasks.rados.rados.0.smithi017.stdout:2968: seq_num 1123 ranges {474261=408518,1582102=449729,2739315=134946} 2024-09-21T09:50:28.380 INFO:tasks.rados.rados.0.smithi017.stdout:2968: writing smithi01785587-22 from 474261 to 882779 tid 1 2024-09-21T09:50:28.386 INFO:tasks.rados.rados.0.smithi017.stdout:2968: writing smithi01785587-22 from 1582102 to 2031831 tid 2 2024-09-21T09:50:28.391 INFO:tasks.rados.rados.0.smithi017.stdout:2968: writing smithi01785587-22 from 2739315 to 2874261 tid 3 2024-09-21T09:50:28.391 INFO:tasks.rados.rados.0.smithi017.stdout:2969: snap_create 2024-09-21T09:50:28.392 INFO:tasks.rados.rados.0.smithi017.stdout:2968: finishing write tid 1 to smithi01785587-22 2024-09-21T09:50:28.392 INFO:tasks.rados.rados.0.smithi017.stdout:2967: expect (ObjNum 1116 snap 368 seq_num 1116) 2024-09-21T09:50:28.568 INFO:tasks.rados.rados.0.smithi017.stdout:2968: finishing write tid 2 to smithi01785587-22 2024-09-21T09:50:28.568 INFO:tasks.rados.rados.0.smithi017.stdout:2968: finishing write tid 3 to smithi01785587-22 2024-09-21T09:50:28.568 INFO:tasks.rados.rados.0.smithi017.stdout:2968: finishing write tid 4 to smithi01785587-22 2024-09-21T09:50:28.568 INFO:tasks.rados.rados.0.smithi017.stdout:2968: finishing write tid 5 to smithi01785587-22 2024-09-21T09:50:28.569 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1543 (ObjNum 1123 snap 369 seq_num 1123) dirty exists 2024-09-21T09:50:28.569 INFO:tasks.rados.rados.0.smithi017.stdout:2968: left oid 22 (ObjNum 1123 snap 369 seq_num 1123) 2024-09-21T09:50:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:28 smithi017 ceph-mon[102913]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T09:50:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:28 smithi017 ceph-mon[102913]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T09:50:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:28 smithi017 ceph-mon[105151]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T09:50:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:28 smithi017 ceph-mon[105151]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T09:50:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:28 smithi134 ceph-mon[90085]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T09:50:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:28 smithi134 ceph-mon[90085]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T09:50:29.351 INFO:tasks.rados.rados.0.smithi017.stdout:2967: done (2 left) 2024-09-21T09:50:29.352 INFO:tasks.rados.rados.0.smithi017.stdout:2968: done (1 left) 2024-09-21T09:50:29.352 INFO:tasks.rados.rados.0.smithi017.stdout:2969: done (0 left) 2024-09-21T09:50:29.352 INFO:tasks.rados.rados.0.smithi017.stdout:2970: delete oid 36 current snap is 370 2024-09-21T09:50:29.356 INFO:tasks.rados.rados.0.smithi017.stdout:2970: done (0 left) 2024-09-21T09:50:29.356 INFO:tasks.rados.rados.0.smithi017.stdout:2971: delete oid 42 current snap is 370 2024-09-21T09:50:29.358 INFO:tasks.rados.rados.0.smithi017.stdout:2971: done (0 left) 2024-09-21T09:50:29.358 INFO:tasks.rados.rados.0.smithi017.stdout:2972: write oid 47 current snap is 370 2024-09-21T09:50:29.359 INFO:tasks.rados.rados.0.smithi017.stdout:2972: seq_num 1126 ranges {524682=750618,1757548=597056} 2024-09-21T09:50:29.365 INFO:tasks.rados.rados.0.smithi017.stdout:2972: writing smithi01785587-47 from 524682 to 1275300 tid 1 2024-09-21T09:50:29.372 INFO:tasks.rados.rados.0.smithi017.stdout:2972: writing smithi01785587-47 from 1757548 to 2354604 tid 2 2024-09-21T09:50:29.372 INFO:tasks.rados.rados.0.smithi017.stdout:2973: write oid 20 current snap is 370 2024-09-21T09:50:29.372 INFO:tasks.rados.rados.0.smithi017.stdout:2972: finishing write tid 1 to smithi01785587-47 2024-09-21T09:50:29.372 INFO:tasks.rados.rados.0.smithi017.stdout:2973: seq_num 1127 ranges {541489=425867,1416030=740714,2732654=208835} 2024-09-21T09:50:29.377 INFO:tasks.rados.rados.0.smithi017.stdout:2973: writing smithi01785587-20 from 541489 to 967356 tid 1 2024-09-21T09:50:29.384 INFO:tasks.rados.rados.0.smithi017.stdout:2973: writing smithi01785587-20 from 1416030 to 2156744 tid 2 2024-09-21T09:50:29.388 INFO:tasks.rados.rados.0.smithi017.stdout:2973: writing smithi01785587-20 from 2732654 to 2941489 tid 3 2024-09-21T09:50:29.388 INFO:tasks.rados.rados.0.smithi017.stdout:2974: read oid 50 snap -1 2024-09-21T09:50:29.388 INFO:tasks.rados.rados.0.smithi017.stdout:2974: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:50:29.389 INFO:tasks.rados.rados.0.smithi017.stdout:2975: write oid 46 current snap is 370 2024-09-21T09:50:29.389 INFO:tasks.rados.rados.0.smithi017.stdout:2972: finishing write tid 2 to smithi01785587-47 2024-09-21T09:50:29.389 INFO:tasks.rados.rados.0.smithi017.stdout:2972: finishing write tid 3 to smithi01785587-47 2024-09-21T09:50:29.389 INFO:tasks.rados.rados.0.smithi017.stdout:2975: seq_num 1128 ranges {558296=501116,1474512=600725,2760077=198219} 2024-09-21T09:50:29.394 INFO:tasks.rados.rados.0.smithi017.stdout:2975: writing smithi01785587-46 from 558296 to 1059412 tid 1 2024-09-21T09:50:29.400 INFO:tasks.rados.rados.0.smithi017.stdout:2975: writing smithi01785587-46 from 1474512 to 2075237 tid 2 2024-09-21T09:50:29.405 INFO:tasks.rados.rados.0.smithi017.stdout:2975: writing smithi01785587-46 from 2760077 to 2958296 tid 3 2024-09-21T09:50:29.405 INFO:tasks.rados.rados.0.smithi017.stdout:2976: snap_create 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:2972: finishing write tid 4 to smithi01785587-47 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 1002 (ObjNum 1126 snap 370 seq_num 1126) dirty exists 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:2972: left oid 47 (ObjNum 1126 snap 370 seq_num 1126) 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:2973: finishing write tid 1 to smithi01785587-20 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:2973: finishing write tid 2 to smithi01785587-20 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:2973: finishing write tid 3 to smithi01785587-20 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:2973: finishing write tid 4 to smithi01785587-20 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:2973: finishing write tid 5 to smithi01785587-20 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 804 (ObjNum 1127 snap 370 seq_num 1127) dirty exists 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:2973: left oid 20 (ObjNum 1127 snap 370 seq_num 1127) 2024-09-21T09:50:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:2974: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:50:29.486 INFO:tasks.rados.rados.0.smithi017.stdout:2975: finishing write tid 1 to smithi01785587-46 2024-09-21T09:50:29.486 INFO:tasks.rados.rados.0.smithi017.stdout:2975: finishing write tid 2 to smithi01785587-46 2024-09-21T09:50:29.486 INFO:tasks.rados.rados.0.smithi017.stdout:2975: finishing write tid 3 to smithi01785587-46 2024-09-21T09:50:29.486 INFO:tasks.rados.rados.0.smithi017.stdout:2975: finishing write tid 4 to smithi01785587-46 2024-09-21T09:50:29.486 INFO:tasks.rados.rados.0.smithi017.stdout:2975: finishing write tid 5 to smithi01785587-46 2024-09-21T09:50:29.486 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1335 (ObjNum 1128 snap 370 seq_num 1128) dirty exists 2024-09-21T09:50:29.486 INFO:tasks.rados.rados.0.smithi017.stdout:2975: left oid 46 (ObjNum 1128 snap 370 seq_num 1128) 2024-09-21T09:50:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:29 smithi017 ceph-mon[102913]: pgmap v783: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:50:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:29 smithi017 ceph-mon[105151]: pgmap v783: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:50:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:29 smithi134 ceph-mon[90085]: pgmap v783: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:50:30.357 INFO:tasks.rados.rados.0.smithi017.stdout:2972: done (4 left) 2024-09-21T09:50:30.357 INFO:tasks.rados.rados.0.smithi017.stdout:2973: done (3 left) 2024-09-21T09:50:30.357 INFO:tasks.rados.rados.0.smithi017.stdout:2974: done (2 left) 2024-09-21T09:50:30.357 INFO:tasks.rados.rados.0.smithi017.stdout:2975: done (1 left) 2024-09-21T09:50:30.357 INFO:tasks.rados.rados.0.smithi017.stdout:2976: done (0 left) 2024-09-21T09:50:30.357 INFO:tasks.rados.rados.0.smithi017.stdout:2977: snap_remove snap 365 2024-09-21T09:50:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:30 smithi134 ceph-mon[90085]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T09:50:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:50:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:50:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:50:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:30 smithi017 ceph-mon[102913]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T09:50:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:30 smithi017 ceph-mon[105151]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T09:50:31.360 INFO:tasks.rados.rados.0.smithi017.stdout:2977: done (0 left) 2024-09-21T09:50:31.360 INFO:tasks.rados.rados.0.smithi017.stdout:2978: read oid 11 snap -1 2024-09-21T09:50:31.360 INFO:tasks.rados.rados.0.smithi017.stdout:2978: expect deleted 2024-09-21T09:50:31.360 INFO:tasks.rados.rados.0.smithi017.stdout:2979: rollback oid 23 current snap is 371 2024-09-21T09:50:31.360 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 23 to 366 2024-09-21T09:50:31.360 INFO:tasks.rados.rados.0.smithi017.stdout:2980: snap_create 2024-09-21T09:50:31.370 INFO:tasks.rados.rados.0.smithi017.stdout:2979: finishing rollback tid 0 to smithi01785587-23 2024-09-21T09:50:31.370 INFO:tasks.rados.rados.0.smithi017.stdout:2979: finishing rollback tid 1 to smithi01785587-23 2024-09-21T09:50:31.370 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1338 (ObjNum 305 snap 0 seq_num 1) dirty dne 2024-09-21T09:50:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:31 smithi017 ceph-mon[102913]: pgmap v786: 89 pgs: 89 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 929 KiB/s wr, 5 op/s 2024-09-21T09:50:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:31 smithi017 ceph-mon[102913]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T09:50:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:31 smithi017 ceph-mon[105151]: pgmap v786: 89 pgs: 89 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 929 KiB/s wr, 5 op/s 2024-09-21T09:50:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:31 smithi017 ceph-mon[105151]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T09:50:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:31 smithi134 ceph-mon[90085]: pgmap v786: 89 pgs: 89 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 929 KiB/s wr, 5 op/s 2024-09-21T09:50:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:31 smithi134 ceph-mon[90085]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T09:50:32.371 INFO:tasks.rados.rados.0.smithi017.stdout:2978: done (2 left) 2024-09-21T09:50:32.371 INFO:tasks.rados.rados.0.smithi017.stdout:2979: done (1 left) 2024-09-21T09:50:32.371 INFO:tasks.rados.rados.0.smithi017.stdout:2980: done (0 left) 2024-09-21T09:50:32.371 INFO:tasks.rados.rados.0.smithi017.stdout:2981: rollback oid 9 current snap is 372 2024-09-21T09:50:32.371 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 9 to 368 2024-09-21T09:50:32.371 INFO:tasks.rados.rados.0.smithi017.stdout:2982: write oid 29 current snap is 372 2024-09-21T09:50:32.372 INFO:tasks.rados.rados.0.smithi017.stdout:2982: seq_num 1129 ranges {575103=576365,1932994=744383} 2024-09-21T09:50:32.377 INFO:tasks.rados.rados.0.smithi017.stdout:2982: writing smithi01785587-29 from 575103 to 1151468 tid 1 2024-09-21T09:50:32.385 INFO:tasks.rados.rados.0.smithi017.stdout:2982: writing smithi01785587-29 from 1932994 to 2677377 tid 2 2024-09-21T09:50:32.385 INFO:tasks.rados.rados.0.smithi017.stdout:2983: write oid 44 current snap is 372 2024-09-21T09:50:32.385 INFO:tasks.rados.rados.0.smithi017.stdout:2983: seq_num 1130 ranges {591910=651614,1875123=604394} 2024-09-21T09:50:32.392 INFO:tasks.rados.rados.0.smithi017.stdout:2983: writing smithi01785587-44 from 591910 to 1243524 tid 1 2024-09-21T09:50:32.399 INFO:tasks.rados.rados.0.smithi017.stdout:2983: writing smithi01785587-44 from 1875123 to 2479517 tid 2 2024-09-21T09:50:32.399 INFO:tasks.rados.rados.0.smithi017.stdout:2984: snap_create 2024-09-21T09:50:32.399 INFO:tasks.rados.rados.0.smithi017.stdout:2981: finishing rollback tid 1 to smithi01785587-9 2024-09-21T09:50:32.399 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:50:32.400 INFO:tasks.rados.rados.0.smithi017.stdout:2982: finishing write tid 1 to smithi01785587-29 2024-09-21T09:50:32.400 INFO:tasks.rados.rados.0.smithi017.stdout:2982: finishing write tid 2 to smithi01785587-29 2024-09-21T09:50:32.400 INFO:tasks.rados.rados.0.smithi017.stdout:2982: finishing write tid 3 to smithi01785587-29 2024-09-21T09:50:32.400 INFO:tasks.rados.rados.0.smithi017.stdout:2982: finishing write tid 4 to smithi01785587-29 2024-09-21T09:50:32.400 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 807 (ObjNum 1129 snap 372 seq_num 1129) dirty exists 2024-09-21T09:50:32.400 INFO:tasks.rados.rados.0.smithi017.stdout:2982: left oid 29 (ObjNum 1129 snap 372 seq_num 1129) 2024-09-21T09:50:32.400 INFO:tasks.rados.rados.0.smithi017.stdout:2983: finishing write tid 1 to smithi01785587-44 2024-09-21T09:50:32.402 INFO:tasks.rados.rados.0.smithi017.stdout:2983: finishing write tid 2 to smithi01785587-44 2024-09-21T09:50:32.403 INFO:tasks.rados.rados.0.smithi017.stdout:2983: finishing write tid 3 to smithi01785587-44 2024-09-21T09:50:32.403 INFO:tasks.rados.rados.0.smithi017.stdout:2983: finishing write tid 4 to smithi01785587-44 2024-09-21T09:50:32.403 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1535 (ObjNum 1130 snap 372 seq_num 1130) dirty exists 2024-09-21T09:50:32.403 INFO:tasks.rados.rados.0.smithi017.stdout:2983: left oid 44 (ObjNum 1130 snap 372 seq_num 1130) 2024-09-21T09:50:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:32 smithi017 ceph-mon[102913]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T09:50:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:32 smithi017 ceph-mon[105151]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T09:50:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:32 smithi134 ceph-mon[90085]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T09:50:33.167 INFO:tasks.rados.rados.0.smithi017.stdout:2981: done (3 left) 2024-09-21T09:50:33.167 INFO:tasks.rados.rados.0.smithi017.stdout:2982: done (2 left) 2024-09-21T09:50:33.167 INFO:tasks.rados.rados.0.smithi017.stdout:2983: done (1 left) 2024-09-21T09:50:33.167 INFO:tasks.rados.rados.0.smithi017.stdout:2984: done (0 left) 2024-09-21T09:50:33.167 INFO:tasks.rados.rados.0.smithi017.stdout:2985: snap_remove snap 368 2024-09-21T09:50:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:33 smithi017 ceph-mon[102913]: pgmap v789: 89 pgs: 89 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.1 MiB/s wr, 18 op/s 2024-09-21T09:50:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:33 smithi017 ceph-mon[102913]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T09:50:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:33 smithi017 ceph-mon[102913]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T09:50:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:33 smithi017 ceph-mon[105151]: pgmap v789: 89 pgs: 89 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.1 MiB/s wr, 18 op/s 2024-09-21T09:50:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:33 smithi017 ceph-mon[105151]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T09:50:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:33 smithi017 ceph-mon[105151]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T09:50:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:33 smithi134 ceph-mon[90085]: pgmap v789: 89 pgs: 89 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.1 MiB/s wr, 18 op/s 2024-09-21T09:50:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:33 smithi134 ceph-mon[90085]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T09:50:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:33 smithi134 ceph-mon[90085]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T09:50:34.170 INFO:tasks.rados.rados.0.smithi017.stdout:2985: done (0 left) 2024-09-21T09:50:34.170 INFO:tasks.rados.rados.0.smithi017.stdout:2986: snap_create 2024-09-21T09:50:35.172 INFO:tasks.rados.rados.0.smithi017.stdout:2986: done (0 left) 2024-09-21T09:50:35.172 INFO:tasks.rados.rados.0.smithi017.stdout:2987: rollback oid 7 current snap is 374 2024-09-21T09:50:35.172 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 366 2024-09-21T09:50:35.172 INFO:tasks.rados.rados.0.smithi017.stdout:2988: write oid 14 current snap is 374 2024-09-21T09:50:35.172 INFO:tasks.rados.rados.0.smithi017.stdout:2988: seq_num 1131 ranges {608717=726863,1933605=748052} 2024-09-21T09:50:35.172 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:50:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:50:34] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:50:35.179 INFO:tasks.rados.rados.0.smithi017.stdout:2988: writing smithi01785587-14 from 608717 to 1335580 tid 1 2024-09-21T09:50:35.188 INFO:tasks.rados.rados.0.smithi017.stdout:2988: writing smithi01785587-14 from 1933605 to 2681657 tid 2 2024-09-21T09:50:35.188 INFO:tasks.rados.rados.0.smithi017.stdout:2989: read oid 39 snap -1 2024-09-21T09:50:35.188 INFO:tasks.rados.rados.0.smithi017.stdout:2989: expect (ObjNum 1097 snap 363 seq_num 1097) 2024-09-21T09:50:35.188 INFO:tasks.rados.rados.0.smithi017.stdout:2990: delete oid 41 current snap is 374 2024-09-21T09:50:35.188 INFO:tasks.rados.rados.0.smithi017.stdout:2987: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:50:35.188 INFO:tasks.rados.rados.0.smithi017.stdout:2987: finishing rollback tid 2 to smithi01785587-7 2024-09-21T09:50:35.189 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1465 (ObjNum 967 snap 325 seq_num 967) dirty exists 2024-09-21T09:50:35.189 INFO:tasks.rados.rados.0.smithi017.stdout:2988: finishing write tid 1 to smithi01785587-14 2024-09-21T09:50:35.191 INFO:tasks.rados.rados.0.smithi017.stdout:2987: done (3 left) 2024-09-21T09:50:35.191 INFO:tasks.rados.rados.0.smithi017.stdout:2990: done (2 left) 2024-09-21T09:50:35.192 INFO:tasks.rados.rados.0.smithi017.stdout:2991: write oid 7 current snap is 374 2024-09-21T09:50:35.192 INFO:tasks.rados.rados.0.smithi017.stdout:2991: seq_num 1133 ranges {642331=593714,1766922=751721} 2024-09-21T09:50:35.199 INFO:tasks.rados.rados.0.smithi017.stdout:2991: writing smithi01785587-7 from 642331 to 1236045 tid 1 2024-09-21T09:50:35.205 INFO:tasks.rados.rados.0.smithi017.stdout:2991: writing smithi01785587-7 from 1766922 to 2518643 tid 2 2024-09-21T09:50:35.205 INFO:tasks.rados.rados.0.smithi017.stdout:2992: write oid 37 current snap is 374 2024-09-21T09:50:35.206 INFO:tasks.rados.rados.0.smithi017.stdout:2988: finishing write tid 2 to smithi01785587-14 2024-09-21T09:50:35.206 INFO:tasks.rados.rados.0.smithi017.stdout:2988: finishing write tid 3 to smithi01785587-14 2024-09-21T09:50:35.206 INFO:tasks.rados.rados.0.smithi017.stdout:2992: seq_num 1134 ranges {659138=668963,2109051=611732} 2024-09-21T09:50:35.213 INFO:tasks.rados.rados.0.smithi017.stdout:2992: writing smithi01785587-37 from 659138 to 1328101 tid 1 2024-09-21T09:50:35.221 INFO:tasks.rados.rados.0.smithi017.stdout:2992: writing smithi01785587-37 from 2109051 to 2720783 tid 2 2024-09-21T09:50:35.221 INFO:tasks.rados.rados.0.smithi017.stdout:2993: snap_create 2024-09-21T09:50:35.221 INFO:tasks.rados.rados.0.smithi017.stdout:2988: finishing write tid 4 to smithi01785587-14 2024-09-21T09:50:35.222 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1541 (ObjNum 1131 snap 374 seq_num 1131) dirty exists 2024-09-21T09:50:35.222 INFO:tasks.rados.rados.0.smithi017.stdout:2988: left oid 14 (ObjNum 1131 snap 374 seq_num 1131) 2024-09-21T09:50:35.222 INFO:tasks.rados.rados.0.smithi017.stdout:2989: expect (ObjNum 1097 snap 363 seq_num 1097) 2024-09-21T09:50:35.304 INFO:tasks.rados.rados.0.smithi017.stdout:2991: finishing write tid 1 to smithi01785587-7 2024-09-21T09:50:35.304 INFO:tasks.rados.rados.0.smithi017.stdout:2991: finishing write tid 2 to smithi01785587-7 2024-09-21T09:50:35.304 INFO:tasks.rados.rados.0.smithi017.stdout:2991: finishing write tid 3 to smithi01785587-7 2024-09-21T09:50:35.304 INFO:tasks.rados.rados.0.smithi017.stdout:2991: finishing write tid 4 to smithi01785587-7 2024-09-21T09:50:35.304 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1468 (ObjNum 1133 snap 374 seq_num 1133) dirty exists 2024-09-21T09:50:35.304 INFO:tasks.rados.rados.0.smithi017.stdout:2991: left oid 7 (ObjNum 1133 snap 374 seq_num 1133) 2024-09-21T09:50:35.304 INFO:tasks.rados.rados.0.smithi017.stdout:2992: finishing write tid 1 to smithi01785587-37 2024-09-21T09:50:35.304 INFO:tasks.rados.rados.0.smithi017.stdout:2992: finishing write tid 2 to smithi01785587-37 2024-09-21T09:50:35.305 INFO:tasks.rados.rados.0.smithi017.stdout:2992: finishing write tid 3 to smithi01785587-37 2024-09-21T09:50:35.305 INFO:tasks.rados.rados.0.smithi017.stdout:2992: finishing write tid 4 to smithi01785587-37 2024-09-21T09:50:35.305 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1518 (ObjNum 1134 snap 374 seq_num 1134) dirty exists 2024-09-21T09:50:35.305 INFO:tasks.rados.rados.0.smithi017.stdout:2992: left oid 37 (ObjNum 1134 snap 374 seq_num 1134) 2024-09-21T09:50:35.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:35 smithi017 ceph-mon[102913]: pgmap v792: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.3 MiB/s wr, 19 op/s 2024-09-21T09:50:35.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:35 smithi017 ceph-mon[102913]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T09:50:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:35 smithi017 ceph-mon[105151]: pgmap v792: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.3 MiB/s wr, 19 op/s 2024-09-21T09:50:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:35 smithi017 ceph-mon[105151]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T09:50:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:35 smithi134 ceph-mon[90085]: pgmap v792: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.3 MiB/s wr, 19 op/s 2024-09-21T09:50:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:35 smithi134 ceph-mon[90085]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T09:50:36.181 INFO:tasks.rados.rados.0.smithi017.stdout:2988: done (4 left) 2024-09-21T09:50:36.181 INFO:tasks.rados.rados.0.smithi017.stdout:2989: done (3 left) 2024-09-21T09:50:36.181 INFO:tasks.rados.rados.0.smithi017.stdout:2991: done (2 left) 2024-09-21T09:50:36.181 INFO:tasks.rados.rados.0.smithi017.stdout:2992: done (1 left) 2024-09-21T09:50:36.181 INFO:tasks.rados.rados.0.smithi017.stdout:2993: done (0 left) 2024-09-21T09:50:36.181 INFO:tasks.rados.rados.0.smithi017.stdout:2994: rollback oid 7 current snap is 375 2024-09-21T09:50:36.182 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 373 2024-09-21T09:50:36.182 INFO:tasks.rados.rados.0.smithi017.stdout:2995: write oid 26 current snap is 375 2024-09-21T09:50:36.182 INFO:tasks.rados.rados.0.smithi017.stdout:2995: seq_num 1135 ranges {675945=744212,2167533=755390} 2024-09-21T09:50:36.189 INFO:tasks.rados.rados.0.smithi017.stdout:2995: writing smithi01785587-26 from 675945 to 1420157 tid 1 2024-09-21T09:50:36.198 INFO:tasks.rados.rados.0.smithi017.stdout:2995: writing smithi01785587-26 from 2167533 to 2922923 tid 2 2024-09-21T09:50:36.198 INFO:tasks.rados.rados.0.smithi017.stdout:2996: snap_create 2024-09-21T09:50:36.198 INFO:tasks.rados.rados.0.smithi017.stdout:2994: finishing rollback tid 0 to smithi01785587-7 2024-09-21T09:50:36.198 INFO:tasks.rados.rados.0.smithi017.stdout:2994: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:50:36.198 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1471 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:50:36.198 INFO:tasks.rados.rados.0.smithi017.stdout:2995: finishing write tid 1 to smithi01785587-26 2024-09-21T09:50:36.202 INFO:tasks.rados.rados.0.smithi017.stdout:2995: finishing write tid 2 to smithi01785587-26 2024-09-21T09:50:36.202 INFO:tasks.rados.rados.0.smithi017.stdout:2995: finishing write tid 3 to smithi01785587-26 2024-09-21T09:50:36.202 INFO:tasks.rados.rados.0.smithi017.stdout:2995: finishing write tid 4 to smithi01785587-26 2024-09-21T09:50:36.202 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1555 (ObjNum 1135 snap 375 seq_num 1135) dirty exists 2024-09-21T09:50:36.202 INFO:tasks.rados.rados.0.smithi017.stdout:2995: left oid 26 (ObjNum 1135 snap 375 seq_num 1135) 2024-09-21T09:50:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:36 smithi017 ceph-mon[102913]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T09:50:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:36 smithi017 ceph-mon[105151]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T09:50:36.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:36 smithi134 ceph-mon[90085]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T09:50:37.185 INFO:tasks.rados.rados.0.smithi017.stdout:2994: done (2 left) 2024-09-21T09:50:37.185 INFO:tasks.rados.rados.0.smithi017.stdout:2995: done (1 left) 2024-09-21T09:50:37.185 INFO:tasks.rados.rados.0.smithi017.stdout:2996: done (0 left) 2024-09-21T09:50:37.185 INFO:tasks.rados.rados.0.smithi017.stdout:2997: rollback oid 2 current snap is 376 2024-09-21T09:50:37.185 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 2 to 374 2024-09-21T09:50:37.185 INFO:tasks.rados.rados.0.smithi017.stdout:2998: snap_create 2024-09-21T09:50:37.190 INFO:tasks.rados.rados.0.smithi017.stdout:2997: finishing rollback tid 0 to smithi01785587-2 2024-09-21T09:50:37.191 INFO:tasks.rados.rados.0.smithi017.stdout:2997: finishing rollback tid 1 to smithi01785587-2 2024-09-21T09:50:37.191 INFO:tasks.rados.rados.0.smithi017.stdout:2997: finishing rollback tid 2 to smithi01785587-2 2024-09-21T09:50:37.191 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 598 (ObjNum 1076 snap 360 seq_num 1076) dirty exists 2024-09-21T09:50:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:37 smithi017 ceph-mon[102913]: pgmap v795: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 877 KiB/s wr, 3 op/s 2024-09-21T09:50:37.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:37 smithi017 ceph-mon[102913]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T09:50:37.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:37 smithi017 ceph-mon[105151]: pgmap v795: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 877 KiB/s wr, 3 op/s 2024-09-21T09:50:37.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:37 smithi017 ceph-mon[105151]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T09:50:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:37 smithi134 ceph-mon[90085]: pgmap v795: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 877 KiB/s wr, 3 op/s 2024-09-21T09:50:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:37 smithi134 ceph-mon[90085]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T09:50:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:2997: done (1 left) 2024-09-21T09:50:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:2998: done (0 left) 2024-09-21T09:50:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:2999: write oid 18 current snap is 377 2024-09-21T09:50:38.168 INFO:tasks.rados.rados.0.smithi017.stdout:2999: seq_num 1136 ranges {692752=419461,1826015=615401,2979461=113291} 2024-09-21T09:50:38.174 INFO:tasks.rados.rados.0.smithi017.stdout:2999: writing smithi01785587-18 from 692752 to 1112213 tid 1 2024-09-21T09:50:38.182 INFO:tasks.rados.rados.0.smithi017.stdout:2999: writing smithi01785587-18 from 1826015 to 2441416 tid 2 2024-09-21T09:50:38.186 INFO:tasks.rados.rados.0.smithi017.stdout:2999: writing smithi01785587-18 from 2979461 to 3092752 tid 3 2024-09-21T09:50:38.186 INFO:tasks.rados.rados.0.smithi017.stdout:3000: delete oid 49 current snap is 377 2024-09-21T09:50:38.186 INFO:tasks.rados.rados.0.smithi017.stdout:2999: finishing write tid 1 to smithi01785587-18 2024-09-21T09:50:38.186 INFO:tasks.rados.rados.0.smithi017.stdout:2999: finishing write tid 2 to smithi01785587-18 2024-09-21T09:50:38.189 INFO:tasks.rados.rados.0.smithi017.stdout:2999: finishing write tid 3 to smithi01785587-18 2024-09-21T09:50:38.189 INFO:tasks.rados.rados.0.smithi017.stdout:2999: finishing write tid 4 to smithi01785587-18 2024-09-21T09:50:38.189 INFO:tasks.rados.rados.0.smithi017.stdout:2999: finishing write tid 5 to smithi01785587-18 2024-09-21T09:50:38.189 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 266 (ObjNum 1136 snap 377 seq_num 1136) dirty exists 2024-09-21T09:50:38.189 INFO:tasks.rados.rados.0.smithi017.stdout:2999: left oid 18 (ObjNum 1136 snap 377 seq_num 1136) 2024-09-21T09:50:38.192 INFO:tasks.rados.rados.0.smithi017.stdout:2999: done (1 left) 2024-09-21T09:50:38.192 INFO:tasks.rados.rados.0.smithi017.stdout:3000: done (0 left) 2024-09-21T09:50:38.192 INFO:tasks.rados.rados.0.smithi017.stdout:3001: snap_remove snap 370 2024-09-21T09:50:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:38 smithi017 ceph-mon[102913]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T09:50:38.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:38 smithi017 ceph-mon[102913]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T09:50:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:38 smithi017 ceph-mon[105151]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T09:50:38.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:38 smithi017 ceph-mon[105151]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T09:50:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:38 smithi134 ceph-mon[90085]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T09:50:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:38 smithi134 ceph-mon[90085]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T09:50:39.199 INFO:tasks.rados.rados.0.smithi017.stdout:3001: done (0 left) 2024-09-21T09:50:39.200 INFO:tasks.rados.rados.0.smithi017.stdout:3002: write oid 42 current snap is 377 2024-09-21T09:50:39.200 INFO:tasks.rados.rados.0.smithi017.stdout:3002: seq_num 1138 ranges {726366=569959,1826626=502717,2801601=324765} 2024-09-21T09:50:39.206 INFO:tasks.rados.rados.0.smithi017.stdout:3002: writing smithi01785587-42 from 726366 to 1296325 tid 1 2024-09-21T09:50:39.212 INFO:tasks.rados.rados.0.smithi017.stdout:3002: writing smithi01785587-42 from 1826626 to 2329343 tid 2 2024-09-21T09:50:39.216 INFO:tasks.rados.rados.0.smithi017.stdout:3002: writing smithi01785587-42 from 2801601 to 3126366 tid 3 2024-09-21T09:50:39.217 INFO:tasks.rados.rados.0.smithi017.stdout:3003: rollback oid 32 current snap is 377 2024-09-21T09:50:39.217 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 371 2024-09-21T09:50:39.217 INFO:tasks.rados.rados.0.smithi017.stdout:3002: finishing write tid 1 to smithi01785587-42 2024-09-21T09:50:39.217 INFO:tasks.rados.rados.0.smithi017.stdout:3002: finishing write tid 2 to smithi01785587-42 2024-09-21T09:50:39.217 INFO:tasks.rados.rados.0.smithi017.stdout:3004: delete oid 41 current snap is 377 2024-09-21T09:50:39.219 INFO:tasks.rados.rados.0.smithi017.stdout:3004: done (2 left) 2024-09-21T09:50:39.219 INFO:tasks.rados.rados.0.smithi017.stdout:3005: read oid 25 snap -1 2024-09-21T09:50:39.219 INFO:tasks.rados.rados.0.smithi017.stdout:3005: expect (ObjNum 1109 snap 366 seq_num 1109) 2024-09-21T09:50:39.219 INFO:tasks.rados.rados.0.smithi017.stdout:3006: write oid 46 current snap is 377 2024-09-21T09:50:39.219 INFO:tasks.rados.rados.0.smithi017.stdout:3006: seq_num 1140 ranges {759980=720457,1943590=506386,2856447=303533} 2024-09-21T09:50:39.228 INFO:tasks.rados.rados.0.smithi017.stdout:3006: writing smithi01785587-46 from 759980 to 1480437 tid 1 2024-09-21T09:50:39.233 INFO:tasks.rados.rados.0.smithi017.stdout:3006: writing smithi01785587-46 from 1943590 to 2449976 tid 2 2024-09-21T09:50:39.237 INFO:tasks.rados.rados.0.smithi017.stdout:3006: writing smithi01785587-46 from 2856447 to 3159980 tid 3 2024-09-21T09:50:39.237 INFO:tasks.rados.rados.0.smithi017.stdout:3007: write oid 9 current snap is 377 2024-09-21T09:50:39.237 INFO:tasks.rados.rados.0.smithi017.stdout:3002: finishing write tid 3 to smithi01785587-42 2024-09-21T09:50:39.237 INFO:tasks.rados.rados.0.smithi017.stdout:3007: seq_num 1141 ranges {776787=512059,1718425=766397,3000223=176564} 2024-09-21T09:50:39.244 INFO:tasks.rados.rados.0.smithi017.stdout:3007: writing smithi01785587-9 from 776787 to 1288846 tid 1 2024-09-21T09:50:39.251 INFO:tasks.rados.rados.0.smithi017.stdout:3007: writing smithi01785587-9 from 1718425 to 2484822 tid 2 2024-09-21T09:50:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:3007: writing smithi01785587-9 from 3000223 to 3176787 tid 3 2024-09-21T09:50:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:3008: rollback oid 37 current snap is 377 2024-09-21T09:50:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 376 2024-09-21T09:50:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:3002: finishing write tid 4 to smithi01785587-42 2024-09-21T09:50:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:3002: finishing write tid 5 to smithi01785587-42 2024-09-21T09:50:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 1006 (ObjNum 1138 snap 377 seq_num 1138) dirty exists 2024-09-21T09:50:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3002: left oid 42 (ObjNum 1138 snap 377 seq_num 1138) 2024-09-21T09:50:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3003: finishing rollback tid 0 to smithi01785587-32 2024-09-21T09:50:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3003: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:50:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3003: finishing rollback tid 2 to smithi01785587-32 2024-09-21T09:50:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1350 (ObjNum 982 snap 335 seq_num 982) dirty exists 2024-09-21T09:50:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3005: expect (ObjNum 1109 snap 366 seq_num 1109) 2024-09-21T09:50:39.345 INFO:tasks.rados.rados.0.smithi017.stdout:3006: finishing write tid 1 to smithi01785587-46 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:3006: finishing write tid 2 to smithi01785587-46 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:3006: finishing write tid 3 to smithi01785587-46 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:3006: finishing write tid 4 to smithi01785587-46 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:3006: finishing write tid 5 to smithi01785587-46 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1355 (ObjNum 1140 snap 377 seq_num 1140) dirty exists 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:3006: left oid 46 (ObjNum 1140 snap 377 seq_num 1140) 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:3007: finishing write tid 1 to smithi01785587-9 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:3002: done (5 left) 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:3003: done (4 left) 2024-09-21T09:50:39.346 INFO:tasks.rados.rados.0.smithi017.stdout:3005: done (3 left) 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3006: done (2 left) 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3009: rollback oid 40 current snap is 377 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 369 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3007: finishing write tid 2 to smithi01785587-9 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3007: finishing write tid 3 to smithi01785587-9 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3007: finishing write tid 4 to smithi01785587-9 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3007: finishing write tid 5 to smithi01785587-9 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 604 (ObjNum 1141 snap 377 seq_num 1141) dirty exists 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3007: left oid 9 (ObjNum 1141 snap 377 seq_num 1141) 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3008: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3008: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3008: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1524 (ObjNum 1134 snap 374 seq_num 1134) dirty exists 2024-09-21T09:50:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3007: done (2 left) 2024-09-21T09:50:39.348 INFO:tasks.rados.rados.0.smithi017.stdout:3008: done (1 left) 2024-09-21T09:50:39.348 INFO:tasks.rados.rados.0.smithi017.stdout:3010: read oid 8 snap -1 2024-09-21T09:50:39.348 INFO:tasks.rados.rados.0.smithi017.stdout:3010: expect (ObjNum 1070 snap 357 seq_num 1070) 2024-09-21T09:50:39.348 INFO:tasks.rados.rados.0.smithi017.stdout:3011: snap_remove snap 354 2024-09-21T09:50:39.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:39 smithi017 ceph-mon[102913]: pgmap v798: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 958 KiB/s wr, 5 op/s 2024-09-21T09:50:39.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:39 smithi017 ceph-mon[105151]: pgmap v798: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 958 KiB/s wr, 5 op/s 2024-09-21T09:50:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:39 smithi134 ceph-mon[90085]: pgmap v798: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 958 KiB/s wr, 5 op/s 2024-09-21T09:50:40.202 INFO:tasks.rados.rados.0.smithi017.stdout:3011: done (2 left) 2024-09-21T09:50:40.202 INFO:tasks.rados.rados.0.smithi017.stdout:3012: read oid 32 snap -1 2024-09-21T09:50:40.202 INFO:tasks.rados.rados.0.smithi017.stdout:3012: expect (ObjNum 982 snap 335 seq_num 982) 2024-09-21T09:50:40.202 INFO:tasks.rados.rados.0.smithi017.stdout:3013: read oid 50 snap -1 2024-09-21T09:50:40.202 INFO:tasks.rados.rados.0.smithi017.stdout:3013: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:50:40.202 INFO:tasks.rados.rados.0.smithi017.stdout:3014: read oid 14 snap -1 2024-09-21T09:50:40.202 INFO:tasks.rados.rados.0.smithi017.stdout:3014: expect (ObjNum 1131 snap 374 seq_num 1131) 2024-09-21T09:50:40.202 INFO:tasks.rados.rados.0.smithi017.stdout:3015: delete oid 9 current snap is 377 2024-09-21T09:50:40.203 INFO:tasks.rados.rados.0.smithi017.stdout:3009: finishing rollback tid 0 to smithi01785587-40 2024-09-21T09:50:40.203 INFO:tasks.rados.rados.0.smithi017.stdout:3009: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:50:40.203 INFO:tasks.rados.rados.0.smithi017.stdout:3009: finishing rollback tid 2 to smithi01785587-40 2024-09-21T09:50:40.203 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1528 (ObjNum 1045 snap 350 seq_num 1045) dirty exists 2024-09-21T09:50:40.203 INFO:tasks.rados.rados.0.smithi017.stdout:3010: expect (ObjNum 1070 snap 357 seq_num 1070) 2024-09-21T09:50:40.394 INFO:tasks.rados.rados.0.smithi017.stdout:3013: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:50:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:40 smithi017 ceph-mon[102913]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T09:50:40.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:40 smithi017 ceph-mon[105151]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T09:50:40.473 INFO:tasks.rados.rados.0.smithi017.stdout:3014: expect (ObjNum 1131 snap 374 seq_num 1131) 2024-09-21T09:50:40.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:40 smithi134 ceph-mon[90085]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T09:50:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:50:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:50:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:50:40.602 INFO:tasks.rados.rados.0.smithi017.stdout:3012: expect (ObjNum 982 snap 335 seq_num 982) 2024-09-21T09:50:40.685 INFO:tasks.rados.rados.0.smithi017.stdout:3009: done (5 left) 2024-09-21T09:50:40.685 INFO:tasks.rados.rados.0.smithi017.stdout:3010: done (4 left) 2024-09-21T09:50:40.685 INFO:tasks.rados.rados.0.smithi017.stdout:3012: done (3 left) 2024-09-21T09:50:40.685 INFO:tasks.rados.rados.0.smithi017.stdout:3013: done (2 left) 2024-09-21T09:50:40.685 INFO:tasks.rados.rados.0.smithi017.stdout:3014: done (1 left) 2024-09-21T09:50:40.685 INFO:tasks.rados.rados.0.smithi017.stdout:3015: done (0 left) 2024-09-21T09:50:40.685 INFO:tasks.rados.rados.0.smithi017.stdout:3016: snap_create 2024-09-21T09:50:41.204 INFO:tasks.rados.rados.0.smithi017.stdout:3016: done (0 left) 2024-09-21T09:50:41.204 INFO:tasks.rados.rados.0.smithi017.stdout:3017: rollback oid 13 current snap is 378 2024-09-21T09:50:41.204 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 13 to 364 2024-09-21T09:50:41.204 INFO:tasks.rados.rados.0.smithi017.stdout:3018: snap_create 2024-09-21T09:50:41.209 INFO:tasks.rados.rados.0.smithi017.stdout:3017: finishing rollback tid 1 to smithi01785587-13 2024-09-21T09:50:41.210 INFO:tasks.rados.rados.0.smithi017.stdout:3017: finishing rollback tid 2 to smithi01785587-13 2024-09-21T09:50:41.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1286 (ObjNum 1095 snap 363 seq_num 1095) dirty exists 2024-09-21T09:50:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:41 smithi134 ceph-mon[90085]: pgmap v801: 89 pgs: 89 active+clean; 183 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 642 KiB/s wr, 5 op/s 2024-09-21T09:50:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:41 smithi134 ceph-mon[90085]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T09:50:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:41.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:41 smithi017 ceph-mon[102913]: pgmap v801: 89 pgs: 89 active+clean; 183 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 642 KiB/s wr, 5 op/s 2024-09-21T09:50:41.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:41 smithi017 ceph-mon[102913]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T09:50:41.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:41.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:41 smithi017 ceph-mon[105151]: pgmap v801: 89 pgs: 89 active+clean; 183 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 642 KiB/s wr, 5 op/s 2024-09-21T09:50:41.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:41 smithi017 ceph-mon[105151]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T09:50:41.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:3017: done (1 left) 2024-09-21T09:50:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:3018: done (0 left) 2024-09-21T09:50:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:3019: read oid 9 snap -1 2024-09-21T09:50:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:3019: expect deleted 2024-09-21T09:50:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:3020: write oid 11 current snap is 379 2024-09-21T09:50:42.214 INFO:tasks.rados.rados.0.smithi017.stdout:3020: seq_num 1143 ranges {410401=662557,1719036=770066,2938716=271685} 2024-09-21T09:50:42.219 INFO:tasks.rados.rados.0.smithi017.stdout:3020: writing smithi01785587-11 from 410401 to 1072958 tid 1 2024-09-21T09:50:42.227 INFO:tasks.rados.rados.0.smithi017.stdout:3020: writing smithi01785587-11 from 1719036 to 2489102 tid 2 2024-09-21T09:50:42.231 INFO:tasks.rados.rados.0.smithi017.stdout:3020: writing smithi01785587-11 from 2938716 to 3210401 tid 3 2024-09-21T09:50:42.231 INFO:tasks.rados.rados.0.smithi017.stdout:3021: snap_remove snap 373 2024-09-21T09:50:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:42 smithi134 ceph-mon[90085]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T09:50:42.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:42 smithi017 ceph-mon[102913]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T09:50:42.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:42 smithi017 ceph-mon[105151]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T09:50:43.171 INFO:tasks.rados.rados.0.smithi017.stdout:3021: done (2 left) 2024-09-21T09:50:43.171 INFO:tasks.rados.rados.0.smithi017.stdout:3022: write oid 27 current snap is 379 2024-09-21T09:50:43.171 INFO:tasks.rados.rados.0.smithi017.stdout:3022: seq_num 1144 ranges {427208=737806,1777518=513724,2966139=261069} 2024-09-21T09:50:43.178 INFO:tasks.rados.rados.0.smithi017.stdout:3022: writing smithi01785587-27 from 427208 to 1165014 tid 1 2024-09-21T09:50:43.185 INFO:tasks.rados.rados.0.smithi017.stdout:3022: writing smithi01785587-27 from 1777518 to 2291242 tid 2 2024-09-21T09:50:43.190 INFO:tasks.rados.rados.0.smithi017.stdout:3022: writing smithi01785587-27 from 2966139 to 3227208 tid 3 2024-09-21T09:50:43.190 INFO:tasks.rados.rados.0.smithi017.stdout:3023: write oid 5 current snap is 379 2024-09-21T09:50:43.190 INFO:tasks.rados.rados.0.smithi017.stdout:3023: seq_num 1145 ranges {444015=413055,1436000=773735,2709915=534100} 2024-09-21T09:50:43.195 INFO:tasks.rados.rados.0.smithi017.stdout:3023: writing smithi01785587-5 from 444015 to 857070 tid 1 2024-09-21T09:50:43.202 INFO:tasks.rados.rados.0.smithi017.stdout:3023: writing smithi01785587-5 from 1436000 to 2209735 tid 2 2024-09-21T09:50:43.208 INFO:tasks.rados.rados.0.smithi017.stdout:3023: writing smithi01785587-5 from 2709915 to 3244015 tid 3 2024-09-21T09:50:43.208 INFO:tasks.rados.rados.0.smithi017.stdout:3024: read oid 1 snap -1 2024-09-21T09:50:43.208 INFO:tasks.rados.rados.0.smithi017.stdout:3024: expect deleted 2024-09-21T09:50:43.208 INFO:tasks.rados.rados.0.smithi017.stdout:3025: snap_remove snap 376 2024-09-21T09:50:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:43 smithi134 ceph-mon[90085]: pgmap v804: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-09-21T09:50:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:43 smithi134 ceph-mon[90085]: osdmap e817: 8 total, 8 up, 8 in 2024-09-21T09:50:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:43 smithi134 ceph-mon[90085]: osdmap e818: 8 total, 8 up, 8 in 2024-09-21T09:50:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:43 smithi017 ceph-mon[102913]: pgmap v804: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-09-21T09:50:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:43 smithi017 ceph-mon[102913]: osdmap e817: 8 total, 8 up, 8 in 2024-09-21T09:50:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:43 smithi017 ceph-mon[102913]: osdmap e818: 8 total, 8 up, 8 in 2024-09-21T09:50:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:43 smithi017 ceph-mon[105151]: pgmap v804: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-09-21T09:50:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:43 smithi017 ceph-mon[105151]: osdmap e817: 8 total, 8 up, 8 in 2024-09-21T09:50:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:43 smithi017 ceph-mon[105151]: osdmap e818: 8 total, 8 up, 8 in 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:3025: done (5 left) 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:3026: snap_create 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:3020: finishing write tid 1 to smithi01785587-11 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:3020: finishing write tid 2 to smithi01785587-11 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:3020: finishing write tid 3 to smithi01785587-11 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:3020: finishing write tid 4 to smithi01785587-11 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:3020: finishing write tid 5 to smithi01785587-11 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1369 (ObjNum 1143 snap 379 seq_num 1143) dirty exists 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:3020: left oid 11 (ObjNum 1143 snap 379 seq_num 1143) 2024-09-21T09:50:44.173 INFO:tasks.rados.rados.0.smithi017.stdout:3022: finishing write tid 1 to smithi01785587-27 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3022: finishing write tid 2 to smithi01785587-27 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3022: finishing write tid 3 to smithi01785587-27 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3022: finishing write tid 4 to smithi01785587-27 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3022: finishing write tid 5 to smithi01785587-27 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 987 (ObjNum 1144 snap 379 seq_num 1144) dirty exists 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3022: left oid 27 (ObjNum 1144 snap 379 seq_num 1144) 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3023: finishing write tid 1 to smithi01785587-5 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3023: finishing write tid 2 to smithi01785587-5 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3023: finishing write tid 3 to smithi01785587-5 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3023: finishing write tid 4 to smithi01785587-5 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:3023: finishing write tid 5 to smithi01785587-5 2024-09-21T09:50:44.174 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1560 (ObjNum 1145 snap 379 seq_num 1145) dirty exists 2024-09-21T09:50:44.175 INFO:tasks.rados.rados.0.smithi017.stdout:3023: left oid 5 (ObjNum 1145 snap 379 seq_num 1145) 2024-09-21T09:50:45.173 INFO:tasks.rados.rados.0.smithi017.stdout:3019: done (5 left) 2024-09-21T09:50:45.173 INFO:tasks.rados.rados.0.smithi017.stdout:3020: done (4 left) 2024-09-21T09:50:45.173 INFO:tasks.rados.rados.0.smithi017.stdout:3022: done (3 left) 2024-09-21T09:50:45.173 INFO:tasks.rados.rados.0.smithi017.stdout:3023: done (2 left) 2024-09-21T09:50:45.173 INFO:tasks.rados.rados.0.smithi017.stdout:3024: done (1 left) 2024-09-21T09:50:45.173 INFO:tasks.rados.rados.0.smithi017.stdout:3026: done (0 left) 2024-09-21T09:50:45.173 INFO:tasks.rados.rados.0.smithi017.stdout:3027: write oid 49 current snap is 380 2024-09-21T09:50:45.174 INFO:tasks.rados.rados.0.smithi017.stdout:3027: seq_num 1146 ranges {460822=488304,1378129=517393,2504632=405962} 2024-09-21T09:50:45.174 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:50:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:50:44] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.51.0" 2024-09-21T09:50:45.178 INFO:tasks.rados.rados.0.smithi017.stdout:3027: writing smithi01785587-49 from 460822 to 949126 tid 1 2024-09-21T09:50:45.183 INFO:tasks.rados.rados.0.smithi017.stdout:3027: writing smithi01785587-49 from 1378129 to 1895522 tid 2 2024-09-21T09:50:45.189 INFO:tasks.rados.rados.0.smithi017.stdout:3027: writing smithi01785587-49 from 2504632 to 2910594 tid 3 2024-09-21T09:50:45.190 INFO:tasks.rados.rados.0.smithi017.stdout:3028: read oid 38 snap -1 2024-09-21T09:50:45.190 INFO:tasks.rados.rados.0.smithi017.stdout:3028: expect deleted 2024-09-21T09:50:45.190 INFO:tasks.rados.rados.0.smithi017.stdout:3029: rollback oid 21 current snap is 380 2024-09-21T09:50:45.190 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 367 2024-09-21T09:50:45.190 INFO:tasks.rados.rados.0.smithi017.stdout:3027: finishing write tid 1 to smithi01785587-49 2024-09-21T09:50:45.190 INFO:tasks.rados.rados.0.smithi017.stdout:3027: finishing write tid 2 to smithi01785587-49 2024-09-21T09:50:45.190 INFO:tasks.rados.rados.0.smithi017.stdout:3030: snap_create 2024-09-21T09:50:45.193 INFO:tasks.rados.rados.0.smithi017.stdout:3027: finishing write tid 3 to smithi01785587-49 2024-09-21T09:50:45.194 INFO:tasks.rados.rados.0.smithi017.stdout:3027: finishing write tid 4 to smithi01785587-49 2024-09-21T09:50:45.194 INFO:tasks.rados.rados.0.smithi017.stdout:3027: finishing write tid 5 to smithi01785587-49 2024-09-21T09:50:45.194 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1487 (ObjNum 1146 snap 380 seq_num 1146) dirty exists 2024-09-21T09:50:45.194 INFO:tasks.rados.rados.0.smithi017.stdout:3027: left oid 49 (ObjNum 1146 snap 380 seq_num 1146) 2024-09-21T09:50:45.194 INFO:tasks.rados.rados.0.smithi017.stdout:3029: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:50:45.194 INFO:tasks.rados.rados.0.smithi017.stdout:3029: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:50:45.195 INFO:tasks.rados.rados.0.smithi017.stdout:3029: finishing rollback tid 2 to smithi01785587-21 2024-09-21T09:50:45.195 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1491 (ObjNum 1090 snap 363 seq_num 1090) dirty exists 2024-09-21T09:50:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:45 smithi017 ceph-mon[102913]: pgmap v807: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-09-21T09:50:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:45 smithi017 ceph-mon[102913]: osdmap e819: 8 total, 8 up, 8 in 2024-09-21T09:50:45.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:45 smithi017 ceph-mon[105151]: pgmap v807: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-09-21T09:50:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:45 smithi017 ceph-mon[105151]: osdmap e819: 8 total, 8 up, 8 in 2024-09-21T09:50:45.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:45 smithi134 ceph-mon[90085]: pgmap v807: 89 pgs: 89 active+clean; 192 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-09-21T09:50:45.496 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:45 smithi134 ceph-mon[90085]: osdmap e819: 8 total, 8 up, 8 in 2024-09-21T09:50:46.183 INFO:tasks.rados.rados.0.smithi017.stdout:3027: done (3 left) 2024-09-21T09:50:46.183 INFO:tasks.rados.rados.0.smithi017.stdout:3028: done (2 left) 2024-09-21T09:50:46.183 INFO:tasks.rados.rados.0.smithi017.stdout:3029: done (1 left) 2024-09-21T09:50:46.183 INFO:tasks.rados.rados.0.smithi017.stdout:3030: done (0 left) 2024-09-21T09:50:46.183 INFO:tasks.rados.rados.0.smithi017.stdout:3031: rollback oid 10 current snap is 381 2024-09-21T09:50:46.184 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 378 2024-09-21T09:50:46.184 INFO:tasks.rados.rados.0.smithi017.stdout:3032: snap_remove snap 375 2024-09-21T09:50:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:46 smithi017 ceph-mon[102913]: osdmap e820: 8 total, 8 up, 8 in 2024-09-21T09:50:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:46 smithi017 ceph-mon[105151]: osdmap e820: 8 total, 8 up, 8 in 2024-09-21T09:50:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:46 smithi134 ceph-mon[90085]: osdmap e820: 8 total, 8 up, 8 in 2024-09-21T09:50:47.184 INFO:tasks.rados.rados.0.smithi017.stdout:3032: done (1 left) 2024-09-21T09:50:47.185 INFO:tasks.rados.rados.0.smithi017.stdout:3033: read oid 47 snap -1 2024-09-21T09:50:47.185 INFO:tasks.rados.rados.0.smithi017.stdout:3033: expect (ObjNum 1126 snap 370 seq_num 1126) 2024-09-21T09:50:47.185 INFO:tasks.rados.rados.0.smithi017.stdout:3034: read oid 31 snap 352 2024-09-21T09:50:47.185 INFO:tasks.rados.rados.0.smithi017.stdout:3034: expect (ObjNum 1023 snap 344 seq_num 1023) 2024-09-21T09:50:47.185 INFO:tasks.rados.rados.0.smithi017.stdout:3035: write oid 20 current snap is 381 2024-09-21T09:50:47.185 INFO:tasks.rados.rados.0.smithi017.stdout:3031: finishing rollback tid 0 to smithi01785587-10 2024-09-21T09:50:47.185 INFO:tasks.rados.rados.0.smithi017.stdout:3035: seq_num 1147 ranges {477629=563553,1836611=777404,3048408=229221} 2024-09-21T09:50:47.193 INFO:tasks.rados.rados.0.smithi017.stdout:3035: writing smithi01785587-20 from 477629 to 1041182 tid 1 2024-09-21T09:50:47.201 INFO:tasks.rados.rados.0.smithi017.stdout:3035: writing smithi01785587-20 from 1836611 to 2614015 tid 2 2024-09-21T09:50:47.205 INFO:tasks.rados.rados.0.smithi017.stdout:3035: writing smithi01785587-20 from 3048408 to 3277629 tid 3 2024-09-21T09:50:47.205 INFO:tasks.rados.rados.0.smithi017.stdout:3036: rollback oid 8 current snap is 381 2024-09-21T09:50:47.205 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 372 2024-09-21T09:50:47.205 INFO:tasks.rados.rados.0.smithi017.stdout:3031: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:50:47.205 INFO:tasks.rados.rados.0.smithi017.stdout:3031: finishing rollback tid 2 to smithi01785587-10 2024-09-21T09:50:47.206 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 266 (ObjNum 985 snap 336 seq_num 985) dirty exists 2024-09-21T09:50:47.206 INFO:tasks.rados.rados.0.smithi017.stdout:3034: expect (ObjNum 1023 snap 344 seq_num 1023) 2024-09-21T09:50:47.312 INFO:tasks.rados.rados.0.smithi017.stdout:3035: finishing write tid 1 to smithi01785587-20 2024-09-21T09:50:47.312 INFO:tasks.rados.rados.0.smithi017.stdout:3033: expect (ObjNum 1126 snap 370 seq_num 1126) 2024-09-21T09:50:47.409 INFO:tasks.rados.rados.0.smithi017.stdout:3035: finishing write tid 2 to smithi01785587-20 2024-09-21T09:50:47.409 INFO:tasks.rados.rados.0.smithi017.stdout:3035: finishing write tid 3 to smithi01785587-20 2024-09-21T09:50:47.409 INFO:tasks.rados.rados.0.smithi017.stdout:3035: finishing write tid 4 to smithi01785587-20 2024-09-21T09:50:47.409 INFO:tasks.rados.rados.0.smithi017.stdout:3035: finishing write tid 5 to smithi01785587-20 2024-09-21T09:50:47.409 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 814 (ObjNum 1147 snap 381 seq_num 1147) dirty exists 2024-09-21T09:50:47.409 INFO:tasks.rados.rados.0.smithi017.stdout:3035: left oid 20 (ObjNum 1147 snap 381 seq_num 1147) 2024-09-21T09:50:47.409 INFO:tasks.rados.rados.0.smithi017.stdout:3036: finishing rollback tid 0 to smithi01785587-8 2024-09-21T09:50:47.409 INFO:tasks.rados.rados.0.smithi017.stdout:3036: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:50:47.409 INFO:tasks.rados.rados.0.smithi017.stdout:3031: done (4 left) 2024-09-21T09:50:47.410 INFO:tasks.rados.rados.0.smithi017.stdout:3033: done (3 left) 2024-09-21T09:50:47.410 INFO:tasks.rados.rados.0.smithi017.stdout:3034: done (2 left) 2024-09-21T09:50:47.410 INFO:tasks.rados.rados.0.smithi017.stdout:3035: done (1 left) 2024-09-21T09:50:47.410 INFO:tasks.rados.rados.0.smithi017.stdout:3037: rollback oid 27 current snap is 381 2024-09-21T09:50:47.410 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 360 2024-09-21T09:50:47.410 INFO:tasks.rados.rados.0.smithi017.stdout:3036: finishing rollback tid 2 to smithi01785587-8 2024-09-21T09:50:47.410 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1385 (ObjNum 1070 snap 357 seq_num 1070) dirty exists 2024-09-21T09:50:47.410 INFO:tasks.rados.rados.0.smithi017.stdout:3036: done (1 left) 2024-09-21T09:50:47.410 INFO:tasks.rados.rados.0.smithi017.stdout:3038: delete oid 22 current snap is 381 2024-09-21T09:50:47.412 INFO:tasks.rados.rados.0.smithi017.stdout:3037: finishing rollback tid 0 to smithi01785587-27 2024-09-21T09:50:47.412 INFO:tasks.rados.rados.0.smithi017.stdout:3038: done (1 left) 2024-09-21T09:50:47.412 INFO:tasks.rados.rados.0.smithi017.stdout:3039: read oid 32 snap -1 2024-09-21T09:50:47.412 INFO:tasks.rados.rados.0.smithi017.stdout:3039: expect (ObjNum 982 snap 335 seq_num 982) 2024-09-21T09:50:47.412 INFO:tasks.rados.rados.0.smithi017.stdout:3040: write oid 48 current snap is 381 2024-09-21T09:50:47.412 INFO:tasks.rados.rados.0.smithi017.stdout:3040: seq_num 1149 ranges {511243=430404,1669928=664720,3103254=207989} 2024-09-21T09:50:47.416 INFO:tasks.rados.rados.0.smithi017.stdout:3040: writing smithi01785587-48 from 511243 to 941647 tid 1 2024-09-21T09:50:47.424 INFO:tasks.rados.rados.0.smithi017.stdout:3040: writing smithi01785587-48 from 1669928 to 2334648 tid 2 2024-09-21T09:50:47.430 INFO:tasks.rados.rados.0.smithi017.stdout:3040: writing smithi01785587-48 from 3103254 to 3311243 tid 3 2024-09-21T09:50:47.430 INFO:tasks.rados.rados.0.smithi017.stdout:3041: read oid 49 snap -1 2024-09-21T09:50:47.430 INFO:tasks.rados.rados.0.smithi017.stdout:3041: expect (ObjNum 1146 snap 380 seq_num 1146) 2024-09-21T09:50:47.430 INFO:tasks.rados.rados.0.smithi017.stdout:3042: snap_create 2024-09-21T09:50:47.430 INFO:tasks.rados.rados.0.smithi017.stdout:3037: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:50:47.430 INFO:tasks.rados.rados.0.smithi017.stdout:3037: finishing rollback tid 2 to smithi01785587-27 2024-09-21T09:50:47.430 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 995 (ObjNum 1009 snap 341 seq_num 1009) dirty exists 2024-09-21T09:50:47.430 INFO:tasks.rados.rados.0.smithi017.stdout:3039: expect (ObjNum 982 snap 335 seq_num 982) 2024-09-21T09:50:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:47 smithi017 ceph-mon[102913]: pgmap v810: 89 pgs: 89 active+clean; 196 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 776 KiB/s wr, 6 op/s 2024-09-21T09:50:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:47 smithi017 ceph-mon[102913]: osdmap e821: 8 total, 8 up, 8 in 2024-09-21T09:50:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:47 smithi017 ceph-mon[105151]: pgmap v810: 89 pgs: 89 active+clean; 196 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 776 KiB/s wr, 6 op/s 2024-09-21T09:50:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:47 smithi017 ceph-mon[105151]: osdmap e821: 8 total, 8 up, 8 in 2024-09-21T09:50:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:47 smithi134 ceph-mon[90085]: pgmap v810: 89 pgs: 89 active+clean; 196 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 776 KiB/s wr, 6 op/s 2024-09-21T09:50:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:47 smithi134 ceph-mon[90085]: osdmap e821: 8 total, 8 up, 8 in 2024-09-21T09:50:47.513 INFO:tasks.rados.rados.0.smithi017.stdout:3040: finishing write tid 1 to smithi01785587-48 2024-09-21T09:50:47.513 INFO:tasks.rados.rados.0.smithi017.stdout:3040: finishing write tid 2 to smithi01785587-48 2024-09-21T09:50:47.513 INFO:tasks.rados.rados.0.smithi017.stdout:3040: finishing write tid 3 to smithi01785587-48 2024-09-21T09:50:47.513 INFO:tasks.rados.rados.0.smithi017.stdout:3040: finishing write tid 4 to smithi01785587-48 2024-09-21T09:50:47.513 INFO:tasks.rados.rados.0.smithi017.stdout:3040: finishing write tid 5 to smithi01785587-48 2024-09-21T09:50:47.513 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 1291 (ObjNum 1149 snap 381 seq_num 1149) dirty exists 2024-09-21T09:50:47.513 INFO:tasks.rados.rados.0.smithi017.stdout:3040: left oid 48 (ObjNum 1149 snap 381 seq_num 1149) 2024-09-21T09:50:47.513 INFO:tasks.rados.rados.0.smithi017.stdout:3041: expect (ObjNum 1146 snap 380 seq_num 1146) 2024-09-21T09:50:48.172 INFO:tasks.rados.rados.0.smithi017.stdout:3037: done (4 left) 2024-09-21T09:50:48.172 INFO:tasks.rados.rados.0.smithi017.stdout:3039: done (3 left) 2024-09-21T09:50:48.172 INFO:tasks.rados.rados.0.smithi017.stdout:3040: done (2 left) 2024-09-21T09:50:48.172 INFO:tasks.rados.rados.0.smithi017.stdout:3041: done (1 left) 2024-09-21T09:50:48.172 INFO:tasks.rados.rados.0.smithi017.stdout:3042: done (0 left) 2024-09-21T09:50:48.172 INFO:tasks.rados.rados.0.smithi017.stdout:3043: snap_create 2024-09-21T09:50:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:48 smithi017 ceph-mon[102913]: osdmap e822: 8 total, 8 up, 8 in 2024-09-21T09:50:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:48 smithi017 ceph-mon[102913]: osdmap e823: 8 total, 8 up, 8 in 2024-09-21T09:50:48.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:48 smithi017 ceph-mon[105151]: osdmap e822: 8 total, 8 up, 8 in 2024-09-21T09:50:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:48 smithi017 ceph-mon[105151]: osdmap e823: 8 total, 8 up, 8 in 2024-09-21T09:50:48.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:48 smithi134 ceph-mon[90085]: osdmap e822: 8 total, 8 up, 8 in 2024-09-21T09:50:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:48 smithi134 ceph-mon[90085]: osdmap e823: 8 total, 8 up, 8 in 2024-09-21T09:50:49.174 INFO:tasks.rados.rados.0.smithi017.stdout:3043: done (0 left) 2024-09-21T09:50:49.174 INFO:tasks.rados.rados.0.smithi017.stdout:3044: read oid 22 snap -1 2024-09-21T09:50:49.174 INFO:tasks.rados.rados.0.smithi017.stdout:3044: expect deleted 2024-09-21T09:50:49.174 INFO:tasks.rados.rados.0.smithi017.stdout:3045: snap_create 2024-09-21T09:50:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:49 smithi017 ceph-mon[102913]: pgmap v813: 89 pgs: 89 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-21T09:50:49.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:49 smithi017 ceph-mon[102913]: osdmap e824: 8 total, 8 up, 8 in 2024-09-21T09:50:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:49 smithi017 ceph-mon[105151]: pgmap v813: 89 pgs: 89 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-21T09:50:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:49 smithi017 ceph-mon[105151]: osdmap e824: 8 total, 8 up, 8 in 2024-09-21T09:50:49.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:49 smithi134 ceph-mon[90085]: pgmap v813: 89 pgs: 89 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-09-21T09:50:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:49 smithi134 ceph-mon[90085]: osdmap e824: 8 total, 8 up, 8 in 2024-09-21T09:50:50.176 INFO:tasks.rados.rados.0.smithi017.stdout:3044: done (1 left) 2024-09-21T09:50:50.176 INFO:tasks.rados.rados.0.smithi017.stdout:3045: done (0 left) 2024-09-21T09:50:50.176 INFO:tasks.rados.rados.0.smithi017.stdout:3046: read oid 29 snap -1 2024-09-21T09:50:50.176 INFO:tasks.rados.rados.0.smithi017.stdout:3046: expect (ObjNum 1129 snap 372 seq_num 1129) 2024-09-21T09:50:50.176 INFO:tasks.rados.rados.0.smithi017.stdout:3047: snap_create 2024-09-21T09:50:50.186 INFO:tasks.rados.rados.0.smithi017.stdout:3046: expect (ObjNum 1129 snap 372 seq_num 1129) 2024-09-21T09:50:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:50:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:50:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:50:51.177 INFO:tasks.rados.rados.0.smithi017.stdout:3046: done (1 left) 2024-09-21T09:50:51.178 INFO:tasks.rados.rados.0.smithi017.stdout:3047: done (0 left) 2024-09-21T09:50:51.178 INFO:tasks.rados.rados.0.smithi017.stdout:3048: write oid 19 current snap is 385 2024-09-21T09:50:51.178 INFO:tasks.rados.rados.0.smithi017.stdout:3048: seq_num 1150 ranges {528050=505653,1612057=524731,2614324=567403} 2024-09-21T09:50:51.183 INFO:tasks.rados.rados.0.smithi017.stdout:3048: writing smithi01785587-19 from 528050 to 1033703 tid 1 2024-09-21T09:50:51.189 INFO:tasks.rados.rados.0.smithi017.stdout:3048: writing smithi01785587-19 from 1612057 to 2136788 tid 2 2024-09-21T09:50:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:3048: writing smithi01785587-19 from 2614324 to 3181727 tid 3 2024-09-21T09:50:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:3049: delete oid 35 current snap is 385 2024-09-21T09:50:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:3048: finishing write tid 1 to smithi01785587-19 2024-09-21T09:50:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:3048: finishing write tid 2 to smithi01785587-19 2024-09-21T09:50:51.199 INFO:tasks.rados.rados.0.smithi017.stdout:3049: done (1 left) 2024-09-21T09:50:51.199 INFO:tasks.rados.rados.0.smithi017.stdout:3050: write oid 45 current snap is 385 2024-09-21T09:50:51.199 INFO:tasks.rados.rados.0.smithi017.stdout:3050: seq_num 1152 ranges {561664=656151,1729021=528400,2669170=692494} 2024-09-21T09:50:51.206 INFO:tasks.rados.rados.0.smithi017.stdout:3050: writing smithi01785587-45 from 561664 to 1217815 tid 1 2024-09-21T09:50:51.212 INFO:tasks.rados.rados.0.smithi017.stdout:3050: writing smithi01785587-45 from 1729021 to 2257421 tid 2 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3050: writing smithi01785587-45 from 2669170 to 3361664 tid 3 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3051: write oid 46 current snap is 385 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3048: finishing write tid 3 to smithi01785587-19 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3048: finishing write tid 4 to smithi01785587-19 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3048: finishing write tid 5 to smithi01785587-19 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 1295 (ObjNum 1150 snap 385 seq_num 1150) dirty exists 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3048: left oid 19 (ObjNum 1150 snap 385 seq_num 1150) 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3050: finishing write tid 1 to smithi01785587-45 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3050: finishing write tid 2 to smithi01785587-45 2024-09-21T09:50:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3051: seq_num 1153 ranges {578471=731400,1787503=672058,3096593=281878} 2024-09-21T09:50:51.226 INFO:tasks.rados.rados.0.smithi017.stdout:3051: writing smithi01785587-46 from 578471 to 1309871 tid 1 2024-09-21T09:50:51.232 INFO:tasks.rados.rados.0.smithi017.stdout:3051: writing smithi01785587-46 from 1787503 to 2459561 tid 2 2024-09-21T09:50:51.237 INFO:tasks.rados.rados.0.smithi017.stdout:3051: writing smithi01785587-46 from 3096593 to 3378471 tid 3 2024-09-21T09:50:51.238 INFO:tasks.rados.rados.0.smithi017.stdout:3048: done (2 left) 2024-09-21T09:50:51.238 INFO:tasks.rados.rados.0.smithi017.stdout:3052: snap_create 2024-09-21T09:50:51.238 INFO:tasks.rados.rados.0.smithi017.stdout:3050: finishing write tid 3 to smithi01785587-45 2024-09-21T09:50:51.238 INFO:tasks.rados.rados.0.smithi017.stdout:3050: finishing write tid 4 to smithi01785587-45 2024-09-21T09:50:51.238 INFO:tasks.rados.rados.0.smithi017.stdout:3050: finishing write tid 5 to smithi01785587-45 2024-09-21T09:50:51.238 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1000 (ObjNum 1152 snap 385 seq_num 1152) dirty exists 2024-09-21T09:50:51.238 INFO:tasks.rados.rados.0.smithi017.stdout:3050: left oid 45 (ObjNum 1152 snap 385 seq_num 1152) 2024-09-21T09:50:51.238 INFO:tasks.rados.rados.0.smithi017.stdout:3051: finishing write tid 1 to smithi01785587-46 2024-09-21T09:50:51.238 INFO:tasks.rados.rados.0.smithi017.stdout:3051: finishing write tid 2 to smithi01785587-46 2024-09-21T09:50:51.240 INFO:tasks.rados.rados.0.smithi017.stdout:3051: finishing write tid 3 to smithi01785587-46 2024-09-21T09:50:51.240 INFO:tasks.rados.rados.0.smithi017.stdout:3051: finishing write tid 4 to smithi01785587-46 2024-09-21T09:50:51.240 INFO:tasks.rados.rados.0.smithi017.stdout:3051: finishing write tid 5 to smithi01785587-46 2024-09-21T09:50:51.240 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1390 (ObjNum 1153 snap 385 seq_num 1153) dirty exists 2024-09-21T09:50:51.240 INFO:tasks.rados.rados.0.smithi017.stdout:3051: left oid 46 (ObjNum 1153 snap 385 seq_num 1153) 2024-09-21T09:50:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:51 smithi017 ceph-mon[102913]: pgmap v816: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:50:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:51 smithi017 ceph-mon[102913]: osdmap e825: 8 total, 8 up, 8 in 2024-09-21T09:50:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:51 smithi017 ceph-mon[105151]: pgmap v816: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:50:51.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:51 smithi017 ceph-mon[105151]: osdmap e825: 8 total, 8 up, 8 in 2024-09-21T09:50:51.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:51 smithi134 ceph-mon[90085]: pgmap v816: 89 pgs: 89 active+clean; 205 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-09-21T09:50:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:51 smithi134 ceph-mon[90085]: osdmap e825: 8 total, 8 up, 8 in 2024-09-21T09:50:52.189 INFO:tasks.rados.rados.0.smithi017.stdout:3050: done (2 left) 2024-09-21T09:50:52.190 INFO:tasks.rados.rados.0.smithi017.stdout:3051: done (1 left) 2024-09-21T09:50:52.190 INFO:tasks.rados.rados.0.smithi017.stdout:3052: done (0 left) 2024-09-21T09:50:52.190 INFO:tasks.rados.rados.0.smithi017.stdout:3053: snap_create 2024-09-21T09:50:52.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:52 smithi017 ceph-mon[102913]: osdmap e826: 8 total, 8 up, 8 in 2024-09-21T09:50:52.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:52 smithi017 ceph-mon[105151]: osdmap e826: 8 total, 8 up, 8 in 2024-09-21T09:50:52.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:52 smithi134 ceph-mon[90085]: osdmap e826: 8 total, 8 up, 8 in 2024-09-21T09:50:53.173 INFO:tasks.rados.rados.0.smithi017.stdout:3053: done (0 left) 2024-09-21T09:50:53.173 INFO:tasks.rados.rados.0.smithi017.stdout:3054: snap_create 2024-09-21T09:50:53.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:53 smithi017 ceph-mon[102913]: pgmap v819: 89 pgs: 89 active+clean; 210 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:50:53.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:53 smithi017 ceph-mon[102913]: osdmap e827: 8 total, 8 up, 8 in 2024-09-21T09:50:53.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:53 smithi017 ceph-mon[102913]: osdmap e828: 8 total, 8 up, 8 in 2024-09-21T09:50:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:53 smithi017 ceph-mon[105151]: pgmap v819: 89 pgs: 89 active+clean; 210 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:50:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:53 smithi017 ceph-mon[105151]: osdmap e827: 8 total, 8 up, 8 in 2024-09-21T09:50:53.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:53 smithi017 ceph-mon[105151]: osdmap e828: 8 total, 8 up, 8 in 2024-09-21T09:50:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:53 smithi134 ceph-mon[90085]: pgmap v819: 89 pgs: 89 active+clean; 210 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:50:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:53 smithi134 ceph-mon[90085]: osdmap e827: 8 total, 8 up, 8 in 2024-09-21T09:50:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:53 smithi134 ceph-mon[90085]: osdmap e828: 8 total, 8 up, 8 in 2024-09-21T09:50:54.175 INFO:tasks.rados.rados.0.smithi017.stdout:3054: done (0 left) 2024-09-21T09:50:54.175 INFO:tasks.rados.rados.0.smithi017.stdout:3055: delete oid 36 current snap is 388 2024-09-21T09:50:54.180 INFO:tasks.rados.rados.0.smithi017.stdout:3055: done (0 left) 2024-09-21T09:50:54.180 INFO:tasks.rados.rados.0.smithi017.stdout:3056: read oid 20 snap -1 2024-09-21T09:50:54.180 INFO:tasks.rados.rados.0.smithi017.stdout:3056: expect (ObjNum 1147 snap 381 seq_num 1147) 2024-09-21T09:50:54.180 INFO:tasks.rados.rados.0.smithi017.stdout:3057: snap_remove snap 381 2024-09-21T09:50:55.172 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:50:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:50:54] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.51.0" 2024-09-21T09:50:55.181 INFO:tasks.rados.rados.0.smithi017.stdout:3057: done (1 left) 2024-09-21T09:50:55.181 INFO:tasks.rados.rados.0.smithi017.stdout:3058: snap_remove snap 382 2024-09-21T09:50:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:55 smithi017 ceph-mon[102913]: pgmap v822: 89 pgs: 89 active+clean; 211 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-21T09:50:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:55 smithi017 ceph-mon[102913]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T09:50:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:55 smithi017 ceph-mon[105151]: pgmap v822: 89 pgs: 89 active+clean; 211 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-21T09:50:55.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:55 smithi017 ceph-mon[105151]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T09:50:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:55 smithi134 ceph-mon[90085]: pgmap v822: 89 pgs: 89 active+clean; 211 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 17 op/s 2024-09-21T09:50:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:55 smithi134 ceph-mon[90085]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T09:50:56.188 INFO:tasks.rados.rados.0.smithi017.stdout:3058: done (1 left) 2024-09-21T09:50:56.188 INFO:tasks.rados.rados.0.smithi017.stdout:3059: read oid 48 snap -1 2024-09-21T09:50:56.188 INFO:tasks.rados.rados.0.smithi017.stdout:3059: expect (ObjNum 1149 snap 381 seq_num 1149) 2024-09-21T09:50:56.188 INFO:tasks.rados.rados.0.smithi017.stdout:3060: write oid 11 current snap is 388 2024-09-21T09:50:56.188 INFO:tasks.rados.rados.0.smithi017.stdout:3060: seq_num 1155 ranges {612085=481898,1788114=675727,3035086=376999} 2024-09-21T09:50:56.194 INFO:tasks.rados.rados.0.smithi017.stdout:3060: writing smithi01785587-11 from 612085 to 1093983 tid 1 2024-09-21T09:50:56.202 INFO:tasks.rados.rados.0.smithi017.stdout:3060: writing smithi01785587-11 from 1788114 to 2463841 tid 2 2024-09-21T09:50:56.207 INFO:tasks.rados.rados.0.smithi017.stdout:3060: writing smithi01785587-11 from 3035086 to 3412085 tid 3 2024-09-21T09:50:56.207 INFO:tasks.rados.rados.0.smithi017.stdout:3061: read oid 50 snap -1 2024-09-21T09:50:56.207 INFO:tasks.rados.rados.0.smithi017.stdout:3061: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:50:56.207 INFO:tasks.rados.rados.0.smithi017.stdout:3062: delete oid 29 current snap is 388 2024-09-21T09:50:56.208 INFO:tasks.rados.rados.0.smithi017.stdout:3056: expect (ObjNum 1147 snap 381 seq_num 1147) 2024-09-21T09:50:56.356 INFO:tasks.rados.rados.0.smithi017.stdout:3060: finishing write tid 1 to smithi01785587-11 2024-09-21T09:50:56.356 INFO:tasks.rados.rados.0.smithi017.stdout:3059: expect (ObjNum 1149 snap 381 seq_num 1149) 2024-09-21T09:50:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:56 smithi017 ceph-mon[105151]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T09:50:56.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:56 smithi017 ceph-mon[102913]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T09:50:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:56 smithi134 ceph-mon[90085]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T09:50:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:56.596 INFO:tasks.rados.rados.0.smithi017.stdout:3060: finishing write tid 2 to smithi01785587-11 2024-09-21T09:50:56.596 INFO:tasks.rados.rados.0.smithi017.stdout:3060: finishing write tid 3 to smithi01785587-11 2024-09-21T09:50:56.596 INFO:tasks.rados.rados.0.smithi017.stdout:3061: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:50:56.676 INFO:tasks.rados.rados.0.smithi017.stdout:3060: finishing write tid 4 to smithi01785587-11 2024-09-21T09:50:56.676 INFO:tasks.rados.rados.0.smithi017.stdout:3060: finishing write tid 5 to smithi01785587-11 2024-09-21T09:50:56.676 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1399 (ObjNum 1155 snap 388 seq_num 1155) dirty exists 2024-09-21T09:50:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:3060: left oid 11 (ObjNum 1155 snap 388 seq_num 1155) 2024-09-21T09:50:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:3056: done (4 left) 2024-09-21T09:50:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:3059: done (3 left) 2024-09-21T09:50:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:3060: done (2 left) 2024-09-21T09:50:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:3061: done (1 left) 2024-09-21T09:50:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:3062: done (0 left) 2024-09-21T09:50:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:3063: read oid 25 snap -1 2024-09-21T09:50:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:3063: expect (ObjNum 1109 snap 366 seq_num 1109) 2024-09-21T09:50:56.677 INFO:tasks.rados.rados.0.smithi017.stdout:3064: snap_create 2024-09-21T09:50:56.684 INFO:tasks.rados.rados.0.smithi017.stdout:3063: expect (ObjNum 1109 snap 366 seq_num 1109) 2024-09-21T09:50:57.186 INFO:tasks.rados.rados.0.smithi017.stdout:3063: done (1 left) 2024-09-21T09:50:57.186 INFO:tasks.rados.rados.0.smithi017.stdout:3064: done (0 left) 2024-09-21T09:50:57.186 INFO:tasks.rados.rados.0.smithi017.stdout:3065: rollback oid 39 current snap is 389 2024-09-21T09:50:57.186 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 366 2024-09-21T09:50:57.186 INFO:tasks.rados.rados.0.smithi017.stdout:3066: delete oid 40 current snap is 389 2024-09-21T09:50:57.191 INFO:tasks.rados.rados.0.smithi017.stdout:3065: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:50:57.193 INFO:tasks.rados.rados.0.smithi017.stdout:3065: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:50:57.193 INFO:tasks.rados.rados.0.smithi017.stdout:3065: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:50:57.193 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1549 (ObjNum 1097 snap 363 seq_num 1097) dirty exists 2024-09-21T09:50:57.193 INFO:tasks.rados.rados.0.smithi017.stdout:3065: done (1 left) 2024-09-21T09:50:57.193 INFO:tasks.rados.rados.0.smithi017.stdout:3066: done (0 left) 2024-09-21T09:50:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:3067: write oid 14 current snap is 389 2024-09-21T09:50:57.194 INFO:tasks.rados.rados.0.smithi017.stdout:3067: seq_num 1158 ranges {662506=423998,1563560=539407,2717355=490285} 2024-09-21T09:50:57.199 INFO:tasks.rados.rados.0.smithi017.stdout:3067: writing smithi01785587-14 from 662506 to 1086504 tid 1 2024-09-21T09:50:57.205 INFO:tasks.rados.rados.0.smithi017.stdout:3067: writing smithi01785587-14 from 1563560 to 2102967 tid 2 2024-09-21T09:50:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:3067: writing smithi01785587-14 from 2717355 to 3207640 tid 3 2024-09-21T09:50:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:3068: read oid 25 snap -1 2024-09-21T09:50:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:3068: expect (ObjNum 1109 snap 366 seq_num 1109) 2024-09-21T09:50:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:3069: read oid 8 snap -1 2024-09-21T09:50:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:3069: expect (ObjNum 1070 snap 357 seq_num 1070) 2024-09-21T09:50:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:3070: write oid 6 current snap is 389 2024-09-21T09:50:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:3067: finishing write tid 1 to smithi01785587-14 2024-09-21T09:50:57.212 INFO:tasks.rados.rados.0.smithi017.stdout:3070: seq_num 1159 ranges {679313=499247,1622042=683065,2744778=701557} 2024-09-21T09:50:57.218 INFO:tasks.rados.rados.0.smithi017.stdout:3070: writing smithi01785587-6 from 679313 to 1178560 tid 1 2024-09-21T09:50:57.224 INFO:tasks.rados.rados.0.smithi017.stdout:3070: writing smithi01785587-6 from 1622042 to 2305107 tid 2 2024-09-21T09:50:57.231 INFO:tasks.rados.rados.0.smithi017.stdout:3070: writing smithi01785587-6 from 2744778 to 3446335 tid 3 2024-09-21T09:50:57.231 INFO:tasks.rados.rados.0.smithi017.stdout:3071: write oid 49 current snap is 389 2024-09-21T09:50:57.231 INFO:tasks.rados.rados.0.smithi017.stdout:3067: finishing write tid 2 to smithi01785587-14 2024-09-21T09:50:57.231 INFO:tasks.rados.rados.0.smithi017.stdout:3071: seq_num 1160 ranges {696120=574496,1680524=543076,2888554=512829} 2024-09-21T09:50:57.237 INFO:tasks.rados.rados.0.smithi017.stdout:3071: writing smithi01785587-49 from 696120 to 1270616 tid 1 2024-09-21T09:50:57.243 INFO:tasks.rados.rados.0.smithi017.stdout:3071: writing smithi01785587-49 from 1680524 to 2223600 tid 2 2024-09-21T09:50:57.249 INFO:tasks.rados.rados.0.smithi017.stdout:3071: writing smithi01785587-49 from 2888554 to 3401383 tid 3 2024-09-21T09:50:57.249 INFO:tasks.rados.rados.0.smithi017.stdout:3072: snap_remove snap 385 2024-09-21T09:50:57.249 INFO:tasks.rados.rados.0.smithi017.stdout:3067: finishing write tid 3 to smithi01785587-14 2024-09-21T09:50:57.250 INFO:tasks.rados.rados.0.smithi017.stdout:3067: finishing write tid 4 to smithi01785587-14 2024-09-21T09:50:57.250 INFO:tasks.rados.rados.0.smithi017.stdout:3067: finishing write tid 5 to smithi01785587-14 2024-09-21T09:50:57.250 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 1569 (ObjNum 1158 snap 389 seq_num 1158) dirty exists 2024-09-21T09:50:57.250 INFO:tasks.rados.rados.0.smithi017.stdout:3067: left oid 14 (ObjNum 1158 snap 389 seq_num 1158) 2024-09-21T09:50:57.250 INFO:tasks.rados.rados.0.smithi017.stdout:3068: expect (ObjNum 1109 snap 366 seq_num 1109) 2024-09-21T09:50:57.338 INFO:tasks.rados.rados.0.smithi017.stdout:3069: expect (ObjNum 1070 snap 357 seq_num 1070) 2024-09-21T09:50:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:57 smithi017 ceph-mon[102913]: pgmap v825: 89 pgs: 89 active+clean; 215 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 849 KiB/s wr, 4 op/s 2024-09-21T09:50:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:57 smithi017 ceph-mon[102913]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T09:50:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:57 smithi017 ceph-mon[105151]: pgmap v825: 89 pgs: 89 active+clean; 215 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 849 KiB/s wr, 4 op/s 2024-09-21T09:50:57.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:57 smithi017 ceph-mon[105151]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T09:50:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:57 smithi134 ceph-mon[90085]: pgmap v825: 89 pgs: 89 active+clean; 215 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 849 KiB/s wr, 4 op/s 2024-09-21T09:50:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:57 smithi134 ceph-mon[90085]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:3070: finishing write tid 1 to smithi01785587-6 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:3070: finishing write tid 2 to smithi01785587-6 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:3070: finishing write tid 3 to smithi01785587-6 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:3070: finishing write tid 4 to smithi01785587-6 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:3070: finishing write tid 5 to smithi01785587-6 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 886 (ObjNum 1159 snap 389 seq_num 1159) dirty exists 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:3070: left oid 6 (ObjNum 1159 snap 389 seq_num 1159) 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:3071: finishing write tid 1 to smithi01785587-49 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:3071: finishing write tid 2 to smithi01785587-49 2024-09-21T09:50:57.524 INFO:tasks.rados.rados.0.smithi017.stdout:3071: finishing write tid 3 to smithi01785587-49 2024-09-21T09:50:57.525 INFO:tasks.rados.rados.0.smithi017.stdout:3071: finishing write tid 4 to smithi01785587-49 2024-09-21T09:50:57.525 INFO:tasks.rados.rados.0.smithi017.stdout:3071: finishing write tid 5 to smithi01785587-49 2024-09-21T09:50:57.525 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1500 (ObjNum 1160 snap 389 seq_num 1160) dirty exists 2024-09-21T09:50:57.525 INFO:tasks.rados.rados.0.smithi017.stdout:3071: left oid 49 (ObjNum 1160 snap 389 seq_num 1160) 2024-09-21T09:50:58.176 INFO:tasks.rados.rados.0.smithi017.stdout:3067: done (5 left) 2024-09-21T09:50:58.176 INFO:tasks.rados.rados.0.smithi017.stdout:3068: done (4 left) 2024-09-21T09:50:58.176 INFO:tasks.rados.rados.0.smithi017.stdout:3069: done (3 left) 2024-09-21T09:50:58.176 INFO:tasks.rados.rados.0.smithi017.stdout:3070: done (2 left) 2024-09-21T09:50:58.176 INFO:tasks.rados.rados.0.smithi017.stdout:3071: done (1 left) 2024-09-21T09:50:58.177 INFO:tasks.rados.rados.0.smithi017.stdout:3072: done (0 left) 2024-09-21T09:50:58.177 INFO:tasks.rados.rados.0.smithi017.stdout:3073: snap_create 2024-09-21T09:50:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:58 smithi017 ceph-mon[102913]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T09:50:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:58 smithi017 ceph-mon[102913]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T09:50:58.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:58 smithi017 ceph-mon[105151]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T09:50:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:58 smithi017 ceph-mon[105151]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T09:50:58.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:58 smithi134 ceph-mon[90085]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T09:50:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:58 smithi134 ceph-mon[90085]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T09:50:59.177 INFO:tasks.rados.rados.0.smithi017.stdout:3073: done (0 left) 2024-09-21T09:50:59.177 INFO:tasks.rados.rados.0.smithi017.stdout:3074: write oid 7 current snap is 390 2024-09-21T09:50:59.177 INFO:tasks.rados.rados.0.smithi017.stdout:3074: seq_num 1161 ranges {712927=649745,2139006=686734} 2024-09-21T09:50:59.184 INFO:tasks.rados.rados.0.smithi017.stdout:3074: writing smithi01785587-7 from 712927 to 1362672 tid 1 2024-09-21T09:50:59.193 INFO:tasks.rados.rados.0.smithi017.stdout:3074: writing smithi01785587-7 from 2139006 to 2825740 tid 2 2024-09-21T09:50:59.193 INFO:tasks.rados.rados.0.smithi017.stdout:3075: read oid 21 snap -1 2024-09-21T09:50:59.194 INFO:tasks.rados.rados.0.smithi017.stdout:3075: expect (ObjNum 1090 snap 363 seq_num 1090) 2024-09-21T09:50:59.194 INFO:tasks.rados.rados.0.smithi017.stdout:3076: rollback oid 17 current snap is 390 2024-09-21T09:50:59.194 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 369 2024-09-21T09:50:59.194 INFO:tasks.rados.rados.0.smithi017.stdout:3074: finishing write tid 1 to smithi01785587-7 2024-09-21T09:50:59.194 INFO:tasks.rados.rados.0.smithi017.stdout:3077: read oid 44 snap -1 2024-09-21T09:50:59.194 INFO:tasks.rados.rados.0.smithi017.stdout:3077: expect (ObjNum 1130 snap 372 seq_num 1130) 2024-09-21T09:50:59.194 INFO:tasks.rados.rados.0.smithi017.stdout:3078: snap_remove snap 380 2024-09-21T09:50:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:59 smithi017 ceph-mon[102913]: pgmap v828: 89 pgs: 89 active+clean; 216 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 765 KiB/s wr, 7 op/s 2024-09-21T09:50:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:50:59 smithi017 ceph-mon[102913]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T09:50:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:59 smithi017 ceph-mon[105151]: pgmap v828: 89 pgs: 89 active+clean; 216 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 765 KiB/s wr, 7 op/s 2024-09-21T09:50:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:50:59 smithi017 ceph-mon[105151]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T09:50:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:59 smithi134 ceph-mon[90085]: pgmap v828: 89 pgs: 89 active+clean; 216 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 765 KiB/s wr, 7 op/s 2024-09-21T09:50:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:50:59 smithi134 ceph-mon[90085]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T09:51:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:3078: done (4 left) 2024-09-21T09:51:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:3079: read oid 9 snap -1 2024-09-21T09:51:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:3079: expect deleted 2024-09-21T09:51:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:3080: rollback oid 6 current snap is 390 2024-09-21T09:51:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 387 2024-09-21T09:51:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:3074: finishing write tid 2 to smithi01785587-7 2024-09-21T09:51:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:3074: finishing write tid 3 to smithi01785587-7 2024-09-21T09:51:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:3074: finishing write tid 4 to smithi01785587-7 2024-09-21T09:51:00.203 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1505 (ObjNum 1161 snap 390 seq_num 1161) dirty exists 2024-09-21T09:51:00.204 INFO:tasks.rados.rados.0.smithi017.stdout:3074: left oid 7 (ObjNum 1161 snap 390 seq_num 1161) 2024-09-21T09:51:00.204 INFO:tasks.rados.rados.0.smithi017.stdout:3077: expect (ObjNum 1130 snap 372 seq_num 1130) 2024-09-21T09:51:00.454 INFO:tasks.rados.rados.0.smithi017.stdout:3075: expect (ObjNum 1090 snap 363 seq_num 1090) 2024-09-21T09:51:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:51:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:51:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:3076: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 0 (ObjNum 305 snap 0 seq_num 1540944041) dirty dne 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:3074: done (5 left) 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:3075: done (4 left) 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:3076: done (3 left) 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:3077: done (2 left) 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:3081: snap_create 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:3080: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:3080: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:51:00.534 INFO:tasks.rados.rados.0.smithi017.stdout:3080: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:51:00.535 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 892 (ObjNum 1010 snap 341 seq_num 1010) dirty exists 2024-09-21T09:51:01.213 INFO:tasks.rados.rados.0.smithi017.stdout:3079: done (2 left) 2024-09-21T09:51:01.213 INFO:tasks.rados.rados.0.smithi017.stdout:3080: done (1 left) 2024-09-21T09:51:01.213 INFO:tasks.rados.rados.0.smithi017.stdout:3081: done (0 left) 2024-09-21T09:51:01.213 INFO:tasks.rados.rados.0.smithi017.stdout:3082: delete oid 11 current snap is 391 2024-09-21T09:51:01.213 INFO:tasks.rados.rados.0.smithi017.stdout:3082: done (0 left) 2024-09-21T09:51:01.214 INFO:tasks.rados.rados.0.smithi017.stdout:3083: snap_remove snap 369 2024-09-21T09:51:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:01 smithi017 ceph-mon[102913]: pgmap v831: 89 pgs: 89 active+clean; 216 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 375 KiB/s wr, 4 op/s 2024-09-21T09:51:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:01 smithi017 ceph-mon[102913]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T09:51:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:01 smithi017 ceph-mon[105151]: pgmap v831: 89 pgs: 89 active+clean; 216 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 375 KiB/s wr, 4 op/s 2024-09-21T09:51:01.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:01 smithi017 ceph-mon[105151]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T09:51:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:01 smithi134 ceph-mon[90085]: pgmap v831: 89 pgs: 89 active+clean; 216 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 375 KiB/s wr, 4 op/s 2024-09-21T09:51:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:01 smithi134 ceph-mon[90085]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T09:51:02.217 INFO:tasks.rados.rados.0.smithi017.stdout:3083: done (0 left) 2024-09-21T09:51:02.217 INFO:tasks.rados.rados.0.smithi017.stdout:3084: rollback oid 30 current snap is 391 2024-09-21T09:51:02.218 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 356 2024-09-21T09:51:02.218 INFO:tasks.rados.rados.0.smithi017.stdout:3085: delete oid 35 current snap is 391 2024-09-21T09:51:02.224 INFO:tasks.rados.rados.0.smithi017.stdout:3085: done (1 left) 2024-09-21T09:51:02.224 INFO:tasks.rados.rados.0.smithi017.stdout:3086: snap_create 2024-09-21T09:51:02.227 INFO:tasks.rados.rados.0.smithi017.stdout:3084: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:51:02.227 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 0 (ObjNum 33 snap 0 seq_num 3571292278) dirty dne 2024-09-21T09:51:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:02 smithi134 ceph-mon[90085]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T09:51:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:02 smithi017 ceph-mon[102913]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T09:51:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:02 smithi017 ceph-mon[105151]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T09:51:03.177 INFO:tasks.rados.rados.0.smithi017.stdout:3084: done (1 left) 2024-09-21T09:51:03.177 INFO:tasks.rados.rados.0.smithi017.stdout:3086: done (0 left) 2024-09-21T09:51:03.177 INFO:tasks.rados.rados.0.smithi017.stdout:3087: read oid 34 snap -1 2024-09-21T09:51:03.177 INFO:tasks.rados.rados.0.smithi017.stdout:3087: expect deleted 2024-09-21T09:51:03.177 INFO:tasks.rados.rados.0.smithi017.stdout:3088: write oid 10 current snap is 392 2024-09-21T09:51:03.177 INFO:tasks.rados.rados.0.smithi017.stdout:3088: seq_num 1164 ranges {763348=591845,1914452=434061,2881893=674270} 2024-09-21T09:51:03.184 INFO:tasks.rados.rados.0.smithi017.stdout:3088: writing smithi01785587-10 from 763348 to 1355193 tid 1 2024-09-21T09:51:03.189 INFO:tasks.rados.rados.0.smithi017.stdout:3088: writing smithi01785587-10 from 1914452 to 2348513 tid 2 2024-09-21T09:51:03.197 INFO:tasks.rados.rados.0.smithi017.stdout:3088: writing smithi01785587-10 from 2881893 to 3556163 tid 3 2024-09-21T09:51:03.197 INFO:tasks.rados.rados.0.smithi017.stdout:3089: rollback oid 26 current snap is 392 2024-09-21T09:51:03.197 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 26 to 364 2024-09-21T09:51:03.197 INFO:tasks.rados.rados.0.smithi017.stdout:3088: finishing write tid 1 to smithi01785587-10 2024-09-21T09:51:03.197 INFO:tasks.rados.rados.0.smithi017.stdout:3088: finishing write tid 2 to smithi01785587-10 2024-09-21T09:51:03.197 INFO:tasks.rados.rados.0.smithi017.stdout:3087: done (2 left) 2024-09-21T09:51:03.197 INFO:tasks.rados.rados.0.smithi017.stdout:3090: read oid 50 snap -1 2024-09-21T09:51:03.197 INFO:tasks.rados.rados.0.smithi017.stdout:3090: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:51:03.197 INFO:tasks.rados.rados.0.smithi017.stdout:3091: write oid 25 current snap is 392 2024-09-21T09:51:03.198 INFO:tasks.rados.rados.0.smithi017.stdout:3091: seq_num 1165 ranges {780155=667094,1972934=694072,3309316=270839} 2024-09-21T09:51:03.205 INFO:tasks.rados.rados.0.smithi017.stdout:3091: writing smithi01785587-25 from 780155 to 1447249 tid 1 2024-09-21T09:51:03.212 INFO:tasks.rados.rados.0.smithi017.stdout:3091: writing smithi01785587-25 from 1972934 to 2667006 tid 2 2024-09-21T09:51:03.217 INFO:tasks.rados.rados.0.smithi017.stdout:3091: writing smithi01785587-25 from 3309316 to 3580155 tid 3 2024-09-21T09:51:03.217 INFO:tasks.rados.rados.0.smithi017.stdout:3092: read oid 43 snap -1 2024-09-21T09:51:03.217 INFO:tasks.rados.rados.0.smithi017.stdout:3092: expect (ObjNum 1098 snap 363 seq_num 1098) 2024-09-21T09:51:03.217 INFO:tasks.rados.rados.0.smithi017.stdout:3093: write oid 9 current snap is 392 2024-09-21T09:51:03.217 INFO:tasks.rados.rados.0.smithi017.stdout:3093: seq_num 1166 ranges {796962=742343,2031416=437730,2936739=413167} 2024-09-21T09:51:03.226 INFO:tasks.rados.rados.0.smithi017.stdout:3093: writing smithi01785587-9 from 796962 to 1539305 tid 1 2024-09-21T09:51:03.232 INFO:tasks.rados.rados.0.smithi017.stdout:3093: writing smithi01785587-9 from 2031416 to 2469146 tid 2 2024-09-21T09:51:03.237 INFO:tasks.rados.rados.0.smithi017.stdout:3093: writing smithi01785587-9 from 2936739 to 3349906 tid 3 2024-09-21T09:51:03.237 INFO:tasks.rados.rados.0.smithi017.stdout:3094: read oid 17 snap 359 2024-09-21T09:51:03.237 INFO:tasks.rados.rados.0.smithi017.stdout:3094: expect deleted 2024-09-21T09:51:03.237 INFO:tasks.rados.rados.0.smithi017.stdout:3095: rollback oid 49 current snap is 392 2024-09-21T09:51:03.237 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 360 2024-09-21T09:51:03.237 INFO:tasks.rados.rados.0.smithi017.stdout:3090: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:51:03.319 INFO:tasks.rados.rados.0.smithi017.stdout:3088: finishing write tid 3 to smithi01785587-10 2024-09-21T09:51:03.319 INFO:tasks.rados.rados.0.smithi017.stdout:3090: done (7 left) 2024-09-21T09:51:03.319 INFO:tasks.rados.rados.0.smithi017.stdout:3096: read oid 37 snap 379 2024-09-21T09:51:03.319 INFO:tasks.rados.rados.0.smithi017.stdout:3096: expect (ObjNum 1134 snap 374 seq_num 1134) 2024-09-21T09:51:03.319 INFO:tasks.rados.rados.0.smithi017.stdout:3097: write oid 12 current snap is 392 2024-09-21T09:51:03.319 INFO:tasks.rados.rados.0.smithi017.stdout:3088: finishing write tid 4 to smithi01785587-10 2024-09-21T09:51:03.319 INFO:tasks.rados.rados.0.smithi017.stdout:3088: finishing write tid 5 to smithi01785587-10 2024-09-21T09:51:03.319 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 277 (ObjNum 1164 snap 392 seq_num 1164) dirty exists 2024-09-21T09:51:03.320 INFO:tasks.rados.rados.0.smithi017.stdout:3088: left oid 10 (ObjNum 1164 snap 392 seq_num 1164) 2024-09-21T09:51:03.320 INFO:tasks.rados.rados.0.smithi017.stdout:3089: finishing rollback tid 0 to smithi01785587-26 2024-09-21T09:51:03.320 INFO:tasks.rados.rados.0.smithi017.stdout:3097: seq_num 1167 ranges {413769=417592,1573545=697741,2847809=624439} 2024-09-21T09:51:03.323 INFO:tasks.rados.rados.0.smithi017.stdout:3097: writing smithi01785587-12 from 413769 to 831361 tid 1 2024-09-21T09:51:03.331 INFO:tasks.rados.rados.0.smithi017.stdout:3097: writing smithi01785587-12 from 1573545 to 2271286 tid 2 2024-09-21T09:51:03.337 INFO:tasks.rados.rados.0.smithi017.stdout:3097: writing smithi01785587-12 from 2847809 to 3472248 tid 3 2024-09-21T09:51:03.338 INFO:tasks.rados.rados.0.smithi017.stdout:3088: done (8 left) 2024-09-21T09:51:03.338 INFO:tasks.rados.rados.0.smithi017.stdout:3098: rollback oid 38 current snap is 392 2024-09-21T09:51:03.338 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 360 2024-09-21T09:51:03.338 INFO:tasks.rados.rados.0.smithi017.stdout:3099: snap_remove snap 367 2024-09-21T09:51:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:03 smithi134 ceph-mon[90085]: pgmap v834: 89 pgs: 89 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 19 op/s 2024-09-21T09:51:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:03 smithi134 ceph-mon[90085]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T09:51:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:03 smithi134 ceph-mon[90085]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T09:51:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:03 smithi017 ceph-mon[102913]: pgmap v834: 89 pgs: 89 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 19 op/s 2024-09-21T09:51:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:03 smithi017 ceph-mon[102913]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T09:51:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:03 smithi017 ceph-mon[102913]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T09:51:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:03 smithi017 ceph-mon[105151]: pgmap v834: 89 pgs: 89 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 19 op/s 2024-09-21T09:51:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:03 smithi017 ceph-mon[105151]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T09:51:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:03 smithi017 ceph-mon[105151]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T09:51:04.220 INFO:tasks.rados.rados.0.smithi017.stdout:3099: done (9 left) 2024-09-21T09:51:04.220 INFO:tasks.rados.rados.0.smithi017.stdout:3100: write oid 11 current snap is 392 2024-09-21T09:51:04.220 INFO:tasks.rados.rados.0.smithi017.stdout:3100: seq_num 1168 ranges {430576=492841,1632027=441399,2475232=435711,3365397=265179} 2024-09-21T09:51:04.226 INFO:tasks.rados.rados.0.smithi017.stdout:3100: writing smithi01785587-11 from 430576 to 923417 tid 1 2024-09-21T09:51:04.233 INFO:tasks.rados.rados.0.smithi017.stdout:3100: writing smithi01785587-11 from 1632027 to 2073426 tid 2 2024-09-21T09:51:04.238 INFO:tasks.rados.rados.0.smithi017.stdout:3100: writing smithi01785587-11 from 2475232 to 2910943 tid 3 2024-09-21T09:51:04.242 INFO:tasks.rados.rados.0.smithi017.stdout:3100: writing smithi01785587-11 from 3365397 to 3630576 tid 4 2024-09-21T09:51:04.242 INFO:tasks.rados.rados.0.smithi017.stdout:3101: write oid 19 current snap is 392 2024-09-21T09:51:04.242 INFO:tasks.rados.rados.0.smithi017.stdout:3101: seq_num 1169 ranges {447383=568090,1690509=701410,2902655=646983} 2024-09-21T09:51:04.248 INFO:tasks.rados.rados.0.smithi017.stdout:3101: writing smithi01785587-19 from 447383 to 1015473 tid 1 2024-09-21T09:51:04.255 INFO:tasks.rados.rados.0.smithi017.stdout:3101: writing smithi01785587-19 from 1690509 to 2391919 tid 2 2024-09-21T09:51:04.262 INFO:tasks.rados.rados.0.smithi017.stdout:3101: writing smithi01785587-19 from 2902655 to 3549638 tid 3 2024-09-21T09:51:04.262 INFO:tasks.rados.rados.0.smithi017.stdout:3102: read oid 35 snap -1 2024-09-21T09:51:04.262 INFO:tasks.rados.rados.0.smithi017.stdout:3102: expect deleted 2024-09-21T09:51:04.262 INFO:tasks.rados.rados.0.smithi017.stdout:3103: rollback oid 46 current snap is 392 2024-09-21T09:51:04.262 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 391 2024-09-21T09:51:04.262 INFO:tasks.rados.rados.0.smithi017.stdout:3089: finishing rollback tid 1 to smithi01785587-26 2024-09-21T09:51:04.262 INFO:tasks.rados.rados.0.smithi017.stdout:3104: write oid 1 current snap is 392 2024-09-21T09:51:04.262 INFO:tasks.rados.rados.0.smithi017.stdout:3104: seq_num 1170 ranges {464190=643339,1748991=445068,2930078=574608} 2024-09-21T09:51:04.268 INFO:tasks.rados.rados.0.smithi017.stdout:3104: writing smithi01785587-1 from 464190 to 1107529 tid 1 2024-09-21T09:51:04.274 INFO:tasks.rados.rados.0.smithi017.stdout:3104: writing smithi01785587-1 from 1748991 to 2194059 tid 2 2024-09-21T09:51:04.281 INFO:tasks.rados.rados.0.smithi017.stdout:3104: writing smithi01785587-1 from 2930078 to 3504686 tid 3 2024-09-21T09:51:04.281 INFO:tasks.rados.rados.0.smithi017.stdout:3105: write oid 31 current snap is 392 2024-09-21T09:51:04.281 INFO:tasks.rados.rados.0.smithi017.stdout:3105: seq_num 1171 ranges {480997=434941,1407473=705079,2557501=785880} 2024-09-21T09:51:04.286 INFO:tasks.rados.rados.0.smithi017.stdout:3105: writing smithi01785587-31 from 480997 to 915938 tid 1 2024-09-21T09:51:04.292 INFO:tasks.rados.rados.0.smithi017.stdout:3105: writing smithi01785587-31 from 1407473 to 2112552 tid 2 2024-09-21T09:51:04.300 INFO:tasks.rados.rados.0.smithi017.stdout:3105: writing smithi01785587-31 from 2557501 to 3343381 tid 3 2024-09-21T09:51:04.300 INFO:tasks.rados.rados.0.smithi017.stdout:3106: read oid 22 snap -1 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3106: expect deleted 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3089: finishing rollback tid 2 to smithi01785587-26 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1579 (ObjNum 1084 snap 362 seq_num 1084) dirty exists 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3091: finishing write tid 1 to smithi01785587-25 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3089: done (15 left) 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3107: rollback oid 5 current snap is 392 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 377 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3091: finishing write tid 2 to smithi01785587-25 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3091: finishing write tid 3 to smithi01785587-25 2024-09-21T09:51:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3091: finishing write tid 4 to smithi01785587-25 2024-09-21T09:51:04.302 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 16 2024-09-21T09:51:04.302 INFO:tasks.rados.rados.0.smithi017.stdout:3091: finishing write tid 5 to smithi01785587-25 2024-09-21T09:51:04.302 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 827 (ObjNum 1165 snap 392 seq_num 1165) dirty exists 2024-09-21T09:51:04.302 INFO:tasks.rados.rados.0.smithi017.stdout:3091: left oid 25 (ObjNum 1165 snap 392 seq_num 1165) 2024-09-21T09:51:04.302 INFO:tasks.rados.rados.0.smithi017.stdout:3091: done (15 left) 2024-09-21T09:51:04.302 INFO:tasks.rados.rados.0.smithi017.stdout:3108: rollback oid 10 current snap is 392 2024-09-21T09:51:04.302 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 10 to 364 2024-09-21T09:51:04.302 INFO:tasks.rados.rados.0.smithi017.stdout:3092: expect (ObjNum 1098 snap 363 seq_num 1098) 2024-09-21T09:51:04.381 INFO:tasks.rados.rados.0.smithi017.stdout:3093: finishing write tid 1 to smithi01785587-9 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:3093: finishing write tid 2 to smithi01785587-9 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:3093: finishing write tid 3 to smithi01785587-9 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:3093: finishing write tid 4 to smithi01785587-9 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:3093: finishing write tid 5 to smithi01785587-9 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 617 (ObjNum 1166 snap 392 seq_num 1166) dirty exists 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:3093: left oid 9 (ObjNum 1166 snap 392 seq_num 1166) 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:3095: finishing rollback tid 0 to smithi01785587-49 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:3095: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1514 (ObjNum 129 snap 0 seq_num 2399459224) dirty dne 2024-09-21T09:51:04.382 INFO:tasks.rados.rados.0.smithi017.stdout:3096: expect (ObjNum 1134 snap 374 seq_num 1134) 2024-09-21T09:51:04.530 INFO:tasks.rados.rados.0.smithi017.stdout:3097: finishing write tid 1 to smithi01785587-12 2024-09-21T09:51:04.530 INFO:tasks.rados.rados.0.smithi017.stdout:3097: finishing write tid 2 to smithi01785587-12 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3098: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 0 (ObjNum 113 snap 0 seq_num 2399479232) dirty dne 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3097: finishing write tid 3 to smithi01785587-12 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3097: finishing write tid 4 to smithi01785587-12 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3092: done (15 left) 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3093: done (14 left) 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3094: done (13 left) 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3095: done (12 left) 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3096: done (11 left) 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3098: done (10 left) 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3109: snap_create 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:3097: finishing write tid 5 to smithi01785587-12 2024-09-21T09:51:04.531 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 1014 (ObjNum 1167 snap 392 seq_num 1167) dirty exists 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3097: left oid 12 (ObjNum 1167 snap 392 seq_num 1167) 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3100: finishing write tid 1 to smithi01785587-11 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3100: finishing write tid 2 to smithi01785587-11 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3100: finishing write tid 3 to smithi01785587-11 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3100: finishing write tid 4 to smithi01785587-11 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3100: finishing write tid 5 to smithi01785587-11 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3100: finishing write tid 6 to smithi01785587-11 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1430 (ObjNum 1168 snap 392 seq_num 1168) dirty exists 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3100: left oid 11 (ObjNum 1168 snap 392 seq_num 1168) 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3101: finishing write tid 1 to smithi01785587-19 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3101: finishing write tid 2 to smithi01785587-19 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3103: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:51:04.532 INFO:tasks.rados.rados.0.smithi017.stdout:3103: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3101: finishing write tid 3 to smithi01785587-19 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3101: finishing write tid 4 to smithi01785587-19 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3103: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1434 (ObjNum 1153 snap 385 seq_num 1153) dirty exists 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3101: finishing write tid 5 to smithi01785587-19 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 1309 (ObjNum 1169 snap 392 seq_num 1169) dirty exists 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3101: left oid 19 (ObjNum 1169 snap 392 seq_num 1169) 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3104: finishing write tid 1 to smithi01785587-1 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3104: finishing write tid 2 to smithi01785587-1 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3104: finishing write tid 3 to smithi01785587-1 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3104: finishing write tid 4 to smithi01785587-1 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3104: finishing write tid 5 to smithi01785587-1 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 904 (ObjNum 1170 snap 392 seq_num 1170) dirty exists 2024-09-21T09:51:04.533 INFO:tasks.rados.rados.0.smithi017.stdout:3104: left oid 1 (ObjNum 1170 snap 392 seq_num 1170) 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3105: finishing write tid 1 to smithi01785587-31 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3105: finishing write tid 2 to smithi01785587-31 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3108: finishing rollback tid 0 to smithi01785587-10 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3108: finishing rollback tid 1 to smithi01785587-10 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3108: finishing rollback tid 2 to smithi01785587-10 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 282 (ObjNum 1072 snap 358 seq_num 1072) dirty exists 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3105: finishing write tid 3 to smithi01785587-31 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3105: finishing write tid 4 to smithi01785587-31 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3105: finishing write tid 5 to smithi01785587-31 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1589 (ObjNum 1171 snap 392 seq_num 1171) dirty exists 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3105: left oid 31 (ObjNum 1171 snap 392 seq_num 1171) 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3107: finishing rollback tid 0 to smithi01785587-5 2024-09-21T09:51:04.534 INFO:tasks.rados.rados.0.smithi017.stdout:3107: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:51:04.535 INFO:tasks.rados.rados.0.smithi017.stdout:3107: finishing rollback tid 2 to smithi01785587-5 2024-09-21T09:51:04.535 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1593 (ObjNum 949 snap 321 seq_num 949) dirty exists 2024-09-21T09:51:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:51:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:51:04] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.51.0" 2024-09-21T09:51:05.221 INFO:tasks.rados.rados.0.smithi017.stdout:3097: done (10 left) 2024-09-21T09:51:05.221 INFO:tasks.rados.rados.0.smithi017.stdout:3100: done (9 left) 2024-09-21T09:51:05.221 INFO:tasks.rados.rados.0.smithi017.stdout:3101: done (8 left) 2024-09-21T09:51:05.221 INFO:tasks.rados.rados.0.smithi017.stdout:3102: done (7 left) 2024-09-21T09:51:05.221 INFO:tasks.rados.rados.0.smithi017.stdout:3103: done (6 left) 2024-09-21T09:51:05.222 INFO:tasks.rados.rados.0.smithi017.stdout:3104: done (5 left) 2024-09-21T09:51:05.222 INFO:tasks.rados.rados.0.smithi017.stdout:3105: done (4 left) 2024-09-21T09:51:05.222 INFO:tasks.rados.rados.0.smithi017.stdout:3106: done (3 left) 2024-09-21T09:51:05.222 INFO:tasks.rados.rados.0.smithi017.stdout:3107: done (2 left) 2024-09-21T09:51:05.222 INFO:tasks.rados.rados.0.smithi017.stdout:3108: done (1 left) 2024-09-21T09:51:05.222 INFO:tasks.rados.rados.0.smithi017.stdout:3109: done (0 left) 2024-09-21T09:51:05.222 INFO:tasks.rados.rados.0.smithi017.stdout:3110: snap_remove snap 359 2024-09-21T09:51:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:05 smithi134 ceph-mon[90085]: pgmap v837: 89 pgs: 89 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-09-21T09:51:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:05 smithi134 ceph-mon[90085]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T09:51:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:05 smithi017 ceph-mon[102913]: pgmap v837: 89 pgs: 89 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-09-21T09:51:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:05 smithi017 ceph-mon[102913]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T09:51:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:05 smithi017 ceph-mon[105151]: pgmap v837: 89 pgs: 89 active+clean; 226 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-09-21T09:51:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:05 smithi017 ceph-mon[105151]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3110: done (0 left) 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3111: read oid 35 snap -1 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3111: expect deleted 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3112: read oid 15 snap 377 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3112: expect deleted 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3113: rollback oid 13 current snap is 393 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 13 to 384 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3114: rollback oid 37 current snap is 393 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 356 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3115: read oid 46 snap -1 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3115: expect (ObjNum 1153 snap 385 seq_num 1153) 2024-09-21T09:51:06.235 INFO:tasks.rados.rados.0.smithi017.stdout:3116: rollback oid 8 current snap is 393 2024-09-21T09:51:06.236 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 366 2024-09-21T09:51:06.236 INFO:tasks.rados.rados.0.smithi017.stdout:3117: write oid 12 current snap is 393 2024-09-21T09:51:06.236 INFO:tasks.rados.rados.0.smithi017.stdout:3117: seq_num 1172 ranges {497804=510190,1465955=448737,2584924=597152} 2024-09-21T09:51:06.241 INFO:tasks.rados.rados.0.smithi017.stdout:3117: writing smithi01785587-12 from 497804 to 1007994 tid 1 2024-09-21T09:51:06.247 INFO:tasks.rados.rados.0.smithi017.stdout:3117: writing smithi01785587-12 from 1465955 to 1914692 tid 2 2024-09-21T09:51:06.255 INFO:tasks.rados.rados.0.smithi017.stdout:3117: writing smithi01785587-12 from 2584924 to 3182076 tid 3 2024-09-21T09:51:06.255 INFO:tasks.rados.rados.0.smithi017.stdout:3118: delete oid 3 current snap is 393 2024-09-21T09:51:06.255 INFO:tasks.rados.rados.0.smithi017.stdout:3113: finishing rollback tid 0 to smithi01785587-13 2024-09-21T09:51:06.255 INFO:tasks.rados.rados.0.smithi017.stdout:3113: finishing rollback tid 1 to smithi01785587-13 2024-09-21T09:51:06.255 INFO:tasks.rados.rados.0.smithi017.stdout:3113: finishing rollback tid 2 to smithi01785587-13 2024-09-21T09:51:06.256 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1313 (ObjNum 1095 snap 363 seq_num 1095) dirty exists 2024-09-21T09:51:06.256 INFO:tasks.rados.rados.0.smithi017.stdout:3117: finishing write tid 1 to smithi01785587-12 2024-09-21T09:51:06.256 INFO:tasks.rados.rados.0.smithi017.stdout:3114: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:51:06.256 INFO:tasks.rados.rados.0.smithi017.stdout:3114: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:51:06.256 INFO:tasks.rados.rados.0.smithi017.stdout:3114: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:51:06.256 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1575 (ObjNum 1068 snap 356 seq_num 1068) dirty exists 2024-09-21T09:51:06.256 INFO:tasks.rados.rados.0.smithi017.stdout:3115: expect (ObjNum 1153 snap 385 seq_num 1153) 2024-09-21T09:51:06.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:06 smithi134 ceph-mon[90085]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T09:51:06.497 INFO:tasks.rados.rados.0.smithi017.stdout:3117: finishing write tid 2 to smithi01785587-12 2024-09-21T09:51:06.497 INFO:tasks.rados.rados.0.smithi017.stdout:3116: finishing rollback tid 0 to smithi01785587-8 2024-09-21T09:51:06.497 INFO:tasks.rados.rados.0.smithi017.stdout:3116: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:51:06.497 INFO:tasks.rados.rados.0.smithi017.stdout:3116: finishing rollback tid 2 to smithi01785587-8 2024-09-21T09:51:06.497 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1438 (ObjNum 1070 snap 357 seq_num 1070) dirty exists 2024-09-21T09:51:06.497 INFO:tasks.rados.rados.0.smithi017.stdout:3117: finishing write tid 3 to smithi01785587-12 2024-09-21T09:51:06.497 INFO:tasks.rados.rados.0.smithi017.stdout:3111: done (7 left) 2024-09-21T09:51:06.498 INFO:tasks.rados.rados.0.smithi017.stdout:3112: done (6 left) 2024-09-21T09:51:06.498 INFO:tasks.rados.rados.0.smithi017.stdout:3113: done (5 left) 2024-09-21T09:51:06.498 INFO:tasks.rados.rados.0.smithi017.stdout:3114: done (4 left) 2024-09-21T09:51:06.498 INFO:tasks.rados.rados.0.smithi017.stdout:3115: done (3 left) 2024-09-21T09:51:06.498 INFO:tasks.rados.rados.0.smithi017.stdout:3116: done (2 left) 2024-09-21T09:51:06.498 INFO:tasks.rados.rados.0.smithi017.stdout:3118: done (1 left) 2024-09-21T09:51:06.498 INFO:tasks.rados.rados.0.smithi017.stdout:3119: snap_remove snap 364 2024-09-21T09:51:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:06 smithi017 ceph-mon[102913]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T09:51:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:06 smithi017 ceph-mon[105151]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T09:51:07.237 INFO:tasks.rados.rados.0.smithi017.stdout:3119: done (1 left) 2024-09-21T09:51:07.237 INFO:tasks.rados.rados.0.smithi017.stdout:3120: read oid 48 snap 356 2024-09-21T09:51:07.237 INFO:tasks.rados.rados.0.smithi017.stdout:3120: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:51:07.237 INFO:tasks.rados.rados.0.smithi017.stdout:3121: write oid 32 current snap is 393 2024-09-21T09:51:07.237 INFO:tasks.rados.rados.0.smithi017.stdout:3121: seq_num 1174 ranges {531418=660688,1982919=452406,3039770=691648} 2024-09-21T09:51:07.245 INFO:tasks.rados.rados.0.smithi017.stdout:3121: writing smithi01785587-32 from 531418 to 1192106 tid 1 2024-09-21T09:51:07.253 INFO:tasks.rados.rados.0.smithi017.stdout:3121: writing smithi01785587-32 from 1982919 to 2435325 tid 2 2024-09-21T09:51:07.261 INFO:tasks.rados.rados.0.smithi017.stdout:3121: writing smithi01785587-32 from 3039770 to 3731418 tid 3 2024-09-21T09:51:07.261 INFO:tasks.rados.rados.0.smithi017.stdout:3122: rollback oid 23 current snap is 393 2024-09-21T09:51:07.261 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 23 to 378 2024-09-21T09:51:07.261 INFO:tasks.rados.rados.0.smithi017.stdout:3117: finishing write tid 4 to smithi01785587-12 2024-09-21T09:51:07.261 INFO:tasks.rados.rados.0.smithi017.stdout:3123: snap_create 2024-09-21T09:51:07.261 INFO:tasks.rados.rados.0.smithi017.stdout:3117: finishing write tid 5 to smithi01785587-12 2024-09-21T09:51:07.261 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 1025 (ObjNum 1172 snap 393 seq_num 1172) dirty exists 2024-09-21T09:51:07.261 INFO:tasks.rados.rados.0.smithi017.stdout:3117: left oid 12 (ObjNum 1172 snap 393 seq_num 1172) 2024-09-21T09:51:07.261 INFO:tasks.rados.rados.0.smithi017.stdout:3120: expect (ObjNum 859 snap 291 seq_num 859) 2024-09-21T09:51:07.374 INFO:tasks.rados.rados.0.smithi017.stdout:3121: finishing write tid 1 to smithi01785587-32 2024-09-21T09:51:07.374 INFO:tasks.rados.rados.0.smithi017.stdout:3121: finishing write tid 2 to smithi01785587-32 2024-09-21T09:51:07.374 INFO:tasks.rados.rados.0.smithi017.stdout:3121: finishing write tid 3 to smithi01785587-32 2024-09-21T09:51:07.374 INFO:tasks.rados.rados.0.smithi017.stdout:3121: finishing write tid 4 to smithi01785587-32 2024-09-21T09:51:07.374 INFO:tasks.rados.rados.0.smithi017.stdout:3121: finishing write tid 5 to smithi01785587-32 2024-09-21T09:51:07.374 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1459 (ObjNum 1174 snap 393 seq_num 1174) dirty exists 2024-09-21T09:51:07.374 INFO:tasks.rados.rados.0.smithi017.stdout:3121: left oid 32 (ObjNum 1174 snap 393 seq_num 1174) 2024-09-21T09:51:07.374 INFO:tasks.rados.rados.0.smithi017.stdout:3122: finishing rollback tid 1 to smithi01785587-23 2024-09-21T09:51:07.374 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1338 (ObjNum 33 snap 0 seq_num 2399950224) dirty dne 2024-09-21T09:51:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:07 smithi134 ceph-mon[90085]: pgmap v840: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 9 op/s 2024-09-21T09:51:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:07 smithi134 ceph-mon[90085]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T09:51:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:07 smithi017 ceph-mon[105151]: pgmap v840: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 9 op/s 2024-09-21T09:51:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:07 smithi017 ceph-mon[105151]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T09:51:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:07 smithi017 ceph-mon[102913]: pgmap v840: 89 pgs: 89 active+clean; 240 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 9 op/s 2024-09-21T09:51:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:07 smithi017 ceph-mon[102913]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T09:51:08.178 INFO:tasks.rados.rados.0.smithi017.stdout:3117: done (4 left) 2024-09-21T09:51:08.178 INFO:tasks.rados.rados.0.smithi017.stdout:3120: done (3 left) 2024-09-21T09:51:08.178 INFO:tasks.rados.rados.0.smithi017.stdout:3121: done (2 left) 2024-09-21T09:51:08.178 INFO:tasks.rados.rados.0.smithi017.stdout:3122: done (1 left) 2024-09-21T09:51:08.178 INFO:tasks.rados.rados.0.smithi017.stdout:3123: done (0 left) 2024-09-21T09:51:08.179 INFO:tasks.rados.rados.0.smithi017.stdout:3124: read oid 8 snap -1 2024-09-21T09:51:08.179 INFO:tasks.rados.rados.0.smithi017.stdout:3124: expect (ObjNum 1070 snap 357 seq_num 1070) 2024-09-21T09:51:08.179 INFO:tasks.rados.rados.0.smithi017.stdout:3125: write oid 31 current snap is 394 2024-09-21T09:51:08.179 INFO:tasks.rados.rados.0.smithi017.stdout:3125: seq_num 1175 ranges {548225=735937,1925048=596064,2950840=547321} 2024-09-21T09:51:08.185 INFO:tasks.rados.rados.0.smithi017.stdout:3125: writing smithi01785587-31 from 548225 to 1284162 tid 1 2024-09-21T09:51:08.192 INFO:tasks.rados.rados.0.smithi017.stdout:3125: writing smithi01785587-31 from 1925048 to 2521112 tid 2 2024-09-21T09:51:08.198 INFO:tasks.rados.rados.0.smithi017.stdout:3125: writing smithi01785587-31 from 2950840 to 3498161 tid 3 2024-09-21T09:51:08.198 INFO:tasks.rados.rados.0.smithi017.stdout:3126: snap_create 2024-09-21T09:51:08.198 INFO:tasks.rados.rados.0.smithi017.stdout:3124: expect (ObjNum 1070 snap 357 seq_num 1070) 2024-09-21T09:51:08.384 INFO:tasks.rados.rados.0.smithi017.stdout:3125: finishing write tid 1 to smithi01785587-31 2024-09-21T09:51:08.384 INFO:tasks.rados.rados.0.smithi017.stdout:3125: finishing write tid 2 to smithi01785587-31 2024-09-21T09:51:08.384 INFO:tasks.rados.rados.0.smithi017.stdout:3125: finishing write tid 3 to smithi01785587-31 2024-09-21T09:51:08.384 INFO:tasks.rados.rados.0.smithi017.stdout:3125: finishing write tid 4 to smithi01785587-31 2024-09-21T09:51:08.384 INFO:tasks.rados.rados.0.smithi017.stdout:3125: finishing write tid 5 to smithi01785587-31 2024-09-21T09:51:08.384 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1620 (ObjNum 1175 snap 394 seq_num 1175) dirty exists 2024-09-21T09:51:08.384 INFO:tasks.rados.rados.0.smithi017.stdout:3125: left oid 31 (ObjNum 1175 snap 394 seq_num 1175) 2024-09-21T09:51:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:08 smithi017 ceph-mon[102913]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T09:51:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:08 smithi017 ceph-mon[102913]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T09:51:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:08 smithi017 ceph-mon[105151]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T09:51:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:08 smithi017 ceph-mon[105151]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T09:51:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:08 smithi134 ceph-mon[90085]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T09:51:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:08 smithi134 ceph-mon[90085]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T09:51:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:3124: done (2 left) 2024-09-21T09:51:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:3125: done (1 left) 2024-09-21T09:51:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:3126: done (0 left) 2024-09-21T09:51:09.179 INFO:tasks.rados.rados.0.smithi017.stdout:3127: snap_remove snap 389 2024-09-21T09:51:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:09 smithi017 ceph-mon[102913]: pgmap v843: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 3.7 MiB/s wr, 22 op/s 2024-09-21T09:51:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:09 smithi017 ceph-mon[102913]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T09:51:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:09 smithi017 ceph-mon[105151]: pgmap v843: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 3.7 MiB/s wr, 22 op/s 2024-09-21T09:51:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:09 smithi017 ceph-mon[105151]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T09:51:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:09 smithi134 ceph-mon[90085]: pgmap v843: 89 pgs: 89 active+clean; 249 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 3.7 MiB/s wr, 22 op/s 2024-09-21T09:51:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:09 smithi134 ceph-mon[90085]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T09:51:10.183 INFO:tasks.rados.rados.0.smithi017.stdout:3127: done (0 left) 2024-09-21T09:51:10.183 INFO:tasks.rados.rados.0.smithi017.stdout:3128: read oid 35 snap -1 2024-09-21T09:51:10.183 INFO:tasks.rados.rados.0.smithi017.stdout:3128: expect deleted 2024-09-21T09:51:10.183 INFO:tasks.rados.rados.0.smithi017.stdout:3129: snap_remove snap 384 2024-09-21T09:51:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:51:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:51:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:51:11.187 INFO:tasks.rados.rados.0.smithi017.stdout:3129: done (1 left) 2024-09-21T09:51:11.187 INFO:tasks.rados.rados.0.smithi017.stdout:3130: write oid 42 current snap is 395 2024-09-21T09:51:11.187 INFO:tasks.rados.rados.0.smithi017.stdout:3130: seq_num 1176 ranges {565032=411186,1583530=456075,2578263=758593} 2024-09-21T09:51:11.193 INFO:tasks.rados.rados.0.smithi017.stdout:3130: writing smithi01785587-42 from 565032 to 976218 tid 1 2024-09-21T09:51:11.200 INFO:tasks.rados.rados.0.smithi017.stdout:3130: writing smithi01785587-42 from 1583530 to 2039605 tid 2 2024-09-21T09:51:11.208 INFO:tasks.rados.rados.0.smithi017.stdout:3130: writing smithi01785587-42 from 2578263 to 3336856 tid 3 2024-09-21T09:51:11.208 INFO:tasks.rados.rados.0.smithi017.stdout:3131: read oid 37 snap -1 2024-09-21T09:51:11.209 INFO:tasks.rados.rados.0.smithi017.stdout:3131: expect (ObjNum 1068 snap 356 seq_num 1068) 2024-09-21T09:51:11.209 INFO:tasks.rados.rados.0.smithi017.stdout:3132: read oid 27 snap -1 2024-09-21T09:51:11.209 INFO:tasks.rados.rados.0.smithi017.stdout:3132: expect (ObjNum 1009 snap 341 seq_num 1009) 2024-09-21T09:51:11.209 INFO:tasks.rados.rados.0.smithi017.stdout:3133: write oid 24 current snap is 395 2024-09-21T09:51:11.209 INFO:tasks.rados.rados.0.smithi017.stdout:3133: seq_num 1177 ranges {581839=486435,1642012=599733,3005686=569865} 2024-09-21T09:51:11.215 INFO:tasks.rados.rados.0.smithi017.stdout:3133: writing smithi01785587-24 from 581839 to 1068274 tid 1 2024-09-21T09:51:11.222 INFO:tasks.rados.rados.0.smithi017.stdout:3133: writing smithi01785587-24 from 1642012 to 2241745 tid 2 2024-09-21T09:51:11.230 INFO:tasks.rados.rados.0.smithi017.stdout:3133: writing smithi01785587-24 from 3005686 to 3575551 tid 3 2024-09-21T09:51:11.230 INFO:tasks.rados.rados.0.smithi017.stdout:3134: read oid 13 snap 356 2024-09-21T09:51:11.230 INFO:tasks.rados.rados.0.smithi017.stdout:3134: expect (ObjNum 1055 snap 352 seq_num 1055) 2024-09-21T09:51:11.230 INFO:tasks.rados.rados.0.smithi017.stdout:3135: rollback oid 38 current snap is 395 2024-09-21T09:51:11.230 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 391 2024-09-21T09:51:11.230 INFO:tasks.rados.rados.0.smithi017.stdout:3130: finishing write tid 1 to smithi01785587-42 2024-09-21T09:51:11.230 INFO:tasks.rados.rados.0.smithi017.stdout:3130: finishing write tid 2 to smithi01785587-42 2024-09-21T09:51:11.230 INFO:tasks.rados.rados.0.smithi017.stdout:3130: finishing write tid 3 to smithi01785587-42 2024-09-21T09:51:11.230 INFO:tasks.rados.rados.0.smithi017.stdout:3128: done (6 left) 2024-09-21T09:51:11.231 INFO:tasks.rados.rados.0.smithi017.stdout:3136: read oid 28 snap 387 2024-09-21T09:51:11.231 INFO:tasks.rados.rados.0.smithi017.stdout:3136: expect (ObjNum 1117 snap 368 seq_num 1117) 2024-09-21T09:51:11.231 INFO:tasks.rados.rados.0.smithi017.stdout:3137: rollback oid 30 current snap is 395 2024-09-21T09:51:11.231 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 374 2024-09-21T09:51:11.231 INFO:tasks.rados.rados.0.smithi017.stdout:3130: finishing write tid 4 to smithi01785587-42 2024-09-21T09:51:11.231 INFO:tasks.rados.rados.0.smithi017.stdout:3130: finishing write tid 5 to smithi01785587-42 2024-09-21T09:51:11.231 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 1032 (ObjNum 1176 snap 395 seq_num 1176) dirty exists 2024-09-21T09:51:11.232 INFO:tasks.rados.rados.0.smithi017.stdout:3130: left oid 42 (ObjNum 1176 snap 395 seq_num 1176) 2024-09-21T09:51:11.232 INFO:tasks.rados.rados.0.smithi017.stdout:3131: expect (ObjNum 1068 snap 356 seq_num 1068) 2024-09-21T09:51:11.359 INFO:tasks.rados.rados.0.smithi017.stdout:3132: expect (ObjNum 1009 snap 341 seq_num 1009) 2024-09-21T09:51:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:11 smithi017 ceph-mon[102913]: pgmap v846: 89 pgs: 89 active+clean; 251 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-21T09:51:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:11 smithi017 ceph-mon[102913]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T09:51:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:11 smithi017 ceph-mon[105151]: pgmap v846: 89 pgs: 89 active+clean; 251 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-21T09:51:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:11 smithi017 ceph-mon[105151]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T09:51:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:11 smithi134 ceph-mon[90085]: pgmap v846: 89 pgs: 89 active+clean; 251 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-21T09:51:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:11 smithi134 ceph-mon[90085]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T09:51:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:11.583 INFO:tasks.rados.rados.0.smithi017.stdout:3133: finishing write tid 1 to smithi01785587-24 2024-09-21T09:51:11.583 INFO:tasks.rados.rados.0.smithi017.stdout:3133: finishing write tid 2 to smithi01785587-24 2024-09-21T09:51:11.583 INFO:tasks.rados.rados.0.smithi017.stdout:3133: finishing write tid 3 to smithi01785587-24 2024-09-21T09:51:11.584 INFO:tasks.rados.rados.0.smithi017.stdout:3133: finishing write tid 4 to smithi01785587-24 2024-09-21T09:51:11.584 INFO:tasks.rados.rados.0.smithi017.stdout:3133: finishing write tid 5 to smithi01785587-24 2024-09-21T09:51:11.584 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 956 (ObjNum 1177 snap 395 seq_num 1177) dirty exists 2024-09-21T09:51:11.584 INFO:tasks.rados.rados.0.smithi017.stdout:3133: left oid 24 (ObjNum 1177 snap 395 seq_num 1177) 2024-09-21T09:51:11.584 INFO:tasks.rados.rados.0.smithi017.stdout:3135: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:51:11.584 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:11.584 INFO:tasks.rados.rados.0.smithi017.stdout:3134: expect (ObjNum 1055 snap 352 seq_num 1055) 2024-09-21T09:51:11.789 INFO:tasks.rados.rados.0.smithi017.stdout:3137: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:51:11.790 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:51:11.790 INFO:tasks.rados.rados.0.smithi017.stdout:3136: expect (ObjNum 1117 snap 368 seq_num 1117) 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3130: done (7 left) 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3131: done (6 left) 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3132: done (5 left) 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3133: done (4 left) 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3134: done (3 left) 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3135: done (2 left) 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3136: done (1 left) 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3137: done (0 left) 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3138: read oid 24 snap -1 2024-09-21T09:51:11.918 INFO:tasks.rados.rados.0.smithi017.stdout:3138: expect (ObjNum 1177 snap 395 seq_num 1177) 2024-09-21T09:51:11.919 INFO:tasks.rados.rados.0.smithi017.stdout:3139: delete oid 4 current snap is 395 2024-09-21T09:51:11.920 INFO:tasks.rados.rados.0.smithi017.stdout:3139: done (1 left) 2024-09-21T09:51:11.920 INFO:tasks.rados.rados.0.smithi017.stdout:3140: snap_remove snap 388 2024-09-21T09:51:12.190 INFO:tasks.rados.rados.0.smithi017.stdout:3140: done (1 left) 2024-09-21T09:51:12.191 INFO:tasks.rados.rados.0.smithi017.stdout:3141: snap_remove snap 379 2024-09-21T09:51:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:12 smithi017 ceph-mon[102913]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T09:51:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:12 smithi017 ceph-mon[105151]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T09:51:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:12 smithi134 ceph-mon[90085]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T09:51:13.179 INFO:tasks.rados.rados.0.smithi017.stdout:3141: done (1 left) 2024-09-21T09:51:13.179 INFO:tasks.rados.rados.0.smithi017.stdout:3142: write oid 18 current snap is 395 2024-09-21T09:51:13.179 INFO:tasks.rados.rados.0.smithi017.stdout:3138: expect (ObjNum 1177 snap 395 seq_num 1177) 2024-09-21T09:51:13.312 INFO:tasks.rados.rados.0.smithi017.stdout:3142: seq_num 1179 ranges {615453=753286,2158976=603402,3460532=354921} 2024-09-21T09:51:13.319 INFO:tasks.rados.rados.0.smithi017.stdout:3142: writing smithi01785587-18 from 615453 to 1368739 tid 1 2024-09-21T09:51:13.327 INFO:tasks.rados.rados.0.smithi017.stdout:3142: writing smithi01785587-18 from 2158976 to 2762378 tid 2 2024-09-21T09:51:13.333 INFO:tasks.rados.rados.0.smithi017.stdout:3142: writing smithi01785587-18 from 3460532 to 3815453 tid 3 2024-09-21T09:51:13.333 INFO:tasks.rados.rados.0.smithi017.stdout:3138: done (1 left) 2024-09-21T09:51:13.333 INFO:tasks.rados.rados.0.smithi017.stdout:3143: delete oid 2 current snap is 395 2024-09-21T09:51:13.333 INFO:tasks.rados.rados.0.smithi017.stdout:3142: finishing write tid 1 to smithi01785587-18 2024-09-21T09:51:13.333 INFO:tasks.rados.rados.0.smithi017.stdout:3142: finishing write tid 2 to smithi01785587-18 2024-09-21T09:51:13.335 INFO:tasks.rados.rados.0.smithi017.stdout:3143: done (1 left) 2024-09-21T09:51:13.335 INFO:tasks.rados.rados.0.smithi017.stdout:3144: delete oid 34 current snap is 395 2024-09-21T09:51:13.335 INFO:tasks.rados.rados.0.smithi017.stdout:3142: finishing write tid 3 to smithi01785587-18 2024-09-21T09:51:13.335 INFO:tasks.rados.rados.0.smithi017.stdout:3142: finishing write tid 4 to smithi01785587-18 2024-09-21T09:51:13.335 INFO:tasks.rados.rados.0.smithi017.stdout:3142: finishing write tid 5 to smithi01785587-18 2024-09-21T09:51:13.335 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 285 (ObjNum 1179 snap 395 seq_num 1179) dirty exists 2024-09-21T09:51:13.335 INFO:tasks.rados.rados.0.smithi017.stdout:3142: left oid 18 (ObjNum 1179 snap 395 seq_num 1179) 2024-09-21T09:51:13.337 INFO:tasks.rados.rados.0.smithi017.stdout:3142: done (1 left) 2024-09-21T09:51:13.337 INFO:tasks.rados.rados.0.smithi017.stdout:3144: done (0 left) 2024-09-21T09:51:13.337 INFO:tasks.rados.rados.0.smithi017.stdout:3145: snap_create 2024-09-21T09:51:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:13 smithi017 ceph-mon[105151]: pgmap v849: 89 pgs: 89 active+clean; 258 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-21T09:51:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:13 smithi017 ceph-mon[105151]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T09:51:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:13 smithi017 ceph-mon[105151]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T09:51:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:13 smithi017 ceph-mon[102913]: pgmap v849: 89 pgs: 89 active+clean; 258 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-21T09:51:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:13 smithi017 ceph-mon[102913]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T09:51:13.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:13 smithi017 ceph-mon[102913]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T09:51:13.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:13 smithi134 ceph-mon[90085]: pgmap v849: 89 pgs: 89 active+clean; 258 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-21T09:51:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:13 smithi134 ceph-mon[90085]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T09:51:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:13 smithi134 ceph-mon[90085]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T09:51:14.198 INFO:tasks.rados.rados.0.smithi017.stdout:3145: done (0 left) 2024-09-21T09:51:14.199 INFO:tasks.rados.rados.0.smithi017.stdout:3146: rollback oid 4 current snap is 396 2024-09-21T09:51:14.199 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 4 to 394 2024-09-21T09:51:14.199 INFO:tasks.rados.rados.0.smithi017.stdout:3147: snap_create 2024-09-21T09:51:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3146: finishing rollback tid 1 to smithi01785587-4 2024-09-21T09:51:14.204 INFO:tasks.rados.rados.0.smithi017.stdout:3146: finishing rollback tid 2 to smithi01785587-4 2024-09-21T09:51:14.204 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1619 (ObjNum 1119 snap 369 seq_num 1119) dirty exists 2024-09-21T09:51:15.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:51:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:51:14] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.51.0" 2024-09-21T09:51:15.201 INFO:tasks.rados.rados.0.smithi017.stdout:3146: done (1 left) 2024-09-21T09:51:15.201 INFO:tasks.rados.rados.0.smithi017.stdout:3147: done (0 left) 2024-09-21T09:51:15.201 INFO:tasks.rados.rados.0.smithi017.stdout:3148: rollback oid 18 current snap is 397 2024-09-21T09:51:15.201 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 18 to 383 2024-09-21T09:51:15.201 INFO:tasks.rados.rados.0.smithi017.stdout:3149: read oid 29 snap 395 2024-09-21T09:51:15.201 INFO:tasks.rados.rados.0.smithi017.stdout:3149: expect deleted 2024-09-21T09:51:15.202 INFO:tasks.rados.rados.0.smithi017.stdout:3150: snap_create 2024-09-21T09:51:15.208 INFO:tasks.rados.rados.0.smithi017.stdout:3148: finishing rollback tid 0 to smithi01785587-18 2024-09-21T09:51:15.209 INFO:tasks.rados.rados.0.smithi017.stdout:3148: finishing rollback tid 1 to smithi01785587-18 2024-09-21T09:51:15.209 INFO:tasks.rados.rados.0.smithi017.stdout:3148: finishing rollback tid 2 to smithi01785587-18 2024-09-21T09:51:15.209 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 289 (ObjNum 1136 snap 377 seq_num 1136) dirty exists 2024-09-21T09:51:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:15 smithi134 ceph-mon[90085]: pgmap v852: 89 pgs: 89 active+clean; 258 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-21T09:51:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:15 smithi134 ceph-mon[90085]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T09:51:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:15 smithi017 ceph-mon[102913]: pgmap v852: 89 pgs: 89 active+clean; 258 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-21T09:51:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:15 smithi017 ceph-mon[102913]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T09:51:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:15 smithi017 ceph-mon[105151]: pgmap v852: 89 pgs: 89 active+clean; 258 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-09-21T09:51:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:15 smithi017 ceph-mon[105151]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T09:51:16.211 INFO:tasks.rados.rados.0.smithi017.stdout:3148: done (2 left) 2024-09-21T09:51:16.211 INFO:tasks.rados.rados.0.smithi017.stdout:3149: done (1 left) 2024-09-21T09:51:16.211 INFO:tasks.rados.rados.0.smithi017.stdout:3150: done (0 left) 2024-09-21T09:51:16.212 INFO:tasks.rados.rados.0.smithi017.stdout:3151: read oid 26 snap -1 2024-09-21T09:51:16.212 INFO:tasks.rados.rados.0.smithi017.stdout:3151: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:51:16.212 INFO:tasks.rados.rados.0.smithi017.stdout:3152: snap_remove snap 392 2024-09-21T09:51:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:16 smithi134 ceph-mon[90085]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T09:51:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:16 smithi017 ceph-mon[102913]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T09:51:16.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:16 smithi017 ceph-mon[105151]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T09:51:17.211 INFO:tasks.rados.rados.0.smithi017.stdout:3152: done (1 left) 2024-09-21T09:51:17.211 INFO:tasks.rados.rados.0.smithi017.stdout:3153: write oid 47 current snap is 398 2024-09-21T09:51:17.211 INFO:tasks.rados.rados.0.smithi017.stdout:3153: seq_num 1182 ranges {665874=579033,1934422=467082,3142801=542578} 2024-09-21T09:51:17.219 INFO:tasks.rados.rados.0.smithi017.stdout:3153: writing smithi01785587-47 from 665874 to 1244907 tid 1 2024-09-21T09:51:17.227 INFO:tasks.rados.rados.0.smithi017.stdout:3153: writing smithi01785587-47 from 1934422 to 2401504 tid 2 2024-09-21T09:51:17.234 INFO:tasks.rados.rados.0.smithi017.stdout:3153: writing smithi01785587-47 from 3142801 to 3685379 tid 3 2024-09-21T09:51:17.234 INFO:tasks.rados.rados.0.smithi017.stdout:3154: snap_remove snap 391 2024-09-21T09:51:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:17 smithi134 ceph-mon[90085]: pgmap v855: 89 pgs: 89 active+clean; 264 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 418 KiB/s wr, 11 op/s 2024-09-21T09:51:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:17 smithi134 ceph-mon[90085]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T09:51:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:17 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:51:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:17 smithi017 ceph-mon[105151]: pgmap v855: 89 pgs: 89 active+clean; 264 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 418 KiB/s wr, 11 op/s 2024-09-21T09:51:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:17 smithi017 ceph-mon[105151]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T09:51:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:17 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:51:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:17 smithi017 ceph-mon[102913]: pgmap v855: 89 pgs: 89 active+clean; 264 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 418 KiB/s wr, 11 op/s 2024-09-21T09:51:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:17 smithi017 ceph-mon[102913]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T09:51:17.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:17 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:51:18.180 INFO:tasks.rados.rados.0.smithi017.stdout:3154: done (2 left) 2024-09-21T09:51:18.180 INFO:tasks.rados.rados.0.smithi017.stdout:3155: snap_remove snap 395 2024-09-21T09:51:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:18 smithi134 ceph-mon[90085]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T09:51:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:18 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:18 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:18 smithi134 ceph-mon[90085]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T09:51:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:18 smithi017 ceph-mon[102913]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T09:51:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:18 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:18 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:18 smithi017 ceph-mon[102913]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T09:51:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:18 smithi017 ceph-mon[105151]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T09:51:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:18 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:18 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:18 smithi017 ceph-mon[105151]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T09:51:19.182 INFO:tasks.rados.rados.0.smithi017.stdout:3155: done (2 left) 2024-09-21T09:51:19.182 INFO:tasks.rados.rados.0.smithi017.stdout:3156: delete oid 23 current snap is 398 2024-09-21T09:51:19.182 INFO:tasks.rados.rados.0.smithi017.stdout:3151: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:51:19.343 INFO:tasks.rados.rados.0.smithi017.stdout:3153: finishing write tid 1 to smithi01785587-47 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3153: finishing write tid 2 to smithi01785587-47 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3153: finishing write tid 3 to smithi01785587-47 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3153: finishing write tid 4 to smithi01785587-47 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3153: finishing write tid 5 to smithi01785587-47 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 1045 (ObjNum 1182 snap 398 seq_num 1182) dirty exists 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3153: left oid 47 (ObjNum 1182 snap 398 seq_num 1182) 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3151: done (2 left) 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3153: done (1 left) 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3156: done (0 left) 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3157: rollback oid 49 current snap is 398 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 372 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:3158: rollback oid 18 current snap is 398 2024-09-21T09:51:19.344 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 18 to 377 2024-09-21T09:51:19.345 INFO:tasks.rados.rados.0.smithi017.stdout:3159: rollback oid 41 current snap is 398 2024-09-21T09:51:19.345 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 378 2024-09-21T09:51:19.345 INFO:tasks.rados.rados.0.smithi017.stdout:3160: read oid 30 snap -1 2024-09-21T09:51:19.345 INFO:tasks.rados.rados.0.smithi017.stdout:3160: expect deleted 2024-09-21T09:51:19.345 INFO:tasks.rados.rados.0.smithi017.stdout:3161: read oid 25 snap -1 2024-09-21T09:51:19.345 INFO:tasks.rados.rados.0.smithi017.stdout:3161: expect (ObjNum 1165 snap 392 seq_num 1165) 2024-09-21T09:51:19.345 INFO:tasks.rados.rados.0.smithi017.stdout:3162: write oid 37 current snap is 398 2024-09-21T09:51:19.345 INFO:tasks.rados.rados.0.smithi017.stdout:3162: seq_num 1184 ranges {699488=729531,1935033=470751,3081294=681475} 2024-09-21T09:51:19.351 INFO:tasks.rados.rados.0.smithi017.stdout:3162: writing smithi01785587-37 from 699488 to 1429019 tid 1 2024-09-21T09:51:19.356 INFO:tasks.rados.rados.0.smithi017.stdout:3162: writing smithi01785587-37 from 1935033 to 2405784 tid 2 2024-09-21T09:51:19.364 INFO:tasks.rados.rados.0.smithi017.stdout:3162: writing smithi01785587-37 from 3081294 to 3762769 tid 3 2024-09-21T09:51:19.364 INFO:tasks.rados.rados.0.smithi017.stdout:3163: write oid 23 current snap is 398 2024-09-21T09:51:19.364 INFO:tasks.rados.rados.0.smithi017.stdout:3163: seq_num 1185 ranges {716295=404780,1593515=614409,2708717=492747,3640519=275776} 2024-09-21T09:51:19.370 INFO:tasks.rados.rados.0.smithi017.stdout:3163: writing smithi01785587-23 from 716295 to 1121075 tid 1 2024-09-21T09:51:19.376 INFO:tasks.rados.rados.0.smithi017.stdout:3163: writing smithi01785587-23 from 1593515 to 2207924 tid 2 2024-09-21T09:51:19.382 INFO:tasks.rados.rados.0.smithi017.stdout:3163: writing smithi01785587-23 from 2708717 to 3201464 tid 3 2024-09-21T09:51:19.386 INFO:tasks.rados.rados.0.smithi017.stdout:3163: writing smithi01785587-23 from 3640519 to 3916295 tid 4 2024-09-21T09:51:19.386 INFO:tasks.rados.rados.0.smithi017.stdout:3164: write oid 7 current snap is 398 2024-09-21T09:51:19.386 INFO:tasks.rados.rados.0.smithi017.stdout:3164: seq_num 1186 ranges {733102=596382,1768350=758067,3136140=704019} 2024-09-21T09:51:19.393 INFO:tasks.rados.rados.0.smithi017.stdout:3164: writing smithi01785587-7 from 733102 to 1329484 tid 1 2024-09-21T09:51:19.400 INFO:tasks.rados.rados.0.smithi017.stdout:3164: writing smithi01785587-7 from 1768350 to 2526417 tid 2 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:3164: writing smithi01785587-7 from 3136140 to 3840159 tid 3 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:3165: read oid 38 snap 387 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:3165: expect deleted 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:3166: delete oid 2 current snap is 398 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:3157: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:3157: finishing rollback tid 2 to smithi01785587-49 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1543 (ObjNum 1106 snap 366 seq_num 1106) dirty exists 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:3158: finishing rollback tid 0 to smithi01785587-18 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:3159: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:51:19.408 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 500 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:19.409 INFO:tasks.rados.rados.0.smithi017.stdout:3158: finishing rollback tid 1 to smithi01785587-18 2024-09-21T09:51:19.409 INFO:tasks.rados.rados.0.smithi017.stdout:3158: finishing rollback tid 2 to smithi01785587-18 2024-09-21T09:51:19.409 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 297 (ObjNum 1136 snap 377 seq_num 1136) dirty exists 2024-09-21T09:51:19.409 INFO:tasks.rados.rados.0.smithi017.stdout:3161: expect (ObjNum 1165 snap 392 seq_num 1165) 2024-09-21T09:51:19.573 INFO:tasks.rados.rados.0.smithi017.stdout:3162: finishing write tid 1 to smithi01785587-37 2024-09-21T09:51:19.573 INFO:tasks.rados.rados.0.smithi017.stdout:3162: finishing write tid 2 to smithi01785587-37 2024-09-21T09:51:19.573 INFO:tasks.rados.rados.0.smithi017.stdout:3162: finishing write tid 3 to smithi01785587-37 2024-09-21T09:51:19.573 INFO:tasks.rados.rados.0.smithi017.stdout:3162: finishing write tid 4 to smithi01785587-37 2024-09-21T09:51:19.573 INFO:tasks.rados.rados.0.smithi017.stdout:3162: finishing write tid 5 to smithi01785587-37 2024-09-21T09:51:19.573 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1616 (ObjNum 1184 snap 398 seq_num 1184) dirty exists 2024-09-21T09:51:19.573 INFO:tasks.rados.rados.0.smithi017.stdout:3162: left oid 37 (ObjNum 1184 snap 398 seq_num 1184) 2024-09-21T09:51:19.573 INFO:tasks.rados.rados.0.smithi017.stdout:3163: finishing write tid 1 to smithi01785587-23 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3157: done (9 left) 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3158: done (8 left) 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3159: done (7 left) 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3160: done (6 left) 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3161: done (5 left) 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3162: done (4 left) 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3166: done (3 left) 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3167: write oid 47 current snap is 398 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3163: finishing write tid 2 to smithi01785587-23 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3163: finishing write tid 3 to smithi01785587-23 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3163: finishing write tid 4 to smithi01785587-23 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3163: finishing write tid 5 to smithi01785587-23 2024-09-21T09:51:19.574 INFO:tasks.rados.rados.0.smithi017.stdout:3163: finishing write tid 6 to smithi01785587-23 2024-09-21T09:51:19.575 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1508 (ObjNum 1185 snap 398 seq_num 1185) dirty exists 2024-09-21T09:51:19.575 INFO:tasks.rados.rados.0.smithi017.stdout:3163: left oid 23 (ObjNum 1185 snap 398 seq_num 1185) 2024-09-21T09:51:19.575 INFO:tasks.rados.rados.0.smithi017.stdout:3164: finishing write tid 1 to smithi01785587-7 2024-09-21T09:51:19.575 INFO:tasks.rados.rados.0.smithi017.stdout:3164: finishing write tid 2 to smithi01785587-7 2024-09-21T09:51:19.575 INFO:tasks.rados.rados.0.smithi017.stdout:3167: seq_num 1188 ranges {766716=746880,2168961=761736,3474633=442916} 2024-09-21T09:51:19.581 INFO:tasks.rados.rados.0.smithi017.stdout:3167: writing smithi01785587-47 from 766716 to 1513596 tid 1 2024-09-21T09:51:19.589 INFO:tasks.rados.rados.0.smithi017.stdout:3167: writing smithi01785587-47 from 2168961 to 2930697 tid 2 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:3167: writing smithi01785587-47 from 3474633 to 3917549 tid 3 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:3163: done (3 left) 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:3168: read oid 35 snap 396 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:3168: expect deleted 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:3169: delete oid 18 current snap is 398 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:3164: finishing write tid 3 to smithi01785587-7 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:3164: finishing write tid 4 to smithi01785587-7 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:3164: finishing write tid 5 to smithi01785587-7 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1548 (ObjNum 1186 snap 398 seq_num 1186) dirty exists 2024-09-21T09:51:19.595 INFO:tasks.rados.rados.0.smithi017.stdout:3164: left oid 7 (ObjNum 1186 snap 398 seq_num 1186) 2024-09-21T09:51:19.596 INFO:tasks.rados.rados.0.smithi017.stdout:3167: finishing write tid 1 to smithi01785587-47 2024-09-21T09:51:19.596 INFO:tasks.rados.rados.0.smithi017.stdout:3167: finishing write tid 2 to smithi01785587-47 2024-09-21T09:51:19.596 INFO:tasks.rados.rados.0.smithi017.stdout:3164: done (4 left) 2024-09-21T09:51:19.596 INFO:tasks.rados.rados.0.smithi017.stdout:3165: done (3 left) 2024-09-21T09:51:19.596 INFO:tasks.rados.rados.0.smithi017.stdout:3168: done (2 left) 2024-09-21T09:51:19.596 INFO:tasks.rados.rados.0.smithi017.stdout:3169: done (1 left) 2024-09-21T09:51:19.597 INFO:tasks.rados.rados.0.smithi017.stdout:3170: read oid 3 snap -1 2024-09-21T09:51:19.597 INFO:tasks.rados.rados.0.smithi017.stdout:3170: expect deleted 2024-09-21T09:51:19.597 INFO:tasks.rados.rados.0.smithi017.stdout:3171: write oid 19 current snap is 398 2024-09-21T09:51:19.597 INFO:tasks.rados.rados.0.smithi017.stdout:3171: seq_num 1190 ranges {400330=497378,1485925=514405} 2024-09-21T09:51:19.601 INFO:tasks.rados.rados.0.smithi017.stdout:3171: writing smithi01785587-19 from 400330 to 897708 tid 1 2024-09-21T09:51:19.607 INFO:tasks.rados.rados.0.smithi017.stdout:3171: writing smithi01785587-19 from 1485925 to 2000330 tid 2 2024-09-21T09:51:19.607 INFO:tasks.rados.rados.0.smithi017.stdout:3172: write oid 2 current snap is 398 2024-09-21T09:51:19.607 INFO:tasks.rados.rados.0.smithi017.stdout:3172: seq_num 1191 ranges {417137=572627,1428054=589083} 2024-09-21T09:51:19.612 INFO:tasks.rados.rados.0.smithi017.stdout:3172: writing smithi01785587-2 from 417137 to 989764 tid 1 2024-09-21T09:51:19.618 INFO:tasks.rados.rados.0.smithi017.stdout:3172: writing smithi01785587-2 from 1428054 to 2017137 tid 2 2024-09-21T09:51:19.618 INFO:tasks.rados.rados.0.smithi017.stdout:3173: read oid 35 snap -1 2024-09-21T09:51:19.618 INFO:tasks.rados.rados.0.smithi017.stdout:3173: expect deleted 2024-09-21T09:51:19.618 INFO:tasks.rados.rados.0.smithi017.stdout:3174: write oid 9 current snap is 398 2024-09-21T09:51:19.618 INFO:tasks.rados.rados.0.smithi017.stdout:3174: seq_num 1192 ranges {433944=647876,1486536=547408} 2024-09-21T09:51:19.624 INFO:tasks.rados.rados.0.smithi017.stdout:3174: writing smithi01785587-9 from 433944 to 1081820 tid 1 2024-09-21T09:51:19.630 INFO:tasks.rados.rados.0.smithi017.stdout:3174: writing smithi01785587-9 from 1486536 to 2033944 tid 2 2024-09-21T09:51:19.630 INFO:tasks.rados.rados.0.smithi017.stdout:3167: finishing write tid 3 to smithi01785587-47 2024-09-21T09:51:19.630 INFO:tasks.rados.rados.0.smithi017.stdout:3167: finishing write tid 4 to smithi01785587-47 2024-09-21T09:51:19.630 INFO:tasks.rados.rados.0.smithi017.stdout:3167: finishing write tid 5 to smithi01785587-47 2024-09-21T09:51:19.630 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 1055 (ObjNum 1188 snap 398 seq_num 1188) dirty exists 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3167: left oid 47 (ObjNum 1188 snap 398 seq_num 1188) 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3171: finishing write tid 1 to smithi01785587-19 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3171: finishing write tid 2 to smithi01785587-19 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3171: finishing write tid 3 to smithi01785587-19 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3171: finishing write tid 4 to smithi01785587-19 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 1345 (ObjNum 1190 snap 398 seq_num 1190) dirty exists 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3171: left oid 19 (ObjNum 1190 snap 398 seq_num 1190) 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3172: finishing write tid 1 to smithi01785587-2 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3175: read oid 32 snap 394 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3175: expect (ObjNum 1174 snap 393 seq_num 1174) 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3167: done (6 left) 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3170: done (5 left) 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3171: done (4 left) 2024-09-21T09:51:19.631 INFO:tasks.rados.rados.0.smithi017.stdout:3176: snap_create 2024-09-21T09:51:19.632 INFO:tasks.rados.rados.0.smithi017.stdout:3172: finishing write tid 2 to smithi01785587-2 2024-09-21T09:51:19.632 INFO:tasks.rados.rados.0.smithi017.stdout:3172: finishing write tid 3 to smithi01785587-2 2024-09-21T09:51:19.632 INFO:tasks.rados.rados.0.smithi017.stdout:3172: finishing write tid 4 to smithi01785587-2 2024-09-21T09:51:19.632 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 633 (ObjNum 1191 snap 398 seq_num 1191) dirty exists 2024-09-21T09:51:19.632 INFO:tasks.rados.rados.0.smithi017.stdout:3172: left oid 2 (ObjNum 1191 snap 398 seq_num 1191) 2024-09-21T09:51:19.632 INFO:tasks.rados.rados.0.smithi017.stdout:3174: finishing write tid 1 to smithi01785587-9 2024-09-21T09:51:19.632 INFO:tasks.rados.rados.0.smithi017.stdout:3174: finishing write tid 2 to smithi01785587-9 2024-09-21T09:51:19.633 INFO:tasks.rados.rados.0.smithi017.stdout:3174: finishing write tid 3 to smithi01785587-9 2024-09-21T09:51:19.633 INFO:tasks.rados.rados.0.smithi017.stdout:3174: finishing write tid 4 to smithi01785587-9 2024-09-21T09:51:19.633 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 637 (ObjNum 1192 snap 398 seq_num 1192) dirty exists 2024-09-21T09:51:19.633 INFO:tasks.rados.rados.0.smithi017.stdout:3174: left oid 9 (ObjNum 1192 snap 398 seq_num 1192) 2024-09-21T09:51:19.636 INFO:tasks.rados.rados.0.smithi017.stdout:3175: expect (ObjNum 1174 snap 393 seq_num 1174) 2024-09-21T09:51:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:19 smithi017 ceph-mon[105151]: pgmap v858: 89 pgs: 89 active+clean; 264 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 418 KiB/s wr, 9 op/s 2024-09-21T09:51:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:19 smithi017 ceph-mon[105151]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T09:51:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:19 smithi017 ceph-mon[102913]: pgmap v858: 89 pgs: 89 active+clean; 264 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 418 KiB/s wr, 9 op/s 2024-09-21T09:51:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:19 smithi017 ceph-mon[102913]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T09:51:19.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:19 smithi134 ceph-mon[90085]: pgmap v858: 89 pgs: 89 active+clean; 264 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 418 KiB/s wr, 9 op/s 2024-09-21T09:51:19.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:19 smithi134 ceph-mon[90085]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T09:51:20.336 INFO:tasks.rados.rados.0.smithi017.stdout:3172: done (4 left) 2024-09-21T09:51:20.336 INFO:tasks.rados.rados.0.smithi017.stdout:3173: done (3 left) 2024-09-21T09:51:20.337 INFO:tasks.rados.rados.0.smithi017.stdout:3174: done (2 left) 2024-09-21T09:51:20.337 INFO:tasks.rados.rados.0.smithi017.stdout:3175: done (1 left) 2024-09-21T09:51:20.337 INFO:tasks.rados.rados.0.smithi017.stdout:3176: done (0 left) 2024-09-21T09:51:20.337 INFO:tasks.rados.rados.0.smithi017.stdout:3177: write oid 1 current snap is 399 2024-09-21T09:51:20.337 INFO:tasks.rados.rados.0.smithi017.stdout:3177: seq_num 1193 ranges {450751=723125,1945018=105733} 2024-09-21T09:51:20.342 INFO:tasks.rados.rados.0.smithi017.stdout:3177: writing smithi01785587-1 from 450751 to 1173876 tid 1 2024-09-21T09:51:20.348 INFO:tasks.rados.rados.0.smithi017.stdout:3177: writing smithi01785587-1 from 1945018 to 2050751 tid 2 2024-09-21T09:51:20.348 INFO:tasks.rados.rados.0.smithi017.stdout:3178: write oid 38 current snap is 399 2024-09-21T09:51:20.348 INFO:tasks.rados.rados.0.smithi017.stdout:3178: seq_num 1194 ranges {467558=514727,1719853=347705} 2024-09-21T09:51:20.353 INFO:tasks.rados.rados.0.smithi017.stdout:3178: writing smithi01785587-38 from 467558 to 982285 tid 1 2024-09-21T09:51:20.360 INFO:tasks.rados.rados.0.smithi017.stdout:3178: writing smithi01785587-38 from 1719853 to 2067558 tid 2 2024-09-21T09:51:20.361 INFO:tasks.rados.rados.0.smithi017.stdout:3179: read oid 48 snap -1 2024-09-21T09:51:20.361 INFO:tasks.rados.rados.0.smithi017.stdout:3179: expect (ObjNum 1149 snap 381 seq_num 1149) 2024-09-21T09:51:20.361 INFO:tasks.rados.rados.0.smithi017.stdout:3180: write oid 13 current snap is 399 2024-09-21T09:51:20.361 INFO:tasks.rados.rados.0.smithi017.stdout:3177: finishing write tid 1 to smithi01785587-1 2024-09-21T09:51:20.361 INFO:tasks.rados.rados.0.smithi017.stdout:3180: seq_num 1195 ranges {484365=589976,1661982=422383} 2024-09-21T09:51:20.366 INFO:tasks.rados.rados.0.smithi017.stdout:3180: writing smithi01785587-13 from 484365 to 1074341 tid 1 2024-09-21T09:51:20.372 INFO:tasks.rados.rados.0.smithi017.stdout:3180: writing smithi01785587-13 from 1661982 to 2084365 tid 2 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3181: read oid 33 snap -1 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3181: expect (ObjNum 1044 snap 350 seq_num 1044) 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3182: snap_create 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3177: finishing write tid 2 to smithi01785587-1 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3177: finishing write tid 3 to smithi01785587-1 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3177: finishing write tid 4 to smithi01785587-1 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 922 (ObjNum 1193 snap 399 seq_num 1193) dirty exists 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3177: left oid 1 (ObjNum 1193 snap 399 seq_num 1193) 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3178: finishing write tid 1 to smithi01785587-38 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3178: finishing write tid 2 to smithi01785587-38 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3178: finishing write tid 3 to smithi01785587-38 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:3178: finishing write tid 4 to smithi01785587-38 2024-09-21T09:51:20.373 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 960 (ObjNum 1194 snap 399 seq_num 1194) dirty exists 2024-09-21T09:51:20.374 INFO:tasks.rados.rados.0.smithi017.stdout:3178: left oid 38 (ObjNum 1194 snap 399 seq_num 1194) 2024-09-21T09:51:20.374 INFO:tasks.rados.rados.0.smithi017.stdout:3179: expect (ObjNum 1149 snap 381 seq_num 1149) 2024-09-21T09:51:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:51:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:51:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:51:20.615 INFO:tasks.rados.rados.0.smithi017.stdout:3180: finishing write tid 1 to smithi01785587-13 2024-09-21T09:51:20.615 INFO:tasks.rados.rados.0.smithi017.stdout:3180: finishing write tid 2 to smithi01785587-13 2024-09-21T09:51:20.615 INFO:tasks.rados.rados.0.smithi017.stdout:3180: finishing write tid 3 to smithi01785587-13 2024-09-21T09:51:20.615 INFO:tasks.rados.rados.0.smithi017.stdout:3180: finishing write tid 4 to smithi01785587-13 2024-09-21T09:51:20.615 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1349 (ObjNum 1195 snap 399 seq_num 1195) dirty exists 2024-09-21T09:51:20.615 INFO:tasks.rados.rados.0.smithi017.stdout:3180: left oid 13 (ObjNum 1195 snap 399 seq_num 1195) 2024-09-21T09:51:20.615 INFO:tasks.rados.rados.0.smithi017.stdout:3181: expect (ObjNum 1044 snap 350 seq_num 1044) 2024-09-21T09:51:21.341 INFO:tasks.rados.rados.0.smithi017.stdout:3177: done (5 left) 2024-09-21T09:51:21.341 INFO:tasks.rados.rados.0.smithi017.stdout:3178: done (4 left) 2024-09-21T09:51:21.342 INFO:tasks.rados.rados.0.smithi017.stdout:3179: done (3 left) 2024-09-21T09:51:21.342 INFO:tasks.rados.rados.0.smithi017.stdout:3180: done (2 left) 2024-09-21T09:51:21.342 INFO:tasks.rados.rados.0.smithi017.stdout:3181: done (1 left) 2024-09-21T09:51:21.342 INFO:tasks.rados.rados.0.smithi017.stdout:3182: done (0 left) 2024-09-21T09:51:21.342 INFO:tasks.rados.rados.0.smithi017.stdout:3183: write oid 19 current snap is 400 2024-09-21T09:51:21.342 INFO:tasks.rados.rados.0.smithi017.stdout:3183: seq_num 1196 ranges {501172=665225,1720464=380708} 2024-09-21T09:51:21.348 INFO:tasks.rados.rados.0.smithi017.stdout:3183: writing smithi01785587-19 from 501172 to 1166397 tid 1 2024-09-21T09:51:21.353 INFO:tasks.rados.rados.0.smithi017.stdout:3183: writing smithi01785587-19 from 1720464 to 2101172 tid 2 2024-09-21T09:51:21.353 INFO:tasks.rados.rados.0.smithi017.stdout:3184: rollback oid 2 current snap is 400 2024-09-21T09:51:21.353 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 2 to 399 2024-09-21T09:51:21.353 INFO:tasks.rados.rados.0.smithi017.stdout:3185: delete oid 15 current snap is 400 2024-09-21T09:51:21.355 INFO:tasks.rados.rados.0.smithi017.stdout:3184: finishing rollback tid 0 to smithi01785587-2 2024-09-21T09:51:21.356 INFO:tasks.rados.rados.0.smithi017.stdout:3184: finishing rollback tid 1 to smithi01785587-2 2024-09-21T09:51:21.356 INFO:tasks.rados.rados.0.smithi017.stdout:3184: finishing rollback tid 2 to smithi01785587-2 2024-09-21T09:51:21.356 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 641 (ObjNum 1191 snap 398 seq_num 1191) dirty exists 2024-09-21T09:51:21.356 INFO:tasks.rados.rados.0.smithi017.stdout:3183: finishing write tid 1 to smithi01785587-19 2024-09-21T09:51:21.356 INFO:tasks.rados.rados.0.smithi017.stdout:3183: finishing write tid 2 to smithi01785587-19 2024-09-21T09:51:21.357 INFO:tasks.rados.rados.0.smithi017.stdout:3183: finishing write tid 3 to smithi01785587-19 2024-09-21T09:51:21.357 INFO:tasks.rados.rados.0.smithi017.stdout:3183: finishing write tid 4 to smithi01785587-19 2024-09-21T09:51:21.357 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 1353 (ObjNum 1196 snap 400 seq_num 1196) dirty exists 2024-09-21T09:51:21.357 INFO:tasks.rados.rados.0.smithi017.stdout:3183: left oid 19 (ObjNum 1196 snap 400 seq_num 1196) 2024-09-21T09:51:21.358 INFO:tasks.rados.rados.0.smithi017.stdout:3183: done (2 left) 2024-09-21T09:51:21.358 INFO:tasks.rados.rados.0.smithi017.stdout:3184: done (1 left) 2024-09-21T09:51:21.358 INFO:tasks.rados.rados.0.smithi017.stdout:3185: done (0 left) 2024-09-21T09:51:21.359 INFO:tasks.rados.rados.0.smithi017.stdout:3186: delete oid 6 current snap is 400 2024-09-21T09:51:21.360 INFO:tasks.rados.rados.0.smithi017.stdout:3186: done (0 left) 2024-09-21T09:51:21.360 INFO:tasks.rados.rados.0.smithi017.stdout:3187: write oid 32 current snap is 400 2024-09-21T09:51:21.360 INFO:tasks.rados.rados.0.smithi017.stdout:3187: seq_num 1199 ranges {551593=490972,1779557=372036} 2024-09-21T09:51:21.366 INFO:tasks.rados.rados.0.smithi017.stdout:3187: writing smithi01785587-32 from 551593 to 1042565 tid 1 2024-09-21T09:51:21.372 INFO:tasks.rados.rados.0.smithi017.stdout:3187: writing smithi01785587-32 from 1779557 to 2151593 tid 2 2024-09-21T09:51:21.372 INFO:tasks.rados.rados.0.smithi017.stdout:3188: rollback oid 35 current snap is 400 2024-09-21T09:51:21.372 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 383 2024-09-21T09:51:21.372 INFO:tasks.rados.rados.0.smithi017.stdout:3187: finishing write tid 1 to smithi01785587-32 2024-09-21T09:51:21.372 INFO:tasks.rados.rados.0.smithi017.stdout:3189: rollback oid 38 current snap is 400 2024-09-21T09:51:21.373 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 38 to 399 2024-09-21T09:51:21.373 INFO:tasks.rados.rados.0.smithi017.stdout:3190: snap_remove snap 366 2024-09-21T09:51:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[102913]: pgmap v861: 89 pgs: 89 active+clean; 262 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 0 B/s wr, 2 op/s 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[102913]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[105151]: pgmap v861: 89 pgs: 89 active+clean; 262 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 0 B/s wr, 2 op/s 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[105151]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:21 smithi134 ceph-mon[90085]: pgmap v861: 89 pgs: 89 active+clean; 262 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 0 B/s wr, 2 op/s 2024-09-21T09:51:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:21 smithi134 ceph-mon[90085]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T09:51:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:22.349 INFO:tasks.rados.rados.0.smithi017.stdout:3188: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:51:22.349 INFO:tasks.rados.rados.0.smithi017.stdout:3188: finishing rollback tid 2 to smithi01785587-35 2024-09-21T09:51:22.349 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 350 (ObjNum 1069 snap 356 seq_num 1069) dirty exists 2024-09-21T09:51:22.349 INFO:tasks.rados.rados.0.smithi017.stdout:3187: finishing write tid 2 to smithi01785587-32 2024-09-21T09:51:22.349 INFO:tasks.rados.rados.0.smithi017.stdout:3189: finishing rollback tid 0 to smithi01785587-38 2024-09-21T09:51:22.349 INFO:tasks.rados.rados.0.smithi017.stdout:3189: finishing rollback tid 1 to smithi01785587-38 2024-09-21T09:51:22.349 INFO:tasks.rados.rados.0.smithi017.stdout:3187: finishing write tid 3 to smithi01785587-32 2024-09-21T09:51:22.349 INFO:tasks.rados.rados.0.smithi017.stdout:3187: finishing write tid 4 to smithi01785587-32 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1512 (ObjNum 1199 snap 400 seq_num 1199) dirty exists 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:3187: left oid 32 (ObjNum 1199 snap 400 seq_num 1199) 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:3189: finishing rollback tid 2 to smithi01785587-38 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 964 (ObjNum 1194 snap 399 seq_num 1194) dirty exists 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:3187: done (3 left) 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:3188: done (2 left) 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:3189: done (1 left) 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:3190: done (0 left) 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:3191: write oid 1 current snap is 400 2024-09-21T09:51:22.350 INFO:tasks.rados.rados.0.smithi017.stdout:3191: seq_num 1200 ranges {568400=566221,1838039=330361} 2024-09-21T09:51:22.356 INFO:tasks.rados.rados.0.smithi017.stdout:3191: writing smithi01785587-1 from 568400 to 1134621 tid 1 2024-09-21T09:51:22.362 INFO:tasks.rados.rados.0.smithi017.stdout:3191: writing smithi01785587-1 from 1838039 to 2168400 tid 2 2024-09-21T09:51:22.362 INFO:tasks.rados.rados.0.smithi017.stdout:3192: snap_remove snap 383 2024-09-21T09:51:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:22 smithi017 ceph-mon[102913]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T09:51:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:51:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:51:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:22 smithi017 ceph-mon[105151]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T09:51:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:51:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:51:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:22.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:22 smithi134 ceph-mon[90085]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T09:51:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:51:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:51:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:51:23.183 INFO:tasks.rados.rados.0.smithi017.stdout:3192: done (1 left) 2024-09-21T09:51:23.183 INFO:tasks.rados.rados.0.smithi017.stdout:3193: read oid 31 snap -1 2024-09-21T09:51:23.183 INFO:tasks.rados.rados.0.smithi017.stdout:3193: expect (ObjNum 1175 snap 394 seq_num 1175) 2024-09-21T09:51:23.184 INFO:tasks.rados.rados.0.smithi017.stdout:3194: snap_create 2024-09-21T09:51:23.184 INFO:tasks.rados.rados.0.smithi017.stdout:3191: finishing write tid 1 to smithi01785587-1 2024-09-21T09:51:23.184 INFO:tasks.rados.rados.0.smithi017.stdout:3191: finishing write tid 2 to smithi01785587-1 2024-09-21T09:51:23.184 INFO:tasks.rados.rados.0.smithi017.stdout:3191: finishing write tid 3 to smithi01785587-1 2024-09-21T09:51:23.184 INFO:tasks.rados.rados.0.smithi017.stdout:3191: finishing write tid 4 to smithi01785587-1 2024-09-21T09:51:23.184 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 932 (ObjNum 1200 snap 400 seq_num 1200) dirty exists 2024-09-21T09:51:23.184 INFO:tasks.rados.rados.0.smithi017.stdout:3191: left oid 1 (ObjNum 1200 snap 400 seq_num 1200) 2024-09-21T09:51:23.198 INFO:tasks.rados.rados.0.smithi017.stdout:3193: expect (ObjNum 1175 snap 394 seq_num 1175) 2024-09-21T09:51:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:23 smithi017 ceph-mon[102913]: pgmap v864: 89 pgs: 89 active+clean; 277 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.6 MiB/s wr, 22 op/s 2024-09-21T09:51:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:23 smithi017 ceph-mon[102913]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T09:51:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:23 smithi017 ceph-mon[102913]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T09:51:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:23 smithi017 ceph-mon[105151]: pgmap v864: 89 pgs: 89 active+clean; 277 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.6 MiB/s wr, 22 op/s 2024-09-21T09:51:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:23 smithi017 ceph-mon[105151]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T09:51:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:23 smithi017 ceph-mon[105151]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T09:51:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:23 smithi134 ceph-mon[90085]: pgmap v864: 89 pgs: 89 active+clean; 277 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.6 MiB/s wr, 22 op/s 2024-09-21T09:51:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:23 smithi134 ceph-mon[90085]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T09:51:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:23 smithi134 ceph-mon[90085]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T09:51:24.184 INFO:tasks.rados.rados.0.smithi017.stdout:3191: done (2 left) 2024-09-21T09:51:24.185 INFO:tasks.rados.rados.0.smithi017.stdout:3193: done (1 left) 2024-09-21T09:51:24.185 INFO:tasks.rados.rados.0.smithi017.stdout:3194: done (0 left) 2024-09-21T09:51:24.185 INFO:tasks.rados.rados.0.smithi017.stdout:3195: write oid 31 current snap is 401 2024-09-21T09:51:24.185 INFO:tasks.rados.rados.0.smithi017.stdout:3195: seq_num 1201 ranges {585207=641470,1896521=288686} 2024-09-21T09:51:24.191 INFO:tasks.rados.rados.0.smithi017.stdout:3195: writing smithi01785587-31 from 585207 to 1226677 tid 1 2024-09-21T09:51:24.197 INFO:tasks.rados.rados.0.smithi017.stdout:3195: writing smithi01785587-31 from 1896521 to 2185207 tid 2 2024-09-21T09:51:24.197 INFO:tasks.rados.rados.0.smithi017.stdout:3196: delete oid 2 current snap is 401 2024-09-21T09:51:24.199 INFO:tasks.rados.rados.0.smithi017.stdout:3196: done (1 left) 2024-09-21T09:51:24.199 INFO:tasks.rados.rados.0.smithi017.stdout:3197: snap_create 2024-09-21T09:51:24.200 INFO:tasks.rados.rados.0.smithi017.stdout:3195: finishing write tid 1 to smithi01785587-31 2024-09-21T09:51:24.201 INFO:tasks.rados.rados.0.smithi017.stdout:3195: finishing write tid 2 to smithi01785587-31 2024-09-21T09:51:24.201 INFO:tasks.rados.rados.0.smithi017.stdout:3195: finishing write tid 3 to smithi01785587-31 2024-09-21T09:51:24.201 INFO:tasks.rados.rados.0.smithi017.stdout:3195: finishing write tid 4 to smithi01785587-31 2024-09-21T09:51:24.202 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1668 (ObjNum 1201 snap 401 seq_num 1201) dirty exists 2024-09-21T09:51:24.202 INFO:tasks.rados.rados.0.smithi017.stdout:3195: left oid 31 (ObjNum 1201 snap 401 seq_num 1201) 2024-09-21T09:51:25.185 INFO:tasks.rados.rados.0.smithi017.stdout:3195: done (1 left) 2024-09-21T09:51:25.186 INFO:tasks.rados.rados.0.smithi017.stdout:3197: done (0 left) 2024-09-21T09:51:25.186 INFO:tasks.rados.rados.0.smithi017.stdout:3198: write oid 13 current snap is 402 2024-09-21T09:51:25.186 INFO:tasks.rados.rados.0.smithi017.stdout:3198: seq_num 1203 ranges {618821=508321,1613485=531077} 2024-09-21T09:51:25.187 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:51:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:51:24] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.51.0" 2024-09-21T09:51:25.191 INFO:tasks.rados.rados.0.smithi017.stdout:3198: writing smithi01785587-13 from 618821 to 1127142 tid 1 2024-09-21T09:51:25.197 INFO:tasks.rados.rados.0.smithi017.stdout:3198: writing smithi01785587-13 from 1613485 to 2144562 tid 2 2024-09-21T09:51:25.197 INFO:tasks.rados.rados.0.smithi017.stdout:3199: snap_create 2024-09-21T09:51:25.197 INFO:tasks.rados.rados.0.smithi017.stdout:3198: finishing write tid 1 to smithi01785587-13 2024-09-21T09:51:25.201 INFO:tasks.rados.rados.0.smithi017.stdout:3198: finishing write tid 2 to smithi01785587-13 2024-09-21T09:51:25.201 INFO:tasks.rados.rados.0.smithi017.stdout:3198: finishing write tid 3 to smithi01785587-13 2024-09-21T09:51:25.201 INFO:tasks.rados.rados.0.smithi017.stdout:3198: finishing write tid 4 to smithi01785587-13 2024-09-21T09:51:25.201 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1363 (ObjNum 1203 snap 402 seq_num 1203) dirty exists 2024-09-21T09:51:25.201 INFO:tasks.rados.rados.0.smithi017.stdout:3198: left oid 13 (ObjNum 1203 snap 402 seq_num 1203) 2024-09-21T09:51:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:25 smithi017 ceph-mon[102913]: pgmap v867: 89 pgs: 89 active+clean; 282 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.7 MiB/s wr, 27 op/s 2024-09-21T09:51:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:25 smithi017 ceph-mon[102913]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T09:51:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:25 smithi017 ceph-mon[105151]: pgmap v867: 89 pgs: 89 active+clean; 282 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.7 MiB/s wr, 27 op/s 2024-09-21T09:51:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:25 smithi017 ceph-mon[105151]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T09:51:25.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:25 smithi134 ceph-mon[90085]: pgmap v867: 89 pgs: 89 active+clean; 282 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 3.7 MiB/s wr, 27 op/s 2024-09-21T09:51:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:25 smithi134 ceph-mon[90085]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T09:51:26.196 INFO:tasks.rados.rados.0.smithi017.stdout:3198: done (1 left) 2024-09-21T09:51:26.196 INFO:tasks.rados.rados.0.smithi017.stdout:3199: done (0 left) 2024-09-21T09:51:26.196 INFO:tasks.rados.rados.0.smithi017.stdout:3200: write oid 38 current snap is 403 2024-09-21T09:51:26.196 INFO:tasks.rados.rados.0.smithi017.stdout:3200: seq_num 1204 ranges {635628=583570,1671967=563661} 2024-09-21T09:51:26.202 INFO:tasks.rados.rados.0.smithi017.stdout:3200: writing smithi01785587-38 from 635628 to 1219198 tid 1 2024-09-21T09:51:26.209 INFO:tasks.rados.rados.0.smithi017.stdout:3200: writing smithi01785587-38 from 1671967 to 2235628 tid 2 2024-09-21T09:51:26.209 INFO:tasks.rados.rados.0.smithi017.stdout:3201: write oid 18 current snap is 403 2024-09-21T09:51:26.209 INFO:tasks.rados.rados.0.smithi017.stdout:3201: seq_num 1205 ranges {652435=658819,1730449=521986} 2024-09-21T09:51:26.216 INFO:tasks.rados.rados.0.smithi017.stdout:3201: writing smithi01785587-18 from 652435 to 1311254 tid 1 2024-09-21T09:51:26.222 INFO:tasks.rados.rados.0.smithi017.stdout:3201: writing smithi01785587-18 from 1730449 to 2252435 tid 2 2024-09-21T09:51:26.222 INFO:tasks.rados.rados.0.smithi017.stdout:3202: rollback oid 40 current snap is 403 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 371 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:3200: finishing write tid 1 to smithi01785587-38 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:3200: finishing write tid 2 to smithi01785587-38 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:3200: finishing write tid 3 to smithi01785587-38 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:3200: finishing write tid 4 to smithi01785587-38 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 970 (ObjNum 1204 snap 403 seq_num 1204) dirty exists 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:3200: left oid 38 (ObjNum 1204 snap 403 seq_num 1204) 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:3200: done (2 left) 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:3203: read oid 7 snap 398 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:3203: expect (ObjNum 1186 snap 398 seq_num 1186) 2024-09-21T09:51:26.223 INFO:tasks.rados.rados.0.smithi017.stdout:3204: read oid 10 snap -1 2024-09-21T09:51:26.224 INFO:tasks.rados.rados.0.smithi017.stdout:3204: expect (ObjNum 1072 snap 358 seq_num 1072) 2024-09-21T09:51:26.224 INFO:tasks.rados.rados.0.smithi017.stdout:3205: rollback oid 15 current snap is 403 2024-09-21T09:51:26.224 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 360 2024-09-21T09:51:26.224 INFO:tasks.rados.rados.0.smithi017.stdout:3206: write oid 1 current snap is 403 2024-09-21T09:51:26.224 INFO:tasks.rados.rados.0.smithi017.stdout:3206: seq_num 1206 ranges {669242=734068,2188931=80311} 2024-09-21T09:51:26.229 INFO:tasks.rados.rados.0.smithi017.stdout:3206: writing smithi01785587-1 from 669242 to 1403310 tid 1 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3206: writing smithi01785587-1 from 2188931 to 2269242 tid 2 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3207: delete oid 19 current snap is 403 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3205: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 0 (ObjNum 129 snap 0 seq_num 2399850240) dirty dne 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3201: finishing write tid 1 to smithi01785587-18 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3201: finishing write tid 2 to smithi01785587-18 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3201: finishing write tid 3 to smithi01785587-18 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3201: finishing write tid 4 to smithi01785587-18 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 307 (ObjNum 1205 snap 403 seq_num 1205) dirty exists 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3201: left oid 18 (ObjNum 1205 snap 403 seq_num 1205) 2024-09-21T09:51:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3202: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:51:26.236 INFO:tasks.rados.rados.0.smithi017.stdout:3204: expect (ObjNum 1072 snap 358 seq_num 1072) 2024-09-21T09:51:26.378 INFO:tasks.rados.rados.0.smithi017.stdout:3202: finishing rollback tid 2 to smithi01785587-40 2024-09-21T09:51:26.378 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1630 (ObjNum 1045 snap 350 seq_num 1045) dirty exists 2024-09-21T09:51:26.378 INFO:tasks.rados.rados.0.smithi017.stdout:3203: expect (ObjNum 1186 snap 398 seq_num 1186) 2024-09-21T09:51:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:26 smithi017 ceph-mon[105151]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T09:51:26.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:26 smithi017 ceph-mon[102913]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T09:51:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:26.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:26 smithi134 ceph-mon[90085]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T09:51:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:26.563 INFO:tasks.rados.rados.0.smithi017.stdout:3206: finishing write tid 1 to smithi01785587-1 2024-09-21T09:51:26.563 INFO:tasks.rados.rados.0.smithi017.stdout:3206: finishing write tid 2 to smithi01785587-1 2024-09-21T09:51:26.563 INFO:tasks.rados.rados.0.smithi017.stdout:3206: finishing write tid 3 to smithi01785587-1 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3206: finishing write tid 4 to smithi01785587-1 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 938 (ObjNum 1206 snap 403 seq_num 1206) dirty exists 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3206: left oid 1 (ObjNum 1206 snap 403 seq_num 1206) 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3201: done (6 left) 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3202: done (5 left) 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3203: done (4 left) 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3204: done (3 left) 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3205: done (2 left) 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3206: done (1 left) 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3207: done (0 left) 2024-09-21T09:51:26.564 INFO:tasks.rados.rados.0.smithi017.stdout:3208: snap_remove snap 390 2024-09-21T09:51:27.202 INFO:tasks.rados.rados.0.smithi017.stdout:3208: done (0 left) 2024-09-21T09:51:27.202 INFO:tasks.rados.rados.0.smithi017.stdout:3209: rollback oid 8 current snap is 403 2024-09-21T09:51:27.203 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 394 2024-09-21T09:51:27.203 INFO:tasks.rados.rados.0.smithi017.stdout:3210: read oid 20 snap -1 2024-09-21T09:51:27.203 INFO:tasks.rados.rados.0.smithi017.stdout:3210: expect (ObjNum 1147 snap 381 seq_num 1147) 2024-09-21T09:51:27.203 INFO:tasks.rados.rados.0.smithi017.stdout:3211: delete oid 9 current snap is 403 2024-09-21T09:51:27.209 INFO:tasks.rados.rados.0.smithi017.stdout:3211: done (2 left) 2024-09-21T09:51:27.209 INFO:tasks.rados.rados.0.smithi017.stdout:3212: snap_remove snap 397 2024-09-21T09:51:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:27 smithi017 ceph-mon[102913]: pgmap v870: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:51:27.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:27 smithi017 ceph-mon[102913]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T09:51:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:27 smithi017 ceph-mon[105151]: pgmap v870: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:51:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:27 smithi017 ceph-mon[105151]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T09:51:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:27 smithi134 ceph-mon[90085]: pgmap v870: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:51:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:27 smithi134 ceph-mon[90085]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T09:51:28.184 INFO:tasks.rados.rados.0.smithi017.stdout:3212: done (2 left) 2024-09-21T09:51:28.184 INFO:tasks.rados.rados.0.smithi017.stdout:3213: read oid 2 snap -1 2024-09-21T09:51:28.184 INFO:tasks.rados.rados.0.smithi017.stdout:3213: expect deleted 2024-09-21T09:51:28.184 INFO:tasks.rados.rados.0.smithi017.stdout:3214: snap_remove snap 387 2024-09-21T09:51:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:28 smithi134 ceph-mon[90085]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T09:51:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:28 smithi134 ceph-mon[90085]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T09:51:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:28 smithi017 ceph-mon[102913]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T09:51:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:28 smithi017 ceph-mon[102913]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T09:51:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:28 smithi017 ceph-mon[105151]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T09:51:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:28 smithi017 ceph-mon[105151]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T09:51:29.186 INFO:tasks.rados.rados.0.smithi017.stdout:3214: done (3 left) 2024-09-21T09:51:29.186 INFO:tasks.rados.rados.0.smithi017.stdout:3215: write oid 31 current snap is 403 2024-09-21T09:51:29.186 INFO:tasks.rados.rados.0.smithi017.stdout:3210: expect (ObjNum 1147 snap 381 seq_num 1147) 2024-09-21T09:51:29.338 INFO:tasks.rados.rados.0.smithi017.stdout:3209: finishing rollback tid 0 to smithi01785587-8 2024-09-21T09:51:29.339 INFO:tasks.rados.rados.0.smithi017.stdout:3209: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:51:29.339 INFO:tasks.rados.rados.0.smithi017.stdout:3209: finishing rollback tid 2 to smithi01785587-8 2024-09-21T09:51:29.339 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1530 (ObjNum 1070 snap 357 seq_num 1070) dirty exists 2024-09-21T09:51:29.339 INFO:tasks.rados.rados.0.smithi017.stdout:3215: seq_num 1209 ranges {719663=676168,1964377=355286} 2024-09-21T09:51:29.346 INFO:tasks.rados.rados.0.smithi017.stdout:3215: writing smithi01785587-31 from 719663 to 1395831 tid 1 2024-09-21T09:51:29.351 INFO:tasks.rados.rados.0.smithi017.stdout:3215: writing smithi01785587-31 from 1964377 to 2319663 tid 2 2024-09-21T09:51:29.351 INFO:tasks.rados.rados.0.smithi017.stdout:3215: finishing write tid 1 to smithi01785587-31 2024-09-21T09:51:29.351 INFO:tasks.rados.rados.0.smithi017.stdout:3209: done (3 left) 2024-09-21T09:51:29.352 INFO:tasks.rados.rados.0.smithi017.stdout:3210: done (2 left) 2024-09-21T09:51:29.352 INFO:tasks.rados.rados.0.smithi017.stdout:3213: done (1 left) 2024-09-21T09:51:29.352 INFO:tasks.rados.rados.0.smithi017.stdout:3216: delete oid 9 current snap is 403 2024-09-21T09:51:29.353 INFO:tasks.rados.rados.0.smithi017.stdout:3216: done (1 left) 2024-09-21T09:51:29.353 INFO:tasks.rados.rados.0.smithi017.stdout:3217: write oid 3 current snap is 403 2024-09-21T09:51:29.353 INFO:tasks.rados.rados.0.smithi017.stdout:3217: seq_num 1211 ranges {753277=426666,1681341=545753} 2024-09-21T09:51:29.359 INFO:tasks.rados.rados.0.smithi017.stdout:3217: writing smithi01785587-3 from 753277 to 1179943 tid 1 2024-09-21T09:51:29.365 INFO:tasks.rados.rados.0.smithi017.stdout:3217: writing smithi01785587-3 from 1681341 to 2227094 tid 2 2024-09-21T09:51:29.365 INFO:tasks.rados.rados.0.smithi017.stdout:3218: rollback oid 34 current snap is 403 2024-09-21T09:51:29.365 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 372 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3215: finishing write tid 2 to smithi01785587-31 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3215: finishing write tid 3 to smithi01785587-31 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3215: finishing write tid 4 to smithi01785587-31 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1684 (ObjNum 1209 snap 403 seq_num 1209) dirty exists 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3215: left oid 31 (ObjNum 1209 snap 403 seq_num 1209) 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3217: finishing write tid 1 to smithi01785587-3 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3215: done (2 left) 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3219: read oid 8 snap -1 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3219: expect (ObjNum 1070 snap 357 seq_num 1070) 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3220: write oid 31 current snap is 403 2024-09-21T09:51:29.366 INFO:tasks.rados.rados.0.smithi017.stdout:3220: seq_num 1212 ranges {770084=501915,2023470=346614} 2024-09-21T09:51:29.372 INFO:tasks.rados.rados.0.smithi017.stdout:3220: writing smithi01785587-31 from 770084 to 1271999 tid 1 2024-09-21T09:51:29.378 INFO:tasks.rados.rados.0.smithi017.stdout:3220: writing smithi01785587-31 from 2023470 to 2370084 tid 2 2024-09-21T09:51:29.379 INFO:tasks.rados.rados.0.smithi017.stdout:3221: read oid 41 snap -1 2024-09-21T09:51:29.379 INFO:tasks.rados.rados.0.smithi017.stdout:3221: expect deleted 2024-09-21T09:51:29.379 INFO:tasks.rados.rados.0.smithi017.stdout:3222: snap_remove snap 356 2024-09-21T09:51:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:29 smithi134 ceph-mon[90085]: pgmap v873: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:51:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:29 smithi134 ceph-mon[90085]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T09:51:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:29 smithi017 ceph-mon[105151]: pgmap v873: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:51:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:29 smithi017 ceph-mon[105151]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T09:51:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:29 smithi017 ceph-mon[102913]: pgmap v873: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:51:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:29 smithi017 ceph-mon[102913]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T09:51:30.214 INFO:tasks.rados.rados.0.smithi017.stdout:3222: done (5 left) 2024-09-21T09:51:30.215 INFO:tasks.rados.rados.0.smithi017.stdout:3223: delete oid 29 current snap is 403 2024-09-21T09:51:30.215 INFO:tasks.rados.rados.0.smithi017.stdout:3217: finishing write tid 2 to smithi01785587-3 2024-09-21T09:51:30.215 INFO:tasks.rados.rados.0.smithi017.stdout:3218: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:51:30.215 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1436 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:30.215 INFO:tasks.rados.rados.0.smithi017.stdout:3217: finishing write tid 3 to smithi01785587-3 2024-09-21T09:51:30.215 INFO:tasks.rados.rados.0.smithi017.stdout:3217: finishing write tid 4 to smithi01785587-3 2024-09-21T09:51:30.215 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1687 (ObjNum 1211 snap 403 seq_num 1211) dirty exists 2024-09-21T09:51:30.215 INFO:tasks.rados.rados.0.smithi017.stdout:3217: left oid 3 (ObjNum 1211 snap 403 seq_num 1211) 2024-09-21T09:51:30.215 INFO:tasks.rados.rados.0.smithi017.stdout:3219: expect (ObjNum 1070 snap 357 seq_num 1070) 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi017.stdout:3220: finishing write tid 1 to smithi01785587-31 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi017.stdout:3220: finishing write tid 2 to smithi01785587-31 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi017.stdout:3220: finishing write tid 3 to smithi01785587-31 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi017.stdout:3220: finishing write tid 4 to smithi01785587-31 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1690 (ObjNum 1212 snap 403 seq_num 1212) dirty exists 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi017.stdout:3220: left oid 31 (ObjNum 1212 snap 403 seq_num 1212) 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi017.stdout:3217: done (5 left) 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi017.stdout:3218: done (4 left) 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi017.stdout:3219: done (3 left) 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi017.stdout:3220: done (2 left) 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi017.stdout:3221: done (1 left) 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi017.stdout:3223: done (0 left) 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi017.stdout:3224: write oid 50 current snap is 403 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi017.stdout:3224: seq_num 1214 ranges {403698=652413,1740434=663264} 2024-09-21T09:51:30.410 INFO:tasks.rados.rados.0.smithi017.stdout:3224: writing smithi01785587-50 from 403698 to 1056111 tid 1 2024-09-21T09:51:30.418 INFO:tasks.rados.rados.0.smithi017.stdout:3224: writing smithi01785587-50 from 1740434 to 2403698 tid 2 2024-09-21T09:51:30.418 INFO:tasks.rados.rados.0.smithi017.stdout:3225: snap_create 2024-09-21T09:51:30.418 INFO:tasks.rados.rados.0.smithi017.stdout:3224: finishing write tid 1 to smithi01785587-50 2024-09-21T09:51:30.421 INFO:tasks.rados.rados.0.smithi017.stdout:3224: finishing write tid 2 to smithi01785587-50 2024-09-21T09:51:30.421 INFO:tasks.rados.rados.0.smithi017.stdout:3224: finishing write tid 3 to smithi01785587-50 2024-09-21T09:51:30.421 INFO:tasks.rados.rados.0.smithi017.stdout:3224: finishing write tid 4 to smithi01785587-50 2024-09-21T09:51:30.421 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 976 (ObjNum 1214 snap 403 seq_num 1214) dirty exists 2024-09-21T09:51:30.422 INFO:tasks.rados.rados.0.smithi017.stdout:3224: left oid 50 (ObjNum 1214 snap 403 seq_num 1214) 2024-09-21T09:51:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:51:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:51:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:51:31.218 INFO:tasks.rados.rados.0.smithi017.stdout:3224: done (1 left) 2024-09-21T09:51:31.219 INFO:tasks.rados.rados.0.smithi017.stdout:3225: done (0 left) 2024-09-21T09:51:31.219 INFO:tasks.rados.rados.0.smithi017.stdout:3226: rollback oid 11 current snap is 404 2024-09-21T09:51:31.219 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 394 2024-09-21T09:51:31.219 INFO:tasks.rados.rados.0.smithi017.stdout:3227: read oid 22 snap 398 2024-09-21T09:51:31.219 INFO:tasks.rados.rados.0.smithi017.stdout:3227: expect deleted 2024-09-21T09:51:31.219 INFO:tasks.rados.rados.0.smithi017.stdout:3228: snap_remove snap 378 2024-09-21T09:51:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:31 smithi134 ceph-mon[90085]: pgmap v876: 89 pgs: 89 active+clean; 286 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 491 KiB/s wr, 7 op/s 2024-09-21T09:51:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:31 smithi134 ceph-mon[90085]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T09:51:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:31 smithi017 ceph-mon[102913]: pgmap v876: 89 pgs: 89 active+clean; 286 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 491 KiB/s wr, 7 op/s 2024-09-21T09:51:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:31 smithi017 ceph-mon[102913]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T09:51:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:31 smithi017 ceph-mon[105151]: pgmap v876: 89 pgs: 89 active+clean; 286 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 491 KiB/s wr, 7 op/s 2024-09-21T09:51:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:31 smithi017 ceph-mon[105151]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T09:51:32.226 INFO:tasks.rados.rados.0.smithi017.stdout:3228: done (2 left) 2024-09-21T09:51:32.226 INFO:tasks.rados.rados.0.smithi017.stdout:3229: snap_remove snap 396 2024-09-21T09:51:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:32 smithi134 ceph-mon[90085]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T09:51:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:32 smithi017 ceph-mon[102913]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T09:51:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:32 smithi017 ceph-mon[105151]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T09:51:33.186 INFO:tasks.rados.rados.0.smithi017.stdout:3229: done (2 left) 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3230: read oid 48 snap -1 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3230: expect (ObjNum 1149 snap 381 seq_num 1149) 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3231: rollback oid 9 current snap is 404 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 9 to 352 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3226: finishing rollback tid 0 to smithi01785587-11 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3226: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3227: done (3 left) 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3232: rollback oid 8 current snap is 404 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 402 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3226: finishing rollback tid 2 to smithi01785587-11 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1564 (ObjNum 1168 snap 392 seq_num 1168) dirty exists 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3226: done (3 left) 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3233: read oid 25 snap -1 2024-09-21T09:51:33.187 INFO:tasks.rados.rados.0.smithi017.stdout:3233: expect (ObjNum 1165 snap 392 seq_num 1165) 2024-09-21T09:51:33.188 INFO:tasks.rados.rados.0.smithi017.stdout:3234: delete oid 12 current snap is 404 2024-09-21T09:51:33.196 INFO:tasks.rados.rados.0.smithi017.stdout:3234: done (4 left) 2024-09-21T09:51:33.196 INFO:tasks.rados.rados.0.smithi017.stdout:3235: delete oid 24 current snap is 404 2024-09-21T09:51:33.199 INFO:tasks.rados.rados.0.smithi017.stdout:3232: finishing rollback tid 0 to smithi01785587-8 2024-09-21T09:51:33.200 INFO:tasks.rados.rados.0.smithi017.stdout:3233: expect (ObjNum 1165 snap 392 seq_num 1165) 2024-09-21T09:51:33.365 INFO:tasks.rados.rados.0.smithi017.stdout:3230: expect (ObjNum 1149 snap 381 seq_num 1149) 2024-09-21T09:51:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:33 smithi134 ceph-mon[90085]: pgmap v879: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-21T09:51:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:33 smithi134 ceph-mon[90085]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T09:51:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:33 smithi134 ceph-mon[90085]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T09:51:33.605 INFO:tasks.rados.rados.0.smithi017.stdout:3231: finishing rollback tid 1 to smithi01785587-9 2024-09-21T09:51:33.605 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 651 (ObjNum 129 snap 0 seq_num 2400769464) dirty dne 2024-09-21T09:51:33.605 INFO:tasks.rados.rados.0.smithi017.stdout:3232: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:51:33.605 INFO:tasks.rados.rados.0.smithi017.stdout:3232: finishing rollback tid 2 to smithi01785587-8 2024-09-21T09:51:33.606 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1574 (ObjNum 1070 snap 357 seq_num 1070) dirty exists 2024-09-21T09:51:33.606 INFO:tasks.rados.rados.0.smithi017.stdout:3230: done (4 left) 2024-09-21T09:51:33.606 INFO:tasks.rados.rados.0.smithi017.stdout:3231: done (3 left) 2024-09-21T09:51:33.606 INFO:tasks.rados.rados.0.smithi017.stdout:3232: done (2 left) 2024-09-21T09:51:33.606 INFO:tasks.rados.rados.0.smithi017.stdout:3233: done (1 left) 2024-09-21T09:51:33.606 INFO:tasks.rados.rados.0.smithi017.stdout:3235: done (0 left) 2024-09-21T09:51:33.606 INFO:tasks.rados.rados.0.smithi017.stdout:3236: snap_remove snap 401 2024-09-21T09:51:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:33 smithi017 ceph-mon[102913]: pgmap v879: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-21T09:51:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:33 smithi017 ceph-mon[102913]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T09:51:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:33 smithi017 ceph-mon[102913]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T09:51:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:33 smithi017 ceph-mon[105151]: pgmap v879: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-21T09:51:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:33 smithi017 ceph-mon[105151]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T09:51:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:33 smithi017 ceph-mon[105151]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T09:51:34.230 INFO:tasks.rados.rados.0.smithi017.stdout:3236: done (0 left) 2024-09-21T09:51:34.230 INFO:tasks.rados.rados.0.smithi017.stdout:3237: snap_create 2024-09-21T09:51:35.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:51:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:51:34] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.51.0" 2024-09-21T09:51:35.232 INFO:tasks.rados.rados.0.smithi017.stdout:3237: done (0 left) 2024-09-21T09:51:35.232 INFO:tasks.rados.rados.0.smithi017.stdout:3238: write oid 30 current snap is 405 2024-09-21T09:51:35.232 INFO:tasks.rados.rados.0.smithi017.stdout:3238: seq_num 1217 ranges {454119=594513,1515880=556760} 2024-09-21T09:51:35.238 INFO:tasks.rados.rados.0.smithi017.stdout:3238: writing smithi01785587-30 from 454119 to 1048632 tid 1 2024-09-21T09:51:35.244 INFO:tasks.rados.rados.0.smithi017.stdout:3238: writing smithi01785587-30 from 1515880 to 2072640 tid 2 2024-09-21T09:51:35.244 INFO:tasks.rados.rados.0.smithi017.stdout:3239: read oid 44 snap 360 2024-09-21T09:51:35.244 INFO:tasks.rados.rados.0.smithi017.stdout:3239: expect (ObjNum 1065 snap 354 seq_num 1065) 2024-09-21T09:51:35.244 INFO:tasks.rados.rados.0.smithi017.stdout:3240: write oid 24 current snap is 405 2024-09-21T09:51:35.244 INFO:tasks.rados.rados.0.smithi017.stdout:3240: seq_num 1218 ranges {470926=669762,1574362=700418} 2024-09-21T09:51:35.250 INFO:tasks.rados.rados.0.smithi017.stdout:3240: writing smithi01785587-24 from 470926 to 1140688 tid 1 2024-09-21T09:51:35.258 INFO:tasks.rados.rados.0.smithi017.stdout:3240: writing smithi01785587-24 from 1574362 to 2274780 tid 2 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:3241: read oid 36 snap -1 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:3241: expect deleted 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:3242: delete oid 16 current snap is 405 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:3238: finishing write tid 1 to smithi01785587-30 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:3238: finishing write tid 2 to smithi01785587-30 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:3238: finishing write tid 3 to smithi01785587-30 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:3238: finishing write tid 4 to smithi01785587-30 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1396 (ObjNum 1217 snap 405 seq_num 1217) dirty exists 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:3238: left oid 30 (ObjNum 1217 snap 405 seq_num 1217) 2024-09-21T09:51:35.259 INFO:tasks.rados.rados.0.smithi017.stdout:3239: expect (ObjNum 1065 snap 354 seq_num 1065) 2024-09-21T09:51:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:35 smithi134 ceph-mon[90085]: pgmap v882: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-21T09:51:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:35 smithi134 ceph-mon[90085]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T09:51:35.554 INFO:tasks.rados.rados.0.smithi017.stdout:3240: finishing write tid 1 to smithi01785587-24 2024-09-21T09:51:35.554 INFO:tasks.rados.rados.0.smithi017.stdout:3240: finishing write tid 2 to smithi01785587-24 2024-09-21T09:51:35.554 INFO:tasks.rados.rados.0.smithi017.stdout:3240: finishing write tid 3 to smithi01785587-24 2024-09-21T09:51:35.554 INFO:tasks.rados.rados.0.smithi017.stdout:3240: finishing write tid 4 to smithi01785587-24 2024-09-21T09:51:35.554 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 991 (ObjNum 1218 snap 405 seq_num 1218) dirty exists 2024-09-21T09:51:35.555 INFO:tasks.rados.rados.0.smithi017.stdout:3240: left oid 24 (ObjNum 1218 snap 405 seq_num 1218) 2024-09-21T09:51:35.555 INFO:tasks.rados.rados.0.smithi017.stdout:3238: done (4 left) 2024-09-21T09:51:35.555 INFO:tasks.rados.rados.0.smithi017.stdout:3239: done (3 left) 2024-09-21T09:51:35.555 INFO:tasks.rados.rados.0.smithi017.stdout:3240: done (2 left) 2024-09-21T09:51:35.555 INFO:tasks.rados.rados.0.smithi017.stdout:3241: done (1 left) 2024-09-21T09:51:35.555 INFO:tasks.rados.rados.0.smithi017.stdout:3242: done (0 left) 2024-09-21T09:51:35.555 INFO:tasks.rados.rados.0.smithi017.stdout:3243: snap_remove snap 360 2024-09-21T09:51:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:35 smithi017 ceph-mon[102913]: pgmap v882: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-21T09:51:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:35 smithi017 ceph-mon[102913]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T09:51:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:35 smithi017 ceph-mon[105151]: pgmap v882: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-21T09:51:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:35 smithi017 ceph-mon[105151]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T09:51:36.244 INFO:tasks.rados.rados.0.smithi017.stdout:3243: done (0 left) 2024-09-21T09:51:36.244 INFO:tasks.rados.rados.0.smithi017.stdout:3244: write oid 44 current snap is 405 2024-09-21T09:51:36.245 INFO:tasks.rados.rados.0.smithi017.stdout:3244: seq_num 1220 ranges {504540=420260,1574973=704087} 2024-09-21T09:51:36.250 INFO:tasks.rados.rados.0.smithi017.stdout:3244: writing smithi01785587-44 from 504540 to 924800 tid 1 2024-09-21T09:51:36.259 INFO:tasks.rados.rados.0.smithi017.stdout:3244: writing smithi01785587-44 from 1574973 to 2279060 tid 2 2024-09-21T09:51:36.259 INFO:tasks.rados.rados.0.smithi017.stdout:3245: write oid 17 current snap is 405 2024-09-21T09:51:36.260 INFO:tasks.rados.rados.0.smithi017.stdout:3244: finishing write tid 1 to smithi01785587-44 2024-09-21T09:51:36.260 INFO:tasks.rados.rados.0.smithi017.stdout:3245: seq_num 1221 ranges {521347=495509,1633455=564098} 2024-09-21T09:51:36.265 INFO:tasks.rados.rados.0.smithi017.stdout:3245: writing smithi01785587-17 from 521347 to 1016856 tid 1 2024-09-21T09:51:36.272 INFO:tasks.rados.rados.0.smithi017.stdout:3245: writing smithi01785587-17 from 1633455 to 2197553 tid 2 2024-09-21T09:51:36.272 INFO:tasks.rados.rados.0.smithi017.stdout:3246: read oid 28 snap -1 2024-09-21T09:51:36.272 INFO:tasks.rados.rados.0.smithi017.stdout:3246: expect (ObjNum 1117 snap 368 seq_num 1117) 2024-09-21T09:51:36.272 INFO:tasks.rados.rados.0.smithi017.stdout:3247: write oid 21 current snap is 405 2024-09-21T09:51:36.272 INFO:tasks.rados.rados.0.smithi017.stdout:3244: finishing write tid 2 to smithi01785587-44 2024-09-21T09:51:36.272 INFO:tasks.rados.rados.0.smithi017.stdout:3244: finishing write tid 3 to smithi01785587-44 2024-09-21T09:51:36.273 INFO:tasks.rados.rados.0.smithi017.stdout:3244: finishing write tid 4 to smithi01785587-44 2024-09-21T09:51:36.273 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1724 (ObjNum 1220 snap 405 seq_num 1220) dirty exists 2024-09-21T09:51:36.273 INFO:tasks.rados.rados.0.smithi017.stdout:3244: left oid 44 (ObjNum 1220 snap 405 seq_num 1220) 2024-09-21T09:51:36.273 INFO:tasks.rados.rados.0.smithi017.stdout:3245: finishing write tid 1 to smithi01785587-17 2024-09-21T09:51:36.273 INFO:tasks.rados.rados.0.smithi017.stdout:3247: seq_num 1222 ranges {538154=570758,1691937=707756} 2024-09-21T09:51:36.278 INFO:tasks.rados.rados.0.smithi017.stdout:3247: writing smithi01785587-21 from 538154 to 1108912 tid 1 2024-09-21T09:51:36.285 INFO:tasks.rados.rados.0.smithi017.stdout:3247: writing smithi01785587-21 from 1691937 to 2399693 tid 2 2024-09-21T09:51:36.286 INFO:tasks.rados.rados.0.smithi017.stdout:3244: done (3 left) 2024-09-21T09:51:36.286 INFO:tasks.rados.rados.0.smithi017.stdout:3248: write oid 32 current snap is 405 2024-09-21T09:51:36.286 INFO:tasks.rados.rados.0.smithi017.stdout:3248: seq_num 1223 ranges {554961=646007,1750419=451414} 2024-09-21T09:51:36.292 INFO:tasks.rados.rados.0.smithi017.stdout:3248: writing smithi01785587-32 from 554961 to 1200968 tid 1 2024-09-21T09:51:36.298 INFO:tasks.rados.rados.0.smithi017.stdout:3248: writing smithi01785587-32 from 1750419 to 2201833 tid 2 2024-09-21T09:51:36.298 INFO:tasks.rados.rados.0.smithi017.stdout:3249: delete oid 9 current snap is 405 2024-09-21T09:51:36.298 INFO:tasks.rados.rados.0.smithi017.stdout:3245: finishing write tid 2 to smithi01785587-17 2024-09-21T09:51:36.298 INFO:tasks.rados.rados.0.smithi017.stdout:3245: finishing write tid 3 to smithi01785587-17 2024-09-21T09:51:36.299 INFO:tasks.rados.rados.0.smithi017.stdout:3245: finishing write tid 4 to smithi01785587-17 2024-09-21T09:51:36.299 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 1052 (ObjNum 1221 snap 405 seq_num 1221) dirty exists 2024-09-21T09:51:36.299 INFO:tasks.rados.rados.0.smithi017.stdout:3245: left oid 17 (ObjNum 1221 snap 405 seq_num 1221) 2024-09-21T09:51:36.299 INFO:tasks.rados.rados.0.smithi017.stdout:3246: expect (ObjNum 1117 snap 368 seq_num 1117) 2024-09-21T09:51:36.427 INFO:tasks.rados.rados.0.smithi017.stdout:3247: finishing write tid 1 to smithi01785587-21 2024-09-21T09:51:36.427 INFO:tasks.rados.rados.0.smithi017.stdout:3247: finishing write tid 2 to smithi01785587-21 2024-09-21T09:51:36.427 INFO:tasks.rados.rados.0.smithi017.stdout:3247: finishing write tid 3 to smithi01785587-21 2024-09-21T09:51:36.427 INFO:tasks.rados.rados.0.smithi017.stdout:3247: finishing write tid 4 to smithi01785587-21 2024-09-21T09:51:36.427 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1581 (ObjNum 1222 snap 405 seq_num 1222) dirty exists 2024-09-21T09:51:36.427 INFO:tasks.rados.rados.0.smithi017.stdout:3247: left oid 21 (ObjNum 1222 snap 405 seq_num 1222) 2024-09-21T09:51:36.427 INFO:tasks.rados.rados.0.smithi017.stdout:3248: finishing write tid 1 to smithi01785587-32 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3248: finishing write tid 2 to smithi01785587-32 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3248: finishing write tid 3 to smithi01785587-32 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3245: done (4 left) 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3246: done (3 left) 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3247: done (2 left) 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3249: done (1 left) 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3250: write oid 45 current snap is 405 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3248: finishing write tid 4 to smithi01785587-32 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1596 (ObjNum 1223 snap 405 seq_num 1223) dirty exists 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3248: left oid 32 (ObjNum 1223 snap 405 seq_num 1223) 2024-09-21T09:51:36.428 INFO:tasks.rados.rados.0.smithi017.stdout:3250: seq_num 1225 ranges {588575=512858,1867383=455083} 2024-09-21T09:51:36.433 INFO:tasks.rados.rados.0.smithi017.stdout:3250: writing smithi01785587-45 from 588575 to 1101433 tid 1 2024-09-21T09:51:36.439 INFO:tasks.rados.rados.0.smithi017.stdout:3250: writing smithi01785587-45 from 1867383 to 2322466 tid 2 2024-09-21T09:51:36.440 INFO:tasks.rados.rados.0.smithi017.stdout:3248: done (1 left) 2024-09-21T09:51:36.440 INFO:tasks.rados.rados.0.smithi017.stdout:3251: snap_remove snap 377 2024-09-21T09:51:36.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:36 smithi134 ceph-mon[90085]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T09:51:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:36 smithi017 ceph-mon[102913]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T09:51:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:36 smithi017 ceph-mon[105151]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T09:51:37.249 INFO:tasks.rados.rados.0.smithi017.stdout:3251: done (1 left) 2024-09-21T09:51:37.249 INFO:tasks.rados.rados.0.smithi017.stdout:3252: write oid 31 current snap is 405 2024-09-21T09:51:37.249 INFO:tasks.rados.rados.0.smithi017.stdout:3252: seq_num 1226 ranges {605382=588107,1925865=679517} 2024-09-21T09:51:37.257 INFO:tasks.rados.rados.0.smithi017.stdout:3252: writing smithi01785587-31 from 605382 to 1193489 tid 1 2024-09-21T09:51:37.265 INFO:tasks.rados.rados.0.smithi017.stdout:3252: writing smithi01785587-31 from 1925865 to 2605382 tid 2 2024-09-21T09:51:37.266 INFO:tasks.rados.rados.0.smithi017.stdout:3253: snap_remove snap 402 2024-09-21T09:51:37.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:37 smithi134 ceph-mon[90085]: pgmap v885: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 281 KiB/s wr, 6 op/s 2024-09-21T09:51:37.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:37 smithi134 ceph-mon[90085]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T09:51:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:37 smithi017 ceph-mon[102913]: pgmap v885: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 281 KiB/s wr, 6 op/s 2024-09-21T09:51:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:37 smithi017 ceph-mon[102913]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T09:51:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:37 smithi017 ceph-mon[105151]: pgmap v885: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 281 KiB/s wr, 6 op/s 2024-09-21T09:51:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:37 smithi017 ceph-mon[105151]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T09:51:38.190 INFO:tasks.rados.rados.0.smithi017.stdout:3250: finishing write tid 1 to smithi01785587-45 2024-09-21T09:51:38.190 INFO:tasks.rados.rados.0.smithi017.stdout:3250: finishing write tid 2 to smithi01785587-45 2024-09-21T09:51:38.190 INFO:tasks.rados.rados.0.smithi017.stdout:3250: finishing write tid 3 to smithi01785587-45 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3250: finishing write tid 4 to smithi01785587-45 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1056 (ObjNum 1225 snap 405 seq_num 1225) dirty exists 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3250: left oid 45 (ObjNum 1225 snap 405 seq_num 1225) 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3252: finishing write tid 1 to smithi01785587-31 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3252: finishing write tid 2 to smithi01785587-31 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3252: finishing write tid 3 to smithi01785587-31 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3252: finishing write tid 4 to smithi01785587-31 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1736 (ObjNum 1226 snap 405 seq_num 1226) dirty exists 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3252: left oid 31 (ObjNum 1226 snap 405 seq_num 1226) 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3250: done (2 left) 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3252: done (1 left) 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3253: done (0 left) 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3254: read oid 45 snap -1 2024-09-21T09:51:38.191 INFO:tasks.rados.rados.0.smithi017.stdout:3254: expect (ObjNum 1225 snap 405 seq_num 1225) 2024-09-21T09:51:38.192 INFO:tasks.rados.rados.0.smithi017.stdout:3255: write oid 12 current snap is 405 2024-09-21T09:51:38.192 INFO:tasks.rados.rados.0.smithi017.stdout:3255: seq_num 1227 ranges {622189=663356,1984347=458752} 2024-09-21T09:51:38.198 INFO:tasks.rados.rados.0.smithi017.stdout:3255: writing smithi01785587-12 from 622189 to 1285545 tid 1 2024-09-21T09:51:38.205 INFO:tasks.rados.rados.0.smithi017.stdout:3255: writing smithi01785587-12 from 1984347 to 2443099 tid 2 2024-09-21T09:51:38.205 INFO:tasks.rados.rados.0.smithi017.stdout:3256: rollback oid 7 current snap is 405 2024-09-21T09:51:38.205 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 371 2024-09-21T09:51:38.205 INFO:tasks.rados.rados.0.smithi017.stdout:3254: expect (ObjNum 1225 snap 405 seq_num 1225) 2024-09-21T09:51:38.425 INFO:tasks.rados.rados.0.smithi017.stdout:3255: finishing write tid 1 to smithi01785587-12 2024-09-21T09:51:38.425 INFO:tasks.rados.rados.0.smithi017.stdout:3255: finishing write tid 2 to smithi01785587-12 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:3255: finishing write tid 3 to smithi01785587-12 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:3254: done (2 left) 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:3257: read oid 4 snap -1 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:3257: expect (ObjNum 1119 snap 369 seq_num 1119) 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:3258: snap_create 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:3255: finishing write tid 4 to smithi01785587-12 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 1098 (ObjNum 1227 snap 405 seq_num 1227) dirty exists 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:3255: left oid 12 (ObjNum 1227 snap 405 seq_num 1227) 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:3256: finishing rollback tid 0 to smithi01785587-7 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:3256: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:51:38.426 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1588 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:38.437 INFO:tasks.rados.rados.0.smithi017.stdout:3257: expect (ObjNum 1119 snap 369 seq_num 1119) 2024-09-21T09:51:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:38 smithi017 ceph-mon[102913]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T09:51:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:38 smithi017 ceph-mon[102913]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T09:51:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:38 smithi017 ceph-mon[105151]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T09:51:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:38 smithi017 ceph-mon[105151]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T09:51:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:38 smithi134 ceph-mon[90085]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T09:51:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:38 smithi134 ceph-mon[90085]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T09:51:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3255: done (3 left) 2024-09-21T09:51:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3256: done (2 left) 2024-09-21T09:51:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3257: done (1 left) 2024-09-21T09:51:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3258: done (0 left) 2024-09-21T09:51:39.256 INFO:tasks.rados.rados.0.smithi017.stdout:3259: snap_create 2024-09-21T09:51:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:39 smithi017 ceph-mon[102913]: pgmap v888: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 756 KiB/s wr, 8 op/s 2024-09-21T09:51:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:39 smithi017 ceph-mon[105151]: pgmap v888: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 756 KiB/s wr, 8 op/s 2024-09-21T09:51:39.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:39 smithi134 ceph-mon[90085]: pgmap v888: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 756 KiB/s wr, 8 op/s 2024-09-21T09:51:40.266 INFO:tasks.rados.rados.0.smithi017.stdout:3259: done (0 left) 2024-09-21T09:51:40.267 INFO:tasks.rados.rados.0.smithi017.stdout:3260: read oid 12 snap -1 2024-09-21T09:51:40.267 INFO:tasks.rados.rados.0.smithi017.stdout:3260: expect (ObjNum 1227 snap 405 seq_num 1227) 2024-09-21T09:51:40.267 INFO:tasks.rados.rados.0.smithi017.stdout:3261: write oid 27 current snap is 407 2024-09-21T09:51:40.267 INFO:tasks.rados.rados.0.smithi017.stdout:3261: seq_num 1228 ranges {638996=738605,1926476=712520} 2024-09-21T09:51:40.274 INFO:tasks.rados.rados.0.smithi017.stdout:3261: writing smithi01785587-27 from 638996 to 1377601 tid 1 2024-09-21T09:51:40.281 INFO:tasks.rados.rados.0.smithi017.stdout:3261: writing smithi01785587-27 from 1926476 to 2638996 tid 2 2024-09-21T09:51:40.281 INFO:tasks.rados.rados.0.smithi017.stdout:3262: rollback oid 3 current snap is 407 2024-09-21T09:51:40.281 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 406 2024-09-21T09:51:40.281 INFO:tasks.rados.rados.0.smithi017.stdout:3260: expect (ObjNum 1227 snap 405 seq_num 1227) 2024-09-21T09:51:40.372 INFO:tasks.rados.rados.0.smithi017.stdout:3261: finishing write tid 1 to smithi01785587-27 2024-09-21T09:51:40.372 INFO:tasks.rados.rados.0.smithi017.stdout:3261: finishing write tid 2 to smithi01785587-27 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:3261: finishing write tid 3 to smithi01785587-27 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:3261: finishing write tid 4 to smithi01785587-27 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 1066 (ObjNum 1228 snap 407 seq_num 1228) dirty exists 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:3261: left oid 27 (ObjNum 1228 snap 407 seq_num 1228) 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:3262: finishing rollback tid 0 to smithi01785587-3 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:3262: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:3260: done (2 left) 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:3261: done (1 left) 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:3263: rollback oid 17 current snap is 407 2024-09-21T09:51:40.373 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 352 2024-09-21T09:51:40.374 INFO:tasks.rados.rados.0.smithi017.stdout:3262: finishing rollback tid 2 to smithi01785587-3 2024-09-21T09:51:40.374 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1744 (ObjNum 1211 snap 403 seq_num 1211) dirty exists 2024-09-21T09:51:40.374 INFO:tasks.rados.rados.0.smithi017.stdout:3262: done (1 left) 2024-09-21T09:51:40.374 INFO:tasks.rados.rados.0.smithi017.stdout:3264: snap_remove snap 400 2024-09-21T09:51:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:40 smithi134 ceph-mon[90085]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T09:51:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:51:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:51:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:51:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:40 smithi017 ceph-mon[102913]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T09:51:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:40 smithi017 ceph-mon[105151]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T09:51:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:41.273 INFO:tasks.rados.rados.0.smithi017.stdout:3264: done (1 left) 2024-09-21T09:51:41.273 INFO:tasks.rados.rados.0.smithi017.stdout:3265: read oid 49 snap -1 2024-09-21T09:51:41.274 INFO:tasks.rados.rados.0.smithi017.stdout:3265: expect (ObjNum 1106 snap 366 seq_num 1106) 2024-09-21T09:51:41.274 INFO:tasks.rados.rados.0.smithi017.stdout:3266: read oid 27 snap -1 2024-09-21T09:51:41.274 INFO:tasks.rados.rados.0.smithi017.stdout:3266: expect (ObjNum 1228 snap 407 seq_num 1228) 2024-09-21T09:51:41.274 INFO:tasks.rados.rados.0.smithi017.stdout:3267: rollback oid 39 current snap is 407 2024-09-21T09:51:41.274 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 404 2024-09-21T09:51:41.274 INFO:tasks.rados.rados.0.smithi017.stdout:3268: read oid 10 snap 403 2024-09-21T09:51:41.274 INFO:tasks.rados.rados.0.smithi017.stdout:3268: expect (ObjNum 1072 snap 358 seq_num 1072) 2024-09-21T09:51:41.274 INFO:tasks.rados.rados.0.smithi017.stdout:3269: read oid 21 snap -1 2024-09-21T09:51:41.274 INFO:tasks.rados.rados.0.smithi017.stdout:3269: expect (ObjNum 1222 snap 405 seq_num 1222) 2024-09-21T09:51:41.275 INFO:tasks.rados.rados.0.smithi017.stdout:3270: snap_create 2024-09-21T09:51:41.275 INFO:tasks.rados.rados.0.smithi017.stdout:3263: finishing rollback tid 0 to smithi01785587-17 2024-09-21T09:51:41.275 INFO:tasks.rados.rados.0.smithi017.stdout:3263: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:51:41.275 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 1069 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:51:41.284 INFO:tasks.rados.rados.0.smithi017.stdout:3267: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:51:41.285 INFO:tasks.rados.rados.0.smithi017.stdout:3268: expect (ObjNum 1072 snap 358 seq_num 1072) 2024-09-21T09:51:41.431 INFO:tasks.rados.rados.0.smithi017.stdout:3267: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:51:41.431 INFO:tasks.rados.rados.0.smithi017.stdout:3265: expect (ObjNum 1106 snap 366 seq_num 1106) 2024-09-21T09:51:41.553 INFO:tasks.rados.rados.0.smithi017.stdout:3266: expect (ObjNum 1228 snap 407 seq_num 1228) 2024-09-21T09:51:41.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:41 smithi017 ceph-mon[102913]: pgmap v891: 89 pgs: 89 active+clean; 285 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 970 KiB/s wr, 4 op/s 2024-09-21T09:51:41.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:41 smithi017 ceph-mon[102913]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T09:51:41.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:41 smithi017 ceph-mon[105151]: pgmap v891: 89 pgs: 89 active+clean; 285 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 970 KiB/s wr, 4 op/s 2024-09-21T09:51:41.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:41 smithi017 ceph-mon[105151]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T09:51:41.734 INFO:tasks.rados.rados.0.smithi017.stdout:3267: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:51:41.734 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1670 (ObjNum 1097 snap 363 seq_num 1097) dirty exists 2024-09-21T09:51:41.735 INFO:tasks.rados.rados.0.smithi017.stdout:3269: expect (ObjNum 1222 snap 405 seq_num 1222) 2024-09-21T09:51:41.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:41 smithi134 ceph-mon[90085]: pgmap v891: 89 pgs: 89 active+clean; 285 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 970 KiB/s wr, 4 op/s 2024-09-21T09:51:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:41 smithi134 ceph-mon[90085]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T09:51:42.280 INFO:tasks.rados.rados.0.smithi017.stdout:3263: done (6 left) 2024-09-21T09:51:42.280 INFO:tasks.rados.rados.0.smithi017.stdout:3265: done (5 left) 2024-09-21T09:51:42.281 INFO:tasks.rados.rados.0.smithi017.stdout:3266: done (4 left) 2024-09-21T09:51:42.281 INFO:tasks.rados.rados.0.smithi017.stdout:3267: done (3 left) 2024-09-21T09:51:42.281 INFO:tasks.rados.rados.0.smithi017.stdout:3268: done (2 left) 2024-09-21T09:51:42.281 INFO:tasks.rados.rados.0.smithi017.stdout:3269: done (1 left) 2024-09-21T09:51:42.281 INFO:tasks.rados.rados.0.smithi017.stdout:3270: done (0 left) 2024-09-21T09:51:42.281 INFO:tasks.rados.rados.0.smithi017.stdout:3271: delete oid 47 current snap is 408 2024-09-21T09:51:42.284 INFO:tasks.rados.rados.0.smithi017.stdout:3271: done (0 left) 2024-09-21T09:51:42.284 INFO:tasks.rados.rados.0.smithi017.stdout:3272: rollback oid 17 current snap is 408 2024-09-21T09:51:42.284 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 407 2024-09-21T09:51:42.284 INFO:tasks.rados.rados.0.smithi017.stdout:3273: read oid 18 snap -1 2024-09-21T09:51:42.284 INFO:tasks.rados.rados.0.smithi017.stdout:3273: expect (ObjNum 1205 snap 403 seq_num 1205) 2024-09-21T09:51:42.285 INFO:tasks.rados.rados.0.smithi017.stdout:3274: read oid 19 snap -1 2024-09-21T09:51:42.285 INFO:tasks.rados.rados.0.smithi017.stdout:3274: expect deleted 2024-09-21T09:51:42.285 INFO:tasks.rados.rados.0.smithi017.stdout:3275: read oid 23 snap -1 2024-09-21T09:51:42.285 INFO:tasks.rados.rados.0.smithi017.stdout:3275: expect (ObjNum 1185 snap 398 seq_num 1185) 2024-09-21T09:51:42.285 INFO:tasks.rados.rados.0.smithi017.stdout:3276: rollback oid 40 current snap is 408 2024-09-21T09:51:42.285 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 40 to 399 2024-09-21T09:51:42.285 INFO:tasks.rados.rados.0.smithi017.stdout:3277: snap_create 2024-09-21T09:51:42.291 INFO:tasks.rados.rados.0.smithi017.stdout:3272: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:51:42.291 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 1069 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:51:42.291 INFO:tasks.rados.rados.0.smithi017.stdout:3276: finishing rollback tid 0 to smithi01785587-40 2024-09-21T09:51:42.293 INFO:tasks.rados.rados.0.smithi017.stdout:3276: finishing rollback tid 1 to smithi01785587-40 2024-09-21T09:51:42.293 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1673 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:51:42.293 INFO:tasks.rados.rados.0.smithi017.stdout:3275: expect (ObjNum 1185 snap 398 seq_num 1185) 2024-09-21T09:51:42.424 INFO:tasks.rados.rados.0.smithi017.stdout:3273: expect (ObjNum 1205 snap 403 seq_num 1205) 2024-09-21T09:51:42.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:42 smithi017 ceph-mon[102913]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T09:51:42.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:42 smithi017 ceph-mon[105151]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T09:51:42.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:42 smithi134 ceph-mon[90085]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T09:51:43.189 INFO:tasks.rados.rados.0.smithi017.stdout:3272: done (5 left) 2024-09-21T09:51:43.189 INFO:tasks.rados.rados.0.smithi017.stdout:3273: done (4 left) 2024-09-21T09:51:43.189 INFO:tasks.rados.rados.0.smithi017.stdout:3274: done (3 left) 2024-09-21T09:51:43.189 INFO:tasks.rados.rados.0.smithi017.stdout:3275: done (2 left) 2024-09-21T09:51:43.189 INFO:tasks.rados.rados.0.smithi017.stdout:3276: done (1 left) 2024-09-21T09:51:43.189 INFO:tasks.rados.rados.0.smithi017.stdout:3277: done (0 left) 2024-09-21T09:51:43.189 INFO:tasks.rados.rados.0.smithi017.stdout:3278: write oid 17 current snap is 409 2024-09-21T09:51:43.189 INFO:tasks.rados.rados.0.smithi017.stdout:3278: seq_num 1230 ranges {672610=489103,1643440=722432} 2024-09-21T09:51:43.195 INFO:tasks.rados.rados.0.smithi017.stdout:3278: writing smithi01785587-17 from 672610 to 1161713 tid 1 2024-09-21T09:51:43.202 INFO:tasks.rados.rados.0.smithi017.stdout:3278: writing smithi01785587-17 from 1643440 to 2365872 tid 2 2024-09-21T09:51:43.202 INFO:tasks.rados.rados.0.smithi017.stdout:3279: write oid 21 current snap is 409 2024-09-21T09:51:43.202 INFO:tasks.rados.rados.0.smithi017.stdout:3279: seq_num 1231 ranges {689417=564352,1701922=466090} 2024-09-21T09:51:43.208 INFO:tasks.rados.rados.0.smithi017.stdout:3279: writing smithi01785587-21 from 689417 to 1253769 tid 1 2024-09-21T09:51:43.214 INFO:tasks.rados.rados.0.smithi017.stdout:3279: writing smithi01785587-21 from 1701922 to 2168012 tid 2 2024-09-21T09:51:43.214 INFO:tasks.rados.rados.0.smithi017.stdout:3280: write oid 47 current snap is 409 2024-09-21T09:51:43.214 INFO:tasks.rados.rados.0.smithi017.stdout:3278: finishing write tid 1 to smithi01785587-17 2024-09-21T09:51:43.214 INFO:tasks.rados.rados.0.smithi017.stdout:3280: seq_num 1232 ranges {706224=755954,2160404=545820} 2024-09-21T09:51:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:3280: writing smithi01785587-47 from 706224 to 1462178 tid 1 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3280: writing smithi01785587-47 from 2160404 to 2706224 tid 2 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3281: delete oid 5 current snap is 409 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3278: finishing write tid 2 to smithi01785587-17 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3278: finishing write tid 3 to smithi01785587-17 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3278: finishing write tid 4 to smithi01785587-17 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 1077 (ObjNum 1230 snap 409 seq_num 1230) dirty exists 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3278: left oid 17 (ObjNum 1230 snap 409 seq_num 1230) 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3279: finishing write tid 1 to smithi01785587-21 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3279: finishing write tid 2 to smithi01785587-21 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3279: finishing write tid 3 to smithi01785587-21 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3279: finishing write tid 4 to smithi01785587-21 2024-09-21T09:51:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1596 (ObjNum 1231 snap 409 seq_num 1231) dirty exists 2024-09-21T09:51:43.230 INFO:tasks.rados.rados.0.smithi017.stdout:3279: left oid 21 (ObjNum 1231 snap 409 seq_num 1231) 2024-09-21T09:51:43.230 INFO:tasks.rados.rados.0.smithi017.stdout:3280: finishing write tid 1 to smithi01785587-47 2024-09-21T09:51:43.232 INFO:tasks.rados.rados.0.smithi017.stdout:3280: finishing write tid 2 to smithi01785587-47 2024-09-21T09:51:43.232 INFO:tasks.rados.rados.0.smithi017.stdout:3280: finishing write tid 3 to smithi01785587-47 2024-09-21T09:51:43.232 INFO:tasks.rados.rados.0.smithi017.stdout:3280: finishing write tid 4 to smithi01785587-47 2024-09-21T09:51:43.233 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 1105 (ObjNum 1232 snap 409 seq_num 1232) dirty exists 2024-09-21T09:51:43.233 INFO:tasks.rados.rados.0.smithi017.stdout:3280: left oid 47 (ObjNum 1232 snap 409 seq_num 1232) 2024-09-21T09:51:43.234 INFO:tasks.rados.rados.0.smithi017.stdout:3278: done (3 left) 2024-09-21T09:51:43.234 INFO:tasks.rados.rados.0.smithi017.stdout:3279: done (2 left) 2024-09-21T09:51:43.234 INFO:tasks.rados.rados.0.smithi017.stdout:3280: done (1 left) 2024-09-21T09:51:43.234 INFO:tasks.rados.rados.0.smithi017.stdout:3281: done (0 left) 2024-09-21T09:51:43.234 INFO:tasks.rados.rados.0.smithi017.stdout:3282: delete oid 11 current snap is 409 2024-09-21T09:51:43.236 INFO:tasks.rados.rados.0.smithi017.stdout:3282: done (0 left) 2024-09-21T09:51:43.236 INFO:tasks.rados.rados.0.smithi017.stdout:3283: snap_create 2024-09-21T09:51:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:43 smithi017 ceph-mon[102913]: pgmap v894: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 287 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 21 op/s 2024-09-21T09:51:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:43 smithi017 ceph-mon[102913]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T09:51:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:43 smithi017 ceph-mon[102913]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T09:51:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:43 smithi017 ceph-mon[105151]: pgmap v894: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 287 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 21 op/s 2024-09-21T09:51:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:43 smithi017 ceph-mon[105151]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T09:51:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:43 smithi017 ceph-mon[105151]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T09:51:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:43 smithi134 ceph-mon[90085]: pgmap v894: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 287 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.8 MiB/s wr, 21 op/s 2024-09-21T09:51:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:43 smithi134 ceph-mon[90085]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T09:51:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:43 smithi134 ceph-mon[90085]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T09:51:44.189 INFO:tasks.rados.rados.0.smithi017.stdout:3283: done (0 left) 2024-09-21T09:51:44.190 INFO:tasks.rados.rados.0.smithi017.stdout:3284: snap_create 2024-09-21T09:51:45.193 INFO:tasks.rados.rados.0.smithi017.stdout:3284: done (0 left) 2024-09-21T09:51:45.193 INFO:tasks.rados.rados.0.smithi017.stdout:3285: read oid 26 snap -1 2024-09-21T09:51:45.193 INFO:tasks.rados.rados.0.smithi017.stdout:3285: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:51:45.193 INFO:tasks.rados.rados.0.smithi017.stdout:3286: snap_remove snap 371 2024-09-21T09:51:45.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:51:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:51:44] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.51.0" 2024-09-21T09:51:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:45 smithi017 ceph-mon[102913]: pgmap v897: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 289 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-09-21T09:51:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:45 smithi017 ceph-mon[102913]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T09:51:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:45 smithi017 ceph-mon[105151]: pgmap v897: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 289 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-09-21T09:51:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:45 smithi017 ceph-mon[105151]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T09:51:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:45 smithi134 ceph-mon[90085]: pgmap v897: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 289 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-09-21T09:51:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:45 smithi134 ceph-mon[90085]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T09:51:46.206 INFO:tasks.rados.rados.0.smithi017.stdout:3286: done (1 left) 2024-09-21T09:51:46.206 INFO:tasks.rados.rados.0.smithi017.stdout:3287: write oid 35 current snap is 411 2024-09-21T09:51:46.206 INFO:tasks.rados.rados.0.smithi017.stdout:3287: seq_num 1235 ranges {756645=581701,1935850=473428} 2024-09-21T09:51:46.214 INFO:tasks.rados.rados.0.smithi017.stdout:3287: writing smithi01785587-35 from 756645 to 1338346 tid 1 2024-09-21T09:51:46.221 INFO:tasks.rados.rados.0.smithi017.stdout:3287: writing smithi01785587-35 from 1935850 to 2409278 tid 2 2024-09-21T09:51:46.221 INFO:tasks.rados.rados.0.smithi017.stdout:3288: rollback oid 46 current snap is 411 2024-09-21T09:51:46.221 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 399 2024-09-21T09:51:46.221 INFO:tasks.rados.rados.0.smithi017.stdout:3285: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3287: finishing write tid 1 to smithi01785587-35 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3287: finishing write tid 2 to smithi01785587-35 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3287: finishing write tid 3 to smithi01785587-35 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3287: finishing write tid 4 to smithi01785587-35 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 368 (ObjNum 1235 snap 411 seq_num 1235) dirty exists 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3287: left oid 35 (ObjNum 1235 snap 411 seq_num 1235) 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3288: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3288: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3288: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1626 (ObjNum 1153 snap 385 seq_num 1153) dirty exists 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3285: done (2 left) 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3287: done (1 left) 2024-09-21T09:51:46.382 INFO:tasks.rados.rados.0.smithi017.stdout:3288: done (0 left) 2024-09-21T09:51:46.383 INFO:tasks.rados.rados.0.smithi017.stdout:3289: rollback oid 46 current snap is 411 2024-09-21T09:51:46.383 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 46 to 386 2024-09-21T09:51:46.383 INFO:tasks.rados.rados.0.smithi017.stdout:3290: delete oid 8 current snap is 411 2024-09-21T09:51:46.385 INFO:tasks.rados.rados.0.smithi017.stdout:3289: finishing rollback tid 0 to smithi01785587-46 2024-09-21T09:51:46.385 INFO:tasks.rados.rados.0.smithi017.stdout:3289: finishing rollback tid 1 to smithi01785587-46 2024-09-21T09:51:46.385 INFO:tasks.rados.rados.0.smithi017.stdout:3289: finishing rollback tid 2 to smithi01785587-46 2024-09-21T09:51:46.385 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1629 (ObjNum 1153 snap 385 seq_num 1153) dirty exists 2024-09-21T09:51:46.386 INFO:tasks.rados.rados.0.smithi017.stdout:3289: done (1 left) 2024-09-21T09:51:46.386 INFO:tasks.rados.rados.0.smithi017.stdout:3290: done (0 left) 2024-09-21T09:51:46.386 INFO:tasks.rados.rados.0.smithi017.stdout:3291: snap_remove snap 393 2024-09-21T09:51:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:46 smithi017 ceph-mon[102913]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T09:51:46.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:46 smithi017 ceph-mon[105151]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T09:51:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:46 smithi134 ceph-mon[90085]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T09:51:47.205 INFO:tasks.rados.rados.0.smithi017.stdout:3291: done (0 left) 2024-09-21T09:51:47.205 INFO:tasks.rados.rados.0.smithi017.stdout:3292: snap_remove snap 386 2024-09-21T09:51:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:47 smithi134 ceph-mon[90085]: pgmap v900: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 287 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 354 KiB/s wr, 11 op/s 2024-09-21T09:51:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:47 smithi134 ceph-mon[90085]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T09:51:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:47 smithi017 ceph-mon[102913]: pgmap v900: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 287 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 354 KiB/s wr, 11 op/s 2024-09-21T09:51:47.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:47 smithi017 ceph-mon[102913]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T09:51:47.719 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:47 smithi017 ceph-mon[105151]: pgmap v900: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 287 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.4 MiB/s rd, 354 KiB/s wr, 11 op/s 2024-09-21T09:51:47.719 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:47 smithi017 ceph-mon[105151]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T09:51:48.192 INFO:tasks.rados.rados.0.smithi017.stdout:3292: done (0 left) 2024-09-21T09:51:48.192 INFO:tasks.rados.rados.0.smithi017.stdout:3293: snap_remove snap 399 2024-09-21T09:51:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:48 smithi134 ceph-mon[90085]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T09:51:48.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:48 smithi134 ceph-mon[90085]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T09:51:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:48 smithi017 ceph-mon[102913]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T09:51:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:48 smithi017 ceph-mon[102913]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T09:51:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:48 smithi017 ceph-mon[105151]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T09:51:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:48 smithi017 ceph-mon[105151]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T09:51:49.202 INFO:tasks.rados.rados.0.smithi017.stdout:3293: done (0 left) 2024-09-21T09:51:49.202 INFO:tasks.rados.rados.0.smithi017.stdout:3294: rollback oid 5 current snap is 411 2024-09-21T09:51:49.202 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 407 2024-09-21T09:51:49.202 INFO:tasks.rados.rados.0.smithi017.stdout:3295: rollback oid 33 current snap is 411 2024-09-21T09:51:49.202 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 405 2024-09-21T09:51:49.202 INFO:tasks.rados.rados.0.smithi017.stdout:3296: rollback oid 15 current snap is 411 2024-09-21T09:51:49.202 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 406 2024-09-21T09:51:49.202 INFO:tasks.rados.rados.0.smithi017.stdout:3297: snap_remove snap 410 2024-09-21T09:51:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:49 smithi134 ceph-mon[90085]: pgmap v903: 89 pgs: 89 active+clean; 291 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 570 KiB/s wr, 13 op/s 2024-09-21T09:51:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:49 smithi134 ceph-mon[90085]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T09:51:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:49 smithi017 ceph-mon[102913]: pgmap v903: 89 pgs: 89 active+clean; 291 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 570 KiB/s wr, 13 op/s 2024-09-21T09:51:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:49 smithi017 ceph-mon[102913]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T09:51:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:49 smithi017 ceph-mon[105151]: pgmap v903: 89 pgs: 89 active+clean; 291 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 570 KiB/s wr, 13 op/s 2024-09-21T09:51:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:49 smithi017 ceph-mon[105151]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T09:51:50.202 INFO:tasks.rados.rados.0.smithi017.stdout:3297: done (3 left) 2024-09-21T09:51:50.202 INFO:tasks.rados.rados.0.smithi017.stdout:3298: write oid 12 current snap is 411 2024-09-21T09:51:50.202 INFO:tasks.rados.rados.0.smithi017.stdout:3296: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:51:50.202 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:51:50.202 INFO:tasks.rados.rados.0.smithi017.stdout:3294: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:51:50.202 INFO:tasks.rados.rados.0.smithi017.stdout:3294: finishing rollback tid 2 to smithi01785587-5 2024-09-21T09:51:50.202 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1774 (ObjNum 949 snap 321 seq_num 949) dirty exists 2024-09-21T09:51:50.203 INFO:tasks.rados.rados.0.smithi017.stdout:3295: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:51:50.203 INFO:tasks.rados.rados.0.smithi017.stdout:3295: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:51:50.203 INFO:tasks.rados.rados.0.smithi017.stdout:3295: finishing rollback tid 2 to smithi01785587-33 2024-09-21T09:51:50.203 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1703 (ObjNum 1044 snap 350 seq_num 1044) dirty exists 2024-09-21T09:51:50.203 INFO:tasks.rados.rados.0.smithi017.stdout:3298: seq_num 1237 ranges {790259=732199,1936461=477097} 2024-09-21T09:51:50.210 INFO:tasks.rados.rados.0.smithi017.stdout:3298: writing smithi01785587-12 from 790259 to 1522458 tid 1 2024-09-21T09:51:50.216 INFO:tasks.rados.rados.0.smithi017.stdout:3298: writing smithi01785587-12 from 1936461 to 2413558 tid 2 2024-09-21T09:51:50.216 INFO:tasks.rados.rados.0.smithi017.stdout:3294: done (3 left) 2024-09-21T09:51:50.216 INFO:tasks.rados.rados.0.smithi017.stdout:3295: done (2 left) 2024-09-21T09:51:50.216 INFO:tasks.rados.rados.0.smithi017.stdout:3296: done (1 left) 2024-09-21T09:51:50.216 INFO:tasks.rados.rados.0.smithi017.stdout:3299: snap_remove snap 408 2024-09-21T09:51:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:51:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:51:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:51:51.219 INFO:tasks.rados.rados.0.smithi017.stdout:3299: done (1 left) 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3300: rollback oid 44 current snap is 411 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 409 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3298: finishing write tid 1 to smithi01785587-12 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3298: finishing write tid 2 to smithi01785587-12 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3298: finishing write tid 3 to smithi01785587-12 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3298: finishing write tid 4 to smithi01785587-12 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 1125 (ObjNum 1237 snap 411 seq_num 1237) dirty exists 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3298: left oid 12 (ObjNum 1237 snap 411 seq_num 1237) 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3298: done (1 left) 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3301: rollback oid 39 current snap is 411 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 409 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3302: write oid 29 current snap is 411 2024-09-21T09:51:51.220 INFO:tasks.rados.rados.0.smithi017.stdout:3302: seq_num 1238 ranges {407066=407448,1594943=620755} 2024-09-21T09:51:51.224 INFO:tasks.rados.rados.0.smithi017.stdout:3302: writing smithi01785587-29 from 407066 to 814514 tid 1 2024-09-21T09:51:51.234 INFO:tasks.rados.rados.0.smithi017.stdout:3302: writing smithi01785587-29 from 1594943 to 2215698 tid 2 2024-09-21T09:51:51.234 INFO:tasks.rados.rados.0.smithi017.stdout:3303: delete oid 45 current snap is 411 2024-09-21T09:51:51.234 INFO:tasks.rados.rados.0.smithi017.stdout:3302: finishing write tid 1 to smithi01785587-29 2024-09-21T09:51:51.234 INFO:tasks.rados.rados.0.smithi017.stdout:3301: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:51:51.235 INFO:tasks.rados.rados.0.smithi017.stdout:3301: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:51:51.235 INFO:tasks.rados.rados.0.smithi017.stdout:3301: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:51:51.235 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1695 (ObjNum 1097 snap 363 seq_num 1097) dirty exists 2024-09-21T09:51:51.235 INFO:tasks.rados.rados.0.smithi017.stdout:3300: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:51:51.235 INFO:tasks.rados.rados.0.smithi017.stdout:3300: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:51:51.235 INFO:tasks.rados.rados.0.smithi017.stdout:3300: finishing rollback tid 2 to smithi01785587-44 2024-09-21T09:51:51.235 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1784 (ObjNum 1220 snap 405 seq_num 1220) dirty exists 2024-09-21T09:51:51.237 INFO:tasks.rados.rados.0.smithi017.stdout:3300: done (3 left) 2024-09-21T09:51:51.237 INFO:tasks.rados.rados.0.smithi017.stdout:3301: done (2 left) 2024-09-21T09:51:51.237 INFO:tasks.rados.rados.0.smithi017.stdout:3303: done (1 left) 2024-09-21T09:51:51.237 INFO:tasks.rados.rados.0.smithi017.stdout:3304: write oid 27 current snap is 411 2024-09-21T09:51:51.237 INFO:tasks.rados.rados.0.smithi017.stdout:3304: seq_num 1240 ranges {440680=674299,1711907=624424} 2024-09-21T09:51:51.243 INFO:tasks.rados.rados.0.smithi017.stdout:3304: writing smithi01785587-27 from 440680 to 1114979 tid 1 2024-09-21T09:51:51.250 INFO:tasks.rados.rados.0.smithi017.stdout:3304: writing smithi01785587-27 from 1711907 to 2336331 tid 2 2024-09-21T09:51:51.250 INFO:tasks.rados.rados.0.smithi017.stdout:3305: delete oid 7 current snap is 411 2024-09-21T09:51:51.250 INFO:tasks.rados.rados.0.smithi017.stdout:3302: finishing write tid 2 to smithi01785587-29 2024-09-21T09:51:51.250 INFO:tasks.rados.rados.0.smithi017.stdout:3302: finishing write tid 3 to smithi01785587-29 2024-09-21T09:51:51.250 INFO:tasks.rados.rados.0.smithi017.stdout:3302: finishing write tid 4 to smithi01785587-29 2024-09-21T09:51:51.250 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 881 (ObjNum 1238 snap 411 seq_num 1238) dirty exists 2024-09-21T09:51:51.250 INFO:tasks.rados.rados.0.smithi017.stdout:3302: left oid 29 (ObjNum 1238 snap 411 seq_num 1238) 2024-09-21T09:51:51.250 INFO:tasks.rados.rados.0.smithi017.stdout:3304: finishing write tid 1 to smithi01785587-27 2024-09-21T09:51:51.252 INFO:tasks.rados.rados.0.smithi017.stdout:3302: done (2 left) 2024-09-21T09:51:51.252 INFO:tasks.rados.rados.0.smithi017.stdout:3305: done (1 left) 2024-09-21T09:51:51.252 INFO:tasks.rados.rados.0.smithi017.stdout:3306: snap_remove snap 404 2024-09-21T09:51:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:51 smithi017 ceph-mon[102913]: pgmap v906: 89 pgs: 89 active+clean; 293 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 866 KiB/s wr, 9 op/s 2024-09-21T09:51:51.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:51 smithi017 ceph-mon[102913]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T09:51:51.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:51 smithi017 ceph-mon[105151]: pgmap v906: 89 pgs: 89 active+clean; 293 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 866 KiB/s wr, 9 op/s 2024-09-21T09:51:51.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:51 smithi017 ceph-mon[105151]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T09:51:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:51 smithi134 ceph-mon[90085]: pgmap v906: 89 pgs: 89 active+clean; 293 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 866 KiB/s wr, 9 op/s 2024-09-21T09:51:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:51 smithi134 ceph-mon[90085]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T09:51:52.225 INFO:tasks.rados.rados.0.smithi017.stdout:3306: done (1 left) 2024-09-21T09:51:52.226 INFO:tasks.rados.rados.0.smithi017.stdout:3307: snap_remove snap 407 2024-09-21T09:51:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:52 smithi134 ceph-mon[90085]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T09:51:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:52 smithi017 ceph-mon[102913]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T09:51:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:52 smithi017 ceph-mon[105151]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T09:51:53.193 INFO:tasks.rados.rados.0.smithi017.stdout:3307: done (1 left) 2024-09-21T09:51:53.194 INFO:tasks.rados.rados.0.smithi017.stdout:3308: snap_create 2024-09-21T09:51:53.194 INFO:tasks.rados.rados.0.smithi017.stdout:3304: finishing write tid 2 to smithi01785587-27 2024-09-21T09:51:53.194 INFO:tasks.rados.rados.0.smithi017.stdout:3304: finishing write tid 3 to smithi01785587-27 2024-09-21T09:51:53.194 INFO:tasks.rados.rados.0.smithi017.stdout:3304: finishing write tid 4 to smithi01785587-27 2024-09-21T09:51:53.194 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 1099 (ObjNum 1240 snap 411 seq_num 1240) dirty exists 2024-09-21T09:51:53.194 INFO:tasks.rados.rados.0.smithi017.stdout:3304: left oid 27 (ObjNum 1240 snap 411 seq_num 1240) 2024-09-21T09:51:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:53 smithi134 ceph-mon[90085]: pgmap v909: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 849 KiB/s wr, 5 op/s 2024-09-21T09:51:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:53 smithi134 ceph-mon[90085]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T09:51:53.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:53 smithi134 ceph-mon[90085]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T09:51:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:53 smithi017 ceph-mon[102913]: pgmap v909: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 849 KiB/s wr, 5 op/s 2024-09-21T09:51:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:53 smithi017 ceph-mon[102913]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T09:51:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:53 smithi017 ceph-mon[102913]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T09:51:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:53 smithi017 ceph-mon[105151]: pgmap v909: 89 pgs: 89 active+clean; 283 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 849 KiB/s wr, 5 op/s 2024-09-21T09:51:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:53 smithi017 ceph-mon[105151]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T09:51:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:53 smithi017 ceph-mon[105151]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T09:51:54.194 INFO:tasks.rados.rados.0.smithi017.stdout:3304: done (1 left) 2024-09-21T09:51:54.194 INFO:tasks.rados.rados.0.smithi017.stdout:3308: done (0 left) 2024-09-21T09:51:54.194 INFO:tasks.rados.rados.0.smithi017.stdout:3309: snap_create 2024-09-21T09:51:55.198 INFO:tasks.rados.rados.0.smithi017.stdout:3309: done (0 left) 2024-09-21T09:51:55.198 INFO:tasks.rados.rados.0.smithi017.stdout:3310: rollback oid 7 current snap is 413 2024-09-21T09:51:55.198 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 411 2024-09-21T09:51:55.198 INFO:tasks.rados.rados.0.smithi017.stdout:3311: read oid 21 snap -1 2024-09-21T09:51:55.198 INFO:tasks.rados.rados.0.smithi017.stdout:3311: expect (ObjNum 1231 snap 409 seq_num 1231) 2024-09-21T09:51:55.198 INFO:tasks.rados.rados.0.smithi017.stdout:3312: read oid 27 snap -1 2024-09-21T09:51:55.198 INFO:tasks.rados.rados.0.smithi017.stdout:3312: expect (ObjNum 1240 snap 411 seq_num 1240) 2024-09-21T09:51:55.198 INFO:tasks.rados.rados.0.smithi017.stdout:3313: write oid 15 current snap is 413 2024-09-21T09:51:55.199 INFO:tasks.rados.rados.0.smithi017.stdout:3313: seq_num 1242 ranges {474294=424797,1428871=628093,2657357=216937} 2024-09-21T09:51:55.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:51:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:51:54] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.51.0" 2024-09-21T09:51:55.203 INFO:tasks.rados.rados.0.smithi017.stdout:3313: writing smithi01785587-15 from 474294 to 899091 tid 1 2024-09-21T09:51:55.211 INFO:tasks.rados.rados.0.smithi017.stdout:3313: writing smithi01785587-15 from 1428871 to 2056964 tid 2 2024-09-21T09:51:55.215 INFO:tasks.rados.rados.0.smithi017.stdout:3313: writing smithi01785587-15 from 2657357 to 2874294 tid 3 2024-09-21T09:51:55.216 INFO:tasks.rados.rados.0.smithi017.stdout:3314: snap_create 2024-09-21T09:51:55.216 INFO:tasks.rados.rados.0.smithi017.stdout:3310: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:51:55.216 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1588 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:55.216 INFO:tasks.rados.rados.0.smithi017.stdout:3311: expect (ObjNum 1231 snap 409 seq_num 1231) 2024-09-21T09:51:55.382 INFO:tasks.rados.rados.0.smithi017.stdout:3312: expect (ObjNum 1240 snap 411 seq_num 1240) 2024-09-21T09:51:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:55 smithi134 ceph-mon[90085]: pgmap v912: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 871 KiB/s wr, 6 op/s 2024-09-21T09:51:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:55 smithi134 ceph-mon[90085]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T09:51:55.611 INFO:tasks.rados.rados.0.smithi017.stdout:3313: finishing write tid 1 to smithi01785587-15 2024-09-21T09:51:55.611 INFO:tasks.rados.rados.0.smithi017.stdout:3313: finishing write tid 2 to smithi01785587-15 2024-09-21T09:51:55.611 INFO:tasks.rados.rados.0.smithi017.stdout:3313: finishing write tid 3 to smithi01785587-15 2024-09-21T09:51:55.611 INFO:tasks.rados.rados.0.smithi017.stdout:3313: finishing write tid 4 to smithi01785587-15 2024-09-21T09:51:55.611 INFO:tasks.rados.rados.0.smithi017.stdout:3313: finishing write tid 5 to smithi01785587-15 2024-09-21T09:51:55.611 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 526 (ObjNum 1242 snap 413 seq_num 1242) dirty exists 2024-09-21T09:51:55.611 INFO:tasks.rados.rados.0.smithi017.stdout:3313: left oid 15 (ObjNum 1242 snap 413 seq_num 1242) 2024-09-21T09:51:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:55 smithi017 ceph-mon[105151]: pgmap v912: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 871 KiB/s wr, 6 op/s 2024-09-21T09:51:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:55 smithi017 ceph-mon[105151]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T09:51:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:55 smithi017 ceph-mon[102913]: pgmap v912: 89 pgs: 89 active+clean; 284 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 871 KiB/s wr, 6 op/s 2024-09-21T09:51:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:55 smithi017 ceph-mon[102913]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T09:51:56.209 INFO:tasks.rados.rados.0.smithi017.stdout:3310: done (4 left) 2024-09-21T09:51:56.209 INFO:tasks.rados.rados.0.smithi017.stdout:3311: done (3 left) 2024-09-21T09:51:56.209 INFO:tasks.rados.rados.0.smithi017.stdout:3312: done (2 left) 2024-09-21T09:51:56.209 INFO:tasks.rados.rados.0.smithi017.stdout:3313: done (1 left) 2024-09-21T09:51:56.209 INFO:tasks.rados.rados.0.smithi017.stdout:3314: done (0 left) 2024-09-21T09:51:56.209 INFO:tasks.rados.rados.0.smithi017.stdout:3315: write oid 20 current snap is 414 2024-09-21T09:51:56.209 INFO:tasks.rados.rados.0.smithi017.stdout:3315: seq_num 1243 ranges {491101=500046,1487353=488104,2401133=489968} 2024-09-21T09:51:56.214 INFO:tasks.rados.rados.0.smithi017.stdout:3315: writing smithi01785587-20 from 491101 to 991147 tid 1 2024-09-21T09:51:56.220 INFO:tasks.rados.rados.0.smithi017.stdout:3315: writing smithi01785587-20 from 1487353 to 1975457 tid 2 2024-09-21T09:51:56.225 INFO:tasks.rados.rados.0.smithi017.stdout:3315: writing smithi01785587-20 from 2401133 to 2891101 tid 3 2024-09-21T09:51:56.225 INFO:tasks.rados.rados.0.smithi017.stdout:3316: read oid 32 snap -1 2024-09-21T09:51:56.225 INFO:tasks.rados.rados.0.smithi017.stdout:3316: expect (ObjNum 1223 snap 405 seq_num 1223) 2024-09-21T09:51:56.225 INFO:tasks.rados.rados.0.smithi017.stdout:3317: snap_create 2024-09-21T09:51:56.225 INFO:tasks.rados.rados.0.smithi017.stdout:3315: finishing write tid 1 to smithi01785587-20 2024-09-21T09:51:56.225 INFO:tasks.rados.rados.0.smithi017.stdout:3315: finishing write tid 2 to smithi01785587-20 2024-09-21T09:51:56.228 INFO:tasks.rados.rados.0.smithi017.stdout:3315: finishing write tid 3 to smithi01785587-20 2024-09-21T09:51:56.228 INFO:tasks.rados.rados.0.smithi017.stdout:3315: finishing write tid 4 to smithi01785587-20 2024-09-21T09:51:56.228 INFO:tasks.rados.rados.0.smithi017.stdout:3315: finishing write tid 5 to smithi01785587-20 2024-09-21T09:51:56.229 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 886 (ObjNum 1243 snap 414 seq_num 1243) dirty exists 2024-09-21T09:51:56.229 INFO:tasks.rados.rados.0.smithi017.stdout:3315: left oid 20 (ObjNum 1243 snap 414 seq_num 1243) 2024-09-21T09:51:56.231 INFO:tasks.rados.rados.0.smithi017.stdout:3316: expect (ObjNum 1223 snap 405 seq_num 1223) 2024-09-21T09:51:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:56 smithi134 ceph-mon[90085]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T09:51:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:56 smithi017 ceph-mon[105151]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T09:51:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:56 smithi017 ceph-mon[102913]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T09:51:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:57.214 INFO:tasks.rados.rados.0.smithi017.stdout:3315: done (2 left) 2024-09-21T09:51:57.214 INFO:tasks.rados.rados.0.smithi017.stdout:3316: done (1 left) 2024-09-21T09:51:57.214 INFO:tasks.rados.rados.0.smithi017.stdout:3317: done (0 left) 2024-09-21T09:51:57.214 INFO:tasks.rados.rados.0.smithi017.stdout:3318: rollback oid 49 current snap is 415 2024-09-21T09:51:57.215 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 49 to 413 2024-09-21T09:51:57.215 INFO:tasks.rados.rados.0.smithi017.stdout:3319: delete oid 35 current snap is 415 2024-09-21T09:51:57.219 INFO:tasks.rados.rados.0.smithi017.stdout:3319: done (1 left) 2024-09-21T09:51:57.219 INFO:tasks.rados.rados.0.smithi017.stdout:3320: rollback oid 15 current snap is 415 2024-09-21T09:51:57.219 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 394 2024-09-21T09:51:57.219 INFO:tasks.rados.rados.0.smithi017.stdout:3321: snap_create 2024-09-21T09:51:57.219 INFO:tasks.rados.rados.0.smithi017.stdout:3318: finishing rollback tid 0 to smithi01785587-49 2024-09-21T09:51:57.219 INFO:tasks.rados.rados.0.smithi017.stdout:3318: finishing rollback tid 1 to smithi01785587-49 2024-09-21T09:51:57.220 INFO:tasks.rados.rados.0.smithi017.stdout:3318: finishing rollback tid 2 to smithi01785587-49 2024-09-21T09:51:57.220 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1626 (ObjNum 1106 snap 366 seq_num 1106) dirty exists 2024-09-21T09:51:57.223 INFO:tasks.rados.rados.0.smithi017.stdout:3320: finishing rollback tid 0 to smithi01785587-15 2024-09-21T09:51:57.223 INFO:tasks.rados.rados.0.smithi017.stdout:3320: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:51:57.223 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 529 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:57 smithi134 ceph-mon[90085]: pgmap v915: 89 pgs: 89 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 879 KiB/s wr, 5 op/s 2024-09-21T09:51:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:57 smithi134 ceph-mon[90085]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T09:51:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:57 smithi017 ceph-mon[102913]: pgmap v915: 89 pgs: 89 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 879 KiB/s wr, 5 op/s 2024-09-21T09:51:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:57 smithi017 ceph-mon[102913]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T09:51:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:57 smithi017 ceph-mon[105151]: pgmap v915: 89 pgs: 89 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 879 KiB/s wr, 5 op/s 2024-09-21T09:51:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:57 smithi017 ceph-mon[105151]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T09:51:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:3318: done (2 left) 2024-09-21T09:51:58.194 INFO:tasks.rados.rados.0.smithi017.stdout:3320: done (1 left) 2024-09-21T09:51:58.195 INFO:tasks.rados.rados.0.smithi017.stdout:3321: done (0 left) 2024-09-21T09:51:58.195 INFO:tasks.rados.rados.0.smithi017.stdout:3322: snap_create 2024-09-21T09:51:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:58 smithi134 ceph-mon[90085]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T09:51:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:58 smithi134 ceph-mon[90085]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T09:51:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:58 smithi017 ceph-mon[102913]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T09:51:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:58 smithi017 ceph-mon[102913]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T09:51:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:58 smithi017 ceph-mon[105151]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T09:51:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:58 smithi017 ceph-mon[105151]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T09:51:59.196 INFO:tasks.rados.rados.0.smithi017.stdout:3322: done (0 left) 2024-09-21T09:51:59.196 INFO:tasks.rados.rados.0.smithi017.stdout:3323: snap_remove snap 405 2024-09-21T09:51:59.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:59 smithi134 ceph-mon[90085]: pgmap v918: 89 pgs: 89 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 561 KiB/s wr, 7 op/s 2024-09-21T09:51:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:51:59 smithi134 ceph-mon[90085]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T09:51:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:59 smithi017 ceph-mon[102913]: pgmap v918: 89 pgs: 89 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 561 KiB/s wr, 7 op/s 2024-09-21T09:51:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:51:59 smithi017 ceph-mon[102913]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T09:51:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:59 smithi017 ceph-mon[105151]: pgmap v918: 89 pgs: 89 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 561 KiB/s wr, 7 op/s 2024-09-21T09:51:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:51:59 smithi017 ceph-mon[105151]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T09:52:00.199 INFO:tasks.rados.rados.0.smithi017.stdout:3323: done (0 left) 2024-09-21T09:52:00.199 INFO:tasks.rados.rados.0.smithi017.stdout:3324: snap_remove snap 416 2024-09-21T09:52:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:52:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:52:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:52:01.201 INFO:tasks.rados.rados.0.smithi017.stdout:3324: done (0 left) 2024-09-21T09:52:01.201 INFO:tasks.rados.rados.0.smithi017.stdout:3325: rollback oid 50 current snap is 417 2024-09-21T09:52:01.201 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 50 to 374 2024-09-21T09:52:01.201 INFO:tasks.rados.rados.0.smithi017.stdout:3326: rollback oid 12 current snap is 417 2024-09-21T09:52:01.201 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 12 to 394 2024-09-21T09:52:01.201 INFO:tasks.rados.rados.0.smithi017.stdout:3327: delete oid 33 current snap is 417 2024-09-21T09:52:01.207 INFO:tasks.rados.rados.0.smithi017.stdout:3326: finishing rollback tid 0 to smithi01785587-12 2024-09-21T09:52:01.208 INFO:tasks.rados.rados.0.smithi017.stdout:3325: finishing rollback tid 0 to smithi01785587-50 2024-09-21T09:52:01.208 INFO:tasks.rados.rados.0.smithi017.stdout:3325: finishing rollback tid 1 to smithi01785587-50 2024-09-21T09:52:01.208 INFO:tasks.rados.rados.0.smithi017.stdout:3326: finishing rollback tid 1 to smithi01785587-12 2024-09-21T09:52:01.208 INFO:tasks.rados.rados.0.smithi017.stdout:3325: finishing rollback tid 2 to smithi01785587-50 2024-09-21T09:52:01.209 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 1023 (ObjNum 1093 snap 363 seq_num 1093) dirty exists 2024-09-21T09:52:01.209 INFO:tasks.rados.rados.0.smithi017.stdout:3326: finishing rollback tid 2 to smithi01785587-12 2024-09-21T09:52:01.209 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 1139 (ObjNum 1172 snap 393 seq_num 1172) dirty exists 2024-09-21T09:52:01.210 INFO:tasks.rados.rados.0.smithi017.stdout:3325: done (2 left) 2024-09-21T09:52:01.210 INFO:tasks.rados.rados.0.smithi017.stdout:3326: done (1 left) 2024-09-21T09:52:01.211 INFO:tasks.rados.rados.0.smithi017.stdout:3327: done (0 left) 2024-09-21T09:52:01.211 INFO:tasks.rados.rados.0.smithi017.stdout:3328: snap_create 2024-09-21T09:52:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:01 smithi134 ceph-mon[90085]: pgmap v921: 89 pgs: 89 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 6 op/s 2024-09-21T09:52:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:01 smithi134 ceph-mon[90085]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T09:52:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:01 smithi017 ceph-mon[102913]: pgmap v921: 89 pgs: 89 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 6 op/s 2024-09-21T09:52:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:01 smithi017 ceph-mon[102913]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T09:52:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:01 smithi017 ceph-mon[105151]: pgmap v921: 89 pgs: 89 active+clean; 290 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 6 op/s 2024-09-21T09:52:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:01 smithi017 ceph-mon[105151]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T09:52:02.211 INFO:tasks.rados.rados.0.smithi017.stdout:3328: done (0 left) 2024-09-21T09:52:02.212 INFO:tasks.rados.rados.0.smithi017.stdout:3329: snap_create 2024-09-21T09:52:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:02 smithi134 ceph-mon[90085]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T09:52:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:02 smithi017 ceph-mon[102913]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T09:52:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:02 smithi017 ceph-mon[105151]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T09:52:03.196 INFO:tasks.rados.rados.0.smithi017.stdout:3329: done (0 left) 2024-09-21T09:52:03.196 INFO:tasks.rados.rados.0.smithi017.stdout:3330: snap_remove snap 403 2024-09-21T09:52:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:03 smithi134 ceph-mon[90085]: pgmap v924: 89 pgs: 89 active+clean; 296 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 361 KiB/s wr, 5 op/s 2024-09-21T09:52:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:03 smithi134 ceph-mon[90085]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T09:52:03.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:03 smithi134 ceph-mon[90085]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T09:52:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:03 smithi017 ceph-mon[102913]: pgmap v924: 89 pgs: 89 active+clean; 296 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 361 KiB/s wr, 5 op/s 2024-09-21T09:52:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:03 smithi017 ceph-mon[102913]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T09:52:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:03 smithi017 ceph-mon[102913]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T09:52:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:03 smithi017 ceph-mon[105151]: pgmap v924: 89 pgs: 89 active+clean; 296 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 361 KiB/s wr, 5 op/s 2024-09-21T09:52:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:03 smithi017 ceph-mon[105151]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T09:52:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:03 smithi017 ceph-mon[105151]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T09:52:04.198 INFO:tasks.rados.rados.0.smithi017.stdout:3330: done (0 left) 2024-09-21T09:52:04.198 INFO:tasks.rados.rados.0.smithi017.stdout:3331: snap_remove snap 418 2024-09-21T09:52:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:52:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:52:04] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-09-21T09:52:05.200 INFO:tasks.rados.rados.0.smithi017.stdout:3331: done (0 left) 2024-09-21T09:52:05.201 INFO:tasks.rados.rados.0.smithi017.stdout:3332: snap_create 2024-09-21T09:52:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:05 smithi134 ceph-mon[90085]: pgmap v927: 89 pgs: 89 active+clean; 296 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 361 KiB/s wr, 3 op/s 2024-09-21T09:52:05.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:05 smithi134 ceph-mon[90085]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T09:52:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:05 smithi017 ceph-mon[105151]: pgmap v927: 89 pgs: 89 active+clean; 296 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 361 KiB/s wr, 3 op/s 2024-09-21T09:52:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:05 smithi017 ceph-mon[105151]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T09:52:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:05 smithi017 ceph-mon[102913]: pgmap v927: 89 pgs: 89 active+clean; 296 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 361 KiB/s wr, 3 op/s 2024-09-21T09:52:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:05 smithi017 ceph-mon[102913]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T09:52:06.214 INFO:tasks.rados.rados.0.smithi017.stdout:3332: done (0 left) 2024-09-21T09:52:06.214 INFO:tasks.rados.rados.0.smithi017.stdout:3333: snap_remove snap 398 2024-09-21T09:52:06.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:06 smithi134 ceph-mon[90085]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T09:52:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:06 smithi017 ceph-mon[102913]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T09:52:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:06 smithi017 ceph-mon[105151]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T09:52:07.215 INFO:tasks.rados.rados.0.smithi017.stdout:3333: done (0 left) 2024-09-21T09:52:07.215 INFO:tasks.rados.rados.0.smithi017.stdout:3334: read oid 42 snap -1 2024-09-21T09:52:07.215 INFO:tasks.rados.rados.0.smithi017.stdout:3334: expect (ObjNum 1176 snap 395 seq_num 1176) 2024-09-21T09:52:07.215 INFO:tasks.rados.rados.0.smithi017.stdout:3335: rollback oid 23 current snap is 420 2024-09-21T09:52:07.215 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 23 to 411 2024-09-21T09:52:07.215 INFO:tasks.rados.rados.0.smithi017.stdout:3336: delete oid 41 current snap is 420 2024-09-21T09:52:07.219 INFO:tasks.rados.rados.0.smithi017.stdout:3336: done (2 left) 2024-09-21T09:52:07.219 INFO:tasks.rados.rados.0.smithi017.stdout:3337: delete oid 22 current snap is 420 2024-09-21T09:52:07.228 INFO:tasks.rados.rados.0.smithi017.stdout:3337: done (2 left) 2024-09-21T09:52:07.228 INFO:tasks.rados.rados.0.smithi017.stdout:3338: read oid 38 snap 409 2024-09-21T09:52:07.228 INFO:tasks.rados.rados.0.smithi017.stdout:3338: expect (ObjNum 1204 snap 403 seq_num 1204) 2024-09-21T09:52:07.229 INFO:tasks.rados.rados.0.smithi017.stdout:3339: delete oid 28 current snap is 420 2024-09-21T09:52:07.230 INFO:tasks.rados.rados.0.smithi017.stdout:3334: expect (ObjNum 1176 snap 395 seq_num 1176) 2024-09-21T09:52:07.406 INFO:tasks.rados.rados.0.smithi017.stdout:3335: finishing rollback tid 0 to smithi01785587-23 2024-09-21T09:52:07.406 INFO:tasks.rados.rados.0.smithi017.stdout:3335: finishing rollback tid 1 to smithi01785587-23 2024-09-21T09:52:07.406 INFO:tasks.rados.rados.0.smithi017.stdout:3335: finishing rollback tid 2 to smithi01785587-23 2024-09-21T09:52:07.406 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1691 (ObjNum 1185 snap 398 seq_num 1185) dirty exists 2024-09-21T09:52:07.406 INFO:tasks.rados.rados.0.smithi017.stdout:3338: expect (ObjNum 1204 snap 403 seq_num 1204) 2024-09-21T09:52:07.486 INFO:tasks.rados.rados.0.smithi017.stdout:3334: done (3 left) 2024-09-21T09:52:07.487 INFO:tasks.rados.rados.0.smithi017.stdout:3335: done (2 left) 2024-09-21T09:52:07.487 INFO:tasks.rados.rados.0.smithi017.stdout:3338: done (1 left) 2024-09-21T09:52:07.487 INFO:tasks.rados.rados.0.smithi017.stdout:3339: done (0 left) 2024-09-21T09:52:07.487 INFO:tasks.rados.rados.0.smithi017.stdout:3340: write oid 23 current snap is 420 2024-09-21T09:52:07.487 INFO:tasks.rados.rados.0.smithi017.stdout:3340: seq_num 1249 ranges {591943=667893,1721892=782758} 2024-09-21T09:52:07.493 INFO:tasks.rados.rados.0.smithi017.stdout:3340: writing smithi01785587-23 from 591943 to 1259836 tid 1 2024-09-21T09:52:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:07 smithi134 ceph-mon[90085]: pgmap v930: 89 pgs: 89 active+clean; 295 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:07 smithi134 ceph-mon[90085]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T09:52:07.500 INFO:tasks.rados.rados.0.smithi017.stdout:3340: writing smithi01785587-23 from 1721892 to 2504650 tid 2 2024-09-21T09:52:07.500 INFO:tasks.rados.rados.0.smithi017.stdout:3341: snap_create 2024-09-21T09:52:07.500 INFO:tasks.rados.rados.0.smithi017.stdout:3340: finishing write tid 1 to smithi01785587-23 2024-09-21T09:52:07.504 INFO:tasks.rados.rados.0.smithi017.stdout:3340: finishing write tid 2 to smithi01785587-23 2024-09-21T09:52:07.504 INFO:tasks.rados.rados.0.smithi017.stdout:3340: finishing write tid 3 to smithi01785587-23 2024-09-21T09:52:07.504 INFO:tasks.rados.rados.0.smithi017.stdout:3340: finishing write tid 4 to smithi01785587-23 2024-09-21T09:52:07.504 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1702 (ObjNum 1249 snap 420 seq_num 1249) dirty exists 2024-09-21T09:52:07.504 INFO:tasks.rados.rados.0.smithi017.stdout:3340: left oid 23 (ObjNum 1249 snap 420 seq_num 1249) 2024-09-21T09:52:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:07 smithi017 ceph-mon[102913]: pgmap v930: 89 pgs: 89 active+clean; 295 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:07 smithi017 ceph-mon[102913]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T09:52:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:07 smithi017 ceph-mon[105151]: pgmap v930: 89 pgs: 89 active+clean; 295 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:07 smithi017 ceph-mon[105151]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T09:52:08.199 INFO:tasks.rados.rados.0.smithi017.stdout:3340: done (1 left) 2024-09-21T09:52:08.199 INFO:tasks.rados.rados.0.smithi017.stdout:3341: done (0 left) 2024-09-21T09:52:08.199 INFO:tasks.rados.rados.0.smithi017.stdout:3342: read oid 33 snap 417 2024-09-21T09:52:08.199 INFO:tasks.rados.rados.0.smithi017.stdout:3342: expect deleted 2024-09-21T09:52:08.199 INFO:tasks.rados.rados.0.smithi017.stdout:3343: read oid 22 snap -1 2024-09-21T09:52:08.199 INFO:tasks.rados.rados.0.smithi017.stdout:3343: expect deleted 2024-09-21T09:52:08.200 INFO:tasks.rados.rados.0.smithi017.stdout:3344: write oid 8 current snap is 421 2024-09-21T09:52:08.200 INFO:tasks.rados.rados.0.smithi017.stdout:3344: seq_num 1250 ranges {608750=743142,1780374=642769} 2024-09-21T09:52:08.206 INFO:tasks.rados.rados.0.smithi017.stdout:3344: writing smithi01785587-8 from 608750 to 1351892 tid 1 2024-09-21T09:52:08.213 INFO:tasks.rados.rados.0.smithi017.stdout:3344: writing smithi01785587-8 from 1780374 to 2423143 tid 2 2024-09-21T09:52:08.213 INFO:tasks.rados.rados.0.smithi017.stdout:3345: rollback oid 42 current snap is 421 2024-09-21T09:52:08.213 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 420 2024-09-21T09:52:08.213 INFO:tasks.rados.rados.0.smithi017.stdout:3344: finishing write tid 1 to smithi01785587-8 2024-09-21T09:52:08.213 INFO:tasks.rados.rados.0.smithi017.stdout:3342: done (3 left) 2024-09-21T09:52:08.213 INFO:tasks.rados.rados.0.smithi017.stdout:3343: done (2 left) 2024-09-21T09:52:08.213 INFO:tasks.rados.rados.0.smithi017.stdout:3346: read oid 38 snap -1 2024-09-21T09:52:08.213 INFO:tasks.rados.rados.0.smithi017.stdout:3346: expect (ObjNum 1204 snap 403 seq_num 1204) 2024-09-21T09:52:08.213 INFO:tasks.rados.rados.0.smithi017.stdout:3347: snap_remove snap 374 2024-09-21T09:52:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:08 smithi134 ceph-mon[90085]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T09:52:08.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:08 smithi134 ceph-mon[90085]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T09:52:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:08 smithi017 ceph-mon[102913]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T09:52:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:08 smithi017 ceph-mon[102913]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T09:52:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:08 smithi017 ceph-mon[105151]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T09:52:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:08 smithi017 ceph-mon[105151]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T09:52:09.201 INFO:tasks.rados.rados.0.smithi017.stdout:3347: done (3 left) 2024-09-21T09:52:09.202 INFO:tasks.rados.rados.0.smithi017.stdout:3348: rollback oid 11 current snap is 421 2024-09-21T09:52:09.202 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 412 2024-09-21T09:52:09.202 INFO:tasks.rados.rados.0.smithi017.stdout:3346: expect (ObjNum 1204 snap 403 seq_num 1204) 2024-09-21T09:52:09.291 INFO:tasks.rados.rados.0.smithi017.stdout:3344: finishing write tid 2 to smithi01785587-8 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:3345: finishing rollback tid 0 to smithi01785587-42 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:3345: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:3345: finishing rollback tid 2 to smithi01785587-42 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 1151 (ObjNum 1176 snap 395 seq_num 1176) dirty exists 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:3344: finishing write tid 3 to smithi01785587-8 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:3344: finishing write tid 4 to smithi01785587-8 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1705 (ObjNum 1250 snap 421 seq_num 1250) dirty exists 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:3344: left oid 8 (ObjNum 1250 snap 421 seq_num 1250) 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:3344: done (3 left) 2024-09-21T09:52:09.292 INFO:tasks.rados.rados.0.smithi017.stdout:3345: done (2 left) 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:3346: done (1 left) 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:3349: read oid 47 snap -1 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:3349: expect (ObjNum 1232 snap 409 seq_num 1232) 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:3350: rollback oid 34 current snap is 421 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 417 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:3348: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1616 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:3348: done (2 left) 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:3351: read oid 44 snap -1 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:3351: expect (ObjNum 1220 snap 405 seq_num 1220) 2024-09-21T09:52:09.293 INFO:tasks.rados.rados.0.smithi017.stdout:3352: read oid 33 snap -1 2024-09-21T09:52:09.294 INFO:tasks.rados.rados.0.smithi017.stdout:3352: expect deleted 2024-09-21T09:52:09.294 INFO:tasks.rados.rados.0.smithi017.stdout:3353: delete oid 30 current snap is 421 2024-09-21T09:52:09.294 INFO:tasks.rados.rados.0.smithi017.stdout:3352: done (4 left) 2024-09-21T09:52:09.294 INFO:tasks.rados.rados.0.smithi017.stdout:3353: done (3 left) 2024-09-21T09:52:09.294 INFO:tasks.rados.rados.0.smithi017.stdout:3354: delete oid 1 current snap is 421 2024-09-21T09:52:09.294 INFO:tasks.rados.rados.0.smithi017.stdout:3349: expect (ObjNum 1232 snap 409 seq_num 1232) 2024-09-21T09:52:09.382 INFO:tasks.rados.rados.0.smithi017.stdout:3350: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:52:09.383 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1436 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:09.383 INFO:tasks.rados.rados.0.smithi017.stdout:3351: expect (ObjNum 1220 snap 405 seq_num 1220) 2024-09-21T09:52:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:09 smithi134 ceph-mon[90085]: pgmap v933: 89 pgs: 89 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:09.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:09 smithi134 ceph-mon[90085]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T09:52:09.563 INFO:tasks.rados.rados.0.smithi017.stdout:3349: done (3 left) 2024-09-21T09:52:09.563 INFO:tasks.rados.rados.0.smithi017.stdout:3350: done (2 left) 2024-09-21T09:52:09.563 INFO:tasks.rados.rados.0.smithi017.stdout:3351: done (1 left) 2024-09-21T09:52:09.563 INFO:tasks.rados.rados.0.smithi017.stdout:3354: done (0 left) 2024-09-21T09:52:09.563 INFO:tasks.rados.rados.0.smithi017.stdout:3355: snap_remove snap 352 2024-09-21T09:52:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:09 smithi017 ceph-mon[102913]: pgmap v933: 89 pgs: 89 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:09 smithi017 ceph-mon[102913]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T09:52:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:09 smithi017 ceph-mon[105151]: pgmap v933: 89 pgs: 89 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:09 smithi017 ceph-mon[105151]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T09:52:10.240 INFO:tasks.rados.rados.0.smithi017.stdout:3355: done (0 left) 2024-09-21T09:52:10.240 INFO:tasks.rados.rados.0.smithi017.stdout:3356: snap_remove snap 394 2024-09-21T09:52:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:52:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:52:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:52:11.241 INFO:tasks.rados.rados.0.smithi017.stdout:3356: done (0 left) 2024-09-21T09:52:11.241 INFO:tasks.rados.rados.0.smithi017.stdout:3357: delete oid 3 current snap is 421 2024-09-21T09:52:11.250 INFO:tasks.rados.rados.0.smithi017.stdout:3357: done (0 left) 2024-09-21T09:52:11.250 INFO:tasks.rados.rados.0.smithi017.stdout:3358: write oid 22 current snap is 421 2024-09-21T09:52:11.250 INFO:tasks.rados.rados.0.smithi017.stdout:3358: seq_num 1254 ranges {675978=644138,1897949=650107} 2024-09-21T09:52:11.257 INFO:tasks.rados.rados.0.smithi017.stdout:3358: writing smithi01785587-22 from 675978 to 1320116 tid 1 2024-09-21T09:52:11.264 INFO:tasks.rados.rados.0.smithi017.stdout:3358: writing smithi01785587-22 from 1897949 to 2548056 tid 2 2024-09-21T09:52:11.264 INFO:tasks.rados.rados.0.smithi017.stdout:3359: write oid 45 current snap is 421 2024-09-21T09:52:11.264 INFO:tasks.rados.rados.0.smithi017.stdout:3358: finishing write tid 1 to smithi01785587-22 2024-09-21T09:52:11.264 INFO:tasks.rados.rados.0.smithi017.stdout:3359: seq_num 1255 ranges {692785=435740,1672784=793765,2897503=195282} 2024-09-21T09:52:11.270 INFO:tasks.rados.rados.0.smithi017.stdout:3359: writing smithi01785587-45 from 692785 to 1128525 tid 1 2024-09-21T09:52:11.278 INFO:tasks.rados.rados.0.smithi017.stdout:3359: writing smithi01785587-45 from 1672784 to 2466549 tid 2 2024-09-21T09:52:11.281 INFO:tasks.rados.rados.0.smithi017.stdout:3359: writing smithi01785587-45 from 2897503 to 3092785 tid 3 2024-09-21T09:52:11.281 INFO:tasks.rados.rados.0.smithi017.stdout:3360: write oid 10 current snap is 421 2024-09-21T09:52:11.281 INFO:tasks.rados.rados.0.smithi017.stdout:3358: finishing write tid 2 to smithi01785587-22 2024-09-21T09:52:11.282 INFO:tasks.rados.rados.0.smithi017.stdout:3358: finishing write tid 3 to smithi01785587-22 2024-09-21T09:52:11.282 INFO:tasks.rados.rados.0.smithi017.stdout:3360: seq_num 1256 ranges {709592=510989,1731266=653776,3041279=68313} 2024-09-21T09:52:11.288 INFO:tasks.rados.rados.0.smithi017.stdout:3360: writing smithi01785587-10 from 709592 to 1220581 tid 1 2024-09-21T09:52:11.295 INFO:tasks.rados.rados.0.smithi017.stdout:3360: writing smithi01785587-10 from 1731266 to 2385042 tid 2 2024-09-21T09:52:11.300 INFO:tasks.rados.rados.0.smithi017.stdout:3360: writing smithi01785587-10 from 3041279 to 3109592 tid 3 2024-09-21T09:52:11.300 INFO:tasks.rados.rados.0.smithi017.stdout:3361: write oid 36 current snap is 421 2024-09-21T09:52:11.300 INFO:tasks.rados.rados.0.smithi017.stdout:3358: finishing write tid 4 to smithi01785587-22 2024-09-21T09:52:11.301 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1763 (ObjNum 1254 snap 421 seq_num 1254) dirty exists 2024-09-21T09:52:11.301 INFO:tasks.rados.rados.0.smithi017.stdout:3358: left oid 22 (ObjNum 1254 snap 421 seq_num 1254) 2024-09-21T09:52:11.301 INFO:tasks.rados.rados.0.smithi017.stdout:3359: finishing write tid 1 to smithi01785587-45 2024-09-21T09:52:11.301 INFO:tasks.rados.rados.0.smithi017.stdout:3361: seq_num 1257 ranges {726399=586238,2073395=797434} 2024-09-21T09:52:11.307 INFO:tasks.rados.rados.0.smithi017.stdout:3361: writing smithi01785587-36 from 726399 to 1312637 tid 1 2024-09-21T09:52:11.316 INFO:tasks.rados.rados.0.smithi017.stdout:3361: writing smithi01785587-36 from 2073395 to 2870829 tid 2 2024-09-21T09:52:11.316 INFO:tasks.rados.rados.0.smithi017.stdout:3358: done (3 left) 2024-09-21T09:52:11.316 INFO:tasks.rados.rados.0.smithi017.stdout:3362: read oid 34 snap 419 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3362: expect deleted 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3363: delete oid 28 current snap is 421 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3359: finishing write tid 2 to smithi01785587-45 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3359: finishing write tid 3 to smithi01785587-45 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3359: finishing write tid 4 to smithi01785587-45 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3359: finishing write tid 5 to smithi01785587-45 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1137 (ObjNum 1255 snap 421 seq_num 1255) dirty exists 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3359: left oid 45 (ObjNum 1255 snap 421 seq_num 1255) 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3360: finishing write tid 1 to smithi01785587-10 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3360: finishing write tid 2 to smithi01785587-10 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3360: finishing write tid 3 to smithi01785587-10 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3360: finishing write tid 4 to smithi01785587-10 2024-09-21T09:52:11.317 INFO:tasks.rados.rados.0.smithi017.stdout:3360: finishing write tid 5 to smithi01785587-10 2024-09-21T09:52:11.318 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 325 (ObjNum 1256 snap 421 seq_num 1256) dirty exists 2024-09-21T09:52:11.318 INFO:tasks.rados.rados.0.smithi017.stdout:3360: left oid 10 (ObjNum 1256 snap 421 seq_num 1256) 2024-09-21T09:52:11.318 INFO:tasks.rados.rados.0.smithi017.stdout:3361: finishing write tid 1 to smithi01785587-36 2024-09-21T09:52:11.319 INFO:tasks.rados.rados.0.smithi017.stdout:3359: done (4 left) 2024-09-21T09:52:11.319 INFO:tasks.rados.rados.0.smithi017.stdout:3360: done (3 left) 2024-09-21T09:52:11.319 INFO:tasks.rados.rados.0.smithi017.stdout:3362: done (2 left) 2024-09-21T09:52:11.319 INFO:tasks.rados.rados.0.smithi017.stdout:3363: done (1 left) 2024-09-21T09:52:11.320 INFO:tasks.rados.rados.0.smithi017.stdout:3364: read oid 50 snap 420 2024-09-21T09:52:11.320 INFO:tasks.rados.rados.0.smithi017.stdout:3364: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:52:11.320 INFO:tasks.rados.rados.0.smithi017.stdout:3365: write oid 16 current snap is 421 2024-09-21T09:52:11.320 INFO:tasks.rados.rados.0.smithi017.stdout:3365: seq_num 1259 ranges {760013=736736,2190359=401103} 2024-09-21T09:52:11.327 INFO:tasks.rados.rados.0.smithi017.stdout:3365: writing smithi01785587-16 from 760013 to 1496749 tid 1 2024-09-21T09:52:11.334 INFO:tasks.rados.rados.0.smithi017.stdout:3365: writing smithi01785587-16 from 2190359 to 2591462 tid 2 2024-09-21T09:52:11.334 INFO:tasks.rados.rados.0.smithi017.stdout:3366: snap_create 2024-09-21T09:52:11.334 INFO:tasks.rados.rados.0.smithi017.stdout:3361: finishing write tid 2 to smithi01785587-36 2024-09-21T09:52:11.334 INFO:tasks.rados.rados.0.smithi017.stdout:3361: finishing write tid 3 to smithi01785587-36 2024-09-21T09:52:11.334 INFO:tasks.rados.rados.0.smithi017.stdout:3361: finishing write tid 4 to smithi01785587-36 2024-09-21T09:52:11.334 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1732 (ObjNum 1257 snap 421 seq_num 1257) dirty exists 2024-09-21T09:52:11.334 INFO:tasks.rados.rados.0.smithi017.stdout:3361: left oid 36 (ObjNum 1257 snap 421 seq_num 1257) 2024-09-21T09:52:11.334 INFO:tasks.rados.rados.0.smithi017.stdout:3364: expect (ObjNum 1093 snap 363 seq_num 1093) 2024-09-21T09:52:11.414 INFO:tasks.rados.rados.0.smithi017.stdout:3365: finishing write tid 1 to smithi01785587-16 2024-09-21T09:52:11.414 INFO:tasks.rados.rados.0.smithi017.stdout:3365: finishing write tid 2 to smithi01785587-16 2024-09-21T09:52:11.414 INFO:tasks.rados.rados.0.smithi017.stdout:3365: finishing write tid 3 to smithi01785587-16 2024-09-21T09:52:11.414 INFO:tasks.rados.rados.0.smithi017.stdout:3365: finishing write tid 4 to smithi01785587-16 2024-09-21T09:52:11.414 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1767 (ObjNum 1259 snap 421 seq_num 1259) dirty exists 2024-09-21T09:52:11.414 INFO:tasks.rados.rados.0.smithi017.stdout:3365: left oid 16 (ObjNum 1259 snap 421 seq_num 1259) 2024-09-21T09:52:11.502 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:11 smithi017 ceph-mon[102913]: pgmap v936: 89 pgs: 89 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:11.502 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:11 smithi017 ceph-mon[102913]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T09:52:11.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:11.503 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:11 smithi017 ceph-mon[105151]: pgmap v936: 89 pgs: 89 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:11.503 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:11 smithi017 ceph-mon[105151]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T09:52:11.503 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:11 smithi134 ceph-mon[90085]: pgmap v936: 89 pgs: 89 active+clean; 294 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:11 smithi134 ceph-mon[90085]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T09:52:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:12.256 INFO:tasks.rados.rados.0.smithi017.stdout:3361: done (3 left) 2024-09-21T09:52:12.256 INFO:tasks.rados.rados.0.smithi017.stdout:3364: done (2 left) 2024-09-21T09:52:12.256 INFO:tasks.rados.rados.0.smithi017.stdout:3365: done (1 left) 2024-09-21T09:52:12.256 INFO:tasks.rados.rados.0.smithi017.stdout:3366: done (0 left) 2024-09-21T09:52:12.256 INFO:tasks.rados.rados.0.smithi017.stdout:3367: snap_remove snap 415 2024-09-21T09:52:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:12 smithi017 ceph-mon[102913]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T09:52:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:12 smithi017 ceph-mon[105151]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T09:52:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:12 smithi134 ceph-mon[90085]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T09:52:13.202 INFO:tasks.rados.rados.0.smithi017.stdout:3367: done (0 left) 2024-09-21T09:52:13.202 INFO:tasks.rados.rados.0.smithi017.stdout:3368: read oid 7 snap 372 2024-09-21T09:52:13.202 INFO:tasks.rados.rados.0.smithi017.stdout:3368: expect deleted 2024-09-21T09:52:13.202 INFO:tasks.rados.rados.0.smithi017.stdout:3369: snap_create 2024-09-21T09:52:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:13 smithi017 ceph-mon[102913]: pgmap v939: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 693 KiB/s wr, 14 op/s 2024-09-21T09:52:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:13 smithi017 ceph-mon[102913]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T09:52:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:13 smithi017 ceph-mon[102913]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T09:52:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:13 smithi017 ceph-mon[105151]: pgmap v939: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 693 KiB/s wr, 14 op/s 2024-09-21T09:52:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:13 smithi017 ceph-mon[105151]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T09:52:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:13 smithi017 ceph-mon[105151]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T09:52:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:13 smithi134 ceph-mon[90085]: pgmap v939: 89 pgs: 89 active+clean; 245 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 693 KiB/s wr, 14 op/s 2024-09-21T09:52:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:13 smithi134 ceph-mon[90085]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T09:52:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:13 smithi134 ceph-mon[90085]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T09:52:14.202 INFO:tasks.rados.rados.0.smithi017.stdout:3368: done (1 left) 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3369: done (0 left) 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3370: read oid 18 snap -1 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3370: expect (ObjNum 1205 snap 403 seq_num 1205) 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3371: read oid 27 snap -1 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3371: expect (ObjNum 1240 snap 411 seq_num 1240) 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3372: read oid 38 snap -1 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3372: expect (ObjNum 1204 snap 403 seq_num 1204) 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3373: read oid 25 snap 421 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3373: expect (ObjNum 1165 snap 392 seq_num 1165) 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3374: read oid 49 snap 406 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3374: expect (ObjNum 1106 snap 366 seq_num 1106) 2024-09-21T09:52:14.203 INFO:tasks.rados.rados.0.smithi017.stdout:3375: snap_remove snap 409 2024-09-21T09:52:15.199 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:52:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:52:14] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.51.0" 2024-09-21T09:52:15.208 INFO:tasks.rados.rados.0.smithi017.stdout:3375: done (5 left) 2024-09-21T09:52:15.208 INFO:tasks.rados.rados.0.smithi017.stdout:3376: rollback oid 20 current snap is 423 2024-09-21T09:52:15.208 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 414 2024-09-21T09:52:15.208 INFO:tasks.rados.rados.0.smithi017.stdout:3374: expect (ObjNum 1106 snap 366 seq_num 1106) 2024-09-21T09:52:15.334 INFO:tasks.rados.rados.0.smithi017.stdout:3372: expect (ObjNum 1204 snap 403 seq_num 1204) 2024-09-21T09:52:15.415 INFO:tasks.rados.rados.0.smithi017.stdout:3371: expect (ObjNum 1240 snap 411 seq_num 1240) 2024-09-21T09:52:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:15 smithi017 ceph-mon[102913]: pgmap v942: 89 pgs: 89 active+clean; 231 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:52:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:15 smithi017 ceph-mon[102913]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T09:52:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:15 smithi017 ceph-mon[105151]: pgmap v942: 89 pgs: 89 active+clean; 231 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:52:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:15 smithi017 ceph-mon[105151]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T09:52:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:15 smithi134 ceph-mon[90085]: pgmap v942: 89 pgs: 89 active+clean; 231 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-09-21T09:52:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:15 smithi134 ceph-mon[90085]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T09:52:15.644 INFO:tasks.rados.rados.0.smithi017.stdout:3373: expect (ObjNum 1165 snap 392 seq_num 1165) 2024-09-21T09:52:15.804 INFO:tasks.rados.rados.0.smithi017.stdout:3370: expect (ObjNum 1205 snap 403 seq_num 1205) 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:3376: finishing rollback tid 0 to smithi01785587-20 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:3376: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:3376: finishing rollback tid 2 to smithi01785587-20 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 906 (ObjNum 1243 snap 414 seq_num 1243) dirty exists 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:3370: done (5 left) 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:3371: done (4 left) 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:3372: done (3 left) 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:3373: done (2 left) 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:3374: done (1 left) 2024-09-21T09:52:15.877 INFO:tasks.rados.rados.0.smithi017.stdout:3376: done (0 left) 2024-09-21T09:52:15.878 INFO:tasks.rados.rados.0.smithi017.stdout:3377: rollback oid 31 current snap is 423 2024-09-21T09:52:15.878 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 31 to 411 2024-09-21T09:52:15.878 INFO:tasks.rados.rados.0.smithi017.stdout:3378: snap_remove snap 417 2024-09-21T09:52:16.215 INFO:tasks.rados.rados.0.smithi017.stdout:3378: done (1 left) 2024-09-21T09:52:16.215 INFO:tasks.rados.rados.0.smithi017.stdout:3379: snap_remove snap 372 2024-09-21T09:52:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:16 smithi134 ceph-mon[90085]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T09:52:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:16 smithi017 ceph-mon[102913]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T09:52:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:16 smithi017 ceph-mon[105151]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T09:52:17.219 INFO:tasks.rados.rados.0.smithi017.stdout:3377: finishing rollback tid 0 to smithi01785587-31 2024-09-21T09:52:17.219 INFO:tasks.rados.rados.0.smithi017.stdout:3377: finishing rollback tid 1 to smithi01785587-31 2024-09-21T09:52:17.219 INFO:tasks.rados.rados.0.smithi017.stdout:3377: finishing rollback tid 2 to smithi01785587-31 2024-09-21T09:52:17.219 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1854 (ObjNum 1226 snap 405 seq_num 1226) dirty exists 2024-09-21T09:52:17.219 INFO:tasks.rados.rados.0.smithi017.stdout:3377: done (1 left) 2024-09-21T09:52:17.219 INFO:tasks.rados.rados.0.smithi017.stdout:3379: done (0 left) 2024-09-21T09:52:17.219 INFO:tasks.rados.rados.0.smithi017.stdout:3380: delete oid 29 current snap is 423 2024-09-21T09:52:17.233 INFO:tasks.rados.rados.0.smithi017.stdout:3380: done (0 left) 2024-09-21T09:52:17.233 INFO:tasks.rados.rados.0.smithi017.stdout:3381: delete oid 38 current snap is 423 2024-09-21T09:52:17.236 INFO:tasks.rados.rados.0.smithi017.stdout:3381: done (0 left) 2024-09-21T09:52:17.236 INFO:tasks.rados.rados.0.smithi017.stdout:3382: write oid 30 current snap is 423 2024-09-21T09:52:17.236 INFO:tasks.rados.rados.0.smithi017.stdout:3382: seq_num 1262 ranges {410434=678836,1565805=548430,2573111=637323} 2024-09-21T09:52:17.242 INFO:tasks.rados.rados.0.smithi017.stdout:3382: writing smithi01785587-30 from 410434 to 1089270 tid 1 2024-09-21T09:52:17.247 INFO:tasks.rados.rados.0.smithi017.stdout:3382: writing smithi01785587-30 from 1565805 to 2114235 tid 2 2024-09-21T09:52:17.253 INFO:tasks.rados.rados.0.smithi017.stdout:3382: writing smithi01785587-30 from 2573111 to 3210434 tid 3 2024-09-21T09:52:17.253 INFO:tasks.rados.rados.0.smithi017.stdout:3383: delete oid 1 current snap is 423 2024-09-21T09:52:17.253 INFO:tasks.rados.rados.0.smithi017.stdout:3382: finishing write tid 1 to smithi01785587-30 2024-09-21T09:52:17.254 INFO:tasks.rados.rados.0.smithi017.stdout:3382: finishing write tid 2 to smithi01785587-30 2024-09-21T09:52:17.257 INFO:tasks.rados.rados.0.smithi017.stdout:3383: done (1 left) 2024-09-21T09:52:17.257 INFO:tasks.rados.rados.0.smithi017.stdout:3384: write oid 18 current snap is 423 2024-09-21T09:52:17.257 INFO:tasks.rados.rados.0.smithi017.stdout:3384: seq_num 1264 ranges {444048=429334,1282769=552099,2627957=616091} 2024-09-21T09:52:17.262 INFO:tasks.rados.rados.0.smithi017.stdout:3384: writing smithi01785587-18 from 444048 to 873382 tid 1 2024-09-21T09:52:17.267 INFO:tasks.rados.rados.0.smithi017.stdout:3384: writing smithi01785587-18 from 1282769 to 1834868 tid 2 2024-09-21T09:52:17.275 INFO:tasks.rados.rados.0.smithi017.stdout:3384: writing smithi01785587-18 from 2627957 to 3244048 tid 3 2024-09-21T09:52:17.275 INFO:tasks.rados.rados.0.smithi017.stdout:3385: write oid 16 current snap is 423 2024-09-21T09:52:17.275 INFO:tasks.rados.rados.0.smithi017.stdout:3385: seq_num 1265 ranges {460855=504583,1624898=412110,2539027=572508} 2024-09-21T09:52:17.280 INFO:tasks.rados.rados.0.smithi017.stdout:3385: writing smithi01785587-16 from 460855 to 965438 tid 1 2024-09-21T09:52:17.286 INFO:tasks.rados.rados.0.smithi017.stdout:3385: writing smithi01785587-16 from 1624898 to 2037008 tid 2 2024-09-21T09:52:17.292 INFO:tasks.rados.rados.0.smithi017.stdout:3385: writing smithi01785587-16 from 2539027 to 3111535 tid 3 2024-09-21T09:52:17.292 INFO:tasks.rados.rados.0.smithi017.stdout:3386: snap_remove snap 419 2024-09-21T09:52:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:17 smithi134 ceph-mon[90085]: pgmap v945: 89 pgs: 89 active+clean; 217 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:52:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:17 smithi134 ceph-mon[90085]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T09:52:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:17 smithi017 ceph-mon[102913]: pgmap v945: 89 pgs: 89 active+clean; 217 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:52:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:17 smithi017 ceph-mon[102913]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T09:52:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:17 smithi017 ceph-mon[105151]: pgmap v945: 89 pgs: 89 active+clean; 217 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:52:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:17 smithi017 ceph-mon[105151]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T09:52:18.203 INFO:tasks.rados.rados.0.smithi017.stdout:3382: finishing write tid 3 to smithi01785587-30 2024-09-21T09:52:18.203 INFO:tasks.rados.rados.0.smithi017.stdout:3382: finishing write tid 4 to smithi01785587-30 2024-09-21T09:52:18.203 INFO:tasks.rados.rados.0.smithi017.stdout:3382: finishing write tid 5 to smithi01785587-30 2024-09-21T09:52:18.203 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1450 (ObjNum 1262 snap 423 seq_num 1262) dirty exists 2024-09-21T09:52:18.203 INFO:tasks.rados.rados.0.smithi017.stdout:3382: left oid 30 (ObjNum 1262 snap 423 seq_num 1262) 2024-09-21T09:52:18.203 INFO:tasks.rados.rados.0.smithi017.stdout:3384: finishing write tid 1 to smithi01785587-18 2024-09-21T09:52:18.203 INFO:tasks.rados.rados.0.smithi017.stdout:3384: finishing write tid 2 to smithi01785587-18 2024-09-21T09:52:18.203 INFO:tasks.rados.rados.0.smithi017.stdout:3384: finishing write tid 3 to smithi01785587-18 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3384: finishing write tid 4 to smithi01785587-18 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3384: finishing write tid 5 to smithi01785587-18 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 336 (ObjNum 1264 snap 423 seq_num 1264) dirty exists 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3384: left oid 18 (ObjNum 1264 snap 423 seq_num 1264) 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3385: finishing write tid 1 to smithi01785587-16 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3385: finishing write tid 2 to smithi01785587-16 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3385: finishing write tid 3 to smithi01785587-16 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3385: finishing write tid 4 to smithi01785587-16 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3385: finishing write tid 5 to smithi01785587-16 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1794 (ObjNum 1265 snap 423 seq_num 1265) dirty exists 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3385: left oid 16 (ObjNum 1265 snap 423 seq_num 1265) 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3382: done (3 left) 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3384: done (2 left) 2024-09-21T09:52:18.204 INFO:tasks.rados.rados.0.smithi017.stdout:3385: done (1 left) 2024-09-21T09:52:18.205 INFO:tasks.rados.rados.0.smithi017.stdout:3386: done (0 left) 2024-09-21T09:52:18.205 INFO:tasks.rados.rados.0.smithi017.stdout:3387: snap_remove snap 412 2024-09-21T09:52:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:18 smithi134 ceph-mon[90085]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T09:52:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:18 smithi134 ceph-mon[90085]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T09:52:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:18 smithi017 ceph-mon[105151]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T09:52:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:18 smithi017 ceph-mon[105151]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T09:52:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:18 smithi017 ceph-mon[102913]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T09:52:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:18 smithi017 ceph-mon[102913]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T09:52:19.204 INFO:tasks.rados.rados.0.smithi017.stdout:3387: done (0 left) 2024-09-21T09:52:19.204 INFO:tasks.rados.rados.0.smithi017.stdout:3388: delete oid 43 current snap is 423 2024-09-21T09:52:19.210 INFO:tasks.rados.rados.0.smithi017.stdout:3388: done (0 left) 2024-09-21T09:52:19.210 INFO:tasks.rados.rados.0.smithi017.stdout:3389: delete oid 27 current snap is 423 2024-09-21T09:52:19.214 INFO:tasks.rados.rados.0.smithi017.stdout:3389: done (0 left) 2024-09-21T09:52:19.214 INFO:tasks.rados.rados.0.smithi017.stdout:3390: snap_create 2024-09-21T09:52:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:19 smithi134 ceph-mon[90085]: pgmap v948: 89 pgs: 89 active+clean; 209 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 896 KiB/s wr, 14 op/s 2024-09-21T09:52:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:19 smithi134 ceph-mon[90085]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T09:52:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:19 smithi017 ceph-mon[102913]: pgmap v948: 89 pgs: 89 active+clean; 209 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 896 KiB/s wr, 14 op/s 2024-09-21T09:52:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:19 smithi017 ceph-mon[102913]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T09:52:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:19 smithi017 ceph-mon[105151]: pgmap v948: 89 pgs: 89 active+clean; 209 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 896 KiB/s wr, 14 op/s 2024-09-21T09:52:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:19 smithi017 ceph-mon[105151]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T09:52:20.205 INFO:tasks.rados.rados.0.smithi017.stdout:3390: done (0 left) 2024-09-21T09:52:20.205 INFO:tasks.rados.rados.0.smithi017.stdout:3391: write oid 4 current snap is 424 2024-09-21T09:52:20.205 INFO:tasks.rados.rados.0.smithi017.stdout:3391: seq_num 1268 ranges {511276=730330,1800344=559437,3021296=289980} 2024-09-21T09:52:20.211 INFO:tasks.rados.rados.0.smithi017.stdout:3391: writing smithi01785587-4 from 511276 to 1241606 tid 1 2024-09-21T09:52:20.218 INFO:tasks.rados.rados.0.smithi017.stdout:3391: writing smithi01785587-4 from 1800344 to 2359781 tid 2 2024-09-21T09:52:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3391: writing smithi01785587-4 from 3021296 to 3311276 tid 3 2024-09-21T09:52:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3392: read oid 29 snap -1 2024-09-21T09:52:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3392: expect deleted 2024-09-21T09:52:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3393: write oid 19 current snap is 424 2024-09-21T09:52:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3391: finishing write tid 1 to smithi01785587-4 2024-09-21T09:52:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3391: finishing write tid 2 to smithi01785587-4 2024-09-21T09:52:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3393: seq_num 1269 ranges {528083=405579,1342473=419448,2248719=733949} 2024-09-21T09:52:20.228 INFO:tasks.rados.rados.0.smithi017.stdout:3393: writing smithi01785587-19 from 528083 to 933662 tid 1 2024-09-21T09:52:20.233 INFO:tasks.rados.rados.0.smithi017.stdout:3393: writing smithi01785587-19 from 1342473 to 1761921 tid 2 2024-09-21T09:52:20.240 INFO:tasks.rados.rados.0.smithi017.stdout:3393: writing smithi01785587-19 from 2248719 to 2982668 tid 3 2024-09-21T09:52:20.240 INFO:tasks.rados.rados.0.smithi017.stdout:3394: read oid 49 snap -1 2024-09-21T09:52:20.240 INFO:tasks.rados.rados.0.smithi017.stdout:3394: expect (ObjNum 1106 snap 366 seq_num 1106) 2024-09-21T09:52:20.240 INFO:tasks.rados.rados.0.smithi017.stdout:3395: rollback oid 30 current snap is 424 2024-09-21T09:52:20.240 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 411 2024-09-21T09:52:20.240 INFO:tasks.rados.rados.0.smithi017.stdout:3392: done (4 left) 2024-09-21T09:52:20.240 INFO:tasks.rados.rados.0.smithi017.stdout:3396: snap_create 2024-09-21T09:52:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:3391: finishing write tid 3 to smithi01785587-4 2024-09-21T09:52:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:3391: finishing write tid 4 to smithi01785587-4 2024-09-21T09:52:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:3391: finishing write tid 5 to smithi01785587-4 2024-09-21T09:52:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1805 (ObjNum 1268 snap 424 seq_num 1268) dirty exists 2024-09-21T09:52:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:3391: left oid 4 (ObjNum 1268 snap 424 seq_num 1268) 2024-09-21T09:52:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:3393: finishing write tid 1 to smithi01785587-19 2024-09-21T09:52:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:3393: finishing write tid 2 to smithi01785587-19 2024-09-21T09:52:20.244 INFO:tasks.rados.rados.0.smithi017.stdout:3394: expect (ObjNum 1106 snap 366 seq_num 1106) 2024-09-21T09:52:20.366 INFO:tasks.rados.rados.0.smithi017.stdout:3393: finishing write tid 3 to smithi01785587-19 2024-09-21T09:52:20.366 INFO:tasks.rados.rados.0.smithi017.stdout:3393: finishing write tid 4 to smithi01785587-19 2024-09-21T09:52:20.366 INFO:tasks.rados.rados.0.smithi017.stdout:3393: finishing write tid 5 to smithi01785587-19 2024-09-21T09:52:20.366 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 1458 (ObjNum 1269 snap 424 seq_num 1269) dirty exists 2024-09-21T09:52:20.366 INFO:tasks.rados.rados.0.smithi017.stdout:3393: left oid 19 (ObjNum 1269 snap 424 seq_num 1269) 2024-09-21T09:52:20.366 INFO:tasks.rados.rados.0.smithi017.stdout:3395: finishing rollback tid 0 to smithi01785587-30 2024-09-21T09:52:20.367 INFO:tasks.rados.rados.0.smithi017.stdout:3395: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:52:20.367 INFO:tasks.rados.rados.0.smithi017.stdout:3395: finishing rollback tid 2 to smithi01785587-30 2024-09-21T09:52:20.367 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1462 (ObjNum 1217 snap 405 seq_num 1217) dirty exists 2024-09-21T09:52:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:52:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:52:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:52:21.232 INFO:tasks.rados.rados.0.smithi017.stdout:3391: done (4 left) 2024-09-21T09:52:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:3393: done (3 left) 2024-09-21T09:52:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:3394: done (2 left) 2024-09-21T09:52:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:3395: done (1 left) 2024-09-21T09:52:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:3396: done (0 left) 2024-09-21T09:52:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:3397: read oid 3 snap 420 2024-09-21T09:52:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:3397: expect (ObjNum 1211 snap 403 seq_num 1211) 2024-09-21T09:52:21.233 INFO:tasks.rados.rados.0.smithi017.stdout:3398: snap_create 2024-09-21T09:52:21.239 INFO:tasks.rados.rados.0.smithi017.stdout:3397: expect (ObjNum 1211 snap 403 seq_num 1211) 2024-09-21T09:52:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:21 smithi134 ceph-mon[90085]: pgmap v951: 89 pgs: 89 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 301 KiB/s wr, 9 op/s 2024-09-21T09:52:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:21 smithi134 ceph-mon[90085]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T09:52:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:21 smithi017 ceph-mon[102913]: pgmap v951: 89 pgs: 89 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 301 KiB/s wr, 9 op/s 2024-09-21T09:52:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:21 smithi017 ceph-mon[102913]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T09:52:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:21 smithi017 ceph-mon[105151]: pgmap v951: 89 pgs: 89 active+clean; 199 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 301 KiB/s wr, 9 op/s 2024-09-21T09:52:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:21 smithi017 ceph-mon[105151]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T09:52:22.238 INFO:tasks.rados.rados.0.smithi017.stdout:3397: done (1 left) 2024-09-21T09:52:22.238 INFO:tasks.rados.rados.0.smithi017.stdout:3398: done (0 left) 2024-09-21T09:52:22.238 INFO:tasks.rados.rados.0.smithi017.stdout:3399: read oid 2 snap -1 2024-09-21T09:52:22.238 INFO:tasks.rados.rados.0.smithi017.stdout:3399: expect deleted 2024-09-21T09:52:22.238 INFO:tasks.rados.rados.0.smithi017.stdout:3400: delete oid 48 current snap is 426 2024-09-21T09:52:22.243 INFO:tasks.rados.rados.0.smithi017.stdout:3399: done (1 left) 2024-09-21T09:52:22.243 INFO:tasks.rados.rados.0.smithi017.stdout:3400: done (0 left) 2024-09-21T09:52:22.243 INFO:tasks.rados.rados.0.smithi017.stdout:3401: delete oid 45 current snap is 426 2024-09-21T09:52:22.247 INFO:tasks.rados.rados.0.smithi017.stdout:3401: done (0 left) 2024-09-21T09:52:22.247 INFO:tasks.rados.rados.0.smithi017.stdout:3402: rollback oid 39 current snap is 426 2024-09-21T09:52:22.247 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 422 2024-09-21T09:52:22.247 INFO:tasks.rados.rados.0.smithi017.stdout:3403: write oid 5 current snap is 426 2024-09-21T09:52:22.247 INFO:tasks.rados.rados.0.smithi017.stdout:3403: seq_num 1272 ranges {578504=747679,2034272=566775,3130988=247516} 2024-09-21T09:52:22.254 INFO:tasks.rados.rados.0.smithi017.stdout:3403: writing smithi01785587-5 from 578504 to 1326183 tid 1 2024-09-21T09:52:22.261 INFO:tasks.rados.rados.0.smithi017.stdout:3403: writing smithi01785587-5 from 2034272 to 2601047 tid 2 2024-09-21T09:52:22.266 INFO:tasks.rados.rados.0.smithi017.stdout:3403: writing smithi01785587-5 from 3130988 to 3378504 tid 3 2024-09-21T09:52:22.266 INFO:tasks.rados.rados.0.smithi017.stdout:3404: snap_create 2024-09-21T09:52:22.266 INFO:tasks.rados.rados.0.smithi017.stdout:3402: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:52:22.266 INFO:tasks.rados.rados.0.smithi017.stdout:3402: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:52:22.266 INFO:tasks.rados.rados.0.smithi017.stdout:3402: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:52:22.267 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1744 (ObjNum 1097 snap 363 seq_num 1097) dirty exists 2024-09-21T09:52:22.267 INFO:tasks.rados.rados.0.smithi017.stdout:3403: finishing write tid 1 to smithi01785587-5 2024-09-21T09:52:22.267 INFO:tasks.rados.rados.0.smithi017.stdout:3403: finishing write tid 2 to smithi01785587-5 2024-09-21T09:52:22.270 INFO:tasks.rados.rados.0.smithi017.stdout:3403: finishing write tid 3 to smithi01785587-5 2024-09-21T09:52:22.270 INFO:tasks.rados.rados.0.smithi017.stdout:3403: finishing write tid 4 to smithi01785587-5 2024-09-21T09:52:22.270 INFO:tasks.rados.rados.0.smithi017.stdout:3403: finishing write tid 5 to smithi01785587-5 2024-09-21T09:52:22.270 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1879 (ObjNum 1272 snap 426 seq_num 1272) dirty exists 2024-09-21T09:52:22.270 INFO:tasks.rados.rados.0.smithi017.stdout:3403: left oid 5 (ObjNum 1272 snap 426 seq_num 1272) 2024-09-21T09:52:22.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:22 smithi134 ceph-mon[90085]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T09:52:22.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:52:22.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:22.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:22.546 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:22 smithi017 ceph-mon[105151]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T09:52:22.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:52:22.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:22.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:22.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:22 smithi017 ceph-mon[102913]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T09:52:22.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:52:22.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:22.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:23.202 INFO:tasks.rados.rados.0.smithi017.stdout:3402: done (2 left) 2024-09-21T09:52:23.202 INFO:tasks.rados.rados.0.smithi017.stdout:3403: done (1 left) 2024-09-21T09:52:23.202 INFO:tasks.rados.rados.0.smithi017.stdout:3404: done (0 left) 2024-09-21T09:52:23.202 INFO:tasks.rados.rados.0.smithi017.stdout:3405: read oid 46 snap -1 2024-09-21T09:52:23.202 INFO:tasks.rados.rados.0.smithi017.stdout:3405: expect (ObjNum 1153 snap 385 seq_num 1153) 2024-09-21T09:52:23.203 INFO:tasks.rados.rados.0.smithi017.stdout:3406: snap_create 2024-09-21T09:52:23.209 INFO:tasks.rados.rados.0.smithi017.stdout:3405: expect (ObjNum 1153 snap 385 seq_num 1153) 2024-09-21T09:52:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:23 smithi134 ceph-mon[90085]: pgmap v954: 89 pgs: 89 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-21T09:52:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:23 smithi134 ceph-mon[90085]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T09:52:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:23 smithi134 ceph-mon[90085]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T09:52:23.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:23 smithi017 ceph-mon[102913]: pgmap v954: 89 pgs: 89 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-21T09:52:23.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:23 smithi017 ceph-mon[102913]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T09:52:23.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:23 smithi017 ceph-mon[102913]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T09:52:23.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:23 smithi017 ceph-mon[105151]: pgmap v954: 89 pgs: 89 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.9 MiB/s wr, 13 op/s 2024-09-21T09:52:23.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:23 smithi017 ceph-mon[105151]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T09:52:23.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:23 smithi017 ceph-mon[105151]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T09:52:24.204 INFO:tasks.rados.rados.0.smithi017.stdout:3405: done (1 left) 2024-09-21T09:52:24.204 INFO:tasks.rados.rados.0.smithi017.stdout:3406: done (0 left) 2024-09-21T09:52:24.204 INFO:tasks.rados.rados.0.smithi017.stdout:3407: rollback oid 32 current snap is 428 2024-09-21T09:52:24.205 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 427 2024-09-21T09:52:24.205 INFO:tasks.rados.rados.0.smithi017.stdout:3408: delete oid 6 current snap is 428 2024-09-21T09:52:24.208 INFO:tasks.rados.rados.0.smithi017.stdout:3408: done (1 left) 2024-09-21T09:52:24.208 INFO:tasks.rados.rados.0.smithi017.stdout:3409: snap_create 2024-09-21T09:52:24.208 INFO:tasks.rados.rados.0.smithi017.stdout:3407: finishing rollback tid 0 to smithi01785587-32 2024-09-21T09:52:24.208 INFO:tasks.rados.rados.0.smithi017.stdout:3407: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:52:24.209 INFO:tasks.rados.rados.0.smithi017.stdout:3407: finishing rollback tid 2 to smithi01785587-32 2024-09-21T09:52:24.209 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1752 (ObjNum 1223 snap 405 seq_num 1223) dirty exists 2024-09-21T09:52:25.048 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:52:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:52:24] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.51.0" 2024-09-21T09:52:25.207 INFO:tasks.rados.rados.0.smithi017.stdout:3407: done (1 left) 2024-09-21T09:52:25.207 INFO:tasks.rados.rados.0.smithi017.stdout:3409: done (0 left) 2024-09-21T09:52:25.207 INFO:tasks.rados.rados.0.smithi017.stdout:3410: read oid 2 snap -1 2024-09-21T09:52:25.207 INFO:tasks.rados.rados.0.smithi017.stdout:3410: expect deleted 2024-09-21T09:52:25.207 INFO:tasks.rados.rados.0.smithi017.stdout:3411: read oid 23 snap 411 2024-09-21T09:52:25.207 INFO:tasks.rados.rados.0.smithi017.stdout:3411: expect (ObjNum 1185 snap 398 seq_num 1185) 2024-09-21T09:52:25.208 INFO:tasks.rados.rados.0.smithi017.stdout:3412: delete oid 47 current snap is 429 2024-09-21T09:52:25.211 INFO:tasks.rados.rados.0.smithi017.stdout:3410: done (2 left) 2024-09-21T09:52:25.211 INFO:tasks.rados.rados.0.smithi017.stdout:3412: done (1 left) 2024-09-21T09:52:25.211 INFO:tasks.rados.rados.0.smithi017.stdout:3413: read oid 48 snap 423 2024-09-21T09:52:25.211 INFO:tasks.rados.rados.0.smithi017.stdout:3413: expect (ObjNum 1149 snap 381 seq_num 1149) 2024-09-21T09:52:25.211 INFO:tasks.rados.rados.0.smithi017.stdout:3414: snap_remove snap 406 2024-09-21T09:52:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:25 smithi017 ceph-mon[102913]: pgmap v957: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 973 KiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:52:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:25 smithi017 ceph-mon[102913]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T09:52:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:25 smithi017 ceph-mon[105151]: pgmap v957: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 973 KiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:52:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:25 smithi017 ceph-mon[105151]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T09:52:25.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:25 smithi134 ceph-mon[90085]: pgmap v957: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 973 KiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-09-21T09:52:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:25 smithi134 ceph-mon[90085]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T09:52:26.214 INFO:tasks.rados.rados.0.smithi017.stdout:3414: done (2 left) 2024-09-21T09:52:26.214 INFO:tasks.rados.rados.0.smithi017.stdout:3415: snap_remove snap 413 2024-09-21T09:52:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:26 smithi134 ceph-mon[90085]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T09:52:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:26 smithi017 ceph-mon[102913]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T09:52:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:26 smithi017 ceph-mon[105151]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T09:52:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:27.217 INFO:tasks.rados.rados.0.smithi017.stdout:3415: done (2 left) 2024-09-21T09:52:27.217 INFO:tasks.rados.rados.0.smithi017.stdout:3416: read oid 18 snap -1 2024-09-21T09:52:27.217 INFO:tasks.rados.rados.0.smithi017.stdout:3416: expect (ObjNum 1264 snap 423 seq_num 1264) 2024-09-21T09:52:27.217 INFO:tasks.rados.rados.0.smithi017.stdout:3417: snap_create 2024-09-21T09:52:27.217 INFO:tasks.rados.rados.0.smithi017.stdout:3411: expect (ObjNum 1185 snap 398 seq_num 1185) 2024-09-21T09:52:27.352 INFO:tasks.rados.rados.0.smithi017.stdout:3413: expect (ObjNum 1149 snap 381 seq_num 1149) 2024-09-21T09:52:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:27 smithi134 ceph-mon[90085]: pgmap v960: 89 pgs: 89 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 382 KiB/s wr, 6 op/s 2024-09-21T09:52:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:27 smithi134 ceph-mon[90085]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T09:52:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:52:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:52:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:27.592 INFO:tasks.rados.rados.0.smithi017.stdout:3416: expect (ObjNum 1264 snap 423 seq_num 1264) 2024-09-21T09:52:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[105151]: pgmap v960: 89 pgs: 89 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 382 KiB/s wr, 6 op/s 2024-09-21T09:52:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[105151]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T09:52:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:52:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:52:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[102913]: pgmap v960: 89 pgs: 89 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 382 KiB/s wr, 6 op/s 2024-09-21T09:52:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[102913]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T09:52:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:52:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:52:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:52:28.205 INFO:tasks.rados.rados.0.smithi017.stdout:3411: done (3 left) 2024-09-21T09:52:28.205 INFO:tasks.rados.rados.0.smithi017.stdout:3413: done (2 left) 2024-09-21T09:52:28.205 INFO:tasks.rados.rados.0.smithi017.stdout:3416: done (1 left) 2024-09-21T09:52:28.205 INFO:tasks.rados.rados.0.smithi017.stdout:3417: done (0 left) 2024-09-21T09:52:28.206 INFO:tasks.rados.rados.0.smithi017.stdout:3418: read oid 6 snap -1 2024-09-21T09:52:28.206 INFO:tasks.rados.rados.0.smithi017.stdout:3418: expect deleted 2024-09-21T09:52:28.206 INFO:tasks.rados.rados.0.smithi017.stdout:3419: delete oid 45 current snap is 430 2024-09-21T09:52:28.211 INFO:tasks.rados.rados.0.smithi017.stdout:3418: done (1 left) 2024-09-21T09:52:28.211 INFO:tasks.rados.rados.0.smithi017.stdout:3419: done (0 left) 2024-09-21T09:52:28.211 INFO:tasks.rados.rados.0.smithi017.stdout:3420: read oid 44 snap -1 2024-09-21T09:52:28.211 INFO:tasks.rados.rados.0.smithi017.stdout:3420: expect (ObjNum 1220 snap 405 seq_num 1220) 2024-09-21T09:52:28.211 INFO:tasks.rados.rados.0.smithi017.stdout:3421: snap_create 2024-09-21T09:52:28.213 INFO:tasks.rados.rados.0.smithi017.stdout:3420: expect (ObjNum 1220 snap 405 seq_num 1220) 2024-09-21T09:52:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:28 smithi134 ceph-mon[90085]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T09:52:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:28 smithi134 ceph-mon[90085]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T09:52:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:28 smithi017 ceph-mon[102913]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T09:52:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:28 smithi017 ceph-mon[102913]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T09:52:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:28 smithi017 ceph-mon[105151]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T09:52:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:28 smithi017 ceph-mon[105151]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T09:52:29.207 INFO:tasks.rados.rados.0.smithi017.stdout:3420: done (1 left) 2024-09-21T09:52:29.207 INFO:tasks.rados.rados.0.smithi017.stdout:3421: done (0 left) 2024-09-21T09:52:29.207 INFO:tasks.rados.rados.0.smithi017.stdout:3422: delete oid 8 current snap is 431 2024-09-21T09:52:29.210 INFO:tasks.rados.rados.0.smithi017.stdout:3422: done (0 left) 2024-09-21T09:52:29.210 INFO:tasks.rados.rados.0.smithi017.stdout:3423: delete oid 35 current snap is 431 2024-09-21T09:52:29.214 INFO:tasks.rados.rados.0.smithi017.stdout:3423: done (0 left) 2024-09-21T09:52:29.214 INFO:tasks.rados.rados.0.smithi017.stdout:3424: write oid 45 current snap is 431 2024-09-21T09:52:29.214 INFO:tasks.rados.rados.0.smithi017.stdout:3424: seq_num 1278 ranges {679346=515526,1868811=577782,3179173=300173} 2024-09-21T09:52:29.220 INFO:tasks.rados.rados.0.smithi017.stdout:3424: writing smithi01785587-45 from 679346 to 1194872 tid 1 2024-09-21T09:52:29.228 INFO:tasks.rados.rados.0.smithi017.stdout:3424: writing smithi01785587-45 from 1868811 to 2446593 tid 2 2024-09-21T09:52:29.234 INFO:tasks.rados.rados.0.smithi017.stdout:3424: writing smithi01785587-45 from 3179173 to 3479346 tid 3 2024-09-21T09:52:29.234 INFO:tasks.rados.rados.0.smithi017.stdout:3425: snap_create 2024-09-21T09:52:29.234 INFO:tasks.rados.rados.0.smithi017.stdout:3424: finishing write tid 1 to smithi01785587-45 2024-09-21T09:52:29.234 INFO:tasks.rados.rados.0.smithi017.stdout:3424: finishing write tid 2 to smithi01785587-45 2024-09-21T09:52:29.237 INFO:tasks.rados.rados.0.smithi017.stdout:3424: finishing write tid 3 to smithi01785587-45 2024-09-21T09:52:29.237 INFO:tasks.rados.rados.0.smithi017.stdout:3424: finishing write tid 4 to smithi01785587-45 2024-09-21T09:52:29.237 INFO:tasks.rados.rados.0.smithi017.stdout:3424: finishing write tid 5 to smithi01785587-45 2024-09-21T09:52:29.237 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1162 (ObjNum 1278 snap 431 seq_num 1278) dirty exists 2024-09-21T09:52:29.237 INFO:tasks.rados.rados.0.smithi017.stdout:3424: left oid 45 (ObjNum 1278 snap 431 seq_num 1278) 2024-09-21T09:52:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:29 smithi134 ceph-mon[90085]: pgmap v963: 89 pgs: 89 active+clean; 153 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 382 KiB/s wr, 11 op/s 2024-09-21T09:52:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:29 smithi134 ceph-mon[90085]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T09:52:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:29 smithi017 ceph-mon[102913]: pgmap v963: 89 pgs: 89 active+clean; 153 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 382 KiB/s wr, 11 op/s 2024-09-21T09:52:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:29 smithi017 ceph-mon[102913]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T09:52:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:29 smithi017 ceph-mon[105151]: pgmap v963: 89 pgs: 89 active+clean; 153 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 382 KiB/s wr, 11 op/s 2024-09-21T09:52:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:29 smithi017 ceph-mon[105151]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T09:52:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:3424: done (1 left) 2024-09-21T09:52:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:3425: done (0 left) 2024-09-21T09:52:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:3426: write oid 37 current snap is 432 2024-09-21T09:52:30.232 INFO:tasks.rados.rados.0.smithi017.stdout:3426: seq_num 1279 ranges {696153=590775,1927293=721440,3206596=289557} 2024-09-21T09:52:30.238 INFO:tasks.rados.rados.0.smithi017.stdout:3426: writing smithi01785587-37 from 696153 to 1286928 tid 1 2024-09-21T09:52:30.246 INFO:tasks.rados.rados.0.smithi017.stdout:3426: writing smithi01785587-37 from 1927293 to 2648733 tid 2 2024-09-21T09:52:30.251 INFO:tasks.rados.rados.0.smithi017.stdout:3426: writing smithi01785587-37 from 3206596 to 3496153 tid 3 2024-09-21T09:52:30.252 INFO:tasks.rados.rados.0.smithi017.stdout:3427: write oid 25 current snap is 432 2024-09-21T09:52:30.252 INFO:tasks.rados.rados.0.smithi017.stdout:3426: finishing write tid 1 to smithi01785587-37 2024-09-21T09:52:30.252 INFO:tasks.rados.rados.0.smithi017.stdout:3426: finishing write tid 2 to smithi01785587-37 2024-09-21T09:52:30.252 INFO:tasks.rados.rados.0.smithi017.stdout:3427: seq_num 1280 ranges {712960=666024,1985775=581451,3234019=278941} 2024-09-21T09:52:30.259 INFO:tasks.rados.rados.0.smithi017.stdout:3427: writing smithi01785587-25 from 712960 to 1378984 tid 1 2024-09-21T09:52:30.265 INFO:tasks.rados.rados.0.smithi017.stdout:3427: writing smithi01785587-25 from 1985775 to 2567226 tid 2 2024-09-21T09:52:30.271 INFO:tasks.rados.rados.0.smithi017.stdout:3427: writing smithi01785587-25 from 3234019 to 3512960 tid 3 2024-09-21T09:52:30.271 INFO:tasks.rados.rados.0.smithi017.stdout:3428: rollback oid 18 current snap is 432 2024-09-21T09:52:30.271 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 18 to 430 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3426: finishing write tid 3 to smithi01785587-37 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3426: finishing write tid 4 to smithi01785587-37 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3426: finishing write tid 5 to smithi01785587-37 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1755 (ObjNum 1279 snap 432 seq_num 1279) dirty exists 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3426: left oid 37 (ObjNum 1279 snap 432 seq_num 1279) 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3427: finishing write tid 1 to smithi01785587-25 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3427: finishing write tid 2 to smithi01785587-25 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3426: done (2 left) 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3429: read oid 13 snap -1 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3429: expect (ObjNum 1203 snap 402 seq_num 1203) 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3430: write oid 3 current snap is 432 2024-09-21T09:52:30.272 INFO:tasks.rados.rados.0.smithi017.stdout:3430: seq_num 1281 ranges {729767=741273,1927904=725109,3145089=384678} 2024-09-21T09:52:30.279 INFO:tasks.rados.rados.0.smithi017.stdout:3430: writing smithi01785587-3 from 729767 to 1471040 tid 1 2024-09-21T09:52:30.286 INFO:tasks.rados.rados.0.smithi017.stdout:3430: writing smithi01785587-3 from 1927904 to 2653013 tid 2 2024-09-21T09:52:30.291 INFO:tasks.rados.rados.0.smithi017.stdout:3430: writing smithi01785587-3 from 3145089 to 3529767 tid 3 2024-09-21T09:52:30.291 INFO:tasks.rados.rados.0.smithi017.stdout:3431: write oid 5 current snap is 432 2024-09-21T09:52:30.291 INFO:tasks.rados.rados.0.smithi017.stdout:3428: finishing rollback tid 0 to smithi01785587-18 2024-09-21T09:52:30.291 INFO:tasks.rados.rados.0.smithi017.stdout:3427: finishing write tid 3 to smithi01785587-25 2024-09-21T09:52:30.291 INFO:tasks.rados.rados.0.smithi017.stdout:3428: finishing rollback tid 1 to smithi01785587-18 2024-09-21T09:52:30.291 INFO:tasks.rados.rados.0.smithi017.stdout:3427: finishing write tid 4 to smithi01785587-25 2024-09-21T09:52:30.291 INFO:tasks.rados.rados.0.smithi017.stdout:3428: finishing rollback tid 2 to smithi01785587-18 2024-09-21T09:52:30.291 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 348 (ObjNum 1264 snap 423 seq_num 1264) dirty exists 2024-09-21T09:52:30.291 INFO:tasks.rados.rados.0.smithi017.stdout:3427: finishing write tid 5 to smithi01785587-25 2024-09-21T09:52:30.292 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 937 (ObjNum 1280 snap 432 seq_num 1280) dirty exists 2024-09-21T09:52:30.292 INFO:tasks.rados.rados.0.smithi017.stdout:3427: left oid 25 (ObjNum 1280 snap 432 seq_num 1280) 2024-09-21T09:52:30.292 INFO:tasks.rados.rados.0.smithi017.stdout:3429: expect (ObjNum 1203 snap 402 seq_num 1203) 2024-09-21T09:52:30.465 INFO:tasks.rados.rados.0.smithi017.stdout:3430: finishing write tid 1 to smithi01785587-3 2024-09-21T09:52:30.465 INFO:tasks.rados.rados.0.smithi017.stdout:3430: finishing write tid 2 to smithi01785587-3 2024-09-21T09:52:30.465 INFO:tasks.rados.rados.0.smithi017.stdout:3430: finishing write tid 3 to smithi01785587-3 2024-09-21T09:52:30.465 INFO:tasks.rados.rados.0.smithi017.stdout:3430: finishing write tid 4 to smithi01785587-3 2024-09-21T09:52:30.465 INFO:tasks.rados.rados.0.smithi017.stdout:3430: finishing write tid 5 to smithi01785587-3 2024-09-21T09:52:30.465 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1897 (ObjNum 1281 snap 432 seq_num 1281) dirty exists 2024-09-21T09:52:30.465 INFO:tasks.rados.rados.0.smithi017.stdout:3430: left oid 3 (ObjNum 1281 snap 432 seq_num 1281) 2024-09-21T09:52:30.466 INFO:tasks.rados.rados.0.smithi017.stdout:3431: seq_num 1282 ranges {746574=416522,1586386=468767,2772512=629544} 2024-09-21T09:52:30.471 INFO:tasks.rados.rados.0.smithi017.stdout:3431: writing smithi01785587-5 from 746574 to 1163096 tid 1 2024-09-21T09:52:30.476 INFO:tasks.rados.rados.0.smithi017.stdout:3431: writing smithi01785587-5 from 1586386 to 2055153 tid 2 2024-09-21T09:52:30.484 INFO:tasks.rados.rados.0.smithi017.stdout:3431: writing smithi01785587-5 from 2772512 to 3402056 tid 3 2024-09-21T09:52:30.484 INFO:tasks.rados.rados.0.smithi017.stdout:3427: done (4 left) 2024-09-21T09:52:30.484 INFO:tasks.rados.rados.0.smithi017.stdout:3428: done (3 left) 2024-09-21T09:52:30.484 INFO:tasks.rados.rados.0.smithi017.stdout:3429: done (2 left) 2024-09-21T09:52:30.484 INFO:tasks.rados.rados.0.smithi017.stdout:3430: done (1 left) 2024-09-21T09:52:30.484 INFO:tasks.rados.rados.0.smithi017.stdout:3432: rollback oid 41 current snap is 432 2024-09-21T09:52:30.484 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 41 to 428 2024-09-21T09:52:30.484 INFO:tasks.rados.rados.0.smithi017.stdout:3431: finishing write tid 1 to smithi01785587-5 2024-09-21T09:52:30.484 INFO:tasks.rados.rados.0.smithi017.stdout:3431: finishing write tid 2 to smithi01785587-5 2024-09-21T09:52:30.485 INFO:tasks.rados.rados.0.smithi017.stdout:3433: read oid 37 snap -1 2024-09-21T09:52:30.485 INFO:tasks.rados.rados.0.smithi017.stdout:3433: expect (ObjNum 1279 snap 432 seq_num 1279) 2024-09-21T09:52:30.485 INFO:tasks.rados.rados.0.smithi017.stdout:3434: delete oid 36 current snap is 432 2024-09-21T09:52:30.485 INFO:tasks.rados.rados.0.smithi017.stdout:3432: finishing rollback tid 1 to smithi01785587-41 2024-09-21T09:52:30.485 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:30.487 INFO:tasks.rados.rados.0.smithi017.stdout:3431: finishing write tid 3 to smithi01785587-5 2024-09-21T09:52:30.487 INFO:tasks.rados.rados.0.smithi017.stdout:3431: finishing write tid 4 to smithi01785587-5 2024-09-21T09:52:30.488 INFO:tasks.rados.rados.0.smithi017.stdout:3431: finishing write tid 5 to smithi01785587-5 2024-09-21T09:52:30.488 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1902 (ObjNum 1282 snap 432 seq_num 1282) dirty exists 2024-09-21T09:52:30.488 INFO:tasks.rados.rados.0.smithi017.stdout:3431: left oid 5 (ObjNum 1282 snap 432 seq_num 1282) 2024-09-21T09:52:30.488 INFO:tasks.rados.rados.0.smithi017.stdout:3433: expect (ObjNum 1279 snap 432 seq_num 1279) 2024-09-21T09:52:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:52:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:52:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:52:30.701 INFO:tasks.rados.rados.0.smithi017.stdout:3431: done (3 left) 2024-09-21T09:52:30.702 INFO:tasks.rados.rados.0.smithi017.stdout:3432: done (2 left) 2024-09-21T09:52:30.702 INFO:tasks.rados.rados.0.smithi017.stdout:3433: done (1 left) 2024-09-21T09:52:30.702 INFO:tasks.rados.rados.0.smithi017.stdout:3434: done (0 left) 2024-09-21T09:52:30.702 INFO:tasks.rados.rados.0.smithi017.stdout:3435: write oid 38 current snap is 432 2024-09-21T09:52:30.702 INFO:tasks.rados.rados.0.smithi017.stdout:3435: seq_num 1284 ranges {780188=567020,2103350=472436,3227358=352830} 2024-09-21T09:52:30.708 INFO:tasks.rados.rados.0.smithi017.stdout:3435: writing smithi01785587-38 from 780188 to 1347208 tid 1 2024-09-21T09:52:30.716 INFO:tasks.rados.rados.0.smithi017.stdout:3435: writing smithi01785587-38 from 2103350 to 2575786 tid 2 2024-09-21T09:52:30.721 INFO:tasks.rados.rados.0.smithi017.stdout:3435: writing smithi01785587-38 from 3227358 to 3580188 tid 3 2024-09-21T09:52:30.721 INFO:tasks.rados.rados.0.smithi017.stdout:3436: read oid 12 snap 424 2024-09-21T09:52:30.721 INFO:tasks.rados.rados.0.smithi017.stdout:3436: expect (ObjNum 1172 snap 393 seq_num 1172) 2024-09-21T09:52:30.721 INFO:tasks.rados.rados.0.smithi017.stdout:3437: write oid 37 current snap is 432 2024-09-21T09:52:30.721 INFO:tasks.rados.rados.0.smithi017.stdout:3435: finishing write tid 1 to smithi01785587-38 2024-09-21T09:52:30.722 INFO:tasks.rados.rados.0.smithi017.stdout:3435: finishing write tid 2 to smithi01785587-38 2024-09-21T09:52:30.722 INFO:tasks.rados.rados.0.smithi017.stdout:3437: seq_num 1285 ranges {796995=758622,2161832=732447} 2024-09-21T09:52:30.730 INFO:tasks.rados.rados.0.smithi017.stdout:3437: writing smithi01785587-37 from 796995 to 1555617 tid 1 2024-09-21T09:52:30.737 INFO:tasks.rados.rados.0.smithi017.stdout:3437: writing smithi01785587-37 from 2161832 to 2894279 tid 2 2024-09-21T09:52:30.737 INFO:tasks.rados.rados.0.smithi017.stdout:3438: snap_remove snap 423 2024-09-21T09:52:31.236 INFO:tasks.rados.rados.0.smithi017.stdout:3438: done (3 left) 2024-09-21T09:52:31.236 INFO:tasks.rados.rados.0.smithi017.stdout:3439: write oid 7 current snap is 432 2024-09-21T09:52:31.236 INFO:tasks.rados.rados.0.smithi017.stdout:3435: finishing write tid 3 to smithi01785587-38 2024-09-21T09:52:31.236 INFO:tasks.rados.rados.0.smithi017.stdout:3435: finishing write tid 4 to smithi01785587-38 2024-09-21T09:52:31.236 INFO:tasks.rados.rados.0.smithi017.stdout:3435: finishing write tid 5 to smithi01785587-38 2024-09-21T09:52:31.237 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 1063 (ObjNum 1284 snap 432 seq_num 1284) dirty exists 2024-09-21T09:52:31.237 INFO:tasks.rados.rados.0.smithi017.stdout:3435: left oid 38 (ObjNum 1284 snap 432 seq_num 1284) 2024-09-21T09:52:31.237 INFO:tasks.rados.rados.0.smithi017.stdout:3436: expect (ObjNum 1172 snap 393 seq_num 1172) 2024-09-21T09:52:31.412 INFO:tasks.rados.rados.0.smithi017.stdout:3437: finishing write tid 1 to smithi01785587-37 2024-09-21T09:52:31.412 INFO:tasks.rados.rados.0.smithi017.stdout:3437: finishing write tid 2 to smithi01785587-37 2024-09-21T09:52:31.412 INFO:tasks.rados.rados.0.smithi017.stdout:3437: finishing write tid 3 to smithi01785587-37 2024-09-21T09:52:31.412 INFO:tasks.rados.rados.0.smithi017.stdout:3437: finishing write tid 4 to smithi01785587-37 2024-09-21T09:52:31.412 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1760 (ObjNum 1285 snap 432 seq_num 1285) dirty exists 2024-09-21T09:52:31.412 INFO:tasks.rados.rados.0.smithi017.stdout:3437: left oid 37 (ObjNum 1285 snap 432 seq_num 1285) 2024-09-21T09:52:31.412 INFO:tasks.rados.rados.0.smithi017.stdout:3439: seq_num 1286 ranges {413802=433871,1420314=476105,2482204=790985} 2024-09-21T09:52:31.416 INFO:tasks.rados.rados.0.smithi017.stdout:3439: writing smithi01785587-7 from 413802 to 847673 tid 1 2024-09-21T09:52:31.422 INFO:tasks.rados.rados.0.smithi017.stdout:3439: writing smithi01785587-7 from 1420314 to 1896419 tid 2 2024-09-21T09:52:31.430 INFO:tasks.rados.rados.0.smithi017.stdout:3439: writing smithi01785587-7 from 2482204 to 3273189 tid 3 2024-09-21T09:52:31.430 INFO:tasks.rados.rados.0.smithi017.stdout:3435: done (3 left) 2024-09-21T09:52:31.430 INFO:tasks.rados.rados.0.smithi017.stdout:3436: done (2 left) 2024-09-21T09:52:31.430 INFO:tasks.rados.rados.0.smithi017.stdout:3437: done (1 left) 2024-09-21T09:52:31.431 INFO:tasks.rados.rados.0.smithi017.stdout:3440: snap_create 2024-09-21T09:52:31.431 INFO:tasks.rados.rados.0.smithi017.stdout:3439: finishing write tid 1 to smithi01785587-7 2024-09-21T09:52:31.431 INFO:tasks.rados.rados.0.smithi017.stdout:3439: finishing write tid 2 to smithi01785587-7 2024-09-21T09:52:31.434 INFO:tasks.rados.rados.0.smithi017.stdout:3439: finishing write tid 3 to smithi01785587-7 2024-09-21T09:52:31.435 INFO:tasks.rados.rados.0.smithi017.stdout:3439: finishing write tid 4 to smithi01785587-7 2024-09-21T09:52:31.435 INFO:tasks.rados.rados.0.smithi017.stdout:3439: finishing write tid 5 to smithi01785587-7 2024-09-21T09:52:31.435 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1675 (ObjNum 1286 snap 432 seq_num 1286) dirty exists 2024-09-21T09:52:31.435 INFO:tasks.rados.rados.0.smithi017.stdout:3439: left oid 7 (ObjNum 1286 snap 432 seq_num 1286) 2024-09-21T09:52:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:31 smithi134 ceph-mon[90085]: pgmap v966: 89 pgs: 89 active+clean; 143 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:52:31.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:31 smithi134 ceph-mon[90085]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T09:52:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:31 smithi017 ceph-mon[102913]: pgmap v966: 89 pgs: 89 active+clean; 143 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:52:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:31 smithi017 ceph-mon[102913]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T09:52:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:31 smithi017 ceph-mon[105151]: pgmap v966: 89 pgs: 89 active+clean; 143 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 0 B/s wr, 5 op/s 2024-09-21T09:52:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:31 smithi017 ceph-mon[105151]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T09:52:32.246 INFO:tasks.rados.rados.0.smithi017.stdout:3439: done (1 left) 2024-09-21T09:52:32.246 INFO:tasks.rados.rados.0.smithi017.stdout:3440: done (0 left) 2024-09-21T09:52:32.246 INFO:tasks.rados.rados.0.smithi017.stdout:3441: write oid 16 current snap is 433 2024-09-21T09:52:32.246 INFO:tasks.rados.rados.0.smithi017.stdout:3441: seq_num 1287 ranges {430609=509120,1478796=736116,2909627=602257} 2024-09-21T09:52:32.250 INFO:tasks.rados.rados.0.smithi017.stdout:3441: writing smithi01785587-16 from 430609 to 939729 tid 1 2024-09-21T09:52:32.258 INFO:tasks.rados.rados.0.smithi017.stdout:3441: writing smithi01785587-16 from 1478796 to 2214912 tid 2 2024-09-21T09:52:32.265 INFO:tasks.rados.rados.0.smithi017.stdout:3441: writing smithi01785587-16 from 2909627 to 3511884 tid 3 2024-09-21T09:52:32.265 INFO:tasks.rados.rados.0.smithi017.stdout:3442: write oid 24 current snap is 433 2024-09-21T09:52:32.265 INFO:tasks.rados.rados.0.smithi017.stdout:3442: seq_num 1288 ranges {447416=584369,1537278=479774,2537050=529882,3524784=122632} 2024-09-21T09:52:32.270 INFO:tasks.rados.rados.0.smithi017.stdout:3442: writing smithi01785587-24 from 447416 to 1031785 tid 1 2024-09-21T09:52:32.276 INFO:tasks.rados.rados.0.smithi017.stdout:3442: writing smithi01785587-24 from 1537278 to 2017052 tid 2 2024-09-21T09:52:32.282 INFO:tasks.rados.rados.0.smithi017.stdout:3442: writing smithi01785587-24 from 2537050 to 3066932 tid 3 2024-09-21T09:52:32.286 INFO:tasks.rados.rados.0.smithi017.stdout:3442: writing smithi01785587-24 from 3524784 to 3647416 tid 4 2024-09-21T09:52:32.286 INFO:tasks.rados.rados.0.smithi017.stdout:3443: rollback oid 35 current snap is 433 2024-09-21T09:52:32.286 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 422 2024-09-21T09:52:32.286 INFO:tasks.rados.rados.0.smithi017.stdout:3441: finishing write tid 1 to smithi01785587-16 2024-09-21T09:52:32.286 INFO:tasks.rados.rados.0.smithi017.stdout:3441: finishing write tid 2 to smithi01785587-16 2024-09-21T09:52:32.286 INFO:tasks.rados.rados.0.smithi017.stdout:3441: finishing write tid 3 to smithi01785587-16 2024-09-21T09:52:32.286 INFO:tasks.rados.rados.0.smithi017.stdout:3441: finishing write tid 4 to smithi01785587-16 2024-09-21T09:52:32.287 INFO:tasks.rados.rados.0.smithi017.stdout:3441: finishing write tid 5 to smithi01785587-16 2024-09-21T09:52:32.287 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1824 (ObjNum 1287 snap 433 seq_num 1287) dirty exists 2024-09-21T09:52:32.287 INFO:tasks.rados.rados.0.smithi017.stdout:3441: left oid 16 (ObjNum 1287 snap 433 seq_num 1287) 2024-09-21T09:52:32.287 INFO:tasks.rados.rados.0.smithi017.stdout:3442: finishing write tid 1 to smithi01785587-24 2024-09-21T09:52:32.287 INFO:tasks.rados.rados.0.smithi017.stdout:3442: finishing write tid 2 to smithi01785587-24 2024-09-21T09:52:32.287 INFO:tasks.rados.rados.0.smithi017.stdout:3442: finishing write tid 3 to smithi01785587-24 2024-09-21T09:52:32.287 INFO:tasks.rados.rados.0.smithi017.stdout:3441: done (2 left) 2024-09-21T09:52:32.287 INFO:tasks.rados.rados.0.smithi017.stdout:3444: write oid 45 current snap is 433 2024-09-21T09:52:32.288 INFO:tasks.rados.rados.0.smithi017.stdout:3444: seq_num 1289 ranges {464223=659618,1879407=739785,3248120=416103} 2024-09-21T09:52:32.291 INFO:tasks.rados.rados.0.smithi017.stdout:3444: writing smithi01785587-45 from 464223 to 1123841 tid 1 2024-09-21T09:52:32.300 INFO:tasks.rados.rados.0.smithi017.stdout:3444: writing smithi01785587-45 from 1879407 to 2619192 tid 2 2024-09-21T09:52:32.306 INFO:tasks.rados.rados.0.smithi017.stdout:3444: writing smithi01785587-45 from 3248120 to 3664223 tid 3 2024-09-21T09:52:32.306 INFO:tasks.rados.rados.0.smithi017.stdout:3445: snap_create 2024-09-21T09:52:32.306 INFO:tasks.rados.rados.0.smithi017.stdout:3442: finishing write tid 4 to smithi01785587-24 2024-09-21T09:52:32.306 INFO:tasks.rados.rados.0.smithi017.stdout:3442: finishing write tid 5 to smithi01785587-24 2024-09-21T09:52:32.306 INFO:tasks.rados.rados.0.smithi017.stdout:3442: finishing write tid 6 to smithi01785587-24 2024-09-21T09:52:32.306 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 1069 (ObjNum 1288 snap 433 seq_num 1288) dirty exists 2024-09-21T09:52:32.306 INFO:tasks.rados.rados.0.smithi017.stdout:3442: left oid 24 (ObjNum 1288 snap 433 seq_num 1288) 2024-09-21T09:52:32.306 INFO:tasks.rados.rados.0.smithi017.stdout:3443: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:52:32.306 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 376 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:32.307 INFO:tasks.rados.rados.0.smithi017.stdout:3444: finishing write tid 1 to smithi01785587-45 2024-09-21T09:52:32.307 INFO:tasks.rados.rados.0.smithi017.stdout:3444: finishing write tid 2 to smithi01785587-45 2024-09-21T09:52:32.308 INFO:tasks.rados.rados.0.smithi017.stdout:3444: finishing write tid 3 to smithi01785587-45 2024-09-21T09:52:32.309 INFO:tasks.rados.rados.0.smithi017.stdout:3444: finishing write tid 4 to smithi01785587-45 2024-09-21T09:52:32.309 INFO:tasks.rados.rados.0.smithi017.stdout:3444: finishing write tid 5 to smithi01785587-45 2024-09-21T09:52:32.309 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1169 (ObjNum 1289 snap 433 seq_num 1289) dirty exists 2024-09-21T09:52:32.309 INFO:tasks.rados.rados.0.smithi017.stdout:3444: left oid 45 (ObjNum 1289 snap 433 seq_num 1289) 2024-09-21T09:52:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:32 smithi134 ceph-mon[90085]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T09:52:32.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:32 smithi017 ceph-mon[102913]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T09:52:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:32 smithi017 ceph-mon[105151]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T09:52:33.207 INFO:tasks.rados.rados.0.smithi017.stdout:3442: done (3 left) 2024-09-21T09:52:33.207 INFO:tasks.rados.rados.0.smithi017.stdout:3443: done (2 left) 2024-09-21T09:52:33.207 INFO:tasks.rados.rados.0.smithi017.stdout:3444: done (1 left) 2024-09-21T09:52:33.207 INFO:tasks.rados.rados.0.smithi017.stdout:3445: done (0 left) 2024-09-21T09:52:33.207 INFO:tasks.rados.rados.0.smithi017.stdout:3446: snap_remove snap 411 2024-09-21T09:52:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:33 smithi134 ceph-mon[90085]: pgmap v969: 89 pgs: 89 active+clean; 142 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 713 KiB/s wr, 12 op/s 2024-09-21T09:52:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:33 smithi134 ceph-mon[90085]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T09:52:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:33 smithi134 ceph-mon[90085]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T09:52:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:33 smithi017 ceph-mon[102913]: pgmap v969: 89 pgs: 89 active+clean; 142 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 713 KiB/s wr, 12 op/s 2024-09-21T09:52:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:33 smithi017 ceph-mon[102913]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T09:52:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:33 smithi017 ceph-mon[102913]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T09:52:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:33 smithi017 ceph-mon[105151]: pgmap v969: 89 pgs: 89 active+clean; 142 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 713 KiB/s wr, 12 op/s 2024-09-21T09:52:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:33 smithi017 ceph-mon[105151]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T09:52:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:33 smithi017 ceph-mon[105151]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T09:52:34.210 INFO:tasks.rados.rados.0.smithi017.stdout:3446: done (0 left) 2024-09-21T09:52:34.210 INFO:tasks.rados.rados.0.smithi017.stdout:3447: read oid 31 snap 414 2024-09-21T09:52:34.210 INFO:tasks.rados.rados.0.smithi017.stdout:3447: expect (ObjNum 1226 snap 405 seq_num 1226) 2024-09-21T09:52:34.210 INFO:tasks.rados.rados.0.smithi017.stdout:3448: write oid 25 current snap is 434 2024-09-21T09:52:34.210 INFO:tasks.rados.rados.0.smithi017.stdout:3448: seq_num 1290 ranges {481030=734867,1937889=483443,2875543=552426} 2024-09-21T09:52:34.217 INFO:tasks.rados.rados.0.smithi017.stdout:3448: writing smithi01785587-25 from 481030 to 1215897 tid 1 2024-09-21T09:52:34.225 INFO:tasks.rados.rados.0.smithi017.stdout:3448: writing smithi01785587-25 from 1937889 to 2421332 tid 2 2024-09-21T09:52:34.230 INFO:tasks.rados.rados.0.smithi017.stdout:3448: writing smithi01785587-25 from 2875543 to 3427969 tid 3 2024-09-21T09:52:34.230 INFO:tasks.rados.rados.0.smithi017.stdout:3449: snap_remove snap 433 2024-09-21T09:52:35.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:52:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:52:34] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.51.0" 2024-09-21T09:52:35.214 INFO:tasks.rados.rados.0.smithi017.stdout:3449: done (2 left) 2024-09-21T09:52:35.214 INFO:tasks.rados.rados.0.smithi017.stdout:3450: read oid 21 snap -1 2024-09-21T09:52:35.214 INFO:tasks.rados.rados.0.smithi017.stdout:3450: expect (ObjNum 1231 snap 409 seq_num 1231) 2024-09-21T09:52:35.214 INFO:tasks.rados.rados.0.smithi017.stdout:3451: snap_create 2024-09-21T09:52:35.214 INFO:tasks.rados.rados.0.smithi017.stdout:3447: expect (ObjNum 1226 snap 405 seq_num 1226) 2024-09-21T09:52:35.407 INFO:tasks.rados.rados.0.smithi017.stdout:3448: finishing write tid 1 to smithi01785587-25 2024-09-21T09:52:35.407 INFO:tasks.rados.rados.0.smithi017.stdout:3448: finishing write tid 2 to smithi01785587-25 2024-09-21T09:52:35.407 INFO:tasks.rados.rados.0.smithi017.stdout:3448: finishing write tid 3 to smithi01785587-25 2024-09-21T09:52:35.407 INFO:tasks.rados.rados.0.smithi017.stdout:3448: finishing write tid 4 to smithi01785587-25 2024-09-21T09:52:35.407 INFO:tasks.rados.rados.0.smithi017.stdout:3448: finishing write tid 5 to smithi01785587-25 2024-09-21T09:52:35.407 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 950 (ObjNum 1290 snap 434 seq_num 1290) dirty exists 2024-09-21T09:52:35.407 INFO:tasks.rados.rados.0.smithi017.stdout:3448: left oid 25 (ObjNum 1290 snap 434 seq_num 1290) 2024-09-21T09:52:35.407 INFO:tasks.rados.rados.0.smithi017.stdout:3450: expect (ObjNum 1231 snap 409 seq_num 1231) 2024-09-21T09:52:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:35 smithi134 ceph-mon[90085]: pgmap v972: 89 pgs: 89 active+clean; 149 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2024-09-21T09:52:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:35 smithi134 ceph-mon[90085]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T09:52:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:35 smithi017 ceph-mon[102913]: pgmap v972: 89 pgs: 89 active+clean; 149 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2024-09-21T09:52:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:35 smithi017 ceph-mon[102913]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T09:52:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:35 smithi017 ceph-mon[105151]: pgmap v972: 89 pgs: 89 active+clean; 149 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.2 MiB/s wr, 20 op/s 2024-09-21T09:52:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:35 smithi017 ceph-mon[105151]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T09:52:36.218 INFO:tasks.rados.rados.0.smithi017.stdout:3447: done (3 left) 2024-09-21T09:52:36.218 INFO:tasks.rados.rados.0.smithi017.stdout:3448: done (2 left) 2024-09-21T09:52:36.218 INFO:tasks.rados.rados.0.smithi017.stdout:3450: done (1 left) 2024-09-21T09:52:36.219 INFO:tasks.rados.rados.0.smithi017.stdout:3451: done (0 left) 2024-09-21T09:52:36.219 INFO:tasks.rados.rados.0.smithi017.stdout:3452: snap_remove snap 425 2024-09-21T09:52:36.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:36 smithi134 ceph-mon[90085]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T09:52:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:36 smithi017 ceph-mon[105151]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T09:52:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:36 smithi017 ceph-mon[102913]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T09:52:37.225 INFO:tasks.rados.rados.0.smithi017.stdout:3452: done (0 left) 2024-09-21T09:52:37.226 INFO:tasks.rados.rados.0.smithi017.stdout:3453: rollback oid 11 current snap is 435 2024-09-21T09:52:37.226 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 11 to 429 2024-09-21T09:52:37.226 INFO:tasks.rados.rados.0.smithi017.stdout:3454: snap_remove snap 414 2024-09-21T09:52:37.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:37 smithi134 ceph-mon[90085]: pgmap v975: 89 pgs: 89 active+clean; 153 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2024-09-21T09:52:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:37 smithi134 ceph-mon[90085]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T09:52:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:37 smithi017 ceph-mon[102913]: pgmap v975: 89 pgs: 89 active+clean; 153 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2024-09-21T09:52:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:37 smithi017 ceph-mon[102913]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T09:52:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:37 smithi017 ceph-mon[105151]: pgmap v975: 89 pgs: 89 active+clean; 153 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2024-09-21T09:52:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:37 smithi017 ceph-mon[105151]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T09:52:38.209 INFO:tasks.rados.rados.0.smithi017.stdout:3454: done (1 left) 2024-09-21T09:52:38.209 INFO:tasks.rados.rados.0.smithi017.stdout:3455: rollback oid 17 current snap is 435 2024-09-21T09:52:38.209 INFO:tasks.rados.rados.0.smithi017.stdout:3453: finishing rollback tid 1 to smithi01785587-11 2024-09-21T09:52:38.210 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:52:38.210 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 434 2024-09-21T09:52:38.210 INFO:tasks.rados.rados.0.smithi017.stdout:3453: done (1 left) 2024-09-21T09:52:38.210 INFO:tasks.rados.rados.0.smithi017.stdout:3456: snap_create 2024-09-21T09:52:38.217 INFO:tasks.rados.rados.0.smithi017.stdout:3455: finishing rollback tid 0 to smithi01785587-17 2024-09-21T09:52:38.217 INFO:tasks.rados.rados.0.smithi017.stdout:3455: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:52:38.218 INFO:tasks.rados.rados.0.smithi017.stdout:3455: finishing rollback tid 2 to smithi01785587-17 2024-09-21T09:52:38.218 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 1177 (ObjNum 1230 snap 409 seq_num 1230) dirty exists 2024-09-21T09:52:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:38 smithi134 ceph-mon[90085]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T09:52:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:38 smithi134 ceph-mon[90085]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T09:52:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:38 smithi017 ceph-mon[102913]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T09:52:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:38 smithi017 ceph-mon[102913]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T09:52:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:38 smithi017 ceph-mon[105151]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T09:52:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:38 smithi017 ceph-mon[105151]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T09:52:39.209 INFO:tasks.rados.rados.0.smithi017.stdout:3455: done (1 left) 2024-09-21T09:52:39.209 INFO:tasks.rados.rados.0.smithi017.stdout:3456: done (0 left) 2024-09-21T09:52:39.209 INFO:tasks.rados.rados.0.smithi017.stdout:3457: snap_remove snap 428 2024-09-21T09:52:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:39 smithi134 ceph-mon[90085]: pgmap v978: 89 pgs: 89 active+clean; 158 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T09:52:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:39 smithi134 ceph-mon[90085]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T09:52:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:39 smithi017 ceph-mon[102913]: pgmap v978: 89 pgs: 89 active+clean; 158 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T09:52:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:39 smithi017 ceph-mon[102913]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T09:52:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:39 smithi017 ceph-mon[105151]: pgmap v978: 89 pgs: 89 active+clean; 158 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T09:52:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:39 smithi017 ceph-mon[105151]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T09:52:40.212 INFO:tasks.rados.rados.0.smithi017.stdout:3457: done (0 left) 2024-09-21T09:52:40.212 INFO:tasks.rados.rados.0.smithi017.stdout:3458: snap_remove snap 431 2024-09-21T09:52:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:52:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:52:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:52:41.218 INFO:tasks.rados.rados.0.smithi017.stdout:3458: done (0 left) 2024-09-21T09:52:41.218 INFO:tasks.rados.rados.0.smithi017.stdout:3459: rollback oid 36 current snap is 436 2024-09-21T09:52:41.218 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 36 to 422 2024-09-21T09:52:41.218 INFO:tasks.rados.rados.0.smithi017.stdout:3460: snap_create 2024-09-21T09:52:41.225 INFO:tasks.rados.rados.0.smithi017.stdout:3459: finishing rollback tid 1 to smithi01785587-36 2024-09-21T09:52:41.225 INFO:tasks.rados.rados.0.smithi017.stdout:3459: finishing rollback tid 2 to smithi01785587-36 2024-09-21T09:52:41.225 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1786 (ObjNum 1257 snap 421 seq_num 1257) dirty exists 2024-09-21T09:52:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:41 smithi134 ceph-mon[90085]: pgmap v981: 89 pgs: 89 active+clean; 158 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 835 KiB/s wr, 5 op/s 2024-09-21T09:52:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:41 smithi134 ceph-mon[90085]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T09:52:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:41.502 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:41 smithi017 ceph-mon[102913]: pgmap v981: 89 pgs: 89 active+clean; 158 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 835 KiB/s wr, 5 op/s 2024-09-21T09:52:41.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:41 smithi017 ceph-mon[102913]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T09:52:41.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:41.503 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:41 smithi017 ceph-mon[105151]: pgmap v981: 89 pgs: 89 active+clean; 158 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 835 KiB/s wr, 5 op/s 2024-09-21T09:52:41.503 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:41 smithi017 ceph-mon[105151]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T09:52:41.503 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:42.223 INFO:tasks.rados.rados.0.smithi017.stdout:3459: done (1 left) 2024-09-21T09:52:42.223 INFO:tasks.rados.rados.0.smithi017.stdout:3460: done (0 left) 2024-09-21T09:52:42.223 INFO:tasks.rados.rados.0.smithi017.stdout:3461: rollback oid 33 current snap is 437 2024-09-21T09:52:42.223 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 434 2024-09-21T09:52:42.223 INFO:tasks.rados.rados.0.smithi017.stdout:3462: write oid 20 current snap is 437 2024-09-21T09:52:42.223 INFO:tasks.rados.rados.0.smithi017.stdout:3462: seq_num 1291 ranges {497837=410116,1596371=743454,2902966=763698} 2024-09-21T09:52:42.227 INFO:tasks.rados.rados.0.smithi017.stdout:3462: writing smithi01785587-20 from 497837 to 907953 tid 1 2024-09-21T09:52:42.235 INFO:tasks.rados.rados.0.smithi017.stdout:3462: writing smithi01785587-20 from 1596371 to 2339825 tid 2 2024-09-21T09:52:42.243 INFO:tasks.rados.rados.0.smithi017.stdout:3462: writing smithi01785587-20 from 2902966 to 3666664 tid 3 2024-09-21T09:52:42.243 INFO:tasks.rados.rados.0.smithi017.stdout:3463: write oid 18 current snap is 437 2024-09-21T09:52:42.243 INFO:tasks.rados.rados.0.smithi017.stdout:3463: seq_num 1292 ranges {514644=485365,1654853=487112,2930389=574970} 2024-09-21T09:52:42.248 INFO:tasks.rados.rados.0.smithi017.stdout:3463: writing smithi01785587-18 from 514644 to 1000009 tid 1 2024-09-21T09:52:42.254 INFO:tasks.rados.rados.0.smithi017.stdout:3463: writing smithi01785587-18 from 1654853 to 2141965 tid 2 2024-09-21T09:52:42.262 INFO:tasks.rados.rados.0.smithi017.stdout:3463: writing smithi01785587-18 from 2930389 to 3505359 tid 3 2024-09-21T09:52:42.262 INFO:tasks.rados.rados.0.smithi017.stdout:3464: snap_create 2024-09-21T09:52:42.262 INFO:tasks.rados.rados.0.smithi017.stdout:3461: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:52:42.262 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:42.262 INFO:tasks.rados.rados.0.smithi017.stdout:3462: finishing write tid 1 to smithi01785587-20 2024-09-21T09:52:42.263 INFO:tasks.rados.rados.0.smithi017.stdout:3462: finishing write tid 2 to smithi01785587-20 2024-09-21T09:52:42.263 INFO:tasks.rados.rados.0.smithi017.stdout:3462: finishing write tid 3 to smithi01785587-20 2024-09-21T09:52:42.263 INFO:tasks.rados.rados.0.smithi017.stdout:3462: finishing write tid 4 to smithi01785587-20 2024-09-21T09:52:42.263 INFO:tasks.rados.rados.0.smithi017.stdout:3462: finishing write tid 5 to smithi01785587-20 2024-09-21T09:52:42.263 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 969 (ObjNum 1291 snap 437 seq_num 1291) dirty exists 2024-09-21T09:52:42.263 INFO:tasks.rados.rados.0.smithi017.stdout:3462: left oid 20 (ObjNum 1291 snap 437 seq_num 1291) 2024-09-21T09:52:42.263 INFO:tasks.rados.rados.0.smithi017.stdout:3463: finishing write tid 1 to smithi01785587-18 2024-09-21T09:52:42.263 INFO:tasks.rados.rados.0.smithi017.stdout:3463: finishing write tid 2 to smithi01785587-18 2024-09-21T09:52:42.265 INFO:tasks.rados.rados.0.smithi017.stdout:3463: finishing write tid 3 to smithi01785587-18 2024-09-21T09:52:42.265 INFO:tasks.rados.rados.0.smithi017.stdout:3463: finishing write tid 4 to smithi01785587-18 2024-09-21T09:52:42.265 INFO:tasks.rados.rados.0.smithi017.stdout:3463: finishing write tid 5 to smithi01785587-18 2024-09-21T09:52:42.265 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 365 (ObjNum 1292 snap 437 seq_num 1292) dirty exists 2024-09-21T09:52:42.265 INFO:tasks.rados.rados.0.smithi017.stdout:3463: left oid 18 (ObjNum 1292 snap 437 seq_num 1292) 2024-09-21T09:52:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:42 smithi134 ceph-mon[90085]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T09:52:42.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:42 smithi017 ceph-mon[102913]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T09:52:42.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:42 smithi017 ceph-mon[105151]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T09:52:43.209 INFO:tasks.rados.rados.0.smithi017.stdout:3461: done (3 left) 2024-09-21T09:52:43.209 INFO:tasks.rados.rados.0.smithi017.stdout:3462: done (2 left) 2024-09-21T09:52:43.209 INFO:tasks.rados.rados.0.smithi017.stdout:3463: done (1 left) 2024-09-21T09:52:43.209 INFO:tasks.rados.rados.0.smithi017.stdout:3464: done (0 left) 2024-09-21T09:52:43.209 INFO:tasks.rados.rados.0.smithi017.stdout:3465: write oid 49 current snap is 438 2024-09-21T09:52:43.209 INFO:tasks.rados.rados.0.smithi017.stdout:3465: seq_num 1293 ranges {531451=676967,1713335=747123,2957812=502595} 2024-09-21T09:52:43.215 INFO:tasks.rados.rados.0.smithi017.stdout:3465: writing smithi01785587-49 from 531451 to 1208418 tid 1 2024-09-21T09:52:43.222 INFO:tasks.rados.rados.0.smithi017.stdout:3465: writing smithi01785587-49 from 1713335 to 2460458 tid 2 2024-09-21T09:52:43.228 INFO:tasks.rados.rados.0.smithi017.stdout:3465: writing smithi01785587-49 from 2957812 to 3460407 tid 3 2024-09-21T09:52:43.228 INFO:tasks.rados.rados.0.smithi017.stdout:3466: write oid 11 current snap is 438 2024-09-21T09:52:43.228 INFO:tasks.rados.rados.0.smithi017.stdout:3465: finishing write tid 1 to smithi01785587-49 2024-09-21T09:52:43.228 INFO:tasks.rados.rados.0.smithi017.stdout:3466: seq_num 1294 ranges {548258=752216,1771817=490781,2985235=713867} 2024-09-21T09:52:43.235 INFO:tasks.rados.rados.0.smithi017.stdout:3466: writing smithi01785587-11 from 548258 to 1300474 tid 1 2024-09-21T09:52:43.240 INFO:tasks.rados.rados.0.smithi017.stdout:3466: writing smithi01785587-11 from 1771817 to 2262598 tid 2 2024-09-21T09:52:43.249 INFO:tasks.rados.rados.0.smithi017.stdout:3466: writing smithi01785587-11 from 2985235 to 3699102 tid 3 2024-09-21T09:52:43.249 INFO:tasks.rados.rados.0.smithi017.stdout:3467: write oid 7 current snap is 438 2024-09-21T09:52:43.249 INFO:tasks.rados.rados.0.smithi017.stdout:3465: finishing write tid 2 to smithi01785587-49 2024-09-21T09:52:43.249 INFO:tasks.rados.rados.0.smithi017.stdout:3467: seq_num 1295 ranges {565065=427465,1430299=634439,2496305=525139,3472104=292961} 2024-09-21T09:52:43.254 INFO:tasks.rados.rados.0.smithi017.stdout:3467: writing smithi01785587-7 from 565065 to 992530 tid 1 2024-09-21T09:52:43.260 INFO:tasks.rados.rados.0.smithi017.stdout:3467: writing smithi01785587-7 from 1430299 to 2064738 tid 2 2024-09-21T09:52:43.265 INFO:tasks.rados.rados.0.smithi017.stdout:3467: writing smithi01785587-7 from 2496305 to 3021444 tid 3 2024-09-21T09:52:43.270 INFO:tasks.rados.rados.0.smithi017.stdout:3467: writing smithi01785587-7 from 3472104 to 3765065 tid 4 2024-09-21T09:52:43.270 INFO:tasks.rados.rados.0.smithi017.stdout:3468: delete oid 28 current snap is 438 2024-09-21T09:52:43.270 INFO:tasks.rados.rados.0.smithi017.stdout:3465: finishing write tid 3 to smithi01785587-49 2024-09-21T09:52:43.270 INFO:tasks.rados.rados.0.smithi017.stdout:3465: finishing write tid 4 to smithi01785587-49 2024-09-21T09:52:43.270 INFO:tasks.rados.rados.0.smithi017.stdout:3465: finishing write tid 5 to smithi01785587-49 2024-09-21T09:52:43.270 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1684 (ObjNum 1293 snap 438 seq_num 1293) dirty exists 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3465: left oid 49 (ObjNum 1293 snap 438 seq_num 1293) 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3466: finishing write tid 1 to smithi01785587-11 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3466: finishing write tid 2 to smithi01785587-11 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3466: finishing write tid 3 to smithi01785587-11 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3466: finishing write tid 4 to smithi01785587-11 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3466: finishing write tid 5 to smithi01785587-11 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1790 (ObjNum 1294 snap 438 seq_num 1294) dirty exists 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3466: left oid 11 (ObjNum 1294 snap 438 seq_num 1294) 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3467: finishing write tid 1 to smithi01785587-7 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3467: finishing write tid 2 to smithi01785587-7 2024-09-21T09:52:43.271 INFO:tasks.rados.rados.0.smithi017.stdout:3467: finishing write tid 3 to smithi01785587-7 2024-09-21T09:52:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:3465: done (3 left) 2024-09-21T09:52:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:3466: done (2 left) 2024-09-21T09:52:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:3468: done (1 left) 2024-09-21T09:52:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:3469: snap_create 2024-09-21T09:52:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:3467: finishing write tid 4 to smithi01785587-7 2024-09-21T09:52:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:3467: finishing write tid 5 to smithi01785587-7 2024-09-21T09:52:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:3467: finishing write tid 6 to smithi01785587-7 2024-09-21T09:52:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1690 (ObjNum 1295 snap 438 seq_num 1295) dirty exists 2024-09-21T09:52:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:3467: left oid 7 (ObjNum 1295 snap 438 seq_num 1295) 2024-09-21T09:52:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:43 smithi134 ceph-mon[90085]: pgmap v984: 89 pgs: 89 active+clean; 141 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:52:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:43 smithi134 ceph-mon[90085]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T09:52:43.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:43 smithi134 ceph-mon[90085]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T09:52:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:43 smithi017 ceph-mon[102913]: pgmap v984: 89 pgs: 89 active+clean; 141 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:52:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:43 smithi017 ceph-mon[102913]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T09:52:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:43 smithi017 ceph-mon[102913]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T09:52:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:43 smithi017 ceph-mon[105151]: pgmap v984: 89 pgs: 89 active+clean; 141 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:52:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:43 smithi017 ceph-mon[105151]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T09:52:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:43 smithi017 ceph-mon[105151]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T09:52:44.227 INFO:tasks.rados.rados.0.smithi017.stdout:3467: done (1 left) 2024-09-21T09:52:44.227 INFO:tasks.rados.rados.0.smithi017.stdout:3469: done (0 left) 2024-09-21T09:52:44.227 INFO:tasks.rados.rados.0.smithi017.stdout:3470: snap_remove snap 422 2024-09-21T09:52:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:52:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:52:44] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.51.0" 2024-09-21T09:52:45.231 INFO:tasks.rados.rados.0.smithi017.stdout:3470: done (0 left) 2024-09-21T09:52:45.232 INFO:tasks.rados.rados.0.smithi017.stdout:3471: snap_remove snap 435 2024-09-21T09:52:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:45 smithi134 ceph-mon[90085]: pgmap v987: 89 pgs: 89 active+clean; 144 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:45 smithi134 ceph-mon[90085]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T09:52:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:45 smithi017 ceph-mon[102913]: pgmap v987: 89 pgs: 89 active+clean; 144 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:45 smithi017 ceph-mon[102913]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T09:52:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:45 smithi017 ceph-mon[105151]: pgmap v987: 89 pgs: 89 active+clean; 144 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:45 smithi017 ceph-mon[105151]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T09:52:46.242 INFO:tasks.rados.rados.0.smithi017.stdout:3471: done (0 left) 2024-09-21T09:52:46.242 INFO:tasks.rados.rados.0.smithi017.stdout:3472: delete oid 7 current snap is 439 2024-09-21T09:52:46.249 INFO:tasks.rados.rados.0.smithi017.stdout:3472: done (0 left) 2024-09-21T09:52:46.249 INFO:tasks.rados.rados.0.smithi017.stdout:3473: write oid 7 current snap is 439 2024-09-21T09:52:46.249 INFO:tasks.rados.rados.0.smithi017.stdout:3473: seq_num 1298 ranges {615486=653212,1889392=498119,2978574=475308} 2024-09-21T09:52:46.255 INFO:tasks.rados.rados.0.smithi017.stdout:3473: writing smithi01785587-7 from 615486 to 1268698 tid 1 2024-09-21T09:52:46.261 INFO:tasks.rados.rados.0.smithi017.stdout:3473: writing smithi01785587-7 from 1889392 to 2387511 tid 2 2024-09-21T09:52:46.267 INFO:tasks.rados.rados.0.smithi017.stdout:3473: writing smithi01785587-7 from 2978574 to 3453882 tid 3 2024-09-21T09:52:46.267 INFO:tasks.rados.rados.0.smithi017.stdout:3474: snap_remove snap 434 2024-09-21T09:52:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:46 smithi134 ceph-mon[90085]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T09:52:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:46 smithi017 ceph-mon[102913]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T09:52:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:46 smithi017 ceph-mon[105151]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T09:52:47.247 INFO:tasks.rados.rados.0.smithi017.stdout:3474: done (1 left) 2024-09-21T09:52:47.248 INFO:tasks.rados.rados.0.smithi017.stdout:3475: write oid 5 current snap is 439 2024-09-21T09:52:47.248 INFO:tasks.rados.rados.0.smithi017.stdout:3473: finishing write tid 1 to smithi01785587-7 2024-09-21T09:52:47.248 INFO:tasks.rados.rados.0.smithi017.stdout:3475: seq_num 1299 ranges {632293=728461,1947874=641777,3005997=686580} 2024-09-21T09:52:47.255 INFO:tasks.rados.rados.0.smithi017.stdout:3475: writing smithi01785587-5 from 632293 to 1360754 tid 1 2024-09-21T09:52:47.264 INFO:tasks.rados.rados.0.smithi017.stdout:3475: writing smithi01785587-5 from 1947874 to 2589651 tid 2 2024-09-21T09:52:47.268 INFO:tasks.rados.rados.0.smithi017.stdout:3475: writing smithi01785587-5 from 3005997 to 3692577 tid 3 2024-09-21T09:52:47.268 INFO:tasks.rados.rados.0.smithi017.stdout:3476: snap_remove snap 427 2024-09-21T09:52:47.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:47 smithi134 ceph-mon[90085]: pgmap v990: 89 pgs: 89 active+clean; 147 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 846 KiB/s wr, 3 op/s 2024-09-21T09:52:47.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:47 smithi134 ceph-mon[90085]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T09:52:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:47 smithi017 ceph-mon[102913]: pgmap v990: 89 pgs: 89 active+clean; 147 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 846 KiB/s wr, 3 op/s 2024-09-21T09:52:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:47 smithi017 ceph-mon[102913]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T09:52:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:47 smithi017 ceph-mon[105151]: pgmap v990: 89 pgs: 89 active+clean; 147 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 846 KiB/s wr, 3 op/s 2024-09-21T09:52:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:47 smithi017 ceph-mon[105151]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T09:52:48.211 INFO:tasks.rados.rados.0.smithi017.stdout:3476: done (2 left) 2024-09-21T09:52:48.211 INFO:tasks.rados.rados.0.smithi017.stdout:3477: snap_create 2024-09-21T09:52:48.211 INFO:tasks.rados.rados.0.smithi017.stdout:3473: finishing write tid 2 to smithi01785587-7 2024-09-21T09:52:48.211 INFO:tasks.rados.rados.0.smithi017.stdout:3473: finishing write tid 3 to smithi01785587-7 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:3473: finishing write tid 4 to smithi01785587-7 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:3473: finishing write tid 5 to smithi01785587-7 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1702 (ObjNum 1298 snap 439 seq_num 1298) dirty exists 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:3473: left oid 7 (ObjNum 1298 snap 439 seq_num 1298) 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:3475: finishing write tid 1 to smithi01785587-5 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:3475: finishing write tid 2 to smithi01785587-5 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:3475: finishing write tid 3 to smithi01785587-5 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:3475: finishing write tid 4 to smithi01785587-5 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:3475: finishing write tid 5 to smithi01785587-5 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1931 (ObjNum 1299 snap 439 seq_num 1299) dirty exists 2024-09-21T09:52:48.212 INFO:tasks.rados.rados.0.smithi017.stdout:3475: left oid 5 (ObjNum 1299 snap 439 seq_num 1299) 2024-09-21T09:52:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:48 smithi017 ceph-mon[102913]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T09:52:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:48 smithi017 ceph-mon[102913]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T09:52:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:48 smithi017 ceph-mon[105151]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T09:52:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:48 smithi017 ceph-mon[105151]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T09:52:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:48 smithi134 ceph-mon[90085]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T09:52:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:48 smithi134 ceph-mon[90085]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T09:52:49.212 INFO:tasks.rados.rados.0.smithi017.stdout:3473: done (2 left) 2024-09-21T09:52:49.213 INFO:tasks.rados.rados.0.smithi017.stdout:3475: done (1 left) 2024-09-21T09:52:49.213 INFO:tasks.rados.rados.0.smithi017.stdout:3477: done (0 left) 2024-09-21T09:52:49.213 INFO:tasks.rados.rados.0.smithi017.stdout:3478: read oid 25 snap -1 2024-09-21T09:52:49.213 INFO:tasks.rados.rados.0.smithi017.stdout:3478: expect (ObjNum 1290 snap 434 seq_num 1290) 2024-09-21T09:52:49.213 INFO:tasks.rados.rados.0.smithi017.stdout:3479: read oid 23 snap -1 2024-09-21T09:52:49.213 INFO:tasks.rados.rados.0.smithi017.stdout:3479: expect (ObjNum 1249 snap 420 seq_num 1249) 2024-09-21T09:52:49.213 INFO:tasks.rados.rados.0.smithi017.stdout:3480: delete oid 45 current snap is 440 2024-09-21T09:52:49.218 INFO:tasks.rados.rados.0.smithi017.stdout:3480: done (2 left) 2024-09-21T09:52:49.218 INFO:tasks.rados.rados.0.smithi017.stdout:3481: write oid 43 current snap is 440 2024-09-21T09:52:49.218 INFO:tasks.rados.rados.0.smithi017.stdout:3481: seq_num 1301 ranges {665907=595312,1781191=645446,3177196=688711} 2024-09-21T09:52:49.225 INFO:tasks.rados.rados.0.smithi017.stdout:3481: writing smithi01785587-43 from 665907 to 1261219 tid 1 2024-09-21T09:52:49.232 INFO:tasks.rados.rados.0.smithi017.stdout:3481: writing smithi01785587-43 from 1781191 to 2426637 tid 2 2024-09-21T09:52:49.240 INFO:tasks.rados.rados.0.smithi017.stdout:3481: writing smithi01785587-43 from 3177196 to 3865907 tid 3 2024-09-21T09:52:49.240 INFO:tasks.rados.rados.0.smithi017.stdout:3482: read oid 33 snap -1 2024-09-21T09:52:49.241 INFO:tasks.rados.rados.0.smithi017.stdout:3482: expect deleted 2024-09-21T09:52:49.241 INFO:tasks.rados.rados.0.smithi017.stdout:3483: write oid 36 current snap is 440 2024-09-21T09:52:49.241 INFO:tasks.rados.rados.0.smithi017.stdout:3479: expect (ObjNum 1249 snap 420 seq_num 1249) 2024-09-21T09:52:49.382 INFO:tasks.rados.rados.0.smithi017.stdout:3478: expect (ObjNum 1290 snap 434 seq_num 1290) 2024-09-21T09:52:49.640 INFO:tasks.rados.rados.0.smithi017.stdout:3481: finishing write tid 1 to smithi01785587-43 2024-09-21T09:52:49.641 INFO:tasks.rados.rados.0.smithi017.stdout:3481: finishing write tid 2 to smithi01785587-43 2024-09-21T09:52:49.641 INFO:tasks.rados.rados.0.smithi017.stdout:3481: finishing write tid 3 to smithi01785587-43 2024-09-21T09:52:49.641 INFO:tasks.rados.rados.0.smithi017.stdout:3481: finishing write tid 4 to smithi01785587-43 2024-09-21T09:52:49.641 INFO:tasks.rados.rados.0.smithi017.stdout:3481: finishing write tid 5 to smithi01785587-43 2024-09-21T09:52:49.641 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 1028 (ObjNum 1301 snap 440 seq_num 1301) dirty exists 2024-09-21T09:52:49.641 INFO:tasks.rados.rados.0.smithi017.stdout:3481: left oid 43 (ObjNum 1301 snap 440 seq_num 1301) 2024-09-21T09:52:49.641 INFO:tasks.rados.rados.0.smithi017.stdout:3483: seq_num 1302 ranges {682714=670561,2123320=505457,3088266=636749} 2024-09-21T09:52:49.648 INFO:tasks.rados.rados.0.smithi017.stdout:3483: writing smithi01785587-36 from 682714 to 1353275 tid 1 2024-09-21T09:52:49.655 INFO:tasks.rados.rados.0.smithi017.stdout:3483: writing smithi01785587-36 from 2123320 to 2628777 tid 2 2024-09-21T09:52:49.661 INFO:tasks.rados.rados.0.smithi017.stdout:3483: writing smithi01785587-36 from 3088266 to 3725015 tid 3 2024-09-21T09:52:49.661 INFO:tasks.rados.rados.0.smithi017.stdout:3478: done (4 left) 2024-09-21T09:52:49.661 INFO:tasks.rados.rados.0.smithi017.stdout:3479: done (3 left) 2024-09-21T09:52:49.661 INFO:tasks.rados.rados.0.smithi017.stdout:3481: done (2 left) 2024-09-21T09:52:49.662 INFO:tasks.rados.rados.0.smithi017.stdout:3482: done (1 left) 2024-09-21T09:52:49.662 INFO:tasks.rados.rados.0.smithi017.stdout:3484: write oid 44 current snap is 440 2024-09-21T09:52:49.662 INFO:tasks.rados.rados.0.smithi017.stdout:3483: finishing write tid 1 to smithi01785587-36 2024-09-21T09:52:49.662 INFO:tasks.rados.rados.0.smithi017.stdout:3483: finishing write tid 2 to smithi01785587-36 2024-09-21T09:52:49.662 INFO:tasks.rados.rados.0.smithi017.stdout:3484: seq_num 1303 ranges {699521=745810,2181802=649115,3515689=383832} 2024-09-21T09:52:49.669 INFO:tasks.rados.rados.0.smithi017.stdout:3484: writing smithi01785587-44 from 699521 to 1445331 tid 1 2024-09-21T09:52:49.676 INFO:tasks.rados.rados.0.smithi017.stdout:3484: writing smithi01785587-44 from 2181802 to 2830917 tid 2 2024-09-21T09:52:49.683 INFO:tasks.rados.rados.0.smithi017.stdout:3484: writing smithi01785587-44 from 3515689 to 3899521 tid 3 2024-09-21T09:52:49.683 INFO:tasks.rados.rados.0.smithi017.stdout:3485: write oid 25 current snap is 440 2024-09-21T09:52:49.683 INFO:tasks.rados.rados.0.smithi017.stdout:3485: seq_num 1304 ranges {716328=421059,1840284=509126,3143112=659293} 2024-09-21T09:52:49.689 INFO:tasks.rados.rados.0.smithi017.stdout:3485: writing smithi01785587-25 from 716328 to 1137387 tid 1 2024-09-21T09:52:49.695 INFO:tasks.rados.rados.0.smithi017.stdout:3485: writing smithi01785587-25 from 1840284 to 2349410 tid 2 2024-09-21T09:52:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:49 smithi017 ceph-mon[102913]: pgmap v993: 89 pgs: 89 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.6 MiB/s wr, 9 op/s 2024-09-21T09:52:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:49 smithi017 ceph-mon[102913]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T09:52:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:49 smithi017 ceph-mon[105151]: pgmap v993: 89 pgs: 89 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.6 MiB/s wr, 9 op/s 2024-09-21T09:52:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:49 smithi017 ceph-mon[105151]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T09:52:49.703 INFO:tasks.rados.rados.0.smithi017.stdout:3485: writing smithi01785587-25 from 3143112 to 3802405 tid 3 2024-09-21T09:52:49.703 INFO:tasks.rados.rados.0.smithi017.stdout:3486: snap_remove snap 430 2024-09-21T09:52:49.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:49 smithi134 ceph-mon[90085]: pgmap v993: 89 pgs: 89 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.6 MiB/s wr, 9 op/s 2024-09-21T09:52:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:49 smithi134 ceph-mon[90085]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T09:52:50.257 INFO:tasks.rados.rados.0.smithi017.stdout:3486: done (3 left) 2024-09-21T09:52:50.257 INFO:tasks.rados.rados.0.smithi017.stdout:3487: read oid 5 snap -1 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3487: expect (ObjNum 1299 snap 439 seq_num 1299) 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3488: read oid 13 snap -1 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3488: expect (ObjNum 1203 snap 402 seq_num 1203) 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3489: read oid 32 snap -1 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3489: expect (ObjNum 1223 snap 405 seq_num 1223) 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3490: rollback oid 43 current snap is 440 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 429 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3483: finishing write tid 3 to smithi01785587-36 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3483: finishing write tid 4 to smithi01785587-36 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3483: finishing write tid 5 to smithi01785587-36 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1805 (ObjNum 1302 snap 440 seq_num 1302) dirty exists 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3483: left oid 36 (ObjNum 1302 snap 440 seq_num 1302) 2024-09-21T09:52:50.258 INFO:tasks.rados.rados.0.smithi017.stdout:3484: finishing write tid 1 to smithi01785587-44 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3484: finishing write tid 2 to smithi01785587-44 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3483: done (6 left) 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3491: read oid 48 snap -1 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3491: expect deleted 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3492: write oid 11 current snap is 440 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3484: finishing write tid 3 to smithi01785587-44 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3484: finishing write tid 4 to smithi01785587-44 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3484: finishing write tid 5 to smithi01785587-44 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1938 (ObjNum 1303 snap 440 seq_num 1303) dirty exists 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3484: left oid 44 (ObjNum 1303 snap 440 seq_num 1303) 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3485: finishing write tid 1 to smithi01785587-25 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3485: finishing write tid 2 to smithi01785587-25 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3485: finishing write tid 3 to smithi01785587-25 2024-09-21T09:52:50.259 INFO:tasks.rados.rados.0.smithi017.stdout:3485: finishing write tid 4 to smithi01785587-25 2024-09-21T09:52:50.260 INFO:tasks.rados.rados.0.smithi017.stdout:3485: finishing write tid 5 to smithi01785587-25 2024-09-21T09:52:50.260 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 988 (ObjNum 1304 snap 440 seq_num 1304) dirty exists 2024-09-21T09:52:50.260 INFO:tasks.rados.rados.0.smithi017.stdout:3485: left oid 25 (ObjNum 1304 snap 440 seq_num 1304) 2024-09-21T09:52:50.260 INFO:tasks.rados.rados.0.smithi017.stdout:3492: seq_num 1305 ranges {733135=496308,1898766=652784,3170535=470565} 2024-09-21T09:52:50.265 INFO:tasks.rados.rados.0.smithi017.stdout:3492: writing smithi01785587-11 from 733135 to 1229443 tid 1 2024-09-21T09:52:50.273 INFO:tasks.rados.rados.0.smithi017.stdout:3492: writing smithi01785587-11 from 1898766 to 2551550 tid 2 2024-09-21T09:52:50.280 INFO:tasks.rados.rados.0.smithi017.stdout:3492: writing smithi01785587-11 from 3170535 to 3641100 tid 3 2024-09-21T09:52:50.280 INFO:tasks.rados.rados.0.smithi017.stdout:3484: done (7 left) 2024-09-21T09:52:50.280 INFO:tasks.rados.rados.0.smithi017.stdout:3485: done (6 left) 2024-09-21T09:52:50.281 INFO:tasks.rados.rados.0.smithi017.stdout:3493: delete oid 44 current snap is 440 2024-09-21T09:52:50.281 INFO:tasks.rados.rados.0.smithi017.stdout:3490: finishing rollback tid 0 to smithi01785587-43 2024-09-21T09:52:50.281 INFO:tasks.rados.rados.0.smithi017.stdout:3489: expect (ObjNum 1223 snap 405 seq_num 1223) 2024-09-21T09:52:50.439 INFO:tasks.rados.rados.0.smithi017.stdout:3490: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:52:50.439 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 1030 (ObjNum 2977 snap 0 seq_num 4181697200) dirty dne 2024-09-21T09:52:50.439 INFO:tasks.rados.rados.0.smithi017.stdout:3488: expect (ObjNum 1203 snap 402 seq_num 1203) 2024-09-21T09:52:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:52:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:52:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:52:50.612 INFO:tasks.rados.rados.0.smithi017.stdout:3492: finishing write tid 1 to smithi01785587-11 2024-09-21T09:52:50.612 INFO:tasks.rados.rados.0.smithi017.stdout:3487: expect (ObjNum 1299 snap 439 seq_num 1299) 2024-09-21T09:52:50.983 INFO:tasks.rados.rados.0.smithi017.stdout:3492: finishing write tid 2 to smithi01785587-11 2024-09-21T09:52:50.983 INFO:tasks.rados.rados.0.smithi017.stdout:3492: finishing write tid 3 to smithi01785587-11 2024-09-21T09:52:50.983 INFO:tasks.rados.rados.0.smithi017.stdout:3492: finishing write tid 4 to smithi01785587-11 2024-09-21T09:52:50.983 INFO:tasks.rados.rados.0.smithi017.stdout:3492: finishing write tid 5 to smithi01785587-11 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1805 (ObjNum 1305 snap 440 seq_num 1305) dirty exists 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3492: left oid 11 (ObjNum 1305 snap 440 seq_num 1305) 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3487: done (6 left) 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3488: done (5 left) 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3489: done (4 left) 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3490: done (3 left) 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3491: done (2 left) 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3492: done (1 left) 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3493: done (0 left) 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3494: read oid 32 snap -1 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3494: expect (ObjNum 1223 snap 405 seq_num 1223) 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3495: write oid 6 current snap is 440 2024-09-21T09:52:50.984 INFO:tasks.rados.rados.0.smithi017.stdout:3495: seq_num 1307 ranges {766749=646806,1899377=656453,3109028=609462} 2024-09-21T09:52:50.990 INFO:tasks.rados.rados.0.smithi017.stdout:3495: writing smithi01785587-6 from 766749 to 1413555 tid 1 2024-09-21T09:52:50.997 INFO:tasks.rados.rados.0.smithi017.stdout:3495: writing smithi01785587-6 from 1899377 to 2555830 tid 2 2024-09-21T09:52:51.003 INFO:tasks.rados.rados.0.smithi017.stdout:3495: writing smithi01785587-6 from 3109028 to 3718490 tid 3 2024-09-21T09:52:51.003 INFO:tasks.rados.rados.0.smithi017.stdout:3496: write oid 44 current snap is 440 2024-09-21T09:52:51.003 INFO:tasks.rados.rados.0.smithi017.stdout:3496: seq_num 1308 ranges {783556=438408,1674212=400111,2736451=420734,3882270=101286} 2024-09-21T09:52:51.010 INFO:tasks.rados.rados.0.smithi017.stdout:3496: writing smithi01785587-44 from 783556 to 1221964 tid 1 2024-09-21T09:52:51.014 INFO:tasks.rados.rados.0.smithi017.stdout:3496: writing smithi01785587-44 from 1674212 to 2074323 tid 2 2024-09-21T09:52:51.020 INFO:tasks.rados.rados.0.smithi017.stdout:3496: writing smithi01785587-44 from 2736451 to 3157185 tid 3 2024-09-21T09:52:51.025 INFO:tasks.rados.rados.0.smithi017.stdout:3496: writing smithi01785587-44 from 3882270 to 3983556 tid 4 2024-09-21T09:52:51.026 INFO:tasks.rados.rados.0.smithi017.stdout:3497: write oid 29 current snap is 440 2024-09-21T09:52:51.026 INFO:tasks.rados.rados.0.smithi017.stdout:3497: seq_num 1309 ranges {400363=513657,1332694=660122} 2024-09-21T09:52:51.030 INFO:tasks.rados.rados.0.smithi017.stdout:3497: writing smithi01785587-29 from 400363 to 914020 tid 1 2024-09-21T09:52:51.036 INFO:tasks.rados.rados.0.smithi017.stdout:3497: writing smithi01785587-29 from 1332694 to 1992816 tid 2 2024-09-21T09:52:51.037 INFO:tasks.rados.rados.0.smithi017.stdout:3498: read oid 8 snap -1 2024-09-21T09:52:51.037 INFO:tasks.rados.rados.0.smithi017.stdout:3498: expect deleted 2024-09-21T09:52:51.037 INFO:tasks.rados.rados.0.smithi017.stdout:3499: snap_create 2024-09-21T09:52:51.037 INFO:tasks.rados.rados.0.smithi017.stdout:3494: expect (ObjNum 1223 snap 405 seq_num 1223) 2024-09-21T09:52:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:3495: finishing write tid 1 to smithi01785587-6 2024-09-21T09:52:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:3495: finishing write tid 2 to smithi01785587-6 2024-09-21T09:52:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:3495: finishing write tid 3 to smithi01785587-6 2024-09-21T09:52:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:3495: finishing write tid 4 to smithi01785587-6 2024-09-21T09:52:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:3495: finishing write tid 5 to smithi01785587-6 2024-09-21T09:52:51.195 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 1034 (ObjNum 1307 snap 440 seq_num 1307) dirty exists 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3495: left oid 6 (ObjNum 1307 snap 440 seq_num 1307) 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3496: finishing write tid 1 to smithi01785587-44 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3496: finishing write tid 2 to smithi01785587-44 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3496: finishing write tid 3 to smithi01785587-44 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3496: finishing write tid 4 to smithi01785587-44 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3496: finishing write tid 5 to smithi01785587-44 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3496: finishing write tid 6 to smithi01785587-44 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1948 (ObjNum 1308 snap 440 seq_num 1308) dirty exists 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3496: left oid 44 (ObjNum 1308 snap 440 seq_num 1308) 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3497: finishing write tid 1 to smithi01785587-29 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3497: finishing write tid 2 to smithi01785587-29 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3497: finishing write tid 3 to smithi01785587-29 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:3497: finishing write tid 4 to smithi01785587-29 2024-09-21T09:52:51.196 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 995 (ObjNum 1309 snap 440 seq_num 1309) dirty exists 2024-09-21T09:52:51.197 INFO:tasks.rados.rados.0.smithi017.stdout:3497: left oid 29 (ObjNum 1309 snap 440 seq_num 1309) 2024-09-21T09:52:51.263 INFO:tasks.rados.rados.0.smithi017.stdout:3494: done (5 left) 2024-09-21T09:52:51.263 INFO:tasks.rados.rados.0.smithi017.stdout:3495: done (4 left) 2024-09-21T09:52:51.263 INFO:tasks.rados.rados.0.smithi017.stdout:3496: done (3 left) 2024-09-21T09:52:51.263 INFO:tasks.rados.rados.0.smithi017.stdout:3497: done (2 left) 2024-09-21T09:52:51.263 INFO:tasks.rados.rados.0.smithi017.stdout:3498: done (1 left) 2024-09-21T09:52:51.263 INFO:tasks.rados.rados.0.smithi017.stdout:3499: done (0 left) 2024-09-21T09:52:51.263 INFO:tasks.rados.rados.0.smithi017.stdout:3500: write oid 42 current snap is 441 2024-09-21T09:52:51.263 INFO:tasks.rados.rados.0.smithi017.stdout:3500: seq_num 1310 ranges {417170=588906,1674823=342347} 2024-09-21T09:52:51.268 INFO:tasks.rados.rados.0.smithi017.stdout:3500: writing smithi01785587-42 from 417170 to 1006076 tid 1 2024-09-21T09:52:51.274 INFO:tasks.rados.rados.0.smithi017.stdout:3500: writing smithi01785587-42 from 1674823 to 2017170 tid 2 2024-09-21T09:52:51.274 INFO:tasks.rados.rados.0.smithi017.stdout:3500: finishing write tid 1 to smithi01785587-42 2024-09-21T09:52:51.274 INFO:tasks.rados.rados.0.smithi017.stdout:3501: read oid 26 snap -1 2024-09-21T09:52:51.274 INFO:tasks.rados.rados.0.smithi017.stdout:3501: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:52:51.274 INFO:tasks.rados.rados.0.smithi017.stdout:3502: write oid 2 current snap is 441 2024-09-21T09:52:51.274 INFO:tasks.rados.rados.0.smithi017.stdout:3502: seq_num 1311 ranges {433977=664155,1733305=300672} 2024-09-21T09:52:51.280 INFO:tasks.rados.rados.0.smithi017.stdout:3502: writing smithi01785587-2 from 433977 to 1098132 tid 1 2024-09-21T09:52:51.286 INFO:tasks.rados.rados.0.smithi017.stdout:3502: writing smithi01785587-2 from 1733305 to 2033977 tid 2 2024-09-21T09:52:51.286 INFO:tasks.rados.rados.0.smithi017.stdout:3503: write oid 36 current snap is 441 2024-09-21T09:52:51.286 INFO:tasks.rados.rados.0.smithi017.stdout:3500: finishing write tid 2 to smithi01785587-42 2024-09-21T09:52:51.286 INFO:tasks.rados.rados.0.smithi017.stdout:3503: seq_num 1312 ranges {450784=739404,1791787=258997} 2024-09-21T09:52:51.292 INFO:tasks.rados.rados.0.smithi017.stdout:3503: writing smithi01785587-36 from 450784 to 1190188 tid 1 2024-09-21T09:52:51.297 INFO:tasks.rados.rados.0.smithi017.stdout:3503: writing smithi01785587-36 from 1791787 to 2050784 tid 2 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:3500: finishing write tid 3 to smithi01785587-42 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:3500: finishing write tid 4 to smithi01785587-42 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 1221 (ObjNum 1310 snap 441 seq_num 1310) dirty exists 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:3500: left oid 42 (ObjNum 1310 snap 441 seq_num 1310) 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:3504: read oid 14 snap -1 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:3504: expect (ObjNum 1158 snap 389 seq_num 1158) 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:3500: done (4 left) 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:3505: rollback oid 29 current snap is 441 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 29 to 424 2024-09-21T09:52:51.298 INFO:tasks.rados.rados.0.smithi017.stdout:3501: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:52:51.458 INFO:tasks.rados.rados.0.smithi017.stdout:3502: finishing write tid 1 to smithi01785587-2 2024-09-21T09:52:51.458 INFO:tasks.rados.rados.0.smithi017.stdout:3502: finishing write tid 2 to smithi01785587-2 2024-09-21T09:52:51.458 INFO:tasks.rados.rados.0.smithi017.stdout:3502: finishing write tid 3 to smithi01785587-2 2024-09-21T09:52:51.458 INFO:tasks.rados.rados.0.smithi017.stdout:3502: finishing write tid 4 to smithi01785587-2 2024-09-21T09:52:51.458 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 705 (ObjNum 1311 snap 441 seq_num 1311) dirty exists 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3502: left oid 2 (ObjNum 1311 snap 441 seq_num 1311) 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3503: finishing write tid 1 to smithi01785587-36 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3505: finishing rollback tid 0 to smithi01785587-29 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3503: finishing write tid 2 to smithi01785587-36 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3503: finishing write tid 3 to smithi01785587-36 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3505: finishing rollback tid 1 to smithi01785587-29 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 998 (ObjNum 177 snap 0 seq_num 2399631568) dirty dne 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3503: finishing write tid 4 to smithi01785587-36 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1813 (ObjNum 1312 snap 441 seq_num 1312) dirty exists 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3503: left oid 36 (ObjNum 1312 snap 441 seq_num 1312) 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3501: done (4 left) 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3502: done (3 left) 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3503: done (2 left) 2024-09-21T09:52:51.459 INFO:tasks.rados.rados.0.smithi017.stdout:3505: done (1 left) 2024-09-21T09:52:51.460 INFO:tasks.rados.rados.0.smithi017.stdout:3506: read oid 4 snap -1 2024-09-21T09:52:51.460 INFO:tasks.rados.rados.0.smithi017.stdout:3506: expect (ObjNum 1268 snap 424 seq_num 1268) 2024-09-21T09:52:51.460 INFO:tasks.rados.rados.0.smithi017.stdout:3507: write oid 29 current snap is 441 2024-09-21T09:52:51.460 INFO:tasks.rados.rados.0.smithi017.stdout:3504: expect (ObjNum 1158 snap 389 seq_num 1158) 2024-09-21T09:52:51.615 INFO:tasks.rados.rados.0.smithi017.stdout:3506: expect (ObjNum 1268 snap 424 seq_num 1268) 2024-09-21T09:52:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:51 smithi017 ceph-mon[102913]: pgmap v996: 89 pgs: 89 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 6 op/s 2024-09-21T09:52:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:51 smithi017 ceph-mon[102913]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T09:52:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:51 smithi017 ceph-mon[105151]: pgmap v996: 89 pgs: 89 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 6 op/s 2024-09-21T09:52:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:51 smithi017 ceph-mon[105151]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T09:52:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:51 smithi134 ceph-mon[90085]: pgmap v996: 89 pgs: 89 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.8 MiB/s wr, 6 op/s 2024-09-21T09:52:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:51 smithi134 ceph-mon[90085]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T09:52:51.778 INFO:tasks.rados.rados.0.smithi017.stdout:3507: seq_num 1313 ranges {467591=414653,1450269=617322} 2024-09-21T09:52:51.783 INFO:tasks.rados.rados.0.smithi017.stdout:3507: writing smithi01785587-29 from 467591 to 882244 tid 1 2024-09-21T09:52:51.790 INFO:tasks.rados.rados.0.smithi017.stdout:3507: writing smithi01785587-29 from 1450269 to 2067591 tid 2 2024-09-21T09:52:51.790 INFO:tasks.rados.rados.0.smithi017.stdout:3504: done (2 left) 2024-09-21T09:52:51.790 INFO:tasks.rados.rados.0.smithi017.stdout:3506: done (1 left) 2024-09-21T09:52:51.790 INFO:tasks.rados.rados.0.smithi017.stdout:3508: read oid 37 snap -1 2024-09-21T09:52:51.790 INFO:tasks.rados.rados.0.smithi017.stdout:3508: expect (ObjNum 1285 snap 432 seq_num 1285) 2024-09-21T09:52:51.790 INFO:tasks.rados.rados.0.smithi017.stdout:3509: snap_remove snap 436 2024-09-21T09:52:52.268 INFO:tasks.rados.rados.0.smithi017.stdout:3509: done (2 left) 2024-09-21T09:52:52.268 INFO:tasks.rados.rados.0.smithi017.stdout:3510: write oid 15 current snap is 441 2024-09-21T09:52:52.268 INFO:tasks.rados.rados.0.smithi017.stdout:3507: finishing write tid 1 to smithi01785587-29 2024-09-21T09:52:52.268 INFO:tasks.rados.rados.0.smithi017.stdout:3510: seq_num 1314 ranges {484398=489902,1392398=411118} 2024-09-21T09:52:52.273 INFO:tasks.rados.rados.0.smithi017.stdout:3510: writing smithi01785587-15 from 484398 to 974300 tid 1 2024-09-21T09:52:52.278 INFO:tasks.rados.rados.0.smithi017.stdout:3510: writing smithi01785587-15 from 1392398 to 1803516 tid 2 2024-09-21T09:52:52.278 INFO:tasks.rados.rados.0.smithi017.stdout:3511: write oid 17 current snap is 441 2024-09-21T09:52:52.278 INFO:tasks.rados.rados.0.smithi017.stdout:3511: seq_num 1315 ranges {501205=565151,1850880=250325} 2024-09-21T09:52:52.285 INFO:tasks.rados.rados.0.smithi017.stdout:3511: writing smithi01785587-17 from 501205 to 1066356 tid 1 2024-09-21T09:52:52.290 INFO:tasks.rados.rados.0.smithi017.stdout:3511: writing smithi01785587-17 from 1850880 to 2101205 tid 2 2024-09-21T09:52:52.290 INFO:tasks.rados.rados.0.smithi017.stdout:3512: write oid 45 current snap is 441 2024-09-21T09:52:52.290 INFO:tasks.rados.rados.0.smithi017.stdout:3512: seq_num 1316 ranges {518012=756753,2025715=92297} 2024-09-21T09:52:52.297 INFO:tasks.rados.rados.0.smithi017.stdout:3512: writing smithi01785587-45 from 518012 to 1274765 tid 1 2024-09-21T09:52:52.302 INFO:tasks.rados.rados.0.smithi017.stdout:3512: writing smithi01785587-45 from 2025715 to 2118012 tid 2 2024-09-21T09:52:52.302 INFO:tasks.rados.rados.0.smithi017.stdout:3513: write oid 32 current snap is 441 2024-09-21T09:52:52.302 INFO:tasks.rados.rados.0.smithi017.stdout:3513: seq_num 1317 ranges {534819=432002,1684197=450622} 2024-09-21T09:52:52.307 INFO:tasks.rados.rados.0.smithi017.stdout:3513: writing smithi01785587-32 from 534819 to 966821 tid 1 2024-09-21T09:52:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:3513: writing smithi01785587-32 from 1684197 to 2134819 tid 2 2024-09-21T09:52:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:3514: rollback oid 21 current snap is 441 2024-09-21T09:52:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 21 to 437 2024-09-21T09:52:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:3507: finishing write tid 2 to smithi01785587-29 2024-09-21T09:52:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:3507: finishing write tid 3 to smithi01785587-29 2024-09-21T09:52:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:3507: finishing write tid 4 to smithi01785587-29 2024-09-21T09:52:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 1001 (ObjNum 1313 snap 441 seq_num 1313) dirty exists 2024-09-21T09:52:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:3507: left oid 29 (ObjNum 1313 snap 441 seq_num 1313) 2024-09-21T09:52:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:3508: expect (ObjNum 1285 snap 432 seq_num 1285) 2024-09-21T09:52:52.585 INFO:tasks.rados.rados.0.smithi017.stdout:3510: finishing write tid 1 to smithi01785587-15 2024-09-21T09:52:52.585 INFO:tasks.rados.rados.0.smithi017.stdout:3510: finishing write tid 2 to smithi01785587-15 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3510: finishing write tid 3 to smithi01785587-15 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3510: finishing write tid 4 to smithi01785587-15 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 541 (ObjNum 1314 snap 441 seq_num 1314) dirty exists 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3510: left oid 15 (ObjNum 1314 snap 441 seq_num 1314) 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3511: finishing write tid 1 to smithi01785587-17 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3507: done (6 left) 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3508: done (5 left) 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3510: done (4 left) 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3515: write oid 46 current snap is 441 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3511: finishing write tid 2 to smithi01785587-17 2024-09-21T09:52:52.586 INFO:tasks.rados.rados.0.smithi017.stdout:3515: seq_num 1318 ranges {551626=507251,1626326=418456} 2024-09-21T09:52:52.591 INFO:tasks.rados.rados.0.smithi017.stdout:3515: writing smithi01785587-46 from 551626 to 1058877 tid 1 2024-09-21T09:52:52.597 INFO:tasks.rados.rados.0.smithi017.stdout:3515: writing smithi01785587-46 from 1626326 to 2044782 tid 2 2024-09-21T09:52:52.597 INFO:tasks.rados.rados.0.smithi017.stdout:3516: rollback oid 43 current snap is 441 2024-09-21T09:52:52.597 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 424 2024-09-21T09:52:52.597 INFO:tasks.rados.rados.0.smithi017.stdout:3511: finishing write tid 3 to smithi01785587-17 2024-09-21T09:52:52.597 INFO:tasks.rados.rados.0.smithi017.stdout:3511: finishing write tid 4 to smithi01785587-17 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 1205 (ObjNum 1315 snap 441 seq_num 1315) dirty exists 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3511: left oid 17 (ObjNum 1315 snap 441 seq_num 1315) 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3511: done (5 left) 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3517: read oid 29 snap -1 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3517: expect (ObjNum 1313 snap 441 seq_num 1313) 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3518: snap_create 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3512: finishing write tid 1 to smithi01785587-45 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3512: finishing write tid 2 to smithi01785587-45 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3512: finishing write tid 3 to smithi01785587-45 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3512: finishing write tid 4 to smithi01785587-45 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1208 (ObjNum 1316 snap 441 seq_num 1316) dirty exists 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3512: left oid 45 (ObjNum 1316 snap 441 seq_num 1316) 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3513: finishing write tid 1 to smithi01785587-32 2024-09-21T09:52:52.598 INFO:tasks.rados.rados.0.smithi017.stdout:3513: finishing write tid 2 to smithi01785587-32 2024-09-21T09:52:52.599 INFO:tasks.rados.rados.0.smithi017.stdout:3513: finishing write tid 3 to smithi01785587-32 2024-09-21T09:52:52.599 INFO:tasks.rados.rados.0.smithi017.stdout:3514: finishing rollback tid 0 to smithi01785587-21 2024-09-21T09:52:52.599 INFO:tasks.rados.rados.0.smithi017.stdout:3513: finishing write tid 4 to smithi01785587-32 2024-09-21T09:52:52.599 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1809 (ObjNum 1317 snap 441 seq_num 1317) dirty exists 2024-09-21T09:52:52.599 INFO:tasks.rados.rados.0.smithi017.stdout:3513: left oid 32 (ObjNum 1317 snap 441 seq_num 1317) 2024-09-21T09:52:52.599 INFO:tasks.rados.rados.0.smithi017.stdout:3514: finishing rollback tid 1 to smithi01785587-21 2024-09-21T09:52:52.599 INFO:tasks.rados.rados.0.smithi017.stdout:3514: finishing rollback tid 2 to smithi01785587-21 2024-09-21T09:52:52.599 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1718 (ObjNum 1231 snap 409 seq_num 1231) dirty exists 2024-09-21T09:52:52.599 INFO:tasks.rados.rados.0.smithi017.stdout:3515: finishing write tid 1 to smithi01785587-46 2024-09-21T09:52:52.600 INFO:tasks.rados.rados.0.smithi017.stdout:3516: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:52:52.600 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 1030 (ObjNum 49 snap 0 seq_num 2384147568) dirty dne 2024-09-21T09:52:52.600 INFO:tasks.rados.rados.0.smithi017.stdout:3515: finishing write tid 2 to smithi01785587-46 2024-09-21T09:52:52.600 INFO:tasks.rados.rados.0.smithi017.stdout:3515: finishing write tid 3 to smithi01785587-46 2024-09-21T09:52:52.600 INFO:tasks.rados.rados.0.smithi017.stdout:3515: finishing write tid 4 to smithi01785587-46 2024-09-21T09:52:52.600 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1813 (ObjNum 1318 snap 441 seq_num 1318) dirty exists 2024-09-21T09:52:52.600 INFO:tasks.rados.rados.0.smithi017.stdout:3515: left oid 46 (ObjNum 1318 snap 441 seq_num 1318) 2024-09-21T09:52:52.605 INFO:tasks.rados.rados.0.smithi017.stdout:3517: expect (ObjNum 1313 snap 441 seq_num 1313) 2024-09-21T09:52:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:52 smithi017 ceph-mon[102913]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T09:52:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:52 smithi017 ceph-mon[105151]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T09:52:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:52 smithi134 ceph-mon[90085]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T09:52:53.212 INFO:tasks.rados.rados.0.smithi017.stdout:3512: done (6 left) 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:3513: done (5 left) 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:3514: done (4 left) 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:3515: done (3 left) 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:3516: done (2 left) 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:3517: done (1 left) 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:3518: done (0 left) 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:3519: rollback oid 36 current snap is 442 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 36 to 438 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:3520: rollback oid 29 current snap is 442 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 29 to 426 2024-09-21T09:52:53.213 INFO:tasks.rados.rados.0.smithi017.stdout:3521: snap_create 2024-09-21T09:52:53.218 INFO:tasks.rados.rados.0.smithi017.stdout:3520: finishing rollback tid 0 to smithi01785587-29 2024-09-21T09:52:53.218 INFO:tasks.rados.rados.0.smithi017.stdout:3520: finishing rollback tid 1 to smithi01785587-29 2024-09-21T09:52:53.218 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 1008 (ObjNum -1895480304 snap 21949 seq_num 304) dirty dne 2024-09-21T09:52:53.220 INFO:tasks.rados.rados.0.smithi017.stdout:3519: finishing rollback tid 0 to smithi01785587-36 2024-09-21T09:52:53.220 INFO:tasks.rados.rados.0.smithi017.stdout:3519: finishing rollback tid 1 to smithi01785587-36 2024-09-21T09:52:53.220 INFO:tasks.rados.rados.0.smithi017.stdout:3519: finishing rollback tid 2 to smithi01785587-36 2024-09-21T09:52:53.220 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1819 (ObjNum 1257 snap 421 seq_num 1257) dirty exists 2024-09-21T09:52:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:53 smithi017 ceph-mon[102913]: pgmap v999: 89 pgs: 89 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-21T09:52:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:53 smithi017 ceph-mon[102913]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T09:52:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:53 smithi017 ceph-mon[102913]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T09:52:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:53 smithi017 ceph-mon[105151]: pgmap v999: 89 pgs: 89 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-21T09:52:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:53 smithi017 ceph-mon[105151]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T09:52:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:53 smithi017 ceph-mon[105151]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T09:52:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:53 smithi134 ceph-mon[90085]: pgmap v999: 89 pgs: 89 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-09-21T09:52:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:53 smithi134 ceph-mon[90085]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T09:52:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:53 smithi134 ceph-mon[90085]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T09:52:54.213 INFO:tasks.rados.rados.0.smithi017.stdout:3519: done (2 left) 2024-09-21T09:52:54.214 INFO:tasks.rados.rados.0.smithi017.stdout:3520: done (1 left) 2024-09-21T09:52:54.214 INFO:tasks.rados.rados.0.smithi017.stdout:3521: done (0 left) 2024-09-21T09:52:54.214 INFO:tasks.rados.rados.0.smithi017.stdout:3522: snap_create 2024-09-21T09:52:55.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:52:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:52:54] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.51.0" 2024-09-21T09:52:55.216 INFO:tasks.rados.rados.0.smithi017.stdout:3522: done (0 left) 2024-09-21T09:52:55.216 INFO:tasks.rados.rados.0.smithi017.stdout:3523: delete oid 2 current snap is 444 2024-09-21T09:52:55.219 INFO:tasks.rados.rados.0.smithi017.stdout:3523: done (0 left) 2024-09-21T09:52:55.219 INFO:tasks.rados.rados.0.smithi017.stdout:3524: rollback oid 20 current snap is 444 2024-09-21T09:52:55.219 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 437 2024-09-21T09:52:55.219 INFO:tasks.rados.rados.0.smithi017.stdout:3525: write oid 39 current snap is 444 2024-09-21T09:52:55.219 INFO:tasks.rados.rados.0.smithi017.stdout:3525: seq_num 1320 ranges {585240=657749,1743290=422125} 2024-09-21T09:52:55.225 INFO:tasks.rados.rados.0.smithi017.stdout:3525: writing smithi01785587-39 from 585240 to 1242989 tid 1 2024-09-21T09:52:55.231 INFO:tasks.rados.rados.0.smithi017.stdout:3525: writing smithi01785587-39 from 1743290 to 2165415 tid 2 2024-09-21T09:52:55.231 INFO:tasks.rados.rados.0.smithi017.stdout:3526: snap_remove snap 443 2024-09-21T09:52:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:55 smithi134 ceph-mon[90085]: pgmap v1002: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.7 MiB/s wr, 28 op/s 2024-09-21T09:52:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:55 smithi134 ceph-mon[90085]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T09:52:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:55 smithi017 ceph-mon[102913]: pgmap v1002: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.7 MiB/s wr, 28 op/s 2024-09-21T09:52:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:55 smithi017 ceph-mon[102913]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T09:52:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:55 smithi017 ceph-mon[105151]: pgmap v1002: 89 pgs: 89 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.7 MiB/s wr, 28 op/s 2024-09-21T09:52:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:55 smithi017 ceph-mon[105151]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T09:52:56.226 INFO:tasks.rados.rados.0.smithi017.stdout:3526: done (2 left) 2024-09-21T09:52:56.226 INFO:tasks.rados.rados.0.smithi017.stdout:3527: write oid 7 current snap is 444 2024-09-21T09:52:56.226 INFO:tasks.rados.rados.0.smithi017.stdout:3524: finishing rollback tid 0 to smithi01785587-20 2024-09-21T09:52:56.226 INFO:tasks.rados.rados.0.smithi017.stdout:3527: seq_num 1321 ranges {602047=732998,1801772=400275} 2024-09-21T09:52:56.233 INFO:tasks.rados.rados.0.smithi017.stdout:3527: writing smithi01785587-7 from 602047 to 1335045 tid 1 2024-09-21T09:52:56.238 INFO:tasks.rados.rados.0.smithi017.stdout:3527: writing smithi01785587-7 from 1801772 to 2202047 tid 2 2024-09-21T09:52:56.238 INFO:tasks.rados.rados.0.smithi017.stdout:3528: rollback oid 26 current snap is 444 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 26 to 440 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3524: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3524: finishing rollback tid 2 to smithi01785587-20 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 1012 (ObjNum 1291 snap 437 seq_num 1291) dirty exists 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3525: finishing write tid 1 to smithi01785587-39 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3525: finishing write tid 2 to smithi01785587-39 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3525: finishing write tid 3 to smithi01785587-39 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3525: finishing write tid 4 to smithi01785587-39 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1823 (ObjNum 1320 snap 444 seq_num 1320) dirty exists 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3525: left oid 39 (ObjNum 1320 snap 444 seq_num 1320) 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3524: done (3 left) 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3525: done (2 left) 2024-09-21T09:52:56.239 INFO:tasks.rados.rados.0.smithi017.stdout:3529: snap_create 2024-09-21T09:52:56.243 INFO:tasks.rados.rados.0.smithi017.stdout:3528: finishing rollback tid 0 to smithi01785587-26 2024-09-21T09:52:56.244 INFO:tasks.rados.rados.0.smithi017.stdout:3527: finishing write tid 1 to smithi01785587-7 2024-09-21T09:52:56.244 INFO:tasks.rados.rados.0.smithi017.stdout:3527: finishing write tid 2 to smithi01785587-7 2024-09-21T09:52:56.244 INFO:tasks.rados.rados.0.smithi017.stdout:3527: finishing write tid 3 to smithi01785587-7 2024-09-21T09:52:56.245 INFO:tasks.rados.rados.0.smithi017.stdout:3527: finishing write tid 4 to smithi01785587-7 2024-09-21T09:52:56.245 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1722 (ObjNum 1321 snap 444 seq_num 1321) dirty exists 2024-09-21T09:52:56.245 INFO:tasks.rados.rados.0.smithi017.stdout:3527: left oid 7 (ObjNum 1321 snap 444 seq_num 1321) 2024-09-21T09:52:56.245 INFO:tasks.rados.rados.0.smithi017.stdout:3528: finishing rollback tid 1 to smithi01785587-26 2024-09-21T09:52:56.245 INFO:tasks.rados.rados.0.smithi017.stdout:3528: finishing rollback tid 2 to smithi01785587-26 2024-09-21T09:52:56.245 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 1855 (ObjNum 1084 snap 362 seq_num 1084) dirty exists 2024-09-21T09:52:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:56 smithi134 ceph-mon[90085]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T09:52:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:56 smithi017 ceph-mon[102913]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T09:52:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:56 smithi017 ceph-mon[105151]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T09:52:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:57.233 INFO:tasks.rados.rados.0.smithi017.stdout:3527: done (2 left) 2024-09-21T09:52:57.233 INFO:tasks.rados.rados.0.smithi017.stdout:3528: done (1 left) 2024-09-21T09:52:57.233 INFO:tasks.rados.rados.0.smithi017.stdout:3529: done (0 left) 2024-09-21T09:52:57.233 INFO:tasks.rados.rados.0.smithi017.stdout:3530: delete oid 23 current snap is 445 2024-09-21T09:52:57.238 INFO:tasks.rados.rados.0.smithi017.stdout:3530: done (0 left) 2024-09-21T09:52:57.238 INFO:tasks.rados.rados.0.smithi017.stdout:3531: read oid 34 snap -1 2024-09-21T09:52:57.238 INFO:tasks.rados.rados.0.smithi017.stdout:3531: expect deleted 2024-09-21T09:52:57.238 INFO:tasks.rados.rados.0.smithi017.stdout:3532: snap_remove snap 429 2024-09-21T09:52:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:57 smithi134 ceph-mon[90085]: pgmap v1005: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.8 MiB/s wr, 28 op/s 2024-09-21T09:52:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:57 smithi134 ceph-mon[90085]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T09:52:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:57 smithi017 ceph-mon[102913]: pgmap v1005: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.8 MiB/s wr, 28 op/s 2024-09-21T09:52:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:57 smithi017 ceph-mon[102913]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T09:52:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:57 smithi017 ceph-mon[105151]: pgmap v1005: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.8 MiB/s wr, 28 op/s 2024-09-21T09:52:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:57 smithi017 ceph-mon[105151]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T09:52:58.215 INFO:tasks.rados.rados.0.smithi017.stdout:3532: done (1 left) 2024-09-21T09:52:58.215 INFO:tasks.rados.rados.0.smithi017.stdout:3533: write oid 22 current snap is 445 2024-09-21T09:52:58.215 INFO:tasks.rados.rados.0.smithi017.stdout:3533: seq_num 1323 ranges {635661=599849,1918736=316925} 2024-09-21T09:52:58.222 INFO:tasks.rados.rados.0.smithi017.stdout:3533: writing smithi01785587-22 from 635661 to 1235510 tid 1 2024-09-21T09:52:58.229 INFO:tasks.rados.rados.0.smithi017.stdout:3533: writing smithi01785587-22 from 1918736 to 2235661 tid 2 2024-09-21T09:52:58.229 INFO:tasks.rados.rados.0.smithi017.stdout:3534: delete oid 7 current snap is 445 2024-09-21T09:52:58.229 INFO:tasks.rados.rados.0.smithi017.stdout:3533: finishing write tid 1 to smithi01785587-22 2024-09-21T09:52:58.231 INFO:tasks.rados.rados.0.smithi017.stdout:3531: done (2 left) 2024-09-21T09:52:58.232 INFO:tasks.rados.rados.0.smithi017.stdout:3534: done (1 left) 2024-09-21T09:52:58.232 INFO:tasks.rados.rados.0.smithi017.stdout:3535: read oid 35 snap -1 2024-09-21T09:52:58.232 INFO:tasks.rados.rados.0.smithi017.stdout:3535: expect deleted 2024-09-21T09:52:58.232 INFO:tasks.rados.rados.0.smithi017.stdout:3536: write oid 31 current snap is 445 2024-09-21T09:52:58.232 INFO:tasks.rados.rados.0.smithi017.stdout:3536: seq_num 1325 ranges {669275=750347,2035700=233575} 2024-09-21T09:52:58.239 INFO:tasks.rados.rados.0.smithi017.stdout:3536: writing smithi01785587-31 from 669275 to 1419622 tid 1 2024-09-21T09:52:58.244 INFO:tasks.rados.rados.0.smithi017.stdout:3536: writing smithi01785587-31 from 2035700 to 2269275 tid 2 2024-09-21T09:52:58.244 INFO:tasks.rados.rados.0.smithi017.stdout:3537: delete oid 2 current snap is 445 2024-09-21T09:52:58.244 INFO:tasks.rados.rados.0.smithi017.stdout:3533: finishing write tid 2 to smithi01785587-22 2024-09-21T09:52:58.244 INFO:tasks.rados.rados.0.smithi017.stdout:3533: finishing write tid 3 to smithi01785587-22 2024-09-21T09:52:58.244 INFO:tasks.rados.rados.0.smithi017.stdout:3533: finishing write tid 4 to smithi01785587-22 2024-09-21T09:52:58.244 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1863 (ObjNum 1323 snap 445 seq_num 1323) dirty exists 2024-09-21T09:52:58.245 INFO:tasks.rados.rados.0.smithi017.stdout:3533: left oid 22 (ObjNum 1323 snap 445 seq_num 1323) 2024-09-21T09:52:58.245 INFO:tasks.rados.rados.0.smithi017.stdout:3536: finishing write tid 1 to smithi01785587-31 2024-09-21T09:52:58.246 INFO:tasks.rados.rados.0.smithi017.stdout:3533: done (3 left) 2024-09-21T09:52:58.246 INFO:tasks.rados.rados.0.smithi017.stdout:3535: done (2 left) 2024-09-21T09:52:58.246 INFO:tasks.rados.rados.0.smithi017.stdout:3537: done (1 left) 2024-09-21T09:52:58.246 INFO:tasks.rados.rados.0.smithi017.stdout:3538: write oid 42 current snap is 445 2024-09-21T09:52:58.246 INFO:tasks.rados.rados.0.smithi017.stdout:3538: seq_num 1327 ranges {702889=500845,1636311=576790} 2024-09-21T09:52:58.252 INFO:tasks.rados.rados.0.smithi017.stdout:3538: writing smithi01785587-42 from 702889 to 1203734 tid 1 2024-09-21T09:52:58.259 INFO:tasks.rados.rados.0.smithi017.stdout:3538: writing smithi01785587-42 from 1636311 to 2213101 tid 2 2024-09-21T09:52:58.259 INFO:tasks.rados.rados.0.smithi017.stdout:3539: read oid 26 snap 421 2024-09-21T09:52:58.259 INFO:tasks.rados.rados.0.smithi017.stdout:3539: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:52:58.259 INFO:tasks.rados.rados.0.smithi017.stdout:3540: rollback oid 5 current snap is 445 2024-09-21T09:52:58.259 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 439 2024-09-21T09:52:58.259 INFO:tasks.rados.rados.0.smithi017.stdout:3536: finishing write tid 2 to smithi01785587-31 2024-09-21T09:52:58.259 INFO:tasks.rados.rados.0.smithi017.stdout:3541: read oid 22 snap 426 2024-09-21T09:52:58.259 INFO:tasks.rados.rados.0.smithi017.stdout:3541: expect (ObjNum 1254 snap 421 seq_num 1254) 2024-09-21T09:52:58.259 INFO:tasks.rados.rados.0.smithi017.stdout:3542: rollback oid 27 current snap is 445 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 442 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3536: finishing write tid 3 to smithi01785587-31 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3536: finishing write tid 4 to smithi01785587-31 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 1960 (ObjNum 1325 snap 445 seq_num 1325) dirty exists 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3536: left oid 31 (ObjNum 1325 snap 445 seq_num 1325) 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3538: finishing write tid 1 to smithi01785587-42 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3536: done (5 left) 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3543: read oid 11 snap -1 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3543: expect (ObjNum 1305 snap 440 seq_num 1305) 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3544: rollback oid 3 current snap is 445 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 424 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3545: write oid 1 current snap is 445 2024-09-21T09:52:58.260 INFO:tasks.rados.rados.0.smithi017.stdout:3545: seq_num 1328 ranges {719696=576094,2094793=224903} 2024-09-21T09:52:58.266 INFO:tasks.rados.rados.0.smithi017.stdout:3545: writing smithi01785587-1 from 719696 to 1295790 tid 1 2024-09-21T09:52:58.273 INFO:tasks.rados.rados.0.smithi017.stdout:3545: writing smithi01785587-1 from 2094793 to 2319696 tid 2 2024-09-21T09:52:58.273 INFO:tasks.rados.rados.0.smithi017.stdout:3546: snap_remove snap 438 2024-09-21T09:52:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:58 smithi134 ceph-mon[90085]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T09:52:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:58 smithi134 ceph-mon[90085]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T09:52:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:58 smithi017 ceph-mon[102913]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T09:52:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:58 smithi017 ceph-mon[102913]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T09:52:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:58 smithi017 ceph-mon[105151]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T09:52:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:58 smithi017 ceph-mon[105151]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T09:52:59.247 INFO:tasks.rados.rados.0.smithi017.stdout:3546: done (8 left) 2024-09-21T09:52:59.247 INFO:tasks.rados.rados.0.smithi017.stdout:3547: rollback oid 44 current snap is 445 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 439 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:3542: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 1147 (ObjNum 33 snap 0 seq_num 2384147832) dirty dne 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:3538: finishing write tid 2 to smithi01785587-42 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:3538: finishing write tid 3 to smithi01785587-42 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:3540: finishing rollback tid 0 to smithi01785587-5 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:3538: finishing write tid 4 to smithi01785587-42 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 1229 (ObjNum 1327 snap 445 seq_num 1327) dirty exists 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:3538: left oid 42 (ObjNum 1327 snap 445 seq_num 1327) 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:3540: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:3540: finishing rollback tid 2 to smithi01785587-5 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 1964 (ObjNum 1299 snap 439 seq_num 1299) dirty exists 2024-09-21T09:52:59.248 INFO:tasks.rados.rados.0.smithi017.stdout:3543: expect (ObjNum 1305 snap 440 seq_num 1305) 2024-09-21T09:52:59.436 INFO:tasks.rados.rados.0.smithi017.stdout:3539: expect (ObjNum 1084 snap 362 seq_num 1084) 2024-09-21T09:52:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:52:59 smithi134 ceph-mon[90085]: pgmap v1008: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.2 MiB/s wr, 24 op/s 2024-09-21T09:52:59.595 INFO:tasks.rados.rados.0.smithi017.stdout:3544: finishing rollback tid 0 to smithi01785587-3 2024-09-21T09:52:59.595 INFO:tasks.rados.rados.0.smithi017.stdout:3544: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:52:59.595 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 1967 (ObjNum 33 snap 0 seq_num 2399872288) dirty dne 2024-09-21T09:52:59.595 INFO:tasks.rados.rados.0.smithi017.stdout:3541: expect (ObjNum 1254 snap 421 seq_num 1254) 2024-09-21T09:52:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:52:59 smithi017 ceph-mon[102913]: pgmap v1008: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.2 MiB/s wr, 24 op/s 2024-09-21T09:52:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:52:59 smithi017 ceph-mon[105151]: pgmap v1008: 89 pgs: 89 active+clean; 189 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.2 MiB/s wr, 24 op/s 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3545: finishing write tid 1 to smithi01785587-1 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3545: finishing write tid 2 to smithi01785587-1 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3538: done (8 left) 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3539: done (7 left) 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3540: done (6 left) 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3541: done (5 left) 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3542: done (4 left) 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3543: done (3 left) 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3544: done (2 left) 2024-09-21T09:52:59.701 INFO:tasks.rados.rados.0.smithi017.stdout:3548: snap_remove snap 426 2024-09-21T09:53:00.254 INFO:tasks.rados.rados.0.smithi017.stdout:3548: done (2 left) 2024-09-21T09:53:00.254 INFO:tasks.rados.rados.0.smithi017.stdout:3549: read oid 19 snap -1 2024-09-21T09:53:00.254 INFO:tasks.rados.rados.0.smithi017.stdout:3549: expect (ObjNum 1269 snap 424 seq_num 1269) 2024-09-21T09:53:00.254 INFO:tasks.rados.rados.0.smithi017.stdout:3550: snap_create 2024-09-21T09:53:00.254 INFO:tasks.rados.rados.0.smithi017.stdout:3545: finishing write tid 3 to smithi01785587-1 2024-09-21T09:53:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:3545: finishing write tid 4 to smithi01785587-1 2024-09-21T09:53:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 1038 (ObjNum 1328 snap 445 seq_num 1328) dirty exists 2024-09-21T09:53:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:3545: left oid 1 (ObjNum 1328 snap 445 seq_num 1328) 2024-09-21T09:53:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:3547: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:53:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:3547: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:53:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:3547: finishing rollback tid 2 to smithi01785587-44 2024-09-21T09:53:00.255 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 1971 (ObjNum 1220 snap 405 seq_num 1220) dirty exists 2024-09-21T09:53:00.264 INFO:tasks.rados.rados.0.smithi017.stdout:3549: expect (ObjNum 1269 snap 424 seq_num 1269) 2024-09-21T09:53:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:00 smithi134 ceph-mon[90085]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T09:53:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:53:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:53:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:00 smithi017 ceph-mon[102913]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T09:53:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:00 smithi017 ceph-mon[105151]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T09:53:01.259 INFO:tasks.rados.rados.0.smithi017.stdout:3545: done (3 left) 2024-09-21T09:53:01.259 INFO:tasks.rados.rados.0.smithi017.stdout:3547: done (2 left) 2024-09-21T09:53:01.259 INFO:tasks.rados.rados.0.smithi017.stdout:3549: done (1 left) 2024-09-21T09:53:01.259 INFO:tasks.rados.rados.0.smithi017.stdout:3550: done (0 left) 2024-09-21T09:53:01.259 INFO:tasks.rados.rados.0.smithi017.stdout:3551: write oid 33 current snap is 446 2024-09-21T09:53:01.259 INFO:tasks.rados.rados.0.smithi017.stdout:3551: seq_num 1329 ranges {736503=651343,2153275=183228} 2024-09-21T09:53:01.266 INFO:tasks.rados.rados.0.smithi017.stdout:3551: writing smithi01785587-33 from 736503 to 1387846 tid 1 2024-09-21T09:53:01.272 INFO:tasks.rados.rados.0.smithi017.stdout:3551: writing smithi01785587-33 from 2153275 to 2336503 tid 2 2024-09-21T09:53:01.272 INFO:tasks.rados.rados.0.smithi017.stdout:3552: write oid 11 current snap is 446 2024-09-21T09:53:01.272 INFO:tasks.rados.rados.0.smithi017.stdout:3552: seq_num 1330 ranges {753310=726592,2095404=257906} 2024-09-21T09:53:01.280 INFO:tasks.rados.rados.0.smithi017.stdout:3552: writing smithi01785587-11 from 753310 to 1479902 tid 1 2024-09-21T09:53:01.286 INFO:tasks.rados.rados.0.smithi017.stdout:3552: writing smithi01785587-11 from 2095404 to 2353310 tid 2 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:3553: rollback oid 15 current snap is 446 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 15 to 445 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:3551: finishing write tid 1 to smithi01785587-33 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:3551: finishing write tid 2 to smithi01785587-33 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:3551: finishing write tid 3 to smithi01785587-33 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:3551: finishing write tid 4 to smithi01785587-33 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1876 (ObjNum 1329 snap 446 seq_num 1329) dirty exists 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:3551: left oid 33 (ObjNum 1329 snap 446 seq_num 1329) 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:3551: done (2 left) 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:3554: write oid 13 current snap is 446 2024-09-21T09:53:01.287 INFO:tasks.rados.rados.0.smithi017.stdout:3554: seq_num 1331 ranges {770117=518194,1870239=499878} 2024-09-21T09:53:01.293 INFO:tasks.rados.rados.0.smithi017.stdout:3554: writing smithi01785587-13 from 770117 to 1288311 tid 1 2024-09-21T09:53:01.300 INFO:tasks.rados.rados.0.smithi017.stdout:3554: writing smithi01785587-13 from 1870239 to 2370117 tid 2 2024-09-21T09:53:01.300 INFO:tasks.rados.rados.0.smithi017.stdout:3555: rollback oid 6 current snap is 446 2024-09-21T09:53:01.300 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 6 to 441 2024-09-21T09:53:01.300 INFO:tasks.rados.rados.0.smithi017.stdout:3553: finishing rollback tid 0 to smithi01785587-15 2024-09-21T09:53:01.300 INFO:tasks.rados.rados.0.smithi017.stdout:3553: finishing rollback tid 1 to smithi01785587-15 2024-09-21T09:53:01.300 INFO:tasks.rados.rados.0.smithi017.stdout:3553: finishing rollback tid 2 to smithi01785587-15 2024-09-21T09:53:01.300 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 545 (ObjNum 1314 snap 441 seq_num 1314) dirty exists 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:3552: finishing write tid 1 to smithi01785587-11 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:3552: finishing write tid 2 to smithi01785587-11 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:3552: finishing write tid 3 to smithi01785587-11 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:3552: finishing write tid 4 to smithi01785587-11 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1843 (ObjNum 1330 snap 446 seq_num 1330) dirty exists 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:3552: left oid 11 (ObjNum 1330 snap 446 seq_num 1330) 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:3554: finishing write tid 1 to smithi01785587-13 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:3552: done (3 left) 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:3553: done (2 left) 2024-09-21T09:53:01.301 INFO:tasks.rados.rados.0.smithi017.stdout:3556: delete oid 31 current snap is 446 2024-09-21T09:53:01.303 INFO:tasks.rados.rados.0.smithi017.stdout:3556: done (2 left) 2024-09-21T09:53:01.303 INFO:tasks.rados.rados.0.smithi017.stdout:3557: write oid 8 current snap is 446 2024-09-21T09:53:01.303 INFO:tasks.rados.rados.0.smithi017.stdout:3557: seq_num 1333 ranges {403731=668692,1587203=587797} 2024-09-21T09:53:01.308 INFO:tasks.rados.rados.0.smithi017.stdout:3557: writing smithi01785587-8 from 403731 to 1072423 tid 1 2024-09-21T09:53:01.315 INFO:tasks.rados.rados.0.smithi017.stdout:3557: writing smithi01785587-8 from 1587203 to 2175000 tid 2 2024-09-21T09:53:01.315 INFO:tasks.rados.rados.0.smithi017.stdout:3558: write oid 40 current snap is 446 2024-09-21T09:53:01.315 INFO:tasks.rados.rados.0.smithi017.stdout:3555: finishing rollback tid 0 to smithi01785587-6 2024-09-21T09:53:01.315 INFO:tasks.rados.rados.0.smithi017.stdout:3554: finishing write tid 2 to smithi01785587-13 2024-09-21T09:53:01.315 INFO:tasks.rados.rados.0.smithi017.stdout:3555: finishing rollback tid 1 to smithi01785587-6 2024-09-21T09:53:01.315 INFO:tasks.rados.rados.0.smithi017.stdout:3555: finishing rollback tid 2 to smithi01785587-6 2024-09-21T09:53:01.315 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 1042 (ObjNum 1307 snap 440 seq_num 1307) dirty exists 2024-09-21T09:53:01.315 INFO:tasks.rados.rados.0.smithi017.stdout:3554: finishing write tid 3 to smithi01785587-13 2024-09-21T09:53:01.315 INFO:tasks.rados.rados.0.smithi017.stdout:3554: finishing write tid 4 to smithi01785587-13 2024-09-21T09:53:01.316 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1492 (ObjNum 1331 snap 446 seq_num 1331) dirty exists 2024-09-21T09:53:01.316 INFO:tasks.rados.rados.0.smithi017.stdout:3554: left oid 13 (ObjNum 1331 snap 446 seq_num 1331) 2024-09-21T09:53:01.316 INFO:tasks.rados.rados.0.smithi017.stdout:3557: finishing write tid 1 to smithi01785587-8 2024-09-21T09:53:01.316 INFO:tasks.rados.rados.0.smithi017.stdout:3558: seq_num 1334 ranges {420538=743941,1929332=491206} 2024-09-21T09:53:01.321 INFO:tasks.rados.rados.0.smithi017.stdout:3558: writing smithi01785587-40 from 420538 to 1164479 tid 1 2024-09-21T09:53:01.328 INFO:tasks.rados.rados.0.smithi017.stdout:3558: writing smithi01785587-40 from 1929332 to 2420538 tid 2 2024-09-21T09:53:01.328 INFO:tasks.rados.rados.0.smithi017.stdout:3554: done (3 left) 2024-09-21T09:53:01.328 INFO:tasks.rados.rados.0.smithi017.stdout:3555: done (2 left) 2024-09-21T09:53:01.328 INFO:tasks.rados.rados.0.smithi017.stdout:3559: read oid 14 snap -1 2024-09-21T09:53:01.328 INFO:tasks.rados.rados.0.smithi017.stdout:3559: expect (ObjNum 1158 snap 389 seq_num 1158) 2024-09-21T09:53:01.328 INFO:tasks.rados.rados.0.smithi017.stdout:3560: snap_remove snap 442 2024-09-21T09:53:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:01 smithi017 ceph-mon[102913]: pgmap v1011: 89 pgs: 89 active+clean; 192 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 982 KiB/s wr, 6 op/s 2024-09-21T09:53:01.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:01 smithi017 ceph-mon[102913]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T09:53:01.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:01 smithi017 ceph-mon[105151]: pgmap v1011: 89 pgs: 89 active+clean; 192 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 982 KiB/s wr, 6 op/s 2024-09-21T09:53:01.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:01 smithi017 ceph-mon[105151]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T09:53:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:01 smithi134 ceph-mon[90085]: pgmap v1011: 89 pgs: 89 active+clean; 192 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 982 KiB/s wr, 6 op/s 2024-09-21T09:53:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:01 smithi134 ceph-mon[90085]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T09:53:02.264 INFO:tasks.rados.rados.0.smithi017.stdout:3560: done (3 left) 2024-09-21T09:53:02.264 INFO:tasks.rados.rados.0.smithi017.stdout:3561: snap_remove snap 441 2024-09-21T09:53:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:02 smithi017 ceph-mon[102913]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T09:53:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:02 smithi017 ceph-mon[105151]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T09:53:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:02 smithi134 ceph-mon[90085]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T09:53:03.217 INFO:tasks.rados.rados.0.smithi017.stdout:3561: done (3 left) 2024-09-21T09:53:03.217 INFO:tasks.rados.rados.0.smithi017.stdout:3562: write oid 10 current snap is 446 2024-09-21T09:53:03.217 INFO:tasks.rados.rados.0.smithi017.stdout:3562: seq_num 1335 ranges {437345=419190,1587814=591466} 2024-09-21T09:53:03.222 INFO:tasks.rados.rados.0.smithi017.stdout:3562: writing smithi01785587-10 from 437345 to 856535 tid 1 2024-09-21T09:53:03.231 INFO:tasks.rados.rados.0.smithi017.stdout:3562: writing smithi01785587-10 from 1587814 to 2179280 tid 2 2024-09-21T09:53:03.231 INFO:tasks.rados.rados.0.smithi017.stdout:3563: write oid 36 current snap is 446 2024-09-21T09:53:03.231 INFO:tasks.rados.rados.0.smithi017.stdout:3557: finishing write tid 2 to smithi01785587-8 2024-09-21T09:53:03.231 INFO:tasks.rados.rados.0.smithi017.stdout:3557: finishing write tid 3 to smithi01785587-8 2024-09-21T09:53:03.231 INFO:tasks.rados.rados.0.smithi017.stdout:3557: finishing write tid 4 to smithi01785587-8 2024-09-21T09:53:03.231 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1846 (ObjNum 1333 snap 446 seq_num 1333) dirty exists 2024-09-21T09:53:03.231 INFO:tasks.rados.rados.0.smithi017.stdout:3557: left oid 8 (ObjNum 1333 snap 446 seq_num 1333) 2024-09-21T09:53:03.231 INFO:tasks.rados.rados.0.smithi017.stdout:3558: finishing write tid 1 to smithi01785587-40 2024-09-21T09:53:03.231 INFO:tasks.rados.rados.0.smithi017.stdout:3559: expect (ObjNum 1158 snap 389 seq_num 1158) 2024-09-21T09:53:03.390 INFO:tasks.rados.rados.0.smithi017.stdout:3558: finishing write tid 2 to smithi01785587-40 2024-09-21T09:53:03.390 INFO:tasks.rados.rados.0.smithi017.stdout:3558: finishing write tid 3 to smithi01785587-40 2024-09-21T09:53:03.391 INFO:tasks.rados.rados.0.smithi017.stdout:3558: finishing write tid 4 to smithi01785587-40 2024-09-21T09:53:03.391 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1844 (ObjNum 1334 snap 446 seq_num 1334) dirty exists 2024-09-21T09:53:03.391 INFO:tasks.rados.rados.0.smithi017.stdout:3558: left oid 40 (ObjNum 1334 snap 446 seq_num 1334) 2024-09-21T09:53:03.391 INFO:tasks.rados.rados.0.smithi017.stdout:3562: finishing write tid 1 to smithi01785587-10 2024-09-21T09:53:03.391 INFO:tasks.rados.rados.0.smithi017.stdout:3562: finishing write tid 2 to smithi01785587-10 2024-09-21T09:53:03.391 INFO:tasks.rados.rados.0.smithi017.stdout:3563: seq_num 1336 ranges {454152=494439,1646296=735124} 2024-09-21T09:53:03.395 INFO:tasks.rados.rados.0.smithi017.stdout:3563: writing smithi01785587-36 from 454152 to 948591 tid 1 2024-09-21T09:53:03.403 INFO:tasks.rados.rados.0.smithi017.stdout:3563: writing smithi01785587-36 from 1646296 to 2381420 tid 2 2024-09-21T09:53:03.403 INFO:tasks.rados.rados.0.smithi017.stdout:3557: done (4 left) 2024-09-21T09:53:03.403 INFO:tasks.rados.rados.0.smithi017.stdout:3558: done (3 left) 2024-09-21T09:53:03.403 INFO:tasks.rados.rados.0.smithi017.stdout:3559: done (2 left) 2024-09-21T09:53:03.404 INFO:tasks.rados.rados.0.smithi017.stdout:3564: delete oid 11 current snap is 446 2024-09-21T09:53:03.404 INFO:tasks.rados.rados.0.smithi017.stdout:3562: finishing write tid 3 to smithi01785587-10 2024-09-21T09:53:03.404 INFO:tasks.rados.rados.0.smithi017.stdout:3562: finishing write tid 4 to smithi01785587-10 2024-09-21T09:53:03.404 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 10 v 349 (ObjNum 1335 snap 446 seq_num 1335) dirty exists 2024-09-21T09:53:03.404 INFO:tasks.rados.rados.0.smithi017.stdout:3562: left oid 10 (ObjNum 1335 snap 446 seq_num 1335) 2024-09-21T09:53:03.404 INFO:tasks.rados.rados.0.smithi017.stdout:3563: finishing write tid 1 to smithi01785587-36 2024-09-21T09:53:03.405 INFO:tasks.rados.rados.0.smithi017.stdout:3562: done (2 left) 2024-09-21T09:53:03.405 INFO:tasks.rados.rados.0.smithi017.stdout:3564: done (1 left) 2024-09-21T09:53:03.406 INFO:tasks.rados.rados.0.smithi017.stdout:3565: write oid 45 current snap is 446 2024-09-21T09:53:03.406 INFO:tasks.rados.rados.0.smithi017.stdout:3565: seq_num 1338 ranges {487766=644937,1646907=738793} 2024-09-21T09:53:03.411 INFO:tasks.rados.rados.0.smithi017.stdout:3565: writing smithi01785587-45 from 487766 to 1132703 tid 1 2024-09-21T09:53:03.419 INFO:tasks.rados.rados.0.smithi017.stdout:3565: writing smithi01785587-45 from 1646907 to 2385700 tid 2 2024-09-21T09:53:03.419 INFO:tasks.rados.rados.0.smithi017.stdout:3566: delete oid 5 current snap is 446 2024-09-21T09:53:03.419 INFO:tasks.rados.rados.0.smithi017.stdout:3563: finishing write tid 2 to smithi01785587-36 2024-09-21T09:53:03.419 INFO:tasks.rados.rados.0.smithi017.stdout:3563: finishing write tid 3 to smithi01785587-36 2024-09-21T09:53:03.419 INFO:tasks.rados.rados.0.smithi017.stdout:3563: finishing write tid 4 to smithi01785587-36 2024-09-21T09:53:03.419 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1854 (ObjNum 1336 snap 446 seq_num 1336) dirty exists 2024-09-21T09:53:03.419 INFO:tasks.rados.rados.0.smithi017.stdout:3563: left oid 36 (ObjNum 1336 snap 446 seq_num 1336) 2024-09-21T09:53:03.419 INFO:tasks.rados.rados.0.smithi017.stdout:3565: finishing write tid 1 to smithi01785587-45 2024-09-21T09:53:03.421 INFO:tasks.rados.rados.0.smithi017.stdout:3563: done (2 left) 2024-09-21T09:53:03.422 INFO:tasks.rados.rados.0.smithi017.stdout:3566: done (1 left) 2024-09-21T09:53:03.422 INFO:tasks.rados.rados.0.smithi017.stdout:3567: snap_create 2024-09-21T09:53:03.423 INFO:tasks.rados.rados.0.smithi017.stdout:3565: finishing write tid 2 to smithi01785587-45 2024-09-21T09:53:03.423 INFO:tasks.rados.rados.0.smithi017.stdout:3565: finishing write tid 3 to smithi01785587-45 2024-09-21T09:53:03.423 INFO:tasks.rados.rados.0.smithi017.stdout:3565: finishing write tid 4 to smithi01785587-45 2024-09-21T09:53:03.423 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1221 (ObjNum 1338 snap 446 seq_num 1338) dirty exists 2024-09-21T09:53:03.423 INFO:tasks.rados.rados.0.smithi017.stdout:3565: left oid 45 (ObjNum 1338 snap 446 seq_num 1338) 2024-09-21T09:53:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:03 smithi017 ceph-mon[102913]: pgmap v1014: 89 pgs: 89 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 19 op/s 2024-09-21T09:53:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:03 smithi017 ceph-mon[102913]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T09:53:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:03 smithi017 ceph-mon[102913]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T09:53:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:03 smithi017 ceph-mon[105151]: pgmap v1014: 89 pgs: 89 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 19 op/s 2024-09-21T09:53:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:03 smithi017 ceph-mon[105151]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T09:53:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:03 smithi017 ceph-mon[105151]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T09:53:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:03 smithi134 ceph-mon[90085]: pgmap v1014: 89 pgs: 89 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 19 op/s 2024-09-21T09:53:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:03 smithi134 ceph-mon[90085]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T09:53:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:03 smithi134 ceph-mon[90085]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T09:53:04.271 INFO:tasks.rados.rados.0.smithi017.stdout:3565: done (1 left) 2024-09-21T09:53:04.271 INFO:tasks.rados.rados.0.smithi017.stdout:3567: done (0 left) 2024-09-21T09:53:04.271 INFO:tasks.rados.rados.0.smithi017.stdout:3568: snap_create 2024-09-21T09:53:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:53:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:53:04] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:53:05.276 INFO:tasks.rados.rados.0.smithi017.stdout:3568: done (0 left) 2024-09-21T09:53:05.277 INFO:tasks.rados.rados.0.smithi017.stdout:3569: write oid 17 current snap is 448 2024-09-21T09:53:05.277 INFO:tasks.rados.rados.0.smithi017.stdout:3569: seq_num 1340 ranges {521380=511788,1480224=742462} 2024-09-21T09:53:05.282 INFO:tasks.rados.rados.0.smithi017.stdout:3569: writing smithi01785587-17 from 521380 to 1033168 tid 1 2024-09-21T09:53:05.289 INFO:tasks.rados.rados.0.smithi017.stdout:3569: writing smithi01785587-17 from 1480224 to 2222686 tid 2 2024-09-21T09:53:05.289 INFO:tasks.rados.rados.0.smithi017.stdout:3570: rollback oid 8 current snap is 448 2024-09-21T09:53:05.289 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 440 2024-09-21T09:53:05.289 INFO:tasks.rados.rados.0.smithi017.stdout:3571: write oid 34 current snap is 448 2024-09-21T09:53:05.289 INFO:tasks.rados.rados.0.smithi017.stdout:3571: seq_num 1341 ranges {538187=587037,1538706=602473} 2024-09-21T09:53:05.296 INFO:tasks.rados.rados.0.smithi017.stdout:3571: writing smithi01785587-34 from 538187 to 1125224 tid 1 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:3571: writing smithi01785587-34 from 1538706 to 2141179 tid 2 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:3572: delete oid 11 current snap is 448 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:3569: finishing write tid 1 to smithi01785587-17 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:3569: finishing write tid 2 to smithi01785587-17 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:3569: finishing write tid 3 to smithi01785587-17 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:3569: finishing write tid 4 to smithi01785587-17 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 1225 (ObjNum 1340 snap 448 seq_num 1340) dirty exists 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:3569: left oid 17 (ObjNum 1340 snap 448 seq_num 1340) 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:3570: finishing rollback tid 0 to smithi01785587-8 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:3570: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:53:05.302 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1858 (ObjNum 33 snap 0 seq_num 2399484768) dirty dne 2024-09-21T09:53:05.303 INFO:tasks.rados.rados.0.smithi017.stdout:3571: finishing write tid 1 to smithi01785587-34 2024-09-21T09:53:05.305 INFO:tasks.rados.rados.0.smithi017.stdout:3569: done (3 left) 2024-09-21T09:53:05.305 INFO:tasks.rados.rados.0.smithi017.stdout:3570: done (2 left) 2024-09-21T09:53:05.305 INFO:tasks.rados.rados.0.smithi017.stdout:3572: done (1 left) 2024-09-21T09:53:05.305 INFO:tasks.rados.rados.0.smithi017.stdout:3573: read oid 27 snap -1 2024-09-21T09:53:05.305 INFO:tasks.rados.rados.0.smithi017.stdout:3573: expect deleted 2024-09-21T09:53:05.305 INFO:tasks.rados.rados.0.smithi017.stdout:3574: snap_remove snap 424 2024-09-21T09:53:05.546 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:05 smithi134 ceph-mon[90085]: pgmap v1017: 89 pgs: 89 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:53:05.546 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:05 smithi134 ceph-mon[90085]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T09:53:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:05 smithi017 ceph-mon[102913]: pgmap v1017: 89 pgs: 89 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:53:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:05 smithi017 ceph-mon[102913]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T09:53:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:05 smithi017 ceph-mon[105151]: pgmap v1017: 89 pgs: 89 active+clean; 201 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-09-21T09:53:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:05 smithi017 ceph-mon[105151]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T09:53:06.284 INFO:tasks.rados.rados.0.smithi017.stdout:3574: done (2 left) 2024-09-21T09:53:06.284 INFO:tasks.rados.rados.0.smithi017.stdout:3575: delete oid 30 current snap is 448 2024-09-21T09:53:06.284 INFO:tasks.rados.rados.0.smithi017.stdout:3571: finishing write tid 2 to smithi01785587-34 2024-09-21T09:53:06.284 INFO:tasks.rados.rados.0.smithi017.stdout:3571: finishing write tid 3 to smithi01785587-34 2024-09-21T09:53:06.284 INFO:tasks.rados.rados.0.smithi017.stdout:3571: finishing write tid 4 to smithi01785587-34 2024-09-21T09:53:06.284 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1743 (ObjNum 1341 snap 448 seq_num 1341) dirty exists 2024-09-21T09:53:06.284 INFO:tasks.rados.rados.0.smithi017.stdout:3571: left oid 34 (ObjNum 1341 snap 448 seq_num 1341) 2024-09-21T09:53:06.292 INFO:tasks.rados.rados.0.smithi017.stdout:3571: done (2 left) 2024-09-21T09:53:06.292 INFO:tasks.rados.rados.0.smithi017.stdout:3573: done (1 left) 2024-09-21T09:53:06.292 INFO:tasks.rados.rados.0.smithi017.stdout:3575: done (0 left) 2024-09-21T09:53:06.292 INFO:tasks.rados.rados.0.smithi017.stdout:3576: read oid 21 snap -1 2024-09-21T09:53:06.292 INFO:tasks.rados.rados.0.smithi017.stdout:3576: expect (ObjNum 1231 snap 409 seq_num 1231) 2024-09-21T09:53:06.292 INFO:tasks.rados.rados.0.smithi017.stdout:3577: delete oid 36 current snap is 448 2024-09-21T09:53:06.294 INFO:tasks.rados.rados.0.smithi017.stdout:3577: done (1 left) 2024-09-21T09:53:06.294 INFO:tasks.rados.rados.0.smithi017.stdout:3578: read oid 31 snap -1 2024-09-21T09:53:06.294 INFO:tasks.rados.rados.0.smithi017.stdout:3578: expect deleted 2024-09-21T09:53:06.294 INFO:tasks.rados.rados.0.smithi017.stdout:3579: write oid 44 current snap is 448 2024-09-21T09:53:06.295 INFO:tasks.rados.rados.0.smithi017.stdout:3579: seq_num 1345 ranges {605415=488033,1656281=493458} 2024-09-21T09:53:06.301 INFO:tasks.rados.rados.0.smithi017.stdout:3579: writing smithi01785587-44 from 605415 to 1093448 tid 1 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3579: writing smithi01785587-44 from 1656281 to 2149739 tid 2 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3580: read oid 24 snap -1 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3580: expect (ObjNum 1288 snap 433 seq_num 1288) 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3581: read oid 17 snap -1 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3581: expect (ObjNum 1340 snap 448 seq_num 1340) 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3582: read oid 32 snap -1 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3582: expect (ObjNum 1317 snap 441 seq_num 1317) 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3583: read oid 9 snap -1 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3583: expect deleted 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3584: rollback oid 47 current snap is 448 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 47 to 440 2024-09-21T09:53:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3576: expect (ObjNum 1231 snap 409 seq_num 1231) 2024-09-21T09:53:06.477 INFO:tasks.rados.rados.0.smithi017.stdout:3579: finishing write tid 1 to smithi01785587-44 2024-09-21T09:53:06.478 INFO:tasks.rados.rados.0.smithi017.stdout:3579: finishing write tid 2 to smithi01785587-44 2024-09-21T09:53:06.478 INFO:tasks.rados.rados.0.smithi017.stdout:3581: expect (ObjNum 1340 snap 448 seq_num 1340) 2024-09-21T09:53:06.613 INFO:tasks.rados.rados.0.smithi017.stdout:3579: finishing write tid 3 to smithi01785587-44 2024-09-21T09:53:06.613 INFO:tasks.rados.rados.0.smithi017.stdout:3579: finishing write tid 4 to smithi01785587-44 2024-09-21T09:53:06.613 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 2015 (ObjNum 1345 snap 448 seq_num 1345) dirty exists 2024-09-21T09:53:06.613 INFO:tasks.rados.rados.0.smithi017.stdout:3579: left oid 44 (ObjNum 1345 snap 448 seq_num 1345) 2024-09-21T09:53:06.613 INFO:tasks.rados.rados.0.smithi017.stdout:3584: finishing rollback tid 1 to smithi01785587-47 2024-09-21T09:53:06.613 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 1183 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:53:06.614 INFO:tasks.rados.rados.0.smithi017.stdout:3582: expect (ObjNum 1317 snap 441 seq_num 1317) 2024-09-21T09:53:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:06 smithi017 ceph-mon[102913]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T09:53:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:06 smithi017 ceph-mon[105151]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T09:53:06.719 INFO:tasks.rados.rados.0.smithi017.stdout:3580: expect (ObjNum 1288 snap 433 seq_num 1288) 2024-09-21T09:53:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:06 smithi134 ceph-mon[90085]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T09:53:06.885 INFO:tasks.rados.rados.0.smithi017.stdout:3576: done (7 left) 2024-09-21T09:53:06.885 INFO:tasks.rados.rados.0.smithi017.stdout:3578: done (6 left) 2024-09-21T09:53:06.885 INFO:tasks.rados.rados.0.smithi017.stdout:3579: done (5 left) 2024-09-21T09:53:06.885 INFO:tasks.rados.rados.0.smithi017.stdout:3580: done (4 left) 2024-09-21T09:53:06.885 INFO:tasks.rados.rados.0.smithi017.stdout:3581: done (3 left) 2024-09-21T09:53:06.885 INFO:tasks.rados.rados.0.smithi017.stdout:3582: done (2 left) 2024-09-21T09:53:06.886 INFO:tasks.rados.rados.0.smithi017.stdout:3583: done (1 left) 2024-09-21T09:53:06.886 INFO:tasks.rados.rados.0.smithi017.stdout:3584: done (0 left) 2024-09-21T09:53:06.886 INFO:tasks.rados.rados.0.smithi017.stdout:3585: read oid 22 snap -1 2024-09-21T09:53:06.886 INFO:tasks.rados.rados.0.smithi017.stdout:3585: expect (ObjNum 1323 snap 445 seq_num 1323) 2024-09-21T09:53:06.886 INFO:tasks.rados.rados.0.smithi017.stdout:3586: snap_remove snap 445 2024-09-21T09:53:07.284 INFO:tasks.rados.rados.0.smithi017.stdout:3586: done (1 left) 2024-09-21T09:53:07.285 INFO:tasks.rados.rados.0.smithi017.stdout:3587: write oid 40 current snap is 448 2024-09-21T09:53:07.285 INFO:tasks.rados.rados.0.smithi017.stdout:3587: seq_num 1346 ranges {622222=679635,1831116=753469} 2024-09-21T09:53:07.292 INFO:tasks.rados.rados.0.smithi017.stdout:3587: writing smithi01785587-40 from 622222 to 1301857 tid 1 2024-09-21T09:53:07.300 INFO:tasks.rados.rados.0.smithi017.stdout:3587: writing smithi01785587-40 from 1831116 to 2584585 tid 2 2024-09-21T09:53:07.300 INFO:tasks.rados.rados.0.smithi017.stdout:3588: rollback oid 3 current snap is 448 2024-09-21T09:53:07.300 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 437 2024-09-21T09:53:07.301 INFO:tasks.rados.rados.0.smithi017.stdout:3585: expect (ObjNum 1323 snap 445 seq_num 1323) 2024-09-21T09:53:07.407 INFO:tasks.rados.rados.0.smithi017.stdout:3587: finishing write tid 1 to smithi01785587-40 2024-09-21T09:53:07.408 INFO:tasks.rados.rados.0.smithi017.stdout:3585: done (2 left) 2024-09-21T09:53:07.408 INFO:tasks.rados.rados.0.smithi017.stdout:3589: snap_remove snap 440 2024-09-21T09:53:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:07 smithi017 ceph-mon[102913]: pgmap v1020: 89 pgs: 89 active+clean; 202 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 755 KiB/s wr, 7 op/s 2024-09-21T09:53:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:07 smithi017 ceph-mon[102913]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T09:53:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:07 smithi017 ceph-mon[105151]: pgmap v1020: 89 pgs: 89 active+clean; 202 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 755 KiB/s wr, 7 op/s 2024-09-21T09:53:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:07 smithi017 ceph-mon[105151]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T09:53:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:07 smithi134 ceph-mon[90085]: pgmap v1020: 89 pgs: 89 active+clean; 202 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 755 KiB/s wr, 7 op/s 2024-09-21T09:53:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:07 smithi134 ceph-mon[90085]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T09:53:08.218 INFO:tasks.rados.rados.0.smithi017.stdout:3589: done (2 left) 2024-09-21T09:53:08.218 INFO:tasks.rados.rados.0.smithi017.stdout:3590: write oid 27 current snap is 448 2024-09-21T09:53:08.218 INFO:tasks.rados.rados.0.smithi017.stdout:3590: seq_num 1347 ranges {639029=754884,2173245=465784} 2024-09-21T09:53:08.226 INFO:tasks.rados.rados.0.smithi017.stdout:3590: writing smithi01785587-27 from 639029 to 1393913 tid 1 2024-09-21T09:53:08.234 INFO:tasks.rados.rados.0.smithi017.stdout:3590: writing smithi01785587-27 from 2173245 to 2639029 tid 2 2024-09-21T09:53:08.234 INFO:tasks.rados.rados.0.smithi017.stdout:3591: snap_remove snap 421 2024-09-21T09:53:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:08 smithi017 ceph-mon[102913]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T09:53:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:08 smithi017 ceph-mon[102913]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T09:53:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:08 smithi017 ceph-mon[105151]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T09:53:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:08 smithi017 ceph-mon[105151]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T09:53:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:08 smithi134 ceph-mon[90085]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T09:53:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:08 smithi134 ceph-mon[90085]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T09:53:09.221 INFO:tasks.rados.rados.0.smithi017.stdout:3588: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3588: finishing rollback tid 2 to smithi01785587-3 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 2023 (ObjNum 1281 snap 432 seq_num 1281) dirty exists 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3587: finishing write tid 2 to smithi01785587-40 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3587: finishing write tid 3 to smithi01785587-40 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3587: finishing write tid 4 to smithi01785587-40 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1868 (ObjNum 1346 snap 448 seq_num 1346) dirty exists 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3587: left oid 40 (ObjNum 1346 snap 448 seq_num 1346) 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3590: finishing write tid 1 to smithi01785587-27 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3590: finishing write tid 2 to smithi01785587-27 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3590: finishing write tid 3 to smithi01785587-27 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3590: finishing write tid 4 to smithi01785587-27 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 1238 (ObjNum 1347 snap 448 seq_num 1347) dirty exists 2024-09-21T09:53:09.222 INFO:tasks.rados.rados.0.smithi017.stdout:3590: left oid 27 (ObjNum 1347 snap 448 seq_num 1347) 2024-09-21T09:53:09.223 INFO:tasks.rados.rados.0.smithi017.stdout:3587: done (3 left) 2024-09-21T09:53:09.223 INFO:tasks.rados.rados.0.smithi017.stdout:3588: done (2 left) 2024-09-21T09:53:09.223 INFO:tasks.rados.rados.0.smithi017.stdout:3590: done (1 left) 2024-09-21T09:53:09.223 INFO:tasks.rados.rados.0.smithi017.stdout:3591: done (0 left) 2024-09-21T09:53:09.223 INFO:tasks.rados.rados.0.smithi017.stdout:3592: write oid 47 current snap is 448 2024-09-21T09:53:09.223 INFO:tasks.rados.rados.0.smithi017.stdout:3592: seq_num 1348 ranges {655836=430133,1831727=757138} 2024-09-21T09:53:09.227 INFO:tasks.rados.rados.0.smithi017.stdout:3592: writing smithi01785587-47 from 655836 to 1085969 tid 1 2024-09-21T09:53:09.237 INFO:tasks.rados.rados.0.smithi017.stdout:3592: writing smithi01785587-47 from 1831727 to 2588865 tid 2 2024-09-21T09:53:09.237 INFO:tasks.rados.rados.0.smithi017.stdout:3593: read oid 10 snap -1 2024-09-21T09:53:09.237 INFO:tasks.rados.rados.0.smithi017.stdout:3593: expect (ObjNum 1335 snap 446 seq_num 1335) 2024-09-21T09:53:09.237 INFO:tasks.rados.rados.0.smithi017.stdout:3594: write oid 38 current snap is 448 2024-09-21T09:53:09.238 INFO:tasks.rados.rados.0.smithi017.stdout:3592: finishing write tid 1 to smithi01785587-47 2024-09-21T09:53:09.238 INFO:tasks.rados.rados.0.smithi017.stdout:3594: seq_num 1349 ranges {672643=505382,1890209=500796} 2024-09-21T09:53:09.244 INFO:tasks.rados.rados.0.smithi017.stdout:3594: writing smithi01785587-38 from 672643 to 1178025 tid 1 2024-09-21T09:53:09.251 INFO:tasks.rados.rados.0.smithi017.stdout:3594: writing smithi01785587-38 from 1890209 to 2391005 tid 2 2024-09-21T09:53:09.251 INFO:tasks.rados.rados.0.smithi017.stdout:3595: write oid 27 current snap is 448 2024-09-21T09:53:09.251 INFO:tasks.rados.rados.0.smithi017.stdout:3595: seq_num 1350 ranges {689450=580631,1948691=740759} 2024-09-21T09:53:09.258 INFO:tasks.rados.rados.0.smithi017.stdout:3595: writing smithi01785587-27 from 689450 to 1270081 tid 1 2024-09-21T09:53:09.266 INFO:tasks.rados.rados.0.smithi017.stdout:3595: writing smithi01785587-27 from 1948691 to 2689450 tid 2 2024-09-21T09:53:09.266 INFO:tasks.rados.rados.0.smithi017.stdout:3596: delete oid 8 current snap is 448 2024-09-21T09:53:09.266 INFO:tasks.rados.rados.0.smithi017.stdout:3593: expect (ObjNum 1335 snap 446 seq_num 1335) 2024-09-21T09:53:09.462 INFO:tasks.rados.rados.0.smithi017.stdout:3592: finishing write tid 2 to smithi01785587-47 2024-09-21T09:53:09.462 INFO:tasks.rados.rados.0.smithi017.stdout:3592: finishing write tid 3 to smithi01785587-47 2024-09-21T09:53:09.462 INFO:tasks.rados.rados.0.smithi017.stdout:3592: finishing write tid 4 to smithi01785587-47 2024-09-21T09:53:09.462 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 1253 (ObjNum 1348 snap 448 seq_num 1348) dirty exists 2024-09-21T09:53:09.462 INFO:tasks.rados.rados.0.smithi017.stdout:3592: left oid 47 (ObjNum 1348 snap 448 seq_num 1348) 2024-09-21T09:53:09.462 INFO:tasks.rados.rados.0.smithi017.stdout:3594: finishing write tid 1 to smithi01785587-38 2024-09-21T09:53:09.462 INFO:tasks.rados.rados.0.smithi017.stdout:3592: done (4 left) 2024-09-21T09:53:09.462 INFO:tasks.rados.rados.0.smithi017.stdout:3593: done (3 left) 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3596: done (2 left) 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3597: snap_create 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3594: finishing write tid 2 to smithi01785587-38 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3594: finishing write tid 3 to smithi01785587-38 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3594: finishing write tid 4 to smithi01785587-38 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 38 v 1109 (ObjNum 1349 snap 448 seq_num 1349) dirty exists 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3594: left oid 38 (ObjNum 1349 snap 448 seq_num 1349) 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3595: finishing write tid 1 to smithi01785587-27 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3595: finishing write tid 2 to smithi01785587-27 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3595: finishing write tid 3 to smithi01785587-27 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3595: finishing write tid 4 to smithi01785587-27 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 1247 (ObjNum 1350 snap 448 seq_num 1350) dirty exists 2024-09-21T09:53:09.463 INFO:tasks.rados.rados.0.smithi017.stdout:3595: left oid 27 (ObjNum 1350 snap 448 seq_num 1350) 2024-09-21T09:53:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:09 smithi017 ceph-mon[102913]: pgmap v1023: 89 pgs: 89 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:53:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:09 smithi017 ceph-mon[102913]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T09:53:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:09 smithi017 ceph-mon[105151]: pgmap v1023: 89 pgs: 89 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:53:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:09 smithi017 ceph-mon[105151]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T09:53:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:09 smithi134 ceph-mon[90085]: pgmap v1023: 89 pgs: 89 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:53:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:09 smithi134 ceph-mon[90085]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T09:53:10.302 INFO:tasks.rados.rados.0.smithi017.stdout:3594: done (2 left) 2024-09-21T09:53:10.302 INFO:tasks.rados.rados.0.smithi017.stdout:3595: done (1 left) 2024-09-21T09:53:10.302 INFO:tasks.rados.rados.0.smithi017.stdout:3597: done (0 left) 2024-09-21T09:53:10.302 INFO:tasks.rados.rados.0.smithi017.stdout:3598: read oid 9 snap -1 2024-09-21T09:53:10.302 INFO:tasks.rados.rados.0.smithi017.stdout:3598: expect deleted 2024-09-21T09:53:10.302 INFO:tasks.rados.rados.0.smithi017.stdout:3599: snap_create 2024-09-21T09:53:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:53:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:53:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:10.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:11.302 INFO:tasks.rados.rados.0.smithi017.stdout:3598: done (1 left) 2024-09-21T09:53:11.302 INFO:tasks.rados.rados.0.smithi017.stdout:3599: done (0 left) 2024-09-21T09:53:11.303 INFO:tasks.rados.rados.0.smithi017.stdout:3600: snap_create 2024-09-21T09:53:11.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:11 smithi017 ceph-mon[102913]: pgmap v1026: 89 pgs: 89 active+clean; 209 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-21T09:53:11.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:11 smithi017 ceph-mon[102913]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T09:53:11.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:11 smithi017 ceph-mon[105151]: pgmap v1026: 89 pgs: 89 active+clean; 209 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-21T09:53:11.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:11 smithi017 ceph-mon[105151]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T09:53:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:11 smithi134 ceph-mon[90085]: pgmap v1026: 89 pgs: 89 active+clean; 209 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-09-21T09:53:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:11 smithi134 ceph-mon[90085]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T09:53:12.313 INFO:tasks.rados.rados.0.smithi017.stdout:3600: done (0 left) 2024-09-21T09:53:12.314 INFO:tasks.rados.rados.0.smithi017.stdout:3601: snap_remove snap 420 2024-09-21T09:53:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:12 smithi017 ceph-mon[102913]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T09:53:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:12 smithi017 ceph-mon[105151]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T09:53:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:12 smithi134 ceph-mon[90085]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T09:53:13.220 INFO:tasks.rados.rados.0.smithi017.stdout:3601: done (0 left) 2024-09-21T09:53:13.397 INFO:tasks.rados.rados.0.smithi017.stdout:3602: write oid 1 current snap is 451 2024-09-21T09:53:13.397 INFO:tasks.rados.rados.0.smithi017.stdout:3602: seq_num 1352 ranges {723064=731129,1949302=764476} 2024-09-21T09:53:13.399 INFO:tasks.rados.rados.0.smithi017.stdout:3602: writing smithi01785587-1 from 723064 to 1454193 tid 1 2024-09-21T09:53:13.399 INFO:tasks.rados.rados.0.smithi017.stdout:3602: writing smithi01785587-1 from 1949302 to 2713778 tid 2 2024-09-21T09:53:13.399 INFO:tasks.rados.rados.0.smithi017.stdout:3603: read oid 2 snap -1 2024-09-21T09:53:13.399 INFO:tasks.rados.rados.0.smithi017.stdout:3603: expect deleted 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3604: write oid 13 current snap is 451 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3602: finishing write tid 1 to smithi01785587-1 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3604: seq_num 1353 ranges {739871=406378,1607784=508134} 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3604: writing smithi01785587-13 from 739871 to 1146249 tid 1 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3604: writing smithi01785587-13 from 1607784 to 2115918 tid 2 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3605: delete oid 34 current snap is 451 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3602: finishing write tid 2 to smithi01785587-1 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3602: finishing write tid 3 to smithi01785587-1 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3602: finishing write tid 4 to smithi01785587-1 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 1060 (ObjNum 1352 snap 451 seq_num 1352) dirty exists 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3602: left oid 1 (ObjNum 1352 snap 451 seq_num 1352) 2024-09-21T09:53:13.400 INFO:tasks.rados.rados.0.smithi017.stdout:3604: finishing write tid 1 to smithi01785587-13 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:3604: finishing write tid 2 to smithi01785587-13 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:3604: finishing write tid 3 to smithi01785587-13 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:3604: finishing write tid 4 to smithi01785587-13 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1524 (ObjNum 1353 snap 451 seq_num 1353) dirty exists 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:3604: left oid 13 (ObjNum 1353 snap 451 seq_num 1353) 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:3602: done (3 left) 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:3603: done (2 left) 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:3604: done (1 left) 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:3605: done (0 left) 2024-09-21T09:53:13.401 INFO:tasks.rados.rados.0.smithi017.stdout:3606: snap_remove snap 446 2024-09-21T09:53:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:13 smithi017 ceph-mon[105151]: pgmap v1029: 89 pgs: 89 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-09-21T09:53:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:13 smithi017 ceph-mon[105151]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T09:53:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:13 smithi017 ceph-mon[105151]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T09:53:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:13 smithi017 ceph-mon[102913]: pgmap v1029: 89 pgs: 89 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-09-21T09:53:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:13 smithi017 ceph-mon[102913]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T09:53:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:13 smithi017 ceph-mon[102913]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T09:53:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:13 smithi134 ceph-mon[90085]: pgmap v1029: 89 pgs: 89 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-09-21T09:53:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:13 smithi134 ceph-mon[90085]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T09:53:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:13 smithi134 ceph-mon[90085]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T09:53:14.220 INFO:tasks.rados.rados.0.smithi017.stdout:3606: done (0 left) 2024-09-21T09:53:14.220 INFO:tasks.rados.rados.0.smithi017.stdout:3607: rollback oid 50 current snap is 451 2024-09-21T09:53:14.221 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 50 to 449 2024-09-21T09:53:14.221 INFO:tasks.rados.rados.0.smithi017.stdout:3608: write oid 16 current snap is 451 2024-09-21T09:53:14.221 INFO:tasks.rados.rados.0.smithi017.stdout:3608: seq_num 1355 ranges {773485=673229,2124748=511803} 2024-09-21T09:53:14.229 INFO:tasks.rados.rados.0.smithi017.stdout:3608: writing smithi01785587-16 from 773485 to 1446714 tid 1 2024-09-21T09:53:14.236 INFO:tasks.rados.rados.0.smithi017.stdout:3608: writing smithi01785587-16 from 2124748 to 2636551 tid 2 2024-09-21T09:53:14.237 INFO:tasks.rados.rados.0.smithi017.stdout:3609: delete oid 22 current snap is 451 2024-09-21T09:53:14.237 INFO:tasks.rados.rados.0.smithi017.stdout:3607: finishing rollback tid 0 to smithi01785587-50 2024-09-21T09:53:14.237 INFO:tasks.rados.rados.0.smithi017.stdout:3607: finishing rollback tid 1 to smithi01785587-50 2024-09-21T09:53:14.237 INFO:tasks.rados.rados.0.smithi017.stdout:3607: finishing rollback tid 2 to smithi01785587-50 2024-09-21T09:53:14.237 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 1117 (ObjNum 1093 snap 363 seq_num 1093) dirty exists 2024-09-21T09:53:14.237 INFO:tasks.rados.rados.0.smithi017.stdout:3608: finishing write tid 1 to smithi01785587-16 2024-09-21T09:53:14.242 INFO:tasks.rados.rados.0.smithi017.stdout:3608: finishing write tid 2 to smithi01785587-16 2024-09-21T09:53:14.242 INFO:tasks.rados.rados.0.smithi017.stdout:3608: finishing write tid 3 to smithi01785587-16 2024-09-21T09:53:14.242 INFO:tasks.rados.rados.0.smithi017.stdout:3608: finishing write tid 4 to smithi01785587-16 2024-09-21T09:53:14.243 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1910 (ObjNum 1355 snap 451 seq_num 1355) dirty exists 2024-09-21T09:53:14.243 INFO:tasks.rados.rados.0.smithi017.stdout:3608: left oid 16 (ObjNum 1355 snap 451 seq_num 1355) 2024-09-21T09:53:14.243 INFO:tasks.rados.rados.0.smithi017.stdout:3607: done (2 left) 2024-09-21T09:53:14.243 INFO:tasks.rados.rados.0.smithi017.stdout:3608: done (1 left) 2024-09-21T09:53:14.243 INFO:tasks.rados.rados.0.smithi017.stdout:3609: done (0 left) 2024-09-21T09:53:14.243 INFO:tasks.rados.rados.0.smithi017.stdout:3610: read oid 48 snap 448 2024-09-21T09:53:14.243 INFO:tasks.rados.rados.0.smithi017.stdout:3610: expect deleted 2024-09-21T09:53:14.243 INFO:tasks.rados.rados.0.smithi017.stdout:3611: snap_remove snap 437 2024-09-21T09:53:15.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:53:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:53:14] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:53:15.228 INFO:tasks.rados.rados.0.smithi017.stdout:3611: done (1 left) 2024-09-21T09:53:15.228 INFO:tasks.rados.rados.0.smithi017.stdout:3612: read oid 17 snap -1 2024-09-21T09:53:15.228 INFO:tasks.rados.rados.0.smithi017.stdout:3612: expect (ObjNum 1340 snap 448 seq_num 1340) 2024-09-21T09:53:15.229 INFO:tasks.rados.rados.0.smithi017.stdout:3613: read oid 2 snap -1 2024-09-21T09:53:15.229 INFO:tasks.rados.rados.0.smithi017.stdout:3613: expect deleted 2024-09-21T09:53:15.229 INFO:tasks.rados.rados.0.smithi017.stdout:3614: write oid 40 current snap is 451 2024-09-21T09:53:15.229 INFO:tasks.rados.rados.0.smithi017.stdout:3614: seq_num 1357 ranges {407099=423727,1441712=515472,2582060=225039} 2024-09-21T09:53:15.234 INFO:tasks.rados.rados.0.smithi017.stdout:3614: writing smithi01785587-40 from 407099 to 830826 tid 1 2024-09-21T09:53:15.241 INFO:tasks.rados.rados.0.smithi017.stdout:3614: writing smithi01785587-40 from 1441712 to 1957184 tid 2 2024-09-21T09:53:15.246 INFO:tasks.rados.rados.0.smithi017.stdout:3614: writing smithi01785587-40 from 2582060 to 2807099 tid 3 2024-09-21T09:53:15.246 INFO:tasks.rados.rados.0.smithi017.stdout:3615: write oid 44 current snap is 451 2024-09-21T09:53:15.246 INFO:tasks.rados.rados.0.smithi017.stdout:3615: seq_num 1358 ranges {423906=498976,1500194=659130,2609483=214423} 2024-09-21T09:53:15.251 INFO:tasks.rados.rados.0.smithi017.stdout:3615: writing smithi01785587-44 from 423906 to 922882 tid 1 2024-09-21T09:53:15.258 INFO:tasks.rados.rados.0.smithi017.stdout:3615: writing smithi01785587-44 from 1500194 to 2159324 tid 2 2024-09-21T09:53:15.262 INFO:tasks.rados.rados.0.smithi017.stdout:3615: writing smithi01785587-44 from 2609483 to 2823906 tid 3 2024-09-21T09:53:15.263 INFO:tasks.rados.rados.0.smithi017.stdout:3616: delete oid 3 current snap is 451 2024-09-21T09:53:15.263 INFO:tasks.rados.rados.0.smithi017.stdout:3612: expect (ObjNum 1340 snap 448 seq_num 1340) 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:3614: finishing write tid 1 to smithi01785587-40 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:3614: finishing write tid 2 to smithi01785587-40 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:3614: finishing write tid 3 to smithi01785587-40 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:3614: finishing write tid 4 to smithi01785587-40 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:3614: finishing write tid 5 to smithi01785587-40 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1895 (ObjNum 1357 snap 451 seq_num 1357) dirty exists 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:3614: left oid 40 (ObjNum 1357 snap 451 seq_num 1357) 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:3615: finishing write tid 1 to smithi01785587-44 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:3610: done (5 left) 2024-09-21T09:53:15.400 INFO:tasks.rados.rados.0.smithi017.stdout:3612: done (4 left) 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3613: done (3 left) 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3614: done (2 left) 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3616: done (1 left) 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3617: read oid 10 snap -1 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3617: expect (ObjNum 1335 snap 446 seq_num 1335) 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3618: rollback oid 42 current snap is 451 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 42 to 450 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3615: finishing write tid 2 to smithi01785587-44 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3615: finishing write tid 3 to smithi01785587-44 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3615: finishing write tid 4 to smithi01785587-44 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3615: finishing write tid 5 to smithi01785587-44 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 2060 (ObjNum 1358 snap 451 seq_num 1358) dirty exists 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3615: left oid 44 (ObjNum 1358 snap 451 seq_num 1358) 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3615: done (2 left) 2024-09-21T09:53:15.401 INFO:tasks.rados.rados.0.smithi017.stdout:3619: snap_create 2024-09-21T09:53:15.402 INFO:tasks.rados.rados.0.smithi017.stdout:3617: expect (ObjNum 1335 snap 446 seq_num 1335) 2024-09-21T09:53:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:15 smithi134 ceph-mon[90085]: pgmap v1032: 89 pgs: 89 active+clean; 205 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-09-21T09:53:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:15 smithi134 ceph-mon[90085]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T09:53:15.595 INFO:tasks.rados.rados.0.smithi017.stdout:3618: finishing rollback tid 0 to smithi01785587-42 2024-09-21T09:53:15.595 INFO:tasks.rados.rados.0.smithi017.stdout:3618: finishing rollback tid 1 to smithi01785587-42 2024-09-21T09:53:15.595 INFO:tasks.rados.rados.0.smithi017.stdout:3618: finishing rollback tid 2 to smithi01785587-42 2024-09-21T09:53:15.595 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 42 v 1263 (ObjNum 1327 snap 445 seq_num 1327) dirty exists 2024-09-21T09:53:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:15 smithi017 ceph-mon[102913]: pgmap v1032: 89 pgs: 89 active+clean; 205 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-09-21T09:53:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:15 smithi017 ceph-mon[102913]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T09:53:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:15 smithi017 ceph-mon[105151]: pgmap v1032: 89 pgs: 89 active+clean; 205 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-09-21T09:53:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:15 smithi017 ceph-mon[105151]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T09:53:16.231 INFO:tasks.rados.rados.0.smithi017.stdout:3617: done (2 left) 2024-09-21T09:53:16.231 INFO:tasks.rados.rados.0.smithi017.stdout:3618: done (1 left) 2024-09-21T09:53:16.231 INFO:tasks.rados.rados.0.smithi017.stdout:3619: done (0 left) 2024-09-21T09:53:16.231 INFO:tasks.rados.rados.0.smithi017.stdout:3620: read oid 2 snap -1 2024-09-21T09:53:16.231 INFO:tasks.rados.rados.0.smithi017.stdout:3620: expect deleted 2024-09-21T09:53:16.231 INFO:tasks.rados.rados.0.smithi017.stdout:3621: snap_remove snap 451 2024-09-21T09:53:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:16 smithi134 ceph-mon[90085]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T09:53:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:16 smithi017 ceph-mon[102913]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T09:53:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:16 smithi017 ceph-mon[105151]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T09:53:17.233 INFO:tasks.rados.rados.0.smithi017.stdout:3621: done (1 left) 2024-09-21T09:53:17.234 INFO:tasks.rados.rados.0.smithi017.stdout:3622: write oid 18 current snap is 452 2024-09-21T09:53:17.234 INFO:tasks.rados.rados.0.smithi017.stdout:3622: seq_num 1360 ranges {457520=649474,1900805=662799} 2024-09-21T09:53:17.240 INFO:tasks.rados.rados.0.smithi017.stdout:3622: writing smithi01785587-18 from 457520 to 1106994 tid 1 2024-09-21T09:53:17.248 INFO:tasks.rados.rados.0.smithi017.stdout:3622: writing smithi01785587-18 from 1900805 to 2563604 tid 2 2024-09-21T09:53:17.248 INFO:tasks.rados.rados.0.smithi017.stdout:3620: done (1 left) 2024-09-21T09:53:17.249 INFO:tasks.rados.rados.0.smithi017.stdout:3623: delete oid 10 current snap is 452 2024-09-21T09:53:17.249 INFO:tasks.rados.rados.0.smithi017.stdout:3622: finishing write tid 1 to smithi01785587-18 2024-09-21T09:53:17.251 INFO:tasks.rados.rados.0.smithi017.stdout:3623: done (1 left) 2024-09-21T09:53:17.251 INFO:tasks.rados.rados.0.smithi017.stdout:3624: read oid 1 snap -1 2024-09-21T09:53:17.251 INFO:tasks.rados.rados.0.smithi017.stdout:3624: expect (ObjNum 1352 snap 451 seq_num 1352) 2024-09-21T09:53:17.251 INFO:tasks.rados.rados.0.smithi017.stdout:3625: write oid 23 current snap is 452 2024-09-21T09:53:17.251 INFO:tasks.rados.rados.0.smithi017.stdout:3625: seq_num 1362 ranges {491134=516325,1734122=666468} 2024-09-21T09:53:17.256 INFO:tasks.rados.rados.0.smithi017.stdout:3625: writing smithi01785587-23 from 491134 to 1007459 tid 1 2024-09-21T09:53:17.265 INFO:tasks.rados.rados.0.smithi017.stdout:3625: writing smithi01785587-23 from 1734122 to 2400590 tid 2 2024-09-21T09:53:17.265 INFO:tasks.rados.rados.0.smithi017.stdout:3626: delete oid 27 current snap is 452 2024-09-21T09:53:17.266 INFO:tasks.rados.rados.0.smithi017.stdout:3622: finishing write tid 2 to smithi01785587-18 2024-09-21T09:53:17.266 INFO:tasks.rados.rados.0.smithi017.stdout:3622: finishing write tid 3 to smithi01785587-18 2024-09-21T09:53:17.266 INFO:tasks.rados.rados.0.smithi017.stdout:3622: finishing write tid 4 to smithi01785587-18 2024-09-21T09:53:17.266 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 391 (ObjNum 1360 snap 452 seq_num 1360) dirty exists 2024-09-21T09:53:17.266 INFO:tasks.rados.rados.0.smithi017.stdout:3622: left oid 18 (ObjNum 1360 snap 452 seq_num 1360) 2024-09-21T09:53:17.266 INFO:tasks.rados.rados.0.smithi017.stdout:3625: finishing write tid 1 to smithi01785587-23 2024-09-21T09:53:17.266 INFO:tasks.rados.rados.0.smithi017.stdout:3624: expect (ObjNum 1352 snap 451 seq_num 1352) 2024-09-21T09:53:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3625: finishing write tid 2 to smithi01785587-23 2024-09-21T09:53:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3625: finishing write tid 3 to smithi01785587-23 2024-09-21T09:53:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3625: finishing write tid 4 to smithi01785587-23 2024-09-21T09:53:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1903 (ObjNum 1362 snap 452 seq_num 1362) dirty exists 2024-09-21T09:53:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3625: left oid 23 (ObjNum 1362 snap 452 seq_num 1362) 2024-09-21T09:53:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3622: done (3 left) 2024-09-21T09:53:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3624: done (2 left) 2024-09-21T09:53:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3625: done (1 left) 2024-09-21T09:53:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3626: done (0 left) 2024-09-21T09:53:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3627: delete oid 5 current snap is 452 2024-09-21T09:53:17.400 INFO:tasks.rados.rados.0.smithi017.stdout:3627: done (0 left) 2024-09-21T09:53:17.400 INFO:tasks.rados.rados.0.smithi017.stdout:3628: write oid 32 current snap is 452 2024-09-21T09:53:17.400 INFO:tasks.rados.rados.0.smithi017.stdout:3628: seq_num 1365 ranges {541555=742072,1793215=530148,2801444=140111} 2024-09-21T09:53:17.407 INFO:tasks.rados.rados.0.smithi017.stdout:3628: writing smithi01785587-32 from 541555 to 1283627 tid 1 2024-09-21T09:53:17.413 INFO:tasks.rados.rados.0.smithi017.stdout:3628: writing smithi01785587-32 from 1793215 to 2323363 tid 2 2024-09-21T09:53:17.417 INFO:tasks.rados.rados.0.smithi017.stdout:3628: writing smithi01785587-32 from 2801444 to 2941555 tid 3 2024-09-21T09:53:17.417 INFO:tasks.rados.rados.0.smithi017.stdout:3629: write oid 41 current snap is 452 2024-09-21T09:53:17.417 INFO:tasks.rados.rados.0.smithi017.stdout:3628: finishing write tid 1 to smithi01785587-32 2024-09-21T09:53:17.417 INFO:tasks.rados.rados.0.smithi017.stdout:3628: finishing write tid 2 to smithi01785587-32 2024-09-21T09:53:17.418 INFO:tasks.rados.rados.0.smithi017.stdout:3629: seq_num 1366 ranges {558362=417321,1451697=673806,2712514=245848} 2024-09-21T09:53:17.422 INFO:tasks.rados.rados.0.smithi017.stdout:3629: writing smithi01785587-41 from 558362 to 975683 tid 1 2024-09-21T09:53:17.428 INFO:tasks.rados.rados.0.smithi017.stdout:3629: writing smithi01785587-41 from 1451697 to 2125503 tid 2 2024-09-21T09:53:17.433 INFO:tasks.rados.rados.0.smithi017.stdout:3629: writing smithi01785587-41 from 2712514 to 2958362 tid 3 2024-09-21T09:53:17.433 INFO:tasks.rados.rados.0.smithi017.stdout:3630: snap_remove snap 448 2024-09-21T09:53:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:17 smithi134 ceph-mon[90085]: pgmap v1035: 89 pgs: 89 active+clean; 187 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-09-21T09:53:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:17 smithi134 ceph-mon[90085]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T09:53:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:17 smithi017 ceph-mon[102913]: pgmap v1035: 89 pgs: 89 active+clean; 187 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-09-21T09:53:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:17 smithi017 ceph-mon[102913]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T09:53:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:17 smithi017 ceph-mon[105151]: pgmap v1035: 89 pgs: 89 active+clean; 187 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-09-21T09:53:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:17 smithi017 ceph-mon[105151]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T09:53:18.222 INFO:tasks.rados.rados.0.smithi017.stdout:3630: done (2 left) 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3631: snap_create 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3628: finishing write tid 3 to smithi01785587-32 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3628: finishing write tid 4 to smithi01785587-32 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3628: finishing write tid 5 to smithi01785587-32 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1908 (ObjNum 1365 snap 452 seq_num 1365) dirty exists 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3628: left oid 32 (ObjNum 1365 snap 452 seq_num 1365) 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3629: finishing write tid 1 to smithi01785587-41 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3629: finishing write tid 2 to smithi01785587-41 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3629: finishing write tid 3 to smithi01785587-41 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3629: finishing write tid 4 to smithi01785587-41 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3629: finishing write tid 5 to smithi01785587-41 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 555 (ObjNum 1366 snap 452 seq_num 1366) dirty exists 2024-09-21T09:53:18.223 INFO:tasks.rados.rados.0.smithi017.stdout:3629: left oid 41 (ObjNum 1366 snap 452 seq_num 1366) 2024-09-21T09:53:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:18 smithi134 ceph-mon[90085]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T09:53:18.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:18 smithi134 ceph-mon[90085]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T09:53:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:18 smithi017 ceph-mon[102913]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T09:53:18.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:18 smithi017 ceph-mon[102913]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T09:53:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:18 smithi017 ceph-mon[105151]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T09:53:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:18 smithi017 ceph-mon[105151]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T09:53:19.222 INFO:tasks.rados.rados.0.smithi017.stdout:3628: done (2 left) 2024-09-21T09:53:19.222 INFO:tasks.rados.rados.0.smithi017.stdout:3629: done (1 left) 2024-09-21T09:53:19.222 INFO:tasks.rados.rados.0.smithi017.stdout:3631: done (0 left) 2024-09-21T09:53:19.222 INFO:tasks.rados.rados.0.smithi017.stdout:3632: snap_create 2024-09-21T09:53:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:19 smithi134 ceph-mon[90085]: pgmap v1038: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 878 KiB/s wr, 7 op/s 2024-09-21T09:53:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:19 smithi134 ceph-mon[90085]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T09:53:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:19 smithi017 ceph-mon[102913]: pgmap v1038: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 878 KiB/s wr, 7 op/s 2024-09-21T09:53:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:19 smithi017 ceph-mon[102913]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T09:53:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:19 smithi017 ceph-mon[105151]: pgmap v1038: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 878 KiB/s wr, 7 op/s 2024-09-21T09:53:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:19 smithi017 ceph-mon[105151]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T09:53:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3632: done (0 left) 2024-09-21T09:53:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3633: write oid 9 current snap is 454 2024-09-21T09:53:20.224 INFO:tasks.rados.rados.0.smithi017.stdout:3633: seq_num 1367 ranges {575169=492570,1793826=417464,2623584=351585} 2024-09-21T09:53:20.229 INFO:tasks.rados.rados.0.smithi017.stdout:3633: writing smithi01785587-9 from 575169 to 1067739 tid 1 2024-09-21T09:53:20.236 INFO:tasks.rados.rados.0.smithi017.stdout:3633: writing smithi01785587-9 from 1793826 to 2211290 tid 2 2024-09-21T09:53:20.240 INFO:tasks.rados.rados.0.smithi017.stdout:3633: writing smithi01785587-9 from 2623584 to 2975169 tid 3 2024-09-21T09:53:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:3634: write oid 7 current snap is 454 2024-09-21T09:53:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:3633: finishing write tid 1 to smithi01785587-9 2024-09-21T09:53:20.241 INFO:tasks.rados.rados.0.smithi017.stdout:3634: seq_num 1368 ranges {591976=567819,1852308=677475} 2024-09-21T09:53:20.247 INFO:tasks.rados.rados.0.smithi017.stdout:3634: writing smithi01785587-7 from 591976 to 1159795 tid 1 2024-09-21T09:53:20.255 INFO:tasks.rados.rados.0.smithi017.stdout:3634: writing smithi01785587-7 from 1852308 to 2529783 tid 2 2024-09-21T09:53:20.255 INFO:tasks.rados.rados.0.smithi017.stdout:3635: delete oid 18 current snap is 454 2024-09-21T09:53:20.255 INFO:tasks.rados.rados.0.smithi017.stdout:3633: finishing write tid 2 to smithi01785587-9 2024-09-21T09:53:20.255 INFO:tasks.rados.rados.0.smithi017.stdout:3633: finishing write tid 3 to smithi01785587-9 2024-09-21T09:53:20.255 INFO:tasks.rados.rados.0.smithi017.stdout:3633: finishing write tid 4 to smithi01785587-9 2024-09-21T09:53:20.255 INFO:tasks.rados.rados.0.smithi017.stdout:3633: finishing write tid 5 to smithi01785587-9 2024-09-21T09:53:20.255 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 718 (ObjNum 1367 snap 454 seq_num 1367) dirty exists 2024-09-21T09:53:20.255 INFO:tasks.rados.rados.0.smithi017.stdout:3633: left oid 9 (ObjNum 1367 snap 454 seq_num 1367) 2024-09-21T09:53:20.255 INFO:tasks.rados.rados.0.smithi017.stdout:3634: finishing write tid 1 to smithi01785587-7 2024-09-21T09:53:20.259 INFO:tasks.rados.rados.0.smithi017.stdout:3633: done (2 left) 2024-09-21T09:53:20.260 INFO:tasks.rados.rados.0.smithi017.stdout:3635: done (1 left) 2024-09-21T09:53:20.260 INFO:tasks.rados.rados.0.smithi017.stdout:3636: delete oid 31 current snap is 454 2024-09-21T09:53:20.260 INFO:tasks.rados.rados.0.smithi017.stdout:3634: finishing write tid 2 to smithi01785587-7 2024-09-21T09:53:20.260 INFO:tasks.rados.rados.0.smithi017.stdout:3634: finishing write tid 3 to smithi01785587-7 2024-09-21T09:53:20.260 INFO:tasks.rados.rados.0.smithi017.stdout:3634: finishing write tid 4 to smithi01785587-7 2024-09-21T09:53:20.260 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1772 (ObjNum 1368 snap 454 seq_num 1368) dirty exists 2024-09-21T09:53:20.260 INFO:tasks.rados.rados.0.smithi017.stdout:3634: left oid 7 (ObjNum 1368 snap 454 seq_num 1368) 2024-09-21T09:53:20.264 INFO:tasks.rados.rados.0.smithi017.stdout:3634: done (1 left) 2024-09-21T09:53:20.264 INFO:tasks.rados.rados.0.smithi017.stdout:3636: done (0 left) 2024-09-21T09:53:20.264 INFO:tasks.rados.rados.0.smithi017.stdout:3637: snap_create 2024-09-21T09:53:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:53:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:53:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:21.249 INFO:tasks.rados.rados.0.smithi017.stdout:3637: done (0 left) 2024-09-21T09:53:21.249 INFO:tasks.rados.rados.0.smithi017.stdout:3638: snap_remove snap 454 2024-09-21T09:53:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:21 smithi134 ceph-mon[90085]: pgmap v1041: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 284 KiB/s wr, 7 op/s 2024-09-21T09:53:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:21 smithi134 ceph-mon[90085]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T09:53:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:21 smithi017 ceph-mon[102913]: pgmap v1041: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 284 KiB/s wr, 7 op/s 2024-09-21T09:53:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:21 smithi017 ceph-mon[102913]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T09:53:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:21 smithi017 ceph-mon[105151]: pgmap v1041: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 284 KiB/s wr, 7 op/s 2024-09-21T09:53:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:21 smithi017 ceph-mon[105151]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T09:53:22.255 INFO:tasks.rados.rados.0.smithi017.stdout:3638: done (0 left) 2024-09-21T09:53:22.255 INFO:tasks.rados.rados.0.smithi017.stdout:3639: rollback oid 39 current snap is 455 2024-09-21T09:53:22.255 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 447 2024-09-21T09:53:22.255 INFO:tasks.rados.rados.0.smithi017.stdout:3640: delete oid 14 current snap is 455 2024-09-21T09:53:22.260 INFO:tasks.rados.rados.0.smithi017.stdout:3640: done (1 left) 2024-09-21T09:53:22.260 INFO:tasks.rados.rados.0.smithi017.stdout:3641: snap_create 2024-09-21T09:53:22.263 INFO:tasks.rados.rados.0.smithi017.stdout:3639: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:53:22.263 INFO:tasks.rados.rados.0.smithi017.stdout:3639: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:53:22.263 INFO:tasks.rados.rados.0.smithi017.stdout:3639: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:53:22.263 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1901 (ObjNum 1320 snap 444 seq_num 1320) dirty exists 2024-09-21T09:53:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:22 smithi017 ceph-mon[102913]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T09:53:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:22 smithi017 ceph-mon[105151]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T09:53:22.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:22 smithi134 ceph-mon[90085]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T09:53:23.222 INFO:tasks.rados.rados.0.smithi017.stdout:3639: done (1 left) 2024-09-21T09:53:23.222 INFO:tasks.rados.rados.0.smithi017.stdout:3641: done (0 left) 2024-09-21T09:53:23.222 INFO:tasks.rados.rados.0.smithi017.stdout:3642: read oid 43 snap 455 2024-09-21T09:53:23.222 INFO:tasks.rados.rados.0.smithi017.stdout:3642: expect deleted 2024-09-21T09:53:23.222 INFO:tasks.rados.rados.0.smithi017.stdout:3643: read oid 9 snap -1 2024-09-21T09:53:23.222 INFO:tasks.rados.rados.0.smithi017.stdout:3643: expect (ObjNum 1367 snap 454 seq_num 1367) 2024-09-21T09:53:23.223 INFO:tasks.rados.rados.0.smithi017.stdout:3644: read oid 10 snap 432 2024-09-21T09:53:23.223 INFO:tasks.rados.rados.0.smithi017.stdout:3644: expect (ObjNum 1256 snap 421 seq_num 1256) 2024-09-21T09:53:23.223 INFO:tasks.rados.rados.0.smithi017.stdout:3645: snap_create 2024-09-21T09:53:23.228 INFO:tasks.rados.rados.0.smithi017.stdout:3643: expect (ObjNum 1367 snap 454 seq_num 1367) 2024-09-21T09:53:23.330 INFO:tasks.rados.rados.0.smithi017.stdout:3644: expect (ObjNum 1256 snap 421 seq_num 1256) 2024-09-21T09:53:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:23 smithi017 ceph-mon[102913]: pgmap v1044: 89 pgs: 89 active+clean; 190 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2024-09-21T09:53:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:23 smithi017 ceph-mon[102913]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T09:53:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:23 smithi017 ceph-mon[102913]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T09:53:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:23 smithi017 ceph-mon[105151]: pgmap v1044: 89 pgs: 89 active+clean; 190 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2024-09-21T09:53:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:23 smithi017 ceph-mon[105151]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T09:53:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:23 smithi017 ceph-mon[105151]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T09:53:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:23 smithi134 ceph-mon[90085]: pgmap v1044: 89 pgs: 89 active+clean; 190 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2024-09-21T09:53:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:23 smithi134 ceph-mon[90085]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T09:53:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:23 smithi134 ceph-mon[90085]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T09:53:24.224 INFO:tasks.rados.rados.0.smithi017.stdout:3642: done (3 left) 2024-09-21T09:53:24.224 INFO:tasks.rados.rados.0.smithi017.stdout:3643: done (2 left) 2024-09-21T09:53:24.224 INFO:tasks.rados.rados.0.smithi017.stdout:3644: done (1 left) 2024-09-21T09:53:24.224 INFO:tasks.rados.rados.0.smithi017.stdout:3645: done (0 left) 2024-09-21T09:53:24.224 INFO:tasks.rados.rados.0.smithi017.stdout:3646: read oid 34 snap -1 2024-09-21T09:53:24.224 INFO:tasks.rados.rados.0.smithi017.stdout:3646: expect deleted 2024-09-21T09:53:24.224 INFO:tasks.rados.rados.0.smithi017.stdout:3647: delete oid 28 current snap is 457 2024-09-21T09:53:24.229 INFO:tasks.rados.rados.0.smithi017.stdout:3646: done (1 left) 2024-09-21T09:53:24.229 INFO:tasks.rados.rados.0.smithi017.stdout:3647: done (0 left) 2024-09-21T09:53:24.229 INFO:tasks.rados.rados.0.smithi017.stdout:3648: write oid 5 current snap is 457 2024-09-21T09:53:24.230 INFO:tasks.rados.rados.0.smithi017.stdout:3648: seq_num 1373 ranges {676011=660417,1744718=428471,2788122=287889} 2024-09-21T09:53:24.236 INFO:tasks.rados.rados.0.smithi017.stdout:3648: writing smithi01785587-5 from 676011 to 1336428 tid 1 2024-09-21T09:53:24.241 INFO:tasks.rados.rados.0.smithi017.stdout:3648: writing smithi01785587-5 from 1744718 to 2173189 tid 2 2024-09-21T09:53:24.247 INFO:tasks.rados.rados.0.smithi017.stdout:3648: writing smithi01785587-5 from 2788122 to 3076011 tid 3 2024-09-21T09:53:24.247 INFO:tasks.rados.rados.0.smithi017.stdout:3649: read oid 15 snap -1 2024-09-21T09:53:24.247 INFO:tasks.rados.rados.0.smithi017.stdout:3649: expect (ObjNum 1314 snap 441 seq_num 1314) 2024-09-21T09:53:24.247 INFO:tasks.rados.rados.0.smithi017.stdout:3650: snap_remove snap 455 2024-09-21T09:53:25.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:53:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:53:24] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:53:25.228 INFO:tasks.rados.rados.0.smithi017.stdout:3650: done (2 left) 2024-09-21T09:53:25.228 INFO:tasks.rados.rados.0.smithi017.stdout:3651: snap_create 2024-09-21T09:53:25.228 INFO:tasks.rados.rados.0.smithi017.stdout:3648: finishing write tid 1 to smithi01785587-5 2024-09-21T09:53:25.228 INFO:tasks.rados.rados.0.smithi017.stdout:3648: finishing write tid 2 to smithi01785587-5 2024-09-21T09:53:25.228 INFO:tasks.rados.rados.0.smithi017.stdout:3648: finishing write tid 3 to smithi01785587-5 2024-09-21T09:53:25.228 INFO:tasks.rados.rados.0.smithi017.stdout:3648: finishing write tid 4 to smithi01785587-5 2024-09-21T09:53:25.229 INFO:tasks.rados.rados.0.smithi017.stdout:3648: finishing write tid 5 to smithi01785587-5 2024-09-21T09:53:25.229 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 2074 (ObjNum 1373 snap 457 seq_num 1373) dirty exists 2024-09-21T09:53:25.229 INFO:tasks.rados.rados.0.smithi017.stdout:3648: left oid 5 (ObjNum 1373 snap 457 seq_num 1373) 2024-09-21T09:53:25.229 INFO:tasks.rados.rados.0.smithi017.stdout:3649: expect (ObjNum 1314 snap 441 seq_num 1314) 2024-09-21T09:53:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:25 smithi134 ceph-mon[90085]: pgmap v1047: 89 pgs: 89 active+clean; 193 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:53:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:25 smithi134 ceph-mon[90085]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T09:53:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:25 smithi017 ceph-mon[102913]: pgmap v1047: 89 pgs: 89 active+clean; 193 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:53:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:25 smithi017 ceph-mon[102913]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T09:53:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:25 smithi017 ceph-mon[105151]: pgmap v1047: 89 pgs: 89 active+clean; 193 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:53:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:25 smithi017 ceph-mon[105151]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T09:53:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3648: done (2 left) 2024-09-21T09:53:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3649: done (1 left) 2024-09-21T09:53:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3651: done (0 left) 2024-09-21T09:53:26.235 INFO:tasks.rados.rados.0.smithi017.stdout:3652: delete oid 6 current snap is 458 2024-09-21T09:53:26.239 INFO:tasks.rados.rados.0.smithi017.stdout:3652: done (0 left) 2024-09-21T09:53:26.239 INFO:tasks.rados.rados.0.smithi017.stdout:3653: delete oid 32 current snap is 458 2024-09-21T09:53:26.242 INFO:tasks.rados.rados.0.smithi017.stdout:3653: done (0 left) 2024-09-21T09:53:26.242 INFO:tasks.rados.rados.0.smithi017.stdout:3654: read oid 33 snap -1 2024-09-21T09:53:26.242 INFO:tasks.rados.rados.0.smithi017.stdout:3654: expect (ObjNum 1329 snap 446 seq_num 1329) 2024-09-21T09:53:26.242 INFO:tasks.rados.rados.0.smithi017.stdout:3655: write oid 16 current snap is 458 2024-09-21T09:53:26.242 INFO:tasks.rados.rados.0.smithi017.stdout:3655: seq_num 1376 ranges {726432=486164,1803811=692151} 2024-09-21T09:53:26.248 INFO:tasks.rados.rados.0.smithi017.stdout:3655: writing smithi01785587-16 from 726432 to 1212596 tid 1 2024-09-21T09:53:26.256 INFO:tasks.rados.rados.0.smithi017.stdout:3655: writing smithi01785587-16 from 1803811 to 2495962 tid 2 2024-09-21T09:53:26.256 INFO:tasks.rados.rados.0.smithi017.stdout:3656: snap_remove snap 449 2024-09-21T09:53:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:26 smithi134 ceph-mon[90085]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T09:53:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:26 smithi017 ceph-mon[102913]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T09:53:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:26 smithi017 ceph-mon[105151]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T09:53:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:27.238 INFO:tasks.rados.rados.0.smithi017.stdout:3656: done (2 left) 2024-09-21T09:53:27.238 INFO:tasks.rados.rados.0.smithi017.stdout:3657: snap_remove snap 439 2024-09-21T09:53:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:27 smithi134 ceph-mon[90085]: pgmap v1050: 89 pgs: 89 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 661 KiB/s rd, 645 KiB/s wr, 5 op/s 2024-09-21T09:53:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:27 smithi134 ceph-mon[90085]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T09:53:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:53:27.546 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:27 smithi017 ceph-mon[102913]: pgmap v1050: 89 pgs: 89 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 661 KiB/s rd, 645 KiB/s wr, 5 op/s 2024-09-21T09:53:27.546 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:27 smithi017 ceph-mon[102913]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T09:53:27.546 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:53:27.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:27 smithi017 ceph-mon[105151]: pgmap v1050: 89 pgs: 89 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 661 KiB/s rd, 645 KiB/s wr, 5 op/s 2024-09-21T09:53:27.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:27 smithi017 ceph-mon[105151]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T09:53:27.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:53:28.224 INFO:tasks.rados.rados.0.smithi017.stdout:3657: done (2 left) 2024-09-21T09:53:28.224 INFO:tasks.rados.rados.0.smithi017.stdout:3658: delete oid 48 current snap is 458 2024-09-21T09:53:28.224 INFO:tasks.rados.rados.0.smithi017.stdout:3654: expect (ObjNum 1329 snap 446 seq_num 1329) 2024-09-21T09:53:28.310 INFO:tasks.rados.rados.0.smithi017.stdout:3655: finishing write tid 1 to smithi01785587-16 2024-09-21T09:53:28.310 INFO:tasks.rados.rados.0.smithi017.stdout:3655: finishing write tid 2 to smithi01785587-16 2024-09-21T09:53:28.310 INFO:tasks.rados.rados.0.smithi017.stdout:3654: done (2 left) 2024-09-21T09:53:28.310 INFO:tasks.rados.rados.0.smithi017.stdout:3658: done (1 left) 2024-09-21T09:53:28.310 INFO:tasks.rados.rados.0.smithi017.stdout:3659: snap_create 2024-09-21T09:53:28.311 INFO:tasks.rados.rados.0.smithi017.stdout:3655: finishing write tid 3 to smithi01785587-16 2024-09-21T09:53:28.311 INFO:tasks.rados.rados.0.smithi017.stdout:3655: finishing write tid 4 to smithi01785587-16 2024-09-21T09:53:28.311 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1927 (ObjNum 1376 snap 458 seq_num 1376) dirty exists 2024-09-21T09:53:28.311 INFO:tasks.rados.rados.0.smithi017.stdout:3655: left oid 16 (ObjNum 1376 snap 458 seq_num 1376) 2024-09-21T09:53:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:28 smithi134 ceph-mon[90085]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T09:53:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:28 smithi134 ceph-mon[90085]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T09:53:28.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:28 smithi017 ceph-mon[102913]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T09:53:28.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:28 smithi017 ceph-mon[102913]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T09:53:28.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:28 smithi017 ceph-mon[105151]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T09:53:28.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:28 smithi017 ceph-mon[105151]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T09:53:29.249 INFO:tasks.rados.rados.0.smithi017.stdout:3655: done (1 left) 2024-09-21T09:53:29.249 INFO:tasks.rados.rados.0.smithi017.stdout:3659: done (0 left) 2024-09-21T09:53:29.249 INFO:tasks.rados.rados.0.smithi017.stdout:3660: read oid 47 snap -1 2024-09-21T09:53:29.250 INFO:tasks.rados.rados.0.smithi017.stdout:3660: expect (ObjNum 1348 snap 448 seq_num 1348) 2024-09-21T09:53:29.250 INFO:tasks.rados.rados.0.smithi017.stdout:3661: write oid 4 current snap is 459 2024-09-21T09:53:29.250 INFO:tasks.rados.rados.0.smithi017.stdout:3661: seq_num 1378 ranges {760046=753015,2037128=695820} 2024-09-21T09:53:29.258 INFO:tasks.rados.rados.0.smithi017.stdout:3661: writing smithi01785587-4 from 760046 to 1513061 tid 1 2024-09-21T09:53:29.266 INFO:tasks.rados.rados.0.smithi017.stdout:3661: writing smithi01785587-4 from 2037128 to 2732948 tid 2 2024-09-21T09:53:29.266 INFO:tasks.rados.rados.0.smithi017.stdout:3662: write oid 13 current snap is 459 2024-09-21T09:53:29.266 INFO:tasks.rados.rados.0.smithi017.stdout:3662: seq_num 1379 ranges {776853=428264,1979257=439478,2836307=340546} 2024-09-21T09:53:29.272 INFO:tasks.rados.rados.0.smithi017.stdout:3662: writing smithi01785587-13 from 776853 to 1205117 tid 1 2024-09-21T09:53:29.279 INFO:tasks.rados.rados.0.smithi017.stdout:3662: writing smithi01785587-13 from 1979257 to 2418735 tid 2 2024-09-21T09:53:29.283 INFO:tasks.rados.rados.0.smithi017.stdout:3662: writing smithi01785587-13 from 2836307 to 3176853 tid 3 2024-09-21T09:53:29.283 INFO:tasks.rados.rados.0.smithi017.stdout:3663: write oid 14 current snap is 459 2024-09-21T09:53:29.284 INFO:tasks.rados.rados.0.smithi017.stdout:3663: seq_num 1380 ranges {793660=503513,2037739=583136} 2024-09-21T09:53:29.290 INFO:tasks.rados.rados.0.smithi017.stdout:3663: writing smithi01785587-14 from 793660 to 1297173 tid 1 2024-09-21T09:53:29.298 INFO:tasks.rados.rados.0.smithi017.stdout:3663: writing smithi01785587-14 from 2037739 to 2620875 tid 2 2024-09-21T09:53:29.298 INFO:tasks.rados.rados.0.smithi017.stdout:3664: write oid 16 current snap is 459 2024-09-21T09:53:29.298 INFO:tasks.rados.rados.0.smithi017.stdout:3660: expect (ObjNum 1348 snap 448 seq_num 1348) 2024-09-21T09:53:29.388 INFO:tasks.rados.rados.0.smithi017.stdout:3661: finishing write tid 1 to smithi01785587-4 2024-09-21T09:53:29.388 INFO:tasks.rados.rados.0.smithi017.stdout:3661: finishing write tid 2 to smithi01785587-4 2024-09-21T09:53:29.388 INFO:tasks.rados.rados.0.smithi017.stdout:3661: finishing write tid 3 to smithi01785587-4 2024-09-21T09:53:29.388 INFO:tasks.rados.rados.0.smithi017.stdout:3661: finishing write tid 4 to smithi01785587-4 2024-09-21T09:53:29.388 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 1941 (ObjNum 1378 snap 459 seq_num 1378) dirty exists 2024-09-21T09:53:29.389 INFO:tasks.rados.rados.0.smithi017.stdout:3661: left oid 4 (ObjNum 1378 snap 459 seq_num 1378) 2024-09-21T09:53:29.389 INFO:tasks.rados.rados.0.smithi017.stdout:3662: finishing write tid 1 to smithi01785587-13 2024-09-21T09:53:29.389 INFO:tasks.rados.rados.0.smithi017.stdout:3662: finishing write tid 2 to smithi01785587-13 2024-09-21T09:53:29.389 INFO:tasks.rados.rados.0.smithi017.stdout:3662: finishing write tid 3 to smithi01785587-13 2024-09-21T09:53:29.389 INFO:tasks.rados.rados.0.smithi017.stdout:3664: seq_num 1381 ranges {410467=578762,1696221=443147,2891153=319314} 2024-09-21T09:53:29.393 INFO:tasks.rados.rados.0.smithi017.stdout:3664: writing smithi01785587-16 from 410467 to 989229 tid 1 2024-09-21T09:53:29.399 INFO:tasks.rados.rados.0.smithi017.stdout:3664: writing smithi01785587-16 from 1696221 to 2139368 tid 2 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3664: writing smithi01785587-16 from 2891153 to 3210467 tid 3 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3660: done (4 left) 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3661: done (3 left) 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3665: snap_create 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3662: finishing write tid 4 to smithi01785587-13 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3662: finishing write tid 5 to smithi01785587-13 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1546 (ObjNum 1379 snap 459 seq_num 1379) dirty exists 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3662: left oid 13 (ObjNum 1379 snap 459 seq_num 1379) 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3663: finishing write tid 1 to smithi01785587-14 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3663: finishing write tid 2 to smithi01785587-14 2024-09-21T09:53:29.406 INFO:tasks.rados.rados.0.smithi017.stdout:3663: finishing write tid 3 to smithi01785587-14 2024-09-21T09:53:29.407 INFO:tasks.rados.rados.0.smithi017.stdout:3663: finishing write tid 4 to smithi01785587-14 2024-09-21T09:53:29.407 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 2093 (ObjNum 1380 snap 459 seq_num 1380) dirty exists 2024-09-21T09:53:29.407 INFO:tasks.rados.rados.0.smithi017.stdout:3663: left oid 14 (ObjNum 1380 snap 459 seq_num 1380) 2024-09-21T09:53:29.407 INFO:tasks.rados.rados.0.smithi017.stdout:3664: finishing write tid 1 to smithi01785587-16 2024-09-21T09:53:29.407 INFO:tasks.rados.rados.0.smithi017.stdout:3664: finishing write tid 2 to smithi01785587-16 2024-09-21T09:53:29.409 INFO:tasks.rados.rados.0.smithi017.stdout:3664: finishing write tid 3 to smithi01785587-16 2024-09-21T09:53:29.409 INFO:tasks.rados.rados.0.smithi017.stdout:3664: finishing write tid 4 to smithi01785587-16 2024-09-21T09:53:29.409 INFO:tasks.rados.rados.0.smithi017.stdout:3664: finishing write tid 5 to smithi01785587-16 2024-09-21T09:53:29.409 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1946 (ObjNum 1381 snap 459 seq_num 1381) dirty exists 2024-09-21T09:53:29.409 INFO:tasks.rados.rados.0.smithi017.stdout:3664: left oid 16 (ObjNum 1381 snap 459 seq_num 1381) 2024-09-21T09:53:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:29 smithi134 ceph-mon[90085]: pgmap v1053: 89 pgs: 89 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 336 KiB/s wr, 7 op/s 2024-09-21T09:53:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:29 smithi017 ceph-mon[102913]: pgmap v1053: 89 pgs: 89 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 336 KiB/s wr, 7 op/s 2024-09-21T09:53:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:29 smithi017 ceph-mon[105151]: pgmap v1053: 89 pgs: 89 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 336 KiB/s wr, 7 op/s 2024-09-21T09:53:30.253 INFO:tasks.rados.rados.0.smithi017.stdout:3662: done (3 left) 2024-09-21T09:53:30.253 INFO:tasks.rados.rados.0.smithi017.stdout:3663: done (2 left) 2024-09-21T09:53:30.253 INFO:tasks.rados.rados.0.smithi017.stdout:3664: done (1 left) 2024-09-21T09:53:30.254 INFO:tasks.rados.rados.0.smithi017.stdout:3665: done (0 left) 2024-09-21T09:53:30.254 INFO:tasks.rados.rados.0.smithi017.stdout:3666: delete oid 14 current snap is 460 2024-09-21T09:53:30.257 INFO:tasks.rados.rados.0.smithi017.stdout:3666: done (0 left) 2024-09-21T09:53:30.257 INFO:tasks.rados.rados.0.smithi017.stdout:3667: rollback oid 7 current snap is 460 2024-09-21T09:53:30.257 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 7 to 459 2024-09-21T09:53:30.257 INFO:tasks.rados.rados.0.smithi017.stdout:3668: write oid 20 current snap is 460 2024-09-21T09:53:30.257 INFO:tasks.rados.rados.0.smithi017.stdout:3668: seq_num 1383 ranges {444081=729260,1813185=446816,2945999=298082} 2024-09-21T09:53:30.263 INFO:tasks.rados.rados.0.smithi017.stdout:3668: writing smithi01785587-20 from 444081 to 1173341 tid 1 2024-09-21T09:53:30.269 INFO:tasks.rados.rados.0.smithi017.stdout:3668: writing smithi01785587-20 from 1813185 to 2260001 tid 2 2024-09-21T09:53:30.275 INFO:tasks.rados.rados.0.smithi017.stdout:3668: writing smithi01785587-20 from 2945999 to 3244081 tid 3 2024-09-21T09:53:30.275 INFO:tasks.rados.rados.0.smithi017.stdout:3669: snap_remove snap 452 2024-09-21T09:53:30.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:30 smithi017 ceph-mon[102913]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T09:53:30.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:30 smithi017 ceph-mon[105151]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T09:53:30.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:30 smithi134 ceph-mon[90085]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T09:53:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:53:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:53:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:31.258 INFO:tasks.rados.rados.0.smithi017.stdout:3669: done (2 left) 2024-09-21T09:53:31.258 INFO:tasks.rados.rados.0.smithi017.stdout:3670: write oid 18 current snap is 460 2024-09-21T09:53:31.258 INFO:tasks.rados.rados.0.smithi017.stdout:3667: finishing rollback tid 0 to smithi01785587-7 2024-09-21T09:53:31.258 INFO:tasks.rados.rados.0.smithi017.stdout:3667: finishing rollback tid 1 to smithi01785587-7 2024-09-21T09:53:31.258 INFO:tasks.rados.rados.0.smithi017.stdout:3667: finishing rollback tid 2 to smithi01785587-7 2024-09-21T09:53:31.258 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 7 v 1782 (ObjNum 1368 snap 454 seq_num 1368) dirty exists 2024-09-21T09:53:31.259 INFO:tasks.rados.rados.0.smithi017.stdout:3668: finishing write tid 1 to smithi01785587-20 2024-09-21T09:53:31.259 INFO:tasks.rados.rados.0.smithi017.stdout:3668: finishing write tid 2 to smithi01785587-20 2024-09-21T09:53:31.259 INFO:tasks.rados.rados.0.smithi017.stdout:3668: finishing write tid 3 to smithi01785587-20 2024-09-21T09:53:31.259 INFO:tasks.rados.rados.0.smithi017.stdout:3668: finishing write tid 4 to smithi01785587-20 2024-09-21T09:53:31.259 INFO:tasks.rados.rados.0.smithi017.stdout:3670: seq_num 1384 ranges {460888=520862,1471667=590474,2573422=455407} 2024-09-21T09:53:31.264 INFO:tasks.rados.rados.0.smithi017.stdout:3670: writing smithi01785587-18 from 460888 to 981750 tid 1 2024-09-21T09:53:31.270 INFO:tasks.rados.rados.0.smithi017.stdout:3670: writing smithi01785587-18 from 1471667 to 2062141 tid 2 2024-09-21T09:53:31.276 INFO:tasks.rados.rados.0.smithi017.stdout:3670: writing smithi01785587-18 from 2573422 to 3028829 tid 3 2024-09-21T09:53:31.276 INFO:tasks.rados.rados.0.smithi017.stdout:3667: done (2 left) 2024-09-21T09:53:31.276 INFO:tasks.rados.rados.0.smithi017.stdout:3671: delete oid 19 current snap is 460 2024-09-21T09:53:31.277 INFO:tasks.rados.rados.0.smithi017.stdout:3668: finishing write tid 5 to smithi01785587-20 2024-09-21T09:53:31.277 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 1065 (ObjNum 1383 snap 460 seq_num 1383) dirty exists 2024-09-21T09:53:31.277 INFO:tasks.rados.rados.0.smithi017.stdout:3668: left oid 20 (ObjNum 1383 snap 460 seq_num 1383) 2024-09-21T09:53:31.277 INFO:tasks.rados.rados.0.smithi017.stdout:3670: finishing write tid 1 to smithi01785587-18 2024-09-21T09:53:31.277 INFO:tasks.rados.rados.0.smithi017.stdout:3670: finishing write tid 2 to smithi01785587-18 2024-09-21T09:53:31.279 INFO:tasks.rados.rados.0.smithi017.stdout:3670: finishing write tid 3 to smithi01785587-18 2024-09-21T09:53:31.279 INFO:tasks.rados.rados.0.smithi017.stdout:3668: done (2 left) 2024-09-21T09:53:31.279 INFO:tasks.rados.rados.0.smithi017.stdout:3671: done (1 left) 2024-09-21T09:53:31.279 INFO:tasks.rados.rados.0.smithi017.stdout:3672: write oid 22 current snap is 460 2024-09-21T09:53:31.279 INFO:tasks.rados.rados.0.smithi017.stdout:3672: seq_num 1386 ranges {494502=671360,1588631=594143,2628268=477951} 2024-09-21T09:53:31.285 INFO:tasks.rados.rados.0.smithi017.stdout:3672: writing smithi01785587-22 from 494502 to 1165862 tid 1 2024-09-21T09:53:31.291 INFO:tasks.rados.rados.0.smithi017.stdout:3672: writing smithi01785587-22 from 1588631 to 2182774 tid 2 2024-09-21T09:53:31.298 INFO:tasks.rados.rados.0.smithi017.stdout:3672: writing smithi01785587-22 from 2628268 to 3106219 tid 3 2024-09-21T09:53:31.298 INFO:tasks.rados.rados.0.smithi017.stdout:3673: read oid 33 snap 432 2024-09-21T09:53:31.298 INFO:tasks.rados.rados.0.smithi017.stdout:3673: expect deleted 2024-09-21T09:53:31.298 INFO:tasks.rados.rados.0.smithi017.stdout:3674: rollback oid 44 current snap is 460 2024-09-21T09:53:31.298 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 459 2024-09-21T09:53:31.299 INFO:tasks.rados.rados.0.smithi017.stdout:3670: finishing write tid 4 to smithi01785587-18 2024-09-21T09:53:31.299 INFO:tasks.rados.rados.0.smithi017.stdout:3670: finishing write tid 5 to smithi01785587-18 2024-09-21T09:53:31.299 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 405 (ObjNum 1384 snap 460 seq_num 1384) dirty exists 2024-09-21T09:53:31.299 INFO:tasks.rados.rados.0.smithi017.stdout:3670: left oid 18 (ObjNum 1384 snap 460 seq_num 1384) 2024-09-21T09:53:31.299 INFO:tasks.rados.rados.0.smithi017.stdout:3672: finishing write tid 1 to smithi01785587-22 2024-09-21T09:53:31.299 INFO:tasks.rados.rados.0.smithi017.stdout:3672: finishing write tid 2 to smithi01785587-22 2024-09-21T09:53:31.299 INFO:tasks.rados.rados.0.smithi017.stdout:3670: done (3 left) 2024-09-21T09:53:31.299 INFO:tasks.rados.rados.0.smithi017.stdout:3675: write oid 47 current snap is 460 2024-09-21T09:53:31.299 INFO:tasks.rados.rados.0.smithi017.stdout:3675: seq_num 1387 ranges {511309=746609,2047113=454154,3055691=255618} 2024-09-21T09:53:31.304 INFO:tasks.rados.rados.0.smithi017.stdout:3675: writing smithi01785587-47 from 511309 to 1257918 tid 1 2024-09-21T09:53:31.311 INFO:tasks.rados.rados.0.smithi017.stdout:3675: writing smithi01785587-47 from 2047113 to 2501267 tid 2 2024-09-21T09:53:31.317 INFO:tasks.rados.rados.0.smithi017.stdout:3675: writing smithi01785587-47 from 3055691 to 3311309 tid 3 2024-09-21T09:53:31.317 INFO:tasks.rados.rados.0.smithi017.stdout:3676: read oid 41 snap -1 2024-09-21T09:53:31.317 INFO:tasks.rados.rados.0.smithi017.stdout:3676: expect (ObjNum 1366 snap 452 seq_num 1366) 2024-09-21T09:53:31.317 INFO:tasks.rados.rados.0.smithi017.stdout:3677: write oid 9 current snap is 460 2024-09-21T09:53:31.317 INFO:tasks.rados.rados.0.smithi017.stdout:3677: seq_num 1388 ranges {528116=421858,1589242=597812,2966761=361355} 2024-09-21T09:53:31.322 INFO:tasks.rados.rados.0.smithi017.stdout:3677: writing smithi01785587-9 from 528116 to 949974 tid 1 2024-09-21T09:53:31.329 INFO:tasks.rados.rados.0.smithi017.stdout:3677: writing smithi01785587-9 from 1589242 to 2187054 tid 2 2024-09-21T09:53:31.336 INFO:tasks.rados.rados.0.smithi017.stdout:3677: writing smithi01785587-9 from 2966761 to 3328116 tid 3 2024-09-21T09:53:31.336 INFO:tasks.rados.rados.0.smithi017.stdout:3678: read oid 40 snap -1 2024-09-21T09:53:31.336 INFO:tasks.rados.rados.0.smithi017.stdout:3678: expect (ObjNum 1357 snap 451 seq_num 1357) 2024-09-21T09:53:31.336 INFO:tasks.rados.rados.0.smithi017.stdout:3679: snap_create 2024-09-21T09:53:31.336 INFO:tasks.rados.rados.0.smithi017.stdout:3672: finishing write tid 3 to smithi01785587-22 2024-09-21T09:53:31.337 INFO:tasks.rados.rados.0.smithi017.stdout:3672: finishing write tid 4 to smithi01785587-22 2024-09-21T09:53:31.337 INFO:tasks.rados.rados.0.smithi017.stdout:3672: finishing write tid 5 to smithi01785587-22 2024-09-21T09:53:31.337 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 1954 (ObjNum 1386 snap 460 seq_num 1386) dirty exists 2024-09-21T09:53:31.337 INFO:tasks.rados.rados.0.smithi017.stdout:3672: left oid 22 (ObjNum 1386 snap 460 seq_num 1386) 2024-09-21T09:53:31.337 INFO:tasks.rados.rados.0.smithi017.stdout:3674: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:53:31.337 INFO:tasks.rados.rados.0.smithi017.stdout:3674: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:53:31.337 INFO:tasks.rados.rados.0.smithi017.stdout:3674: finishing rollback tid 2 to smithi01785587-44 2024-09-21T09:53:31.337 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 2101 (ObjNum 1358 snap 451 seq_num 1358) dirty exists 2024-09-21T09:53:31.337 INFO:tasks.rados.rados.0.smithi017.stdout:3675: finishing write tid 1 to smithi01785587-47 2024-09-21T09:53:31.338 INFO:tasks.rados.rados.0.smithi017.stdout:3675: finishing write tid 2 to smithi01785587-47 2024-09-21T09:53:31.338 INFO:tasks.rados.rados.0.smithi017.stdout:3675: finishing write tid 3 to smithi01785587-47 2024-09-21T09:53:31.338 INFO:tasks.rados.rados.0.smithi017.stdout:3675: finishing write tid 4 to smithi01785587-47 2024-09-21T09:53:31.338 INFO:tasks.rados.rados.0.smithi017.stdout:3675: finishing write tid 5 to smithi01785587-47 2024-09-21T09:53:31.338 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 1274 (ObjNum 1387 snap 460 seq_num 1387) dirty exists 2024-09-21T09:53:31.338 INFO:tasks.rados.rados.0.smithi017.stdout:3675: left oid 47 (ObjNum 1387 snap 460 seq_num 1387) 2024-09-21T09:53:31.338 INFO:tasks.rados.rados.0.smithi017.stdout:3676: expect (ObjNum 1366 snap 452 seq_num 1366) 2024-09-21T09:53:31.436 INFO:tasks.rados.rados.0.smithi017.stdout:3677: finishing write tid 1 to smithi01785587-9 2024-09-21T09:53:31.436 INFO:tasks.rados.rados.0.smithi017.stdout:3677: finishing write tid 2 to smithi01785587-9 2024-09-21T09:53:31.436 INFO:tasks.rados.rados.0.smithi017.stdout:3677: finishing write tid 3 to smithi01785587-9 2024-09-21T09:53:31.436 INFO:tasks.rados.rados.0.smithi017.stdout:3678: expect (ObjNum 1357 snap 451 seq_num 1357) 2024-09-21T09:53:31.558 INFO:tasks.rados.rados.0.smithi017.stdout:3677: finishing write tid 4 to smithi01785587-9 2024-09-21T09:53:31.559 INFO:tasks.rados.rados.0.smithi017.stdout:3677: finishing write tid 5 to smithi01785587-9 2024-09-21T09:53:31.559 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 723 (ObjNum 1388 snap 460 seq_num 1388) dirty exists 2024-09-21T09:53:31.559 INFO:tasks.rados.rados.0.smithi017.stdout:3677: left oid 9 (ObjNum 1388 snap 460 seq_num 1388) 2024-09-21T09:53:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[102913]: pgmap v1056: 89 pgs: 89 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-09-21T09:53:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[102913]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T09:53:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[105151]: pgmap v1056: 89 pgs: 89 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-09-21T09:53:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[105151]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T09:53:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:31 smithi134 ceph-mon[90085]: pgmap v1056: 89 pgs: 89 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-09-21T09:53:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:31 smithi134 ceph-mon[90085]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T09:53:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:32.274 INFO:tasks.rados.rados.0.smithi017.stdout:3672: done (7 left) 2024-09-21T09:53:32.274 INFO:tasks.rados.rados.0.smithi017.stdout:3673: done (6 left) 2024-09-21T09:53:32.274 INFO:tasks.rados.rados.0.smithi017.stdout:3674: done (5 left) 2024-09-21T09:53:32.274 INFO:tasks.rados.rados.0.smithi017.stdout:3675: done (4 left) 2024-09-21T09:53:32.274 INFO:tasks.rados.rados.0.smithi017.stdout:3676: done (3 left) 2024-09-21T09:53:32.274 INFO:tasks.rados.rados.0.smithi017.stdout:3677: done (2 left) 2024-09-21T09:53:32.275 INFO:tasks.rados.rados.0.smithi017.stdout:3678: done (1 left) 2024-09-21T09:53:32.275 INFO:tasks.rados.rados.0.smithi017.stdout:3679: done (0 left) 2024-09-21T09:53:32.275 INFO:tasks.rados.rados.0.smithi017.stdout:3680: write oid 16 current snap is 461 2024-09-21T09:53:32.275 INFO:tasks.rados.rados.0.smithi017.stdout:3680: seq_num 1389 ranges {544923=497107,1647724=457823,2710537=428120} 2024-09-21T09:53:32.279 INFO:tasks.rados.rados.0.smithi017.stdout:3680: writing smithi01785587-16 from 544923 to 1042030 tid 1 2024-09-21T09:53:32.285 INFO:tasks.rados.rados.0.smithi017.stdout:3680: writing smithi01785587-16 from 1647724 to 2105547 tid 2 2024-09-21T09:53:32.291 INFO:tasks.rados.rados.0.smithi017.stdout:3680: writing smithi01785587-16 from 2710537 to 3138657 tid 3 2024-09-21T09:53:32.291 INFO:tasks.rados.rados.0.smithi017.stdout:3681: snap_remove snap 447 2024-09-21T09:53:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:32 smithi017 ceph-mon[105151]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T09:53:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:53:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:53:32.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:32 smithi017 ceph-mon[102913]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T09:53:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:53:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:53:32.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:32 smithi134 ceph-mon[90085]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T09:53:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:53:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:53:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:53:33.224 INFO:tasks.rados.rados.0.smithi017.stdout:3681: done (1 left) 2024-09-21T09:53:33.224 INFO:tasks.rados.rados.0.smithi017.stdout:3682: delete oid 28 current snap is 461 2024-09-21T09:53:33.224 INFO:tasks.rados.rados.0.smithi017.stdout:3680: finishing write tid 1 to smithi01785587-16 2024-09-21T09:53:33.225 INFO:tasks.rados.rados.0.smithi017.stdout:3680: finishing write tid 2 to smithi01785587-16 2024-09-21T09:53:33.225 INFO:tasks.rados.rados.0.smithi017.stdout:3680: finishing write tid 3 to smithi01785587-16 2024-09-21T09:53:33.225 INFO:tasks.rados.rados.0.smithi017.stdout:3680: finishing write tid 4 to smithi01785587-16 2024-09-21T09:53:33.225 INFO:tasks.rados.rados.0.smithi017.stdout:3680: finishing write tid 5 to smithi01785587-16 2024-09-21T09:53:33.225 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 1959 (ObjNum 1389 snap 461 seq_num 1389) dirty exists 2024-09-21T09:53:33.225 INFO:tasks.rados.rados.0.smithi017.stdout:3680: left oid 16 (ObjNum 1389 snap 461 seq_num 1389) 2024-09-21T09:53:33.234 INFO:tasks.rados.rados.0.smithi017.stdout:3680: done (1 left) 2024-09-21T09:53:33.234 INFO:tasks.rados.rados.0.smithi017.stdout:3682: done (0 left) 2024-09-21T09:53:33.234 INFO:tasks.rados.rados.0.smithi017.stdout:3683: snap_create 2024-09-21T09:53:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:33 smithi017 ceph-mon[102913]: pgmap v1059: 89 pgs: 89 active+clean; 200 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-09-21T09:53:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:33 smithi017 ceph-mon[102913]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T09:53:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:33 smithi017 ceph-mon[102913]: osdmap e988: 8 total, 8 up, 8 in 2024-09-21T09:53:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:33 smithi017 ceph-mon[105151]: pgmap v1059: 89 pgs: 89 active+clean; 200 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-09-21T09:53:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:33 smithi017 ceph-mon[105151]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T09:53:33.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:33 smithi017 ceph-mon[105151]: osdmap e988: 8 total, 8 up, 8 in 2024-09-21T09:53:33.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:33 smithi134 ceph-mon[90085]: pgmap v1059: 89 pgs: 89 active+clean; 200 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-09-21T09:53:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:33 smithi134 ceph-mon[90085]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T09:53:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:33 smithi134 ceph-mon[90085]: osdmap e988: 8 total, 8 up, 8 in 2024-09-21T09:53:34.227 INFO:tasks.rados.rados.0.smithi017.stdout:3683: done (0 left) 2024-09-21T09:53:34.227 INFO:tasks.rados.rados.0.smithi017.stdout:3684: rollback oid 30 current snap is 462 2024-09-21T09:53:34.227 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 432 2024-09-21T09:53:34.227 INFO:tasks.rados.rados.0.smithi017.stdout:3685: snap_remove snap 456 2024-09-21T09:53:35.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:53:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:53:34] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.51.0" 2024-09-21T09:53:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:3685: done (1 left) 2024-09-21T09:53:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:3686: read oid 9 snap 457 2024-09-21T09:53:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:3686: expect (ObjNum 1367 snap 454 seq_num 1367) 2024-09-21T09:53:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:3687: rollback oid 3 current snap is 462 2024-09-21T09:53:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 459 2024-09-21T09:53:35.230 INFO:tasks.rados.rados.0.smithi017.stdout:3684: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:53:35.231 INFO:tasks.rados.rados.0.smithi017.stdout:3684: finishing rollback tid 2 to smithi01785587-30 2024-09-21T09:53:35.231 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1558 (ObjNum 1217 snap 405 seq_num 1217) dirty exists 2024-09-21T09:53:35.231 INFO:tasks.rados.rados.0.smithi017.stdout:3684: done (2 left) 2024-09-21T09:53:35.231 INFO:tasks.rados.rados.0.smithi017.stdout:3688: rollback oid 33 current snap is 462 2024-09-21T09:53:35.231 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 459 2024-09-21T09:53:35.231 INFO:tasks.rados.rados.0.smithi017.stdout:3689: snap_remove snap 450 2024-09-21T09:53:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:35 smithi134 ceph-mon[90085]: pgmap v1062: 89 pgs: 89 active+clean; 198 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:53:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:35 smithi134 ceph-mon[90085]: osdmap e989: 8 total, 8 up, 8 in 2024-09-21T09:53:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:35 smithi017 ceph-mon[102913]: pgmap v1062: 89 pgs: 89 active+clean; 198 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:53:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:35 smithi017 ceph-mon[102913]: osdmap e989: 8 total, 8 up, 8 in 2024-09-21T09:53:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:35 smithi017 ceph-mon[105151]: pgmap v1062: 89 pgs: 89 active+clean; 198 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:53:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:35 smithi017 ceph-mon[105151]: osdmap e989: 8 total, 8 up, 8 in 2024-09-21T09:53:36.245 INFO:tasks.rados.rados.0.smithi017.stdout:3687: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:53:36.245 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 2062 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:53:36.245 INFO:tasks.rados.rados.0.smithi017.stdout:3686: expect (ObjNum 1367 snap 454 seq_num 1367) 2024-09-21T09:53:36.349 INFO:tasks.rados.rados.0.smithi017.stdout:3688: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:53:36.349 INFO:tasks.rados.rados.0.smithi017.stdout:3688: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:53:36.349 INFO:tasks.rados.rados.0.smithi017.stdout:3688: finishing rollback tid 2 to smithi01785587-33 2024-09-21T09:53:36.349 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 1970 (ObjNum 1329 snap 446 seq_num 1329) dirty exists 2024-09-21T09:53:36.349 INFO:tasks.rados.rados.0.smithi017.stdout:3686: done (3 left) 2024-09-21T09:53:36.349 INFO:tasks.rados.rados.0.smithi017.stdout:3687: done (2 left) 2024-09-21T09:53:36.349 INFO:tasks.rados.rados.0.smithi017.stdout:3688: done (1 left) 2024-09-21T09:53:36.349 INFO:tasks.rados.rados.0.smithi017.stdout:3689: done (0 left) 2024-09-21T09:53:36.349 INFO:tasks.rados.rados.0.smithi017.stdout:3690: rollback oid 8 current snap is 462 2024-09-21T09:53:36.350 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 444 2024-09-21T09:53:36.350 INFO:tasks.rados.rados.0.smithi017.stdout:3691: read oid 14 snap 444 2024-09-21T09:53:36.350 INFO:tasks.rados.rados.0.smithi017.stdout:3691: expect (ObjNum 1158 snap 389 seq_num 1158) 2024-09-21T09:53:36.350 INFO:tasks.rados.rados.0.smithi017.stdout:3692: write oid 35 current snap is 462 2024-09-21T09:53:36.350 INFO:tasks.rados.rados.0.smithi017.stdout:3692: seq_num 1391 ranges {578537=647605,1764688=745139,3049030=329507} 2024-09-21T09:53:36.355 INFO:tasks.rados.rados.0.smithi017.stdout:3692: writing smithi01785587-35 from 578537 to 1226142 tid 1 2024-09-21T09:53:36.362 INFO:tasks.rados.rados.0.smithi017.stdout:3692: writing smithi01785587-35 from 1764688 to 2509827 tid 2 2024-09-21T09:53:36.367 INFO:tasks.rados.rados.0.smithi017.stdout:3692: writing smithi01785587-35 from 3049030 to 3378537 tid 3 2024-09-21T09:53:36.367 INFO:tasks.rados.rados.0.smithi017.stdout:3693: write oid 11 current snap is 462 2024-09-21T09:53:36.367 INFO:tasks.rados.rados.0.smithi017.stdout:3690: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:53:36.368 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 0 (ObjNum 129 snap 0 seq_num 2400830776) dirty dne 2024-09-21T09:53:36.368 INFO:tasks.rados.rados.0.smithi017.stdout:3693: seq_num 1392 ranges {595344=439207,1823170=605150,3076453=318891} 2024-09-21T09:53:36.373 INFO:tasks.rados.rados.0.smithi017.stdout:3693: writing smithi01785587-11 from 595344 to 1034551 tid 1 2024-09-21T09:53:36.380 INFO:tasks.rados.rados.0.smithi017.stdout:3693: writing smithi01785587-11 from 1823170 to 2428320 tid 2 2024-09-21T09:53:36.386 INFO:tasks.rados.rados.0.smithi017.stdout:3693: writing smithi01785587-11 from 3076453 to 3395344 tid 3 2024-09-21T09:53:36.386 INFO:tasks.rados.rados.0.smithi017.stdout:3690: done (3 left) 2024-09-21T09:53:36.386 INFO:tasks.rados.rados.0.smithi017.stdout:3694: write oid 49 current snap is 462 2024-09-21T09:53:36.386 INFO:tasks.rados.rados.0.smithi017.stdout:3694: seq_num 1393 ranges {612151=514456,1881652=748808,3103876=308275} 2024-09-21T09:53:36.392 INFO:tasks.rados.rados.0.smithi017.stdout:3694: writing smithi01785587-49 from 612151 to 1126607 tid 1 2024-09-21T09:53:36.400 INFO:tasks.rados.rados.0.smithi017.stdout:3694: writing smithi01785587-49 from 1881652 to 2630460 tid 2 2024-09-21T09:53:36.405 INFO:tasks.rados.rados.0.smithi017.stdout:3694: writing smithi01785587-49 from 3103876 to 3412151 tid 3 2024-09-21T09:53:36.405 INFO:tasks.rados.rados.0.smithi017.stdout:3695: read oid 2 snap 444 2024-09-21T09:53:36.405 INFO:tasks.rados.rados.0.smithi017.stdout:3695: expect deleted 2024-09-21T09:53:36.405 INFO:tasks.rados.rados.0.smithi017.stdout:3696: snap_remove snap 459 2024-09-21T09:53:36.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:36 smithi134 ceph-mon[90085]: osdmap e990: 8 total, 8 up, 8 in 2024-09-21T09:53:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:36 smithi017 ceph-mon[102913]: osdmap e990: 8 total, 8 up, 8 in 2024-09-21T09:53:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:36 smithi017 ceph-mon[105151]: osdmap e990: 8 total, 8 up, 8 in 2024-09-21T09:53:37.247 INFO:tasks.rados.rados.0.smithi017.stdout:3696: done (5 left) 2024-09-21T09:53:37.247 INFO:tasks.rados.rados.0.smithi017.stdout:3697: write oid 3 current snap is 462 2024-09-21T09:53:37.248 INFO:tasks.rados.rados.0.smithi017.stdout:3691: expect (ObjNum 1158 snap 389 seq_num 1158) 2024-09-21T09:53:37.409 INFO:tasks.rados.rados.0.smithi017.stdout:3692: finishing write tid 1 to smithi01785587-35 2024-09-21T09:53:37.409 INFO:tasks.rados.rados.0.smithi017.stdout:3692: finishing write tid 2 to smithi01785587-35 2024-09-21T09:53:37.409 INFO:tasks.rados.rados.0.smithi017.stdout:3692: finishing write tid 3 to smithi01785587-35 2024-09-21T09:53:37.409 INFO:tasks.rados.rados.0.smithi017.stdout:3692: finishing write tid 4 to smithi01785587-35 2024-09-21T09:53:37.409 INFO:tasks.rados.rados.0.smithi017.stdout:3692: finishing write tid 5 to smithi01785587-35 2024-09-21T09:53:37.409 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 402 (ObjNum 1391 snap 462 seq_num 1391) dirty exists 2024-09-21T09:53:37.409 INFO:tasks.rados.rados.0.smithi017.stdout:3692: left oid 35 (ObjNum 1391 snap 462 seq_num 1391) 2024-09-21T09:53:37.409 INFO:tasks.rados.rados.0.smithi017.stdout:3693: finishing write tid 1 to smithi01785587-11 2024-09-21T09:53:37.409 INFO:tasks.rados.rados.0.smithi017.stdout:3697: seq_num 1394 ranges {628958=589705,1940134=608819,3131299=297659} 2024-09-21T09:53:37.415 INFO:tasks.rados.rados.0.smithi017.stdout:3697: writing smithi01785587-3 from 628958 to 1218663 tid 1 2024-09-21T09:53:37.422 INFO:tasks.rados.rados.0.smithi017.stdout:3697: writing smithi01785587-3 from 1940134 to 2548953 tid 2 2024-09-21T09:53:37.428 INFO:tasks.rados.rados.0.smithi017.stdout:3697: writing smithi01785587-3 from 3131299 to 3428958 tid 3 2024-09-21T09:53:37.428 INFO:tasks.rados.rados.0.smithi017.stdout:3691: done (5 left) 2024-09-21T09:53:37.428 INFO:tasks.rados.rados.0.smithi017.stdout:3692: done (4 left) 2024-09-21T09:53:37.428 INFO:tasks.rados.rados.0.smithi017.stdout:3698: delete oid 32 current snap is 462 2024-09-21T09:53:37.428 INFO:tasks.rados.rados.0.smithi017.stdout:3693: finishing write tid 2 to smithi01785587-11 2024-09-21T09:53:37.428 INFO:tasks.rados.rados.0.smithi017.stdout:3693: finishing write tid 3 to smithi01785587-11 2024-09-21T09:53:37.428 INFO:tasks.rados.rados.0.smithi017.stdout:3693: finishing write tid 4 to smithi01785587-11 2024-09-21T09:53:37.428 INFO:tasks.rados.rados.0.smithi017.stdout:3693: finishing write tid 5 to smithi01785587-11 2024-09-21T09:53:37.428 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 11 v 1937 (ObjNum 1392 snap 462 seq_num 1392) dirty exists 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:3693: left oid 11 (ObjNum 1392 snap 462 seq_num 1392) 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:3694: finishing write tid 1 to smithi01785587-49 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:3694: finishing write tid 2 to smithi01785587-49 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:3694: finishing write tid 3 to smithi01785587-49 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:3694: finishing write tid 4 to smithi01785587-49 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:3694: finishing write tid 5 to smithi01785587-49 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1791 (ObjNum 1393 snap 462 seq_num 1393) dirty exists 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:3694: left oid 49 (ObjNum 1393 snap 462 seq_num 1393) 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:3697: finishing write tid 1 to smithi01785587-3 2024-09-21T09:53:37.429 INFO:tasks.rados.rados.0.smithi017.stdout:3697: finishing write tid 2 to smithi01785587-3 2024-09-21T09:53:37.430 INFO:tasks.rados.rados.0.smithi017.stdout:3693: done (4 left) 2024-09-21T09:53:37.430 INFO:tasks.rados.rados.0.smithi017.stdout:3694: done (3 left) 2024-09-21T09:53:37.430 INFO:tasks.rados.rados.0.smithi017.stdout:3695: done (2 left) 2024-09-21T09:53:37.430 INFO:tasks.rados.rados.0.smithi017.stdout:3698: done (1 left) 2024-09-21T09:53:37.430 INFO:tasks.rados.rados.0.smithi017.stdout:3699: delete oid 17 current snap is 462 2024-09-21T09:53:37.431 INFO:tasks.rados.rados.0.smithi017.stdout:3697: finishing write tid 3 to smithi01785587-3 2024-09-21T09:53:37.431 INFO:tasks.rados.rados.0.smithi017.stdout:3697: finishing write tid 4 to smithi01785587-3 2024-09-21T09:53:37.431 INFO:tasks.rados.rados.0.smithi017.stdout:3697: finishing write tid 5 to smithi01785587-3 2024-09-21T09:53:37.431 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 2122 (ObjNum 1394 snap 462 seq_num 1394) dirty exists 2024-09-21T09:53:37.431 INFO:tasks.rados.rados.0.smithi017.stdout:3697: left oid 3 (ObjNum 1394 snap 462 seq_num 1394) 2024-09-21T09:53:37.432 INFO:tasks.rados.rados.0.smithi017.stdout:3697: done (1 left) 2024-09-21T09:53:37.432 INFO:tasks.rados.rados.0.smithi017.stdout:3699: done (0 left) 2024-09-21T09:53:37.432 INFO:tasks.rados.rados.0.smithi017.stdout:3700: read oid 39 snap -1 2024-09-21T09:53:37.433 INFO:tasks.rados.rados.0.smithi017.stdout:3700: expect (ObjNum 1320 snap 444 seq_num 1320) 2024-09-21T09:53:37.433 INFO:tasks.rados.rados.0.smithi017.stdout:3701: write oid 45 current snap is 462 2024-09-21T09:53:37.433 INFO:tasks.rados.rados.0.smithi017.stdout:3701: seq_num 1397 ranges {679379=415452,1599227=756146,3097215=382164} 2024-09-21T09:53:37.438 INFO:tasks.rados.rados.0.smithi017.stdout:3701: writing smithi01785587-45 from 679379 to 1094831 tid 1 2024-09-21T09:53:37.445 INFO:tasks.rados.rados.0.smithi017.stdout:3701: writing smithi01785587-45 from 1599227 to 2355373 tid 2 2024-09-21T09:53:37.452 INFO:tasks.rados.rados.0.smithi017.stdout:3701: writing smithi01785587-45 from 3097215 to 3479379 tid 3 2024-09-21T09:53:37.452 INFO:tasks.rados.rados.0.smithi017.stdout:3702: delete oid 43 current snap is 462 2024-09-21T09:53:37.452 INFO:tasks.rados.rados.0.smithi017.stdout:3700: expect (ObjNum 1320 snap 444 seq_num 1320) 2024-09-21T09:53:37.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:37 smithi134 ceph-mon[90085]: pgmap v1065: 89 pgs: 89 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:53:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:37 smithi134 ceph-mon[90085]: osdmap e991: 8 total, 8 up, 8 in 2024-09-21T09:53:37.549 INFO:tasks.rados.rados.0.smithi017.stdout:3701: finishing write tid 1 to smithi01785587-45 2024-09-21T09:53:37.549 INFO:tasks.rados.rados.0.smithi017.stdout:3701: finishing write tid 2 to smithi01785587-45 2024-09-21T09:53:37.549 INFO:tasks.rados.rados.0.smithi017.stdout:3701: finishing write tid 3 to smithi01785587-45 2024-09-21T09:53:37.549 INFO:tasks.rados.rados.0.smithi017.stdout:3701: finishing write tid 4 to smithi01785587-45 2024-09-21T09:53:37.549 INFO:tasks.rados.rados.0.smithi017.stdout:3701: finishing write tid 5 to smithi01785587-45 2024-09-21T09:53:37.549 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1278 (ObjNum 1397 snap 462 seq_num 1397) dirty exists 2024-09-21T09:53:37.549 INFO:tasks.rados.rados.0.smithi017.stdout:3701: left oid 45 (ObjNum 1397 snap 462 seq_num 1397) 2024-09-21T09:53:37.550 INFO:tasks.rados.rados.0.smithi017.stdout:3700: done (2 left) 2024-09-21T09:53:37.550 INFO:tasks.rados.rados.0.smithi017.stdout:3701: done (1 left) 2024-09-21T09:53:37.550 INFO:tasks.rados.rados.0.smithi017.stdout:3702: done (0 left) 2024-09-21T09:53:37.550 INFO:tasks.rados.rados.0.smithi017.stdout:3703: delete oid 15 current snap is 462 2024-09-21T09:53:37.551 INFO:tasks.rados.rados.0.smithi017.stdout:3703: done (0 left) 2024-09-21T09:53:37.551 INFO:tasks.rados.rados.0.smithi017.stdout:3704: delete oid 40 current snap is 462 2024-09-21T09:53:37.553 INFO:tasks.rados.rados.0.smithi017.stdout:3704: done (0 left) 2024-09-21T09:53:37.553 INFO:tasks.rados.rados.0.smithi017.stdout:3705: snap_remove snap 432 2024-09-21T09:53:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:37 smithi017 ceph-mon[102913]: pgmap v1065: 89 pgs: 89 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:53:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:37 smithi017 ceph-mon[102913]: osdmap e991: 8 total, 8 up, 8 in 2024-09-21T09:53:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:37 smithi017 ceph-mon[105151]: pgmap v1065: 89 pgs: 89 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:53:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:37 smithi017 ceph-mon[105151]: osdmap e991: 8 total, 8 up, 8 in 2024-09-21T09:53:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:3705: done (0 left) 2024-09-21T09:53:38.226 INFO:tasks.rados.rados.0.smithi017.stdout:3706: snap_remove snap 458 2024-09-21T09:53:38.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:38 smithi017 ceph-mon[102913]: osdmap e992: 8 total, 8 up, 8 in 2024-09-21T09:53:38.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:38 smithi017 ceph-mon[102913]: osdmap e993: 8 total, 8 up, 8 in 2024-09-21T09:53:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:38 smithi017 ceph-mon[105151]: osdmap e992: 8 total, 8 up, 8 in 2024-09-21T09:53:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:38 smithi017 ceph-mon[105151]: osdmap e993: 8 total, 8 up, 8 in 2024-09-21T09:53:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:38 smithi134 ceph-mon[90085]: osdmap e992: 8 total, 8 up, 8 in 2024-09-21T09:53:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:38 smithi134 ceph-mon[90085]: osdmap e993: 8 total, 8 up, 8 in 2024-09-21T09:53:39.229 INFO:tasks.rados.rados.0.smithi017.stdout:3706: done (0 left) 2024-09-21T09:53:39.229 INFO:tasks.rados.rados.0.smithi017.stdout:3707: write oid 31 current snap is 462 2024-09-21T09:53:39.229 INFO:tasks.rados.rados.0.smithi017.stdout:3707: seq_num 1401 ranges {746607=432801,1833155=763484,3206907=339700} 2024-09-21T09:53:39.236 INFO:tasks.rados.rados.0.smithi017.stdout:3707: writing smithi01785587-31 from 746607 to 1179408 tid 1 2024-09-21T09:53:39.245 INFO:tasks.rados.rados.0.smithi017.stdout:3707: writing smithi01785587-31 from 1833155 to 2596639 tid 2 2024-09-21T09:53:39.250 INFO:tasks.rados.rados.0.smithi017.stdout:3707: writing smithi01785587-31 from 3206907 to 3546607 tid 3 2024-09-21T09:53:39.250 INFO:tasks.rados.rados.0.smithi017.stdout:3708: snap_create 2024-09-21T09:53:39.250 INFO:tasks.rados.rados.0.smithi017.stdout:3707: finishing write tid 1 to smithi01785587-31 2024-09-21T09:53:39.251 INFO:tasks.rados.rados.0.smithi017.stdout:3707: finishing write tid 2 to smithi01785587-31 2024-09-21T09:53:39.254 INFO:tasks.rados.rados.0.smithi017.stdout:3707: finishing write tid 3 to smithi01785587-31 2024-09-21T09:53:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:3707: finishing write tid 4 to smithi01785587-31 2024-09-21T09:53:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:3707: finishing write tid 5 to smithi01785587-31 2024-09-21T09:53:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 2138 (ObjNum 1401 snap 462 seq_num 1401) dirty exists 2024-09-21T09:53:39.255 INFO:tasks.rados.rados.0.smithi017.stdout:3707: left oid 31 (ObjNum 1401 snap 462 seq_num 1401) 2024-09-21T09:53:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:39 smithi017 ceph-mon[102913]: pgmap v1068: 89 pgs: 89 active+clean; 203 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-21T09:53:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:39 smithi017 ceph-mon[102913]: osdmap e994: 8 total, 8 up, 8 in 2024-09-21T09:53:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:39 smithi017 ceph-mon[105151]: pgmap v1068: 89 pgs: 89 active+clean; 203 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-21T09:53:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:39 smithi017 ceph-mon[105151]: osdmap e994: 8 total, 8 up, 8 in 2024-09-21T09:53:39.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:39 smithi134 ceph-mon[90085]: pgmap v1068: 89 pgs: 89 active+clean; 203 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-09-21T09:53:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:39 smithi134 ceph-mon[90085]: osdmap e994: 8 total, 8 up, 8 in 2024-09-21T09:53:40.262 INFO:tasks.rados.rados.0.smithi017.stdout:3707: done (1 left) 2024-09-21T09:53:40.262 INFO:tasks.rados.rados.0.smithi017.stdout:3708: done (0 left) 2024-09-21T09:53:40.263 INFO:tasks.rados.rados.0.smithi017.stdout:3709: rollback oid 14 current snap is 463 2024-09-21T09:53:40.263 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 461 2024-09-21T09:53:40.263 INFO:tasks.rados.rados.0.smithi017.stdout:3710: write oid 3 current snap is 463 2024-09-21T09:53:40.263 INFO:tasks.rados.rados.0.smithi017.stdout:3710: seq_num 1402 ranges {763414=508050,1891637=623495,3234330=329084} 2024-09-21T09:53:40.269 INFO:tasks.rados.rados.0.smithi017.stdout:3710: writing smithi01785587-3 from 763414 to 1271464 tid 1 2024-09-21T09:53:40.276 INFO:tasks.rados.rados.0.smithi017.stdout:3710: writing smithi01785587-3 from 1891637 to 2515132 tid 2 2024-09-21T09:53:40.282 INFO:tasks.rados.rados.0.smithi017.stdout:3710: writing smithi01785587-3 from 3234330 to 3563414 tid 3 2024-09-21T09:53:40.282 INFO:tasks.rados.rados.0.smithi017.stdout:3711: delete oid 10 current snap is 463 2024-09-21T09:53:40.282 INFO:tasks.rados.rados.0.smithi017.stdout:3709: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:53:40.283 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 2095 (ObjNum 33 snap 0 seq_num 2400082544) dirty dne 2024-09-21T09:53:40.283 INFO:tasks.rados.rados.0.smithi017.stdout:3710: finishing write tid 1 to smithi01785587-3 2024-09-21T09:53:40.283 INFO:tasks.rados.rados.0.smithi017.stdout:3710: finishing write tid 2 to smithi01785587-3 2024-09-21T09:53:40.285 INFO:tasks.rados.rados.0.smithi017.stdout:3709: done (2 left) 2024-09-21T09:53:40.285 INFO:tasks.rados.rados.0.smithi017.stdout:3711: done (1 left) 2024-09-21T09:53:40.285 INFO:tasks.rados.rados.0.smithi017.stdout:3712: snap_create 2024-09-21T09:53:40.287 INFO:tasks.rados.rados.0.smithi017.stdout:3710: finishing write tid 3 to smithi01785587-3 2024-09-21T09:53:40.287 INFO:tasks.rados.rados.0.smithi017.stdout:3710: finishing write tid 4 to smithi01785587-3 2024-09-21T09:53:40.287 INFO:tasks.rados.rados.0.smithi017.stdout:3710: finishing write tid 5 to smithi01785587-3 2024-09-21T09:53:40.287 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 2144 (ObjNum 1402 snap 463 seq_num 1402) dirty exists 2024-09-21T09:53:40.287 INFO:tasks.rados.rados.0.smithi017.stdout:3710: left oid 3 (ObjNum 1402 snap 463 seq_num 1402) 2024-09-21T09:53:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:53:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:53:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:41.267 INFO:tasks.rados.rados.0.smithi017.stdout:3710: done (1 left) 2024-09-21T09:53:41.267 INFO:tasks.rados.rados.0.smithi017.stdout:3712: done (0 left) 2024-09-21T09:53:41.267 INFO:tasks.rados.rados.0.smithi017.stdout:3713: read oid 18 snap -1 2024-09-21T09:53:41.267 INFO:tasks.rados.rados.0.smithi017.stdout:3713: expect (ObjNum 1384 snap 460 seq_num 1384) 2024-09-21T09:53:41.268 INFO:tasks.rados.rados.0.smithi017.stdout:3714: rollback oid 32 current snap is 464 2024-09-21T09:53:41.268 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 32 to 460 2024-09-21T09:53:41.268 INFO:tasks.rados.rados.0.smithi017.stdout:3715: write oid 8 current snap is 464 2024-09-21T09:53:41.268 INFO:tasks.rados.rados.0.smithi017.stdout:3715: seq_num 1404 ranges {797028=658548,1892248=510811,3172823=424205} 2024-09-21T09:53:41.275 INFO:tasks.rados.rados.0.smithi017.stdout:3715: writing smithi01785587-8 from 797028 to 1455576 tid 1 2024-09-21T09:53:41.281 INFO:tasks.rados.rados.0.smithi017.stdout:3715: writing smithi01785587-8 from 1892248 to 2403059 tid 2 2024-09-21T09:53:41.288 INFO:tasks.rados.rados.0.smithi017.stdout:3715: writing smithi01785587-8 from 3172823 to 3597028 tid 3 2024-09-21T09:53:41.288 INFO:tasks.rados.rados.0.smithi017.stdout:3716: read oid 16 snap -1 2024-09-21T09:53:41.288 INFO:tasks.rados.rados.0.smithi017.stdout:3716: expect (ObjNum 1389 snap 461 seq_num 1389) 2024-09-21T09:53:41.288 INFO:tasks.rados.rados.0.smithi017.stdout:3717: delete oid 24 current snap is 464 2024-09-21T09:53:41.288 INFO:tasks.rados.rados.0.smithi017.stdout:3714: finishing rollback tid 1 to smithi01785587-32 2024-09-21T09:53:41.288 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 32 v 1912 (ObjNum 129 snap 0 seq_num 2399434032) dirty dne 2024-09-21T09:53:41.288 INFO:tasks.rados.rados.0.smithi017.stdout:3715: finishing write tid 1 to smithi01785587-8 2024-09-21T09:53:41.288 INFO:tasks.rados.rados.0.smithi017.stdout:3713: expect (ObjNum 1384 snap 460 seq_num 1384) 2024-09-21T09:53:41.397 INFO:tasks.rados.rados.0.smithi017.stdout:3715: finishing write tid 2 to smithi01785587-8 2024-09-21T09:53:41.397 INFO:tasks.rados.rados.0.smithi017.stdout:3715: finishing write tid 3 to smithi01785587-8 2024-09-21T09:53:41.398 INFO:tasks.rados.rados.0.smithi017.stdout:3715: finishing write tid 4 to smithi01785587-8 2024-09-21T09:53:41.398 INFO:tasks.rados.rados.0.smithi017.stdout:3715: finishing write tid 5 to smithi01785587-8 2024-09-21T09:53:41.398 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1949 (ObjNum 1404 snap 464 seq_num 1404) dirty exists 2024-09-21T09:53:41.398 INFO:tasks.rados.rados.0.smithi017.stdout:3715: left oid 8 (ObjNum 1404 snap 464 seq_num 1404) 2024-09-21T09:53:41.398 INFO:tasks.rados.rados.0.smithi017.stdout:3716: expect (ObjNum 1389 snap 461 seq_num 1389) 2024-09-21T09:53:41.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:41 smithi017 ceph-mon[102913]: pgmap v1071: 89 pgs: 89 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 736 KiB/s wr, 7 op/s 2024-09-21T09:53:41.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:41 smithi017 ceph-mon[102913]: osdmap e995: 8 total, 8 up, 8 in 2024-09-21T09:53:41.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:41 smithi017 ceph-mon[105151]: pgmap v1071: 89 pgs: 89 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 736 KiB/s wr, 7 op/s 2024-09-21T09:53:41.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:41 smithi017 ceph-mon[105151]: osdmap e995: 8 total, 8 up, 8 in 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:3713: done (4 left) 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:3714: done (3 left) 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:3715: done (2 left) 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:3716: done (1 left) 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:3717: done (0 left) 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:3718: rollback oid 5 current snap is 464 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 457 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:3719: rollback oid 20 current snap is 464 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 20 to 463 2024-09-21T09:53:41.724 INFO:tasks.rados.rados.0.smithi017.stdout:3720: write oid 23 current snap is 464 2024-09-21T09:53:41.725 INFO:tasks.rados.rados.0.smithi017.stdout:3720: seq_num 1406 ranges {430642=409046,1609212=514480,2827669=768803} 2024-09-21T09:53:41.730 INFO:tasks.rados.rados.0.smithi017.stdout:3720: writing smithi01785587-23 from 430642 to 839688 tid 1 2024-09-21T09:53:41.735 INFO:tasks.rados.rados.0.smithi017.stdout:3720: writing smithi01785587-23 from 1609212 to 2123692 tid 2 2024-09-21T09:53:41.744 INFO:tasks.rados.rados.0.smithi017.stdout:3720: writing smithi01785587-23 from 2827669 to 3596472 tid 3 2024-09-21T09:53:41.744 INFO:tasks.rados.rados.0.smithi017.stdout:3721: read oid 34 snap -1 2024-09-21T09:53:41.744 INFO:tasks.rados.rados.0.smithi017.stdout:3721: expect deleted 2024-09-21T09:53:41.744 INFO:tasks.rados.rados.0.smithi017.stdout:3722: write oid 44 current snap is 464 2024-09-21T09:53:41.744 INFO:tasks.rados.rados.0.smithi017.stdout:3719: finishing rollback tid 0 to smithi01785587-20 2024-09-21T09:53:41.744 INFO:tasks.rados.rados.0.smithi017.stdout:3718: finishing rollback tid 0 to smithi01785587-5 2024-09-21T09:53:41.744 INFO:tasks.rados.rados.0.smithi017.stdout:3719: finishing rollback tid 1 to smithi01785587-20 2024-09-21T09:53:41.744 INFO:tasks.rados.rados.0.smithi017.stdout:3722: seq_num 1407 ranges {447449=600648,1784047=774491,2971445=676004} 2024-09-21T09:53:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:41 smithi134 ceph-mon[90085]: pgmap v1071: 89 pgs: 89 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 736 KiB/s wr, 7 op/s 2024-09-21T09:53:41.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:41 smithi134 ceph-mon[90085]: osdmap e995: 8 total, 8 up, 8 in 2024-09-21T09:53:41.749 INFO:tasks.rados.rados.0.smithi017.stdout:3722: writing smithi01785587-44 from 447449 to 1048097 tid 1 2024-09-21T09:53:41.758 INFO:tasks.rados.rados.0.smithi017.stdout:3722: writing smithi01785587-44 from 1784047 to 2558538 tid 2 2024-09-21T09:53:41.764 INFO:tasks.rados.rados.0.smithi017.stdout:3722: writing smithi01785587-44 from 2971445 to 3647449 tid 3 2024-09-21T09:53:41.764 INFO:tasks.rados.rados.0.smithi017.stdout:3723: delete oid 7 current snap is 464 2024-09-21T09:53:41.764 INFO:tasks.rados.rados.0.smithi017.stdout:3718: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:53:41.764 INFO:tasks.rados.rados.0.smithi017.stdout:3718: finishing rollback tid 2 to smithi01785587-5 2024-09-21T09:53:41.764 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 2148 (ObjNum 1373 snap 457 seq_num 1373) dirty exists 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:3719: finishing rollback tid 2 to smithi01785587-20 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 1085 (ObjNum 1383 snap 460 seq_num 1383) dirty exists 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:3720: finishing write tid 1 to smithi01785587-23 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:3720: finishing write tid 2 to smithi01785587-23 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:3720: finishing write tid 3 to smithi01785587-23 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:3720: finishing write tid 4 to smithi01785587-23 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:3720: finishing write tid 5 to smithi01785587-23 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1954 (ObjNum 1406 snap 464 seq_num 1406) dirty exists 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:3720: left oid 23 (ObjNum 1406 snap 464 seq_num 1406) 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:3722: finishing write tid 1 to smithi01785587-44 2024-09-21T09:53:41.765 INFO:tasks.rados.rados.0.smithi017.stdout:3722: finishing write tid 2 to smithi01785587-44 2024-09-21T09:53:41.766 INFO:tasks.rados.rados.0.smithi017.stdout:3718: done (5 left) 2024-09-21T09:53:41.766 INFO:tasks.rados.rados.0.smithi017.stdout:3719: done (4 left) 2024-09-21T09:53:41.766 INFO:tasks.rados.rados.0.smithi017.stdout:3720: done (3 left) 2024-09-21T09:53:41.767 INFO:tasks.rados.rados.0.smithi017.stdout:3721: done (2 left) 2024-09-21T09:53:41.767 INFO:tasks.rados.rados.0.smithi017.stdout:3723: done (1 left) 2024-09-21T09:53:41.767 INFO:tasks.rados.rados.0.smithi017.stdout:3724: rollback oid 17 current snap is 464 2024-09-21T09:53:41.767 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 17 to 453 2024-09-21T09:53:41.767 INFO:tasks.rados.rados.0.smithi017.stdout:3725: snap_remove snap 457 2024-09-21T09:53:42.273 INFO:tasks.rados.rados.0.smithi017.stdout:3725: done (2 left) 2024-09-21T09:53:42.286 INFO:tasks.rados.rados.0.smithi017.stdout:3726: read oid 46 snap -1 2024-09-21T09:53:42.286 INFO:tasks.rados.rados.0.smithi017.stdout:3726: expect (ObjNum 1318 snap 441 seq_num 1318) 2024-09-21T09:53:42.286 INFO:tasks.rados.rados.0.smithi017.stdout:3727: snap_remove snap 444 2024-09-21T09:53:42.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:42 smithi017 ceph-mon[102913]: osdmap e996: 8 total, 8 up, 8 in 2024-09-21T09:53:42.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:42 smithi017 ceph-mon[105151]: osdmap e996: 8 total, 8 up, 8 in 2024-09-21T09:53:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:42 smithi134 ceph-mon[90085]: osdmap e996: 8 total, 8 up, 8 in 2024-09-21T09:53:43.227 INFO:tasks.rados.rados.0.smithi017.stdout:3727: done (3 left) 2024-09-21T09:53:43.228 INFO:tasks.rados.rados.0.smithi017.stdout:3728: delete oid 21 current snap is 464 2024-09-21T09:53:43.228 INFO:tasks.rados.rados.0.smithi017.stdout:3722: finishing write tid 3 to smithi01785587-44 2024-09-21T09:53:43.228 INFO:tasks.rados.rados.0.smithi017.stdout:3722: finishing write tid 4 to smithi01785587-44 2024-09-21T09:53:43.228 INFO:tasks.rados.rados.0.smithi017.stdout:3722: finishing write tid 5 to smithi01785587-44 2024-09-21T09:53:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 2153 (ObjNum 1407 snap 464 seq_num 1407) dirty exists 2024-09-21T09:53:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3722: left oid 44 (ObjNum 1407 snap 464 seq_num 1407) 2024-09-21T09:53:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3724: finishing rollback tid 1 to smithi01785587-17 2024-09-21T09:53:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3724: finishing rollback tid 2 to smithi01785587-17 2024-09-21T09:53:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 1288 (ObjNum 1340 snap 448 seq_num 1340) dirty exists 2024-09-21T09:53:43.229 INFO:tasks.rados.rados.0.smithi017.stdout:3726: expect (ObjNum 1318 snap 441 seq_num 1318) 2024-09-21T09:53:43.392 INFO:tasks.rados.rados.0.smithi017.stdout:3722: done (3 left) 2024-09-21T09:53:43.392 INFO:tasks.rados.rados.0.smithi017.stdout:3724: done (2 left) 2024-09-21T09:53:43.392 INFO:tasks.rados.rados.0.smithi017.stdout:3726: done (1 left) 2024-09-21T09:53:43.392 INFO:tasks.rados.rados.0.smithi017.stdout:3728: done (0 left) 2024-09-21T09:53:43.392 INFO:tasks.rados.rados.0.smithi017.stdout:3729: snap_create 2024-09-21T09:53:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:43 smithi017 ceph-mon[102913]: pgmap v1074: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-09-21T09:53:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:43 smithi017 ceph-mon[102913]: osdmap e997: 8 total, 8 up, 8 in 2024-09-21T09:53:43.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:43 smithi017 ceph-mon[102913]: osdmap e998: 8 total, 8 up, 8 in 2024-09-21T09:53:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:43 smithi017 ceph-mon[105151]: pgmap v1074: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-09-21T09:53:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:43 smithi017 ceph-mon[105151]: osdmap e997: 8 total, 8 up, 8 in 2024-09-21T09:53:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:43 smithi017 ceph-mon[105151]: osdmap e998: 8 total, 8 up, 8 in 2024-09-21T09:53:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:43 smithi134 ceph-mon[90085]: pgmap v1074: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-09-21T09:53:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:43 smithi134 ceph-mon[90085]: osdmap e997: 8 total, 8 up, 8 in 2024-09-21T09:53:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:43 smithi134 ceph-mon[90085]: osdmap e998: 8 total, 8 up, 8 in 2024-09-21T09:53:44.277 INFO:tasks.rados.rados.0.smithi017.stdout:3729: done (0 left) 2024-09-21T09:53:44.278 INFO:tasks.rados.rados.0.smithi017.stdout:3730: rollback oid 3 current snap is 465 2024-09-21T09:53:44.278 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 3 to 464 2024-09-21T09:53:44.278 INFO:tasks.rados.rados.0.smithi017.stdout:3731: rollback oid 16 current snap is 465 2024-09-21T09:53:44.278 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 16 to 462 2024-09-21T09:53:44.278 INFO:tasks.rados.rados.0.smithi017.stdout:3732: rollback oid 33 current snap is 465 2024-09-21T09:53:44.278 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 453 2024-09-21T09:53:44.278 INFO:tasks.rados.rados.0.smithi017.stdout:3733: write oid 27 current snap is 465 2024-09-21T09:53:44.278 INFO:tasks.rados.rados.0.smithi017.stdout:3733: seq_num 1410 ranges {497870=426395,1443140=521818,2537361=530244,3583943=113927} 2024-09-21T09:53:44.283 INFO:tasks.rados.rados.0.smithi017.stdout:3733: writing smithi01785587-27 from 497870 to 924265 tid 1 2024-09-21T09:53:44.289 INFO:tasks.rados.rados.0.smithi017.stdout:3733: writing smithi01785587-27 from 1443140 to 1964958 tid 2 2024-09-21T09:53:44.295 INFO:tasks.rados.rados.0.smithi017.stdout:3733: writing smithi01785587-27 from 2537361 to 3067605 tid 3 2024-09-21T09:53:44.299 INFO:tasks.rados.rados.0.smithi017.stdout:3733: writing smithi01785587-27 from 3583943 to 3697870 tid 4 2024-09-21T09:53:44.299 INFO:tasks.rados.rados.0.smithi017.stdout:3734: snap_create 2024-09-21T09:53:44.299 INFO:tasks.rados.rados.0.smithi017.stdout:3730: finishing rollback tid 0 to smithi01785587-3 2024-09-21T09:53:44.299 INFO:tasks.rados.rados.0.smithi017.stdout:3730: finishing rollback tid 1 to smithi01785587-3 2024-09-21T09:53:44.299 INFO:tasks.rados.rados.0.smithi017.stdout:3730: finishing rollback tid 2 to smithi01785587-3 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 2171 (ObjNum 1402 snap 463 seq_num 1402) dirty exists 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:3733: finishing write tid 1 to smithi01785587-27 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:3731: finishing rollback tid 0 to smithi01785587-16 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:3731: finishing rollback tid 1 to smithi01785587-16 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:3731: finishing rollback tid 2 to smithi01785587-16 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 2016 (ObjNum 1389 snap 461 seq_num 1389) dirty exists 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:3732: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:3732: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:3732: finishing rollback tid 2 to smithi01785587-33 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 2020 (ObjNum 1329 snap 446 seq_num 1329) dirty exists 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:3733: finishing write tid 2 to smithi01785587-27 2024-09-21T09:53:44.300 INFO:tasks.rados.rados.0.smithi017.stdout:3733: finishing write tid 3 to smithi01785587-27 2024-09-21T09:53:44.302 INFO:tasks.rados.rados.0.smithi017.stdout:3733: finishing write tid 4 to smithi01785587-27 2024-09-21T09:53:44.302 INFO:tasks.rados.rados.0.smithi017.stdout:3733: finishing write tid 5 to smithi01785587-27 2024-09-21T09:53:44.302 INFO:tasks.rados.rados.0.smithi017.stdout:3733: finishing write tid 6 to smithi01785587-27 2024-09-21T09:53:44.302 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 1301 (ObjNum 1410 snap 465 seq_num 1410) dirty exists 2024-09-21T09:53:44.302 INFO:tasks.rados.rados.0.smithi017.stdout:3733: left oid 27 (ObjNum 1410 snap 465 seq_num 1410) 2024-09-21T09:53:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:53:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:53:44] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:53:45.280 INFO:tasks.rados.rados.0.smithi017.stdout:3730: done (4 left) 2024-09-21T09:53:45.280 INFO:tasks.rados.rados.0.smithi017.stdout:3731: done (3 left) 2024-09-21T09:53:45.280 INFO:tasks.rados.rados.0.smithi017.stdout:3732: done (2 left) 2024-09-21T09:53:45.281 INFO:tasks.rados.rados.0.smithi017.stdout:3733: done (1 left) 2024-09-21T09:53:45.281 INFO:tasks.rados.rados.0.smithi017.stdout:3734: done (0 left) 2024-09-21T09:53:45.281 INFO:tasks.rados.rados.0.smithi017.stdout:3735: snap_create 2024-09-21T09:53:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:45 smithi017 ceph-mon[102913]: pgmap v1077: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.2 MiB/s wr, 11 op/s 2024-09-21T09:53:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:45 smithi017 ceph-mon[102913]: osdmap e999: 8 total, 8 up, 8 in 2024-09-21T09:53:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:45 smithi017 ceph-mon[105151]: pgmap v1077: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.2 MiB/s wr, 11 op/s 2024-09-21T09:53:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:45 smithi017 ceph-mon[105151]: osdmap e999: 8 total, 8 up, 8 in 2024-09-21T09:53:45.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:45 smithi134 ceph-mon[90085]: pgmap v1077: 89 pgs: 89 active+clean; 181 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.2 MiB/s wr, 11 op/s 2024-09-21T09:53:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:45 smithi134 ceph-mon[90085]: osdmap e999: 8 total, 8 up, 8 in 2024-09-21T09:53:46.290 INFO:tasks.rados.rados.0.smithi017.stdout:3735: done (0 left) 2024-09-21T09:53:46.291 INFO:tasks.rados.rados.0.smithi017.stdout:3736: write oid 6 current snap is 467 2024-09-21T09:53:46.291 INFO:tasks.rados.rados.0.smithi017.stdout:3736: seq_num 1411 ranges {514677=501644,1501622=781829,2964784=741516} 2024-09-21T09:53:46.296 INFO:tasks.rados.rados.0.smithi017.stdout:3736: writing smithi01785587-6 from 514677 to 1016321 tid 1 2024-09-21T09:53:46.303 INFO:tasks.rados.rados.0.smithi017.stdout:3736: writing smithi01785587-6 from 1501622 to 2283451 tid 2 2024-09-21T09:53:46.311 INFO:tasks.rados.rados.0.smithi017.stdout:3736: writing smithi01785587-6 from 2964784 to 3706300 tid 3 2024-09-21T09:53:46.312 INFO:tasks.rados.rados.0.smithi017.stdout:3737: read oid 23 snap -1 2024-09-21T09:53:46.312 INFO:tasks.rados.rados.0.smithi017.stdout:3737: expect (ObjNum 1406 snap 464 seq_num 1406) 2024-09-21T09:53:46.312 INFO:tasks.rados.rados.0.smithi017.stdout:3738: snap_remove snap 462 2024-09-21T09:53:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:46 smithi017 ceph-mon[102913]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-21T09:53:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:46 smithi017 ceph-mon[105151]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-21T09:53:46.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:46 smithi134 ceph-mon[90085]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:3738: done (2 left) 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:3739: rollback oid 48 current snap is 467 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 48 to 460 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:3736: finishing write tid 1 to smithi01785587-6 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:3736: finishing write tid 2 to smithi01785587-6 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:3736: finishing write tid 3 to smithi01785587-6 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:3736: finishing write tid 4 to smithi01785587-6 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:3736: finishing write tid 5 to smithi01785587-6 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 1091 (ObjNum 1411 snap 467 seq_num 1411) dirty exists 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:3736: left oid 6 (ObjNum 1411 snap 467 seq_num 1411) 2024-09-21T09:53:47.296 INFO:tasks.rados.rados.0.smithi017.stdout:3737: expect (ObjNum 1406 snap 464 seq_num 1406) 2024-09-21T09:53:47.473 INFO:tasks.rados.rados.0.smithi017.stdout:3739: finishing rollback tid 1 to smithi01785587-48 2024-09-21T09:53:47.474 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 0 (ObjNum 65 snap 0 seq_num 2399961104) dirty dne 2024-09-21T09:53:47.474 INFO:tasks.rados.rados.0.smithi017.stdout:3736: done (2 left) 2024-09-21T09:53:47.474 INFO:tasks.rados.rados.0.smithi017.stdout:3737: done (1 left) 2024-09-21T09:53:47.474 INFO:tasks.rados.rados.0.smithi017.stdout:3739: done (0 left) 2024-09-21T09:53:47.474 INFO:tasks.rados.rados.0.smithi017.stdout:3740: snap_remove snap 466 2024-09-21T09:53:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:47 smithi134 ceph-mon[90085]: pgmap v1080: 89 pgs: 89 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 858 KiB/s wr, 9 op/s 2024-09-21T09:53:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:47 smithi134 ceph-mon[90085]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-21T09:53:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:47 smithi017 ceph-mon[102913]: pgmap v1080: 89 pgs: 89 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 858 KiB/s wr, 9 op/s 2024-09-21T09:53:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:47 smithi017 ceph-mon[102913]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-21T09:53:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:47 smithi017 ceph-mon[105151]: pgmap v1080: 89 pgs: 89 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 858 KiB/s wr, 9 op/s 2024-09-21T09:53:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:47 smithi017 ceph-mon[105151]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-21T09:53:48.230 INFO:tasks.rados.rados.0.smithi017.stdout:3740: done (0 left) 2024-09-21T09:53:48.230 INFO:tasks.rados.rados.0.smithi017.stdout:3741: rollback oid 39 current snap is 467 2024-09-21T09:53:48.230 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 453 2024-09-21T09:53:48.230 INFO:tasks.rados.rados.0.smithi017.stdout:3742: snap_create 2024-09-21T09:53:48.237 INFO:tasks.rados.rados.0.smithi017.stdout:3741: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:53:48.237 INFO:tasks.rados.rados.0.smithi017.stdout:3741: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:53:48.237 INFO:tasks.rados.rados.0.smithi017.stdout:3741: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:53:48.238 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1937 (ObjNum 1320 snap 444 seq_num 1320) dirty exists 2024-09-21T09:53:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:48 smithi017 ceph-mon[102913]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-21T09:53:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:48 smithi017 ceph-mon[102913]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-21T09:53:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:48 smithi017 ceph-mon[105151]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-21T09:53:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:48 smithi017 ceph-mon[105151]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-21T09:53:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:48 smithi134 ceph-mon[90085]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-21T09:53:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:48 smithi134 ceph-mon[90085]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-21T09:53:49.233 INFO:tasks.rados.rados.0.smithi017.stdout:3741: done (1 left) 2024-09-21T09:53:49.233 INFO:tasks.rados.rados.0.smithi017.stdout:3742: done (0 left) 2024-09-21T09:53:49.233 INFO:tasks.rados.rados.0.smithi017.stdout:3743: snap_remove snap 461 2024-09-21T09:53:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:49 smithi017 ceph-mon[102913]: pgmap v1083: 89 pgs: 89 active+clean; 146 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:53:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:49 smithi017 ceph-mon[102913]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-21T09:53:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:49 smithi017 ceph-mon[105151]: pgmap v1083: 89 pgs: 89 active+clean; 146 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:53:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:49 smithi017 ceph-mon[105151]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-21T09:53:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:49 smithi134 ceph-mon[90085]: pgmap v1083: 89 pgs: 89 active+clean; 146 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:53:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:49 smithi134 ceph-mon[90085]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-21T09:53:50.234 INFO:tasks.rados.rados.0.smithi017.stdout:3743: done (0 left) 2024-09-21T09:53:50.234 INFO:tasks.rados.rados.0.smithi017.stdout:3744: delete oid 41 current snap is 468 2024-09-21T09:53:50.240 INFO:tasks.rados.rados.0.smithi017.stdout:3744: done (0 left) 2024-09-21T09:53:50.241 INFO:tasks.rados.rados.0.smithi017.stdout:3745: write oid 36 current snap is 468 2024-09-21T09:53:50.241 INFO:tasks.rados.rados.0.smithi017.stdout:3745: seq_num 1413 ranges {548291=652142,1902233=785498,3303277=445014} 2024-09-21T09:53:50.247 INFO:tasks.rados.rados.0.smithi017.stdout:3745: writing smithi01785587-36 from 548291 to 1200433 tid 1 2024-09-21T09:53:50.255 INFO:tasks.rados.rados.0.smithi017.stdout:3745: writing smithi01785587-36 from 1902233 to 2687731 tid 2 2024-09-21T09:53:50.262 INFO:tasks.rados.rados.0.smithi017.stdout:3745: writing smithi01785587-36 from 3303277 to 3748291 tid 3 2024-09-21T09:53:50.262 INFO:tasks.rados.rados.0.smithi017.stdout:3746: write oid 9 current snap is 468 2024-09-21T09:53:50.262 INFO:tasks.rados.rados.0.smithi017.stdout:3746: seq_num 1414 ranges {565098=727391,1960715=529156,2930700=691685} 2024-09-21T09:53:50.269 INFO:tasks.rados.rados.0.smithi017.stdout:3746: writing smithi01785587-9 from 565098 to 1292489 tid 1 2024-09-21T09:53:50.275 INFO:tasks.rados.rados.0.smithi017.stdout:3746: writing smithi01785587-9 from 1960715 to 2489871 tid 2 2024-09-21T09:53:50.282 INFO:tasks.rados.rados.0.smithi017.stdout:3746: writing smithi01785587-9 from 2930700 to 3622385 tid 3 2024-09-21T09:53:50.282 INFO:tasks.rados.rados.0.smithi017.stdout:3747: write oid 3 current snap is 468 2024-09-21T09:53:50.282 INFO:tasks.rados.rados.0.smithi017.stdout:3745: finishing write tid 1 to smithi01785587-36 2024-09-21T09:53:50.282 INFO:tasks.rados.rados.0.smithi017.stdout:3747: seq_num 1415 ranges {581905=518993,1735550=672814,2958123=502957} 2024-09-21T09:53:50.288 INFO:tasks.rados.rados.0.smithi017.stdout:3747: writing smithi01785587-3 from 581905 to 1100898 tid 1 2024-09-21T09:53:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3747: writing smithi01785587-3 from 1735550 to 2408364 tid 2 2024-09-21T09:53:50.301 INFO:tasks.rados.rados.0.smithi017.stdout:3747: writing smithi01785587-3 from 2958123 to 3461080 tid 3 2024-09-21T09:53:50.301 INFO:tasks.rados.rados.0.smithi017.stdout:3748: write oid 27 current snap is 468 2024-09-21T09:53:50.302 INFO:tasks.rados.rados.0.smithi017.stdout:3745: finishing write tid 2 to smithi01785587-36 2024-09-21T09:53:50.302 INFO:tasks.rados.rados.0.smithi017.stdout:3745: finishing write tid 3 to smithi01785587-36 2024-09-21T09:53:50.302 INFO:tasks.rados.rados.0.smithi017.stdout:3748: seq_num 1416 ranges {598712=594242,1677679=532825,2985546=430582} 2024-09-21T09:53:50.308 INFO:tasks.rados.rados.0.smithi017.stdout:3748: writing smithi01785587-27 from 598712 to 1192954 tid 1 2024-09-21T09:53:50.314 INFO:tasks.rados.rados.0.smithi017.stdout:3748: writing smithi01785587-27 from 1677679 to 2210504 tid 2 2024-09-21T09:53:50.321 INFO:tasks.rados.rados.0.smithi017.stdout:3748: writing smithi01785587-27 from 2985546 to 3416128 tid 3 2024-09-21T09:53:50.321 INFO:tasks.rados.rados.0.smithi017.stdout:3749: delete oid 4 current snap is 468 2024-09-21T09:53:50.321 INFO:tasks.rados.rados.0.smithi017.stdout:3745: finishing write tid 4 to smithi01785587-36 2024-09-21T09:53:50.321 INFO:tasks.rados.rados.0.smithi017.stdout:3745: finishing write tid 5 to smithi01785587-36 2024-09-21T09:53:50.321 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 1945 (ObjNum 1413 snap 468 seq_num 1413) dirty exists 2024-09-21T09:53:50.321 INFO:tasks.rados.rados.0.smithi017.stdout:3745: left oid 36 (ObjNum 1413 snap 468 seq_num 1413) 2024-09-21T09:53:50.321 INFO:tasks.rados.rados.0.smithi017.stdout:3746: finishing write tid 1 to smithi01785587-9 2024-09-21T09:53:50.321 INFO:tasks.rados.rados.0.smithi017.stdout:3746: finishing write tid 2 to smithi01785587-9 2024-09-21T09:53:50.321 INFO:tasks.rados.rados.0.smithi017.stdout:3746: finishing write tid 3 to smithi01785587-9 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3746: finishing write tid 4 to smithi01785587-9 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3746: finishing write tid 5 to smithi01785587-9 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 736 (ObjNum 1414 snap 468 seq_num 1414) dirty exists 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3746: left oid 9 (ObjNum 1414 snap 468 seq_num 1414) 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3747: finishing write tid 1 to smithi01785587-3 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3747: finishing write tid 2 to smithi01785587-3 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3747: finishing write tid 3 to smithi01785587-3 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3747: finishing write tid 4 to smithi01785587-3 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3747: finishing write tid 5 to smithi01785587-3 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 2186 (ObjNum 1415 snap 468 seq_num 1415) dirty exists 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3747: left oid 3 (ObjNum 1415 snap 468 seq_num 1415) 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3748: finishing write tid 1 to smithi01785587-27 2024-09-21T09:53:50.322 INFO:tasks.rados.rados.0.smithi017.stdout:3748: finishing write tid 2 to smithi01785587-27 2024-09-21T09:53:50.323 INFO:tasks.rados.rados.0.smithi017.stdout:3745: done (4 left) 2024-09-21T09:53:50.323 INFO:tasks.rados.rados.0.smithi017.stdout:3746: done (3 left) 2024-09-21T09:53:50.323 INFO:tasks.rados.rados.0.smithi017.stdout:3747: done (2 left) 2024-09-21T09:53:50.323 INFO:tasks.rados.rados.0.smithi017.stdout:3749: done (1 left) 2024-09-21T09:53:50.323 INFO:tasks.rados.rados.0.smithi017.stdout:3750: snap_remove snap 460 2024-09-21T09:53:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:53:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:53:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:51.306 INFO:tasks.rados.rados.0.smithi017.stdout:3750: done (1 left) 2024-09-21T09:53:51.306 INFO:tasks.rados.rados.0.smithi017.stdout:3751: read oid 16 snap -1 2024-09-21T09:53:51.306 INFO:tasks.rados.rados.0.smithi017.stdout:3751: expect (ObjNum 1389 snap 461 seq_num 1389) 2024-09-21T09:53:51.306 INFO:tasks.rados.rados.0.smithi017.stdout:3752: read oid 25 snap -1 2024-09-21T09:53:51.306 INFO:tasks.rados.rados.0.smithi017.stdout:3752: expect (ObjNum 1304 snap 440 seq_num 1304) 2024-09-21T09:53:51.306 INFO:tasks.rados.rados.0.smithi017.stdout:3753: delete oid 42 current snap is 468 2024-09-21T09:53:51.306 INFO:tasks.rados.rados.0.smithi017.stdout:3748: finishing write tid 3 to smithi01785587-27 2024-09-21T09:53:51.307 INFO:tasks.rados.rados.0.smithi017.stdout:3748: finishing write tid 4 to smithi01785587-27 2024-09-21T09:53:51.307 INFO:tasks.rados.rados.0.smithi017.stdout:3748: finishing write tid 5 to smithi01785587-27 2024-09-21T09:53:51.307 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 1310 (ObjNum 1416 snap 468 seq_num 1416) dirty exists 2024-09-21T09:53:51.307 INFO:tasks.rados.rados.0.smithi017.stdout:3748: left oid 27 (ObjNum 1416 snap 468 seq_num 1416) 2024-09-21T09:53:51.318 INFO:tasks.rados.rados.0.smithi017.stdout:3748: done (3 left) 2024-09-21T09:53:51.318 INFO:tasks.rados.rados.0.smithi017.stdout:3753: done (2 left) 2024-09-21T09:53:51.318 INFO:tasks.rados.rados.0.smithi017.stdout:3754: read oid 21 snap -1 2024-09-21T09:53:51.318 INFO:tasks.rados.rados.0.smithi017.stdout:3754: expect deleted 2024-09-21T09:53:51.318 INFO:tasks.rados.rados.0.smithi017.stdout:3755: write oid 28 current snap is 468 2024-09-21T09:53:51.318 INFO:tasks.rados.rados.0.smithi017.stdout:3755: seq_num 1419 ranges {649133=419989,1853125=680152,2951462=664398} 2024-09-21T09:53:51.325 INFO:tasks.rados.rados.0.smithi017.stdout:3755: writing smithi01785587-28 from 649133 to 1069122 tid 1 2024-09-21T09:53:51.333 INFO:tasks.rados.rados.0.smithi017.stdout:3755: writing smithi01785587-28 from 1853125 to 2533277 tid 2 2024-09-21T09:53:51.339 INFO:tasks.rados.rados.0.smithi017.stdout:3755: writing smithi01785587-28 from 2951462 to 3615860 tid 3 2024-09-21T09:53:51.339 INFO:tasks.rados.rados.0.smithi017.stdout:3756: delete oid 38 current snap is 468 2024-09-21T09:53:51.339 INFO:tasks.rados.rados.0.smithi017.stdout:3751: expect (ObjNum 1389 snap 461 seq_num 1389) 2024-09-21T09:53:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:51 smithi134 ceph-mon[90085]: pgmap v1086: 89 pgs: 89 active+clean; 145 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 803 KiB/s wr, 5 op/s 2024-09-21T09:53:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:51 smithi134 ceph-mon[90085]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-21T09:53:51.666 INFO:tasks.rados.rados.0.smithi017.stdout:3755: finishing write tid 1 to smithi01785587-28 2024-09-21T09:53:51.666 INFO:tasks.rados.rados.0.smithi017.stdout:3752: expect (ObjNum 1304 snap 440 seq_num 1304) 2024-09-21T09:53:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:51 smithi017 ceph-mon[102913]: pgmap v1086: 89 pgs: 89 active+clean; 145 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 803 KiB/s wr, 5 op/s 2024-09-21T09:53:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:51 smithi017 ceph-mon[102913]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-21T09:53:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:51 smithi017 ceph-mon[105151]: pgmap v1086: 89 pgs: 89 active+clean; 145 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 803 KiB/s wr, 5 op/s 2024-09-21T09:53:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:51 smithi017 ceph-mon[105151]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-21T09:53:51.975 INFO:tasks.rados.rados.0.smithi017.stdout:3755: finishing write tid 2 to smithi01785587-28 2024-09-21T09:53:51.975 INFO:tasks.rados.rados.0.smithi017.stdout:3755: finishing write tid 3 to smithi01785587-28 2024-09-21T09:53:51.975 INFO:tasks.rados.rados.0.smithi017.stdout:3755: finishing write tid 4 to smithi01785587-28 2024-09-21T09:53:51.975 INFO:tasks.rados.rados.0.smithi017.stdout:3755: finishing write tid 5 to smithi01785587-28 2024-09-21T09:53:51.975 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 2040 (ObjNum 1419 snap 468 seq_num 1419) dirty exists 2024-09-21T09:53:51.975 INFO:tasks.rados.rados.0.smithi017.stdout:3755: left oid 28 (ObjNum 1419 snap 468 seq_num 1419) 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3751: done (4 left) 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3752: done (3 left) 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3754: done (2 left) 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3755: done (1 left) 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3756: done (0 left) 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3757: read oid 2 snap -1 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3757: expect deleted 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3758: read oid 13 snap 453 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3758: expect (ObjNum 1353 snap 451 seq_num 1353) 2024-09-21T09:53:51.976 INFO:tasks.rados.rados.0.smithi017.stdout:3759: snap_remove snap 463 2024-09-21T09:53:52.315 INFO:tasks.rados.rados.0.smithi017.stdout:3758: expect (ObjNum 1353 snap 451 seq_num 1353) 2024-09-21T09:53:52.491 INFO:tasks.rados.rados.0.smithi017.stdout:3757: done (2 left) 2024-09-21T09:53:52.491 INFO:tasks.rados.rados.0.smithi017.stdout:3758: done (1 left) 2024-09-21T09:53:52.491 INFO:tasks.rados.rados.0.smithi017.stdout:3759: done (0 left) 2024-09-21T09:53:52.491 INFO:tasks.rados.rados.0.smithi017.stdout:3760: delete oid 34 current snap is 468 2024-09-21T09:53:52.494 INFO:tasks.rados.rados.0.smithi017.stdout:3760: done (0 left) 2024-09-21T09:53:52.494 INFO:tasks.rados.rados.0.smithi017.stdout:3761: write oid 15 current snap is 468 2024-09-21T09:53:52.494 INFO:tasks.rados.rados.0.smithi017.stdout:3761: seq_num 1422 ranges {699554=762089,2028571=543832,3150084=614567} 2024-09-21T09:53:52.502 INFO:tasks.rados.rados.0.smithi017.stdout:3761: writing smithi01785587-15 from 699554 to 1461643 tid 1 2024-09-21T09:53:52.508 INFO:tasks.rados.rados.0.smithi017.stdout:3761: writing smithi01785587-15 from 2028571 to 2572403 tid 2 2024-09-21T09:53:52.514 INFO:tasks.rados.rados.0.smithi017.stdout:3761: writing smithi01785587-15 from 3150084 to 3764651 tid 3 2024-09-21T09:53:52.515 INFO:tasks.rados.rados.0.smithi017.stdout:3762: write oid 30 current snap is 468 2024-09-21T09:53:52.515 INFO:tasks.rados.rados.0.smithi017.stdout:3762: seq_num 1423 ranges {716361=437338,1687053=687490,2777507=425839,3877756=38605} 2024-09-21T09:53:52.521 INFO:tasks.rados.rados.0.smithi017.stdout:3762: writing smithi01785587-30 from 716361 to 1153699 tid 1 2024-09-21T09:53:52.527 INFO:tasks.rados.rados.0.smithi017.stdout:3762: writing smithi01785587-30 from 1687053 to 2374543 tid 2 2024-09-21T09:53:52.532 INFO:tasks.rados.rados.0.smithi017.stdout:3762: writing smithi01785587-30 from 2777507 to 3203346 tid 3 2024-09-21T09:53:52.536 INFO:tasks.rados.rados.0.smithi017.stdout:3762: writing smithi01785587-30 from 3877756 to 3916361 tid 4 2024-09-21T09:53:52.536 INFO:tasks.rados.rados.0.smithi017.stdout:3763: snap_remove snap 465 2024-09-21T09:53:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:52 smithi017 ceph-mon[102913]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-21T09:53:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:52 smithi017 ceph-mon[105151]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-21T09:53:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:52 smithi134 ceph-mon[90085]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-21T09:53:53.232 INFO:tasks.rados.rados.0.smithi017.stdout:3763: done (2 left) 2024-09-21T09:53:53.232 INFO:tasks.rados.rados.0.smithi017.stdout:3764: snap_create 2024-09-21T09:53:53.232 INFO:tasks.rados.rados.0.smithi017.stdout:3761: finishing write tid 1 to smithi01785587-15 2024-09-21T09:53:53.232 INFO:tasks.rados.rados.0.smithi017.stdout:3761: finishing write tid 2 to smithi01785587-15 2024-09-21T09:53:53.232 INFO:tasks.rados.rados.0.smithi017.stdout:3761: finishing write tid 3 to smithi01785587-15 2024-09-21T09:53:53.232 INFO:tasks.rados.rados.0.smithi017.stdout:3761: finishing write tid 4 to smithi01785587-15 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:3761: finishing write tid 5 to smithi01785587-15 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 577 (ObjNum 1422 snap 468 seq_num 1422) dirty exists 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:3761: left oid 15 (ObjNum 1422 snap 468 seq_num 1422) 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:3762: finishing write tid 1 to smithi01785587-30 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:3762: finishing write tid 2 to smithi01785587-30 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:3762: finishing write tid 3 to smithi01785587-30 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:3762: finishing write tid 4 to smithi01785587-30 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:3762: finishing write tid 5 to smithi01785587-30 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:3762: finishing write tid 6 to smithi01785587-30 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1594 (ObjNum 1423 snap 468 seq_num 1423) dirty exists 2024-09-21T09:53:53.233 INFO:tasks.rados.rados.0.smithi017.stdout:3762: left oid 30 (ObjNum 1423 snap 468 seq_num 1423) 2024-09-21T09:53:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:53 smithi017 ceph-mon[102913]: pgmap v1089: 89 pgs: 89 active+clean; 147 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 884 KiB/s wr, 6 op/s 2024-09-21T09:53:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:53 smithi017 ceph-mon[102913]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-21T09:53:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:53 smithi017 ceph-mon[102913]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-21T09:53:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:53 smithi017 ceph-mon[105151]: pgmap v1089: 89 pgs: 89 active+clean; 147 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 884 KiB/s wr, 6 op/s 2024-09-21T09:53:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:53 smithi017 ceph-mon[105151]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-21T09:53:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:53 smithi017 ceph-mon[105151]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-21T09:53:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:53 smithi134 ceph-mon[90085]: pgmap v1089: 89 pgs: 89 active+clean; 147 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 884 KiB/s wr, 6 op/s 2024-09-21T09:53:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:53 smithi134 ceph-mon[90085]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-21T09:53:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:53 smithi134 ceph-mon[90085]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-21T09:53:54.238 INFO:tasks.rados.rados.0.smithi017.stdout:3761: done (2 left) 2024-09-21T09:53:54.239 INFO:tasks.rados.rados.0.smithi017.stdout:3762: done (1 left) 2024-09-21T09:53:54.239 INFO:tasks.rados.rados.0.smithi017.stdout:3764: done (0 left) 2024-09-21T09:53:54.239 INFO:tasks.rados.rados.0.smithi017.stdout:3765: read oid 2 snap -1 2024-09-21T09:53:54.239 INFO:tasks.rados.rados.0.smithi017.stdout:3765: expect deleted 2024-09-21T09:53:54.239 INFO:tasks.rados.rados.0.smithi017.stdout:3766: snap_create 2024-09-21T09:53:55.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:53:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:53:54] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:53:55.240 INFO:tasks.rados.rados.0.smithi017.stdout:3765: done (1 left) 2024-09-21T09:53:55.241 INFO:tasks.rados.rados.0.smithi017.stdout:3766: done (0 left) 2024-09-21T09:53:55.241 INFO:tasks.rados.rados.0.smithi017.stdout:3767: snap_create 2024-09-21T09:53:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:55 smithi134 ceph-mon[90085]: pgmap v1092: 89 pgs: 89 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-21T09:53:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:55 smithi134 ceph-mon[90085]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-21T09:53:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:55 smithi017 ceph-mon[102913]: pgmap v1092: 89 pgs: 89 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-21T09:53:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:55 smithi017 ceph-mon[102913]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-21T09:53:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:55 smithi017 ceph-mon[105151]: pgmap v1092: 89 pgs: 89 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-09-21T09:53:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:55 smithi017 ceph-mon[105151]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-21T09:53:56.251 INFO:tasks.rados.rados.0.smithi017.stdout:3767: done (0 left) 2024-09-21T09:53:56.251 INFO:tasks.rados.rados.0.smithi017.stdout:3768: snap_remove snap 470 2024-09-21T09:53:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:56 smithi017 ceph-mon[102913]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-21T09:53:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:56 smithi017 ceph-mon[105151]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-21T09:53:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:56 smithi134 ceph-mon[90085]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-21T09:53:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:57.253 INFO:tasks.rados.rados.0.smithi017.stdout:3768: done (0 left) 2024-09-21T09:53:57.254 INFO:tasks.rados.rados.0.smithi017.stdout:3769: snap_remove snap 468 2024-09-21T09:53:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:57 smithi017 ceph-mon[105151]: pgmap v1095: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-09-21T09:53:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:57 smithi017 ceph-mon[105151]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-21T09:53:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:57 smithi017 ceph-mon[102913]: pgmap v1095: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-09-21T09:53:57.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:57 smithi017 ceph-mon[102913]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-21T09:53:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:57 smithi134 ceph-mon[90085]: pgmap v1095: 89 pgs: 89 active+clean; 165 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-09-21T09:53:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:57 smithi134 ceph-mon[90085]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-21T09:53:58.233 INFO:tasks.rados.rados.0.smithi017.stdout:3769: done (0 left) 2024-09-21T09:53:58.233 INFO:tasks.rados.rados.0.smithi017.stdout:3770: write oid 39 current snap is 471 2024-09-21T09:53:58.234 INFO:tasks.rados.rados.0.smithi017.stdout:3770: seq_num 1424 ranges {733168=512587,2029182=547501,3088577=637111} 2024-09-21T09:53:58.240 INFO:tasks.rados.rados.0.smithi017.stdout:3770: writing smithi01785587-39 from 733168 to 1245755 tid 1 2024-09-21T09:53:58.248 INFO:tasks.rados.rados.0.smithi017.stdout:3770: writing smithi01785587-39 from 2029182 to 2576683 tid 2 2024-09-21T09:53:58.255 INFO:tasks.rados.rados.0.smithi017.stdout:3770: writing smithi01785587-39 from 3088577 to 3725688 tid 3 2024-09-21T09:53:58.255 INFO:tasks.rados.rados.0.smithi017.stdout:3771: write oid 30 current snap is 471 2024-09-21T09:53:58.255 INFO:tasks.rados.rados.0.smithi017.stdout:3770: finishing write tid 1 to smithi01785587-39 2024-09-21T09:53:58.255 INFO:tasks.rados.rados.0.smithi017.stdout:3770: finishing write tid 2 to smithi01785587-39 2024-09-21T09:53:58.255 INFO:tasks.rados.rados.0.smithi017.stdout:3771: seq_num 1425 ranges {749975=587836,2087664=691159,3516000=433975} 2024-09-21T09:53:58.262 INFO:tasks.rados.rados.0.smithi017.stdout:3771: writing smithi01785587-30 from 749975 to 1337811 tid 1 2024-09-21T09:53:58.270 INFO:tasks.rados.rados.0.smithi017.stdout:3771: writing smithi01785587-30 from 2087664 to 2778823 tid 2 2024-09-21T09:53:58.277 INFO:tasks.rados.rados.0.smithi017.stdout:3771: writing smithi01785587-30 from 3516000 to 3949975 tid 3 2024-09-21T09:53:58.277 INFO:tasks.rados.rados.0.smithi017.stdout:3772: write oid 1 current snap is 471 2024-09-21T09:53:58.277 INFO:tasks.rados.rados.0.smithi017.stdout:3772: seq_num 1426 ranges {766782=663085,2146146=551170,3143423=776008} 2024-09-21T09:53:58.284 INFO:tasks.rados.rados.0.smithi017.stdout:3772: writing smithi01785587-1 from 766782 to 1429867 tid 1 2024-09-21T09:53:58.293 INFO:tasks.rados.rados.0.smithi017.stdout:3772: writing smithi01785587-1 from 2146146 to 2697316 tid 2 2024-09-21T09:53:58.298 INFO:tasks.rados.rados.0.smithi017.stdout:3772: writing smithi01785587-1 from 3143423 to 3919431 tid 3 2024-09-21T09:53:58.299 INFO:tasks.rados.rados.0.smithi017.stdout:3773: snap_remove snap 464 2024-09-21T09:53:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:58 smithi017 ceph-mon[102913]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-21T09:53:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:58 smithi017 ceph-mon[102913]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-21T09:53:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:58 smithi017 ceph-mon[105151]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-21T09:53:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:58 smithi017 ceph-mon[105151]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-21T09:53:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:58 smithi134 ceph-mon[90085]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-21T09:53:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:58 smithi134 ceph-mon[90085]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-21T09:53:59.268 INFO:tasks.rados.rados.0.smithi017.stdout:3770: finishing write tid 3 to smithi01785587-39 2024-09-21T09:53:59.268 INFO:tasks.rados.rados.0.smithi017.stdout:3770: finishing write tid 4 to smithi01785587-39 2024-09-21T09:53:59.268 INFO:tasks.rados.rados.0.smithi017.stdout:3770: finishing write tid 5 to smithi01785587-39 2024-09-21T09:53:59.268 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1958 (ObjNum 1424 snap 471 seq_num 1424) dirty exists 2024-09-21T09:53:59.268 INFO:tasks.rados.rados.0.smithi017.stdout:3770: left oid 39 (ObjNum 1424 snap 471 seq_num 1424) 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3771: finishing write tid 1 to smithi01785587-30 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3771: finishing write tid 2 to smithi01785587-30 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3771: finishing write tid 3 to smithi01785587-30 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3771: finishing write tid 4 to smithi01785587-30 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3771: finishing write tid 5 to smithi01785587-30 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1601 (ObjNum 1425 snap 471 seq_num 1425) dirty exists 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3771: left oid 30 (ObjNum 1425 snap 471 seq_num 1425) 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3772: finishing write tid 1 to smithi01785587-1 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3772: finishing write tid 2 to smithi01785587-1 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3772: finishing write tid 3 to smithi01785587-1 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3772: finishing write tid 4 to smithi01785587-1 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3772: finishing write tid 5 to smithi01785587-1 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 1096 (ObjNum 1426 snap 471 seq_num 1426) dirty exists 2024-09-21T09:53:59.269 INFO:tasks.rados.rados.0.smithi017.stdout:3772: left oid 1 (ObjNum 1426 snap 471 seq_num 1426) 2024-09-21T09:53:59.270 INFO:tasks.rados.rados.0.smithi017.stdout:3770: done (3 left) 2024-09-21T09:53:59.270 INFO:tasks.rados.rados.0.smithi017.stdout:3771: done (2 left) 2024-09-21T09:53:59.270 INFO:tasks.rados.rados.0.smithi017.stdout:3772: done (1 left) 2024-09-21T09:53:59.270 INFO:tasks.rados.rados.0.smithi017.stdout:3773: done (0 left) 2024-09-21T09:53:59.270 INFO:tasks.rados.rados.0.smithi017.stdout:3774: snap_create 2024-09-21T09:53:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:53:59 smithi017 ceph-mon[102913]: pgmap v1098: 89 pgs: 89 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-21T09:53:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:53:59 smithi017 ceph-mon[105151]: pgmap v1098: 89 pgs: 89 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-21T09:53:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:53:59 smithi134 ceph-mon[90085]: pgmap v1098: 89 pgs: 89 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-21T09:54:00.278 INFO:tasks.rados.rados.0.smithi017.stdout:3774: done (0 left) 2024-09-21T09:54:00.278 INFO:tasks.rados.rados.0.smithi017.stdout:3775: write oid 24 current snap is 472 2024-09-21T09:54:00.278 INFO:tasks.rados.rados.0.smithi017.stdout:3775: seq_num 1427 ranges {783589=738334,2204628=694828,3570846=412743} 2024-09-21T09:54:00.286 INFO:tasks.rados.rados.0.smithi017.stdout:3775: writing smithi01785587-24 from 783589 to 1521923 tid 1 2024-09-21T09:54:00.294 INFO:tasks.rados.rados.0.smithi017.stdout:3775: writing smithi01785587-24 from 2204628 to 2899456 tid 2 2024-09-21T09:54:00.301 INFO:tasks.rados.rados.0.smithi017.stdout:3775: writing smithi01785587-24 from 3570846 to 3983589 tid 3 2024-09-21T09:54:00.301 INFO:tasks.rados.rados.0.smithi017.stdout:3776: rollback oid 4 current snap is 472 2024-09-21T09:54:00.301 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 4 to 453 2024-09-21T09:54:00.301 INFO:tasks.rados.rados.0.smithi017.stdout:3775: finishing write tid 1 to smithi01785587-24 2024-09-21T09:54:00.301 INFO:tasks.rados.rados.0.smithi017.stdout:3775: finishing write tid 2 to smithi01785587-24 2024-09-21T09:54:00.301 INFO:tasks.rados.rados.0.smithi017.stdout:3777: write oid 12 current snap is 472 2024-09-21T09:54:00.301 INFO:tasks.rados.rados.0.smithi017.stdout:3777: seq_num 1428 ranges {400396=413583,1463110=438486} 2024-09-21T09:54:00.305 INFO:tasks.rados.rados.0.smithi017.stdout:3777: writing smithi01785587-12 from 400396 to 813979 tid 1 2024-09-21T09:54:00.311 INFO:tasks.rados.rados.0.smithi017.stdout:3777: writing smithi01785587-12 from 1463110 to 1901596 tid 2 2024-09-21T09:54:00.312 INFO:tasks.rados.rados.0.smithi017.stdout:3778: write oid 29 current snap is 472 2024-09-21T09:54:00.312 INFO:tasks.rados.rados.0.smithi017.stdout:3775: finishing write tid 3 to smithi01785587-24 2024-09-21T09:54:00.312 INFO:tasks.rados.rados.0.smithi017.stdout:3775: finishing write tid 4 to smithi01785587-24 2024-09-21T09:54:00.312 INFO:tasks.rados.rados.0.smithi017.stdout:3776: finishing rollback tid 1 to smithi01785587-4 2024-09-21T09:54:00.312 INFO:tasks.rados.rados.0.smithi017.stdout:3778: seq_num 1429 ranges {417203=488832,1405239=611964} 2024-09-21T09:54:00.316 INFO:tasks.rados.rados.0.smithi017.stdout:3778: writing smithi01785587-29 from 417203 to 906035 tid 1 2024-09-21T09:54:00.323 INFO:tasks.rados.rados.0.smithi017.stdout:3778: writing smithi01785587-29 from 1405239 to 2017203 tid 2 2024-09-21T09:54:00.323 INFO:tasks.rados.rados.0.smithi017.stdout:3779: write oid 26 current snap is 472 2024-09-21T09:54:00.323 INFO:tasks.rados.rados.0.smithi017.stdout:3775: finishing write tid 5 to smithi01785587-24 2024-09-21T09:54:00.323 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 1173 (ObjNum 1427 snap 472 seq_num 1427) dirty exists 2024-09-21T09:54:00.323 INFO:tasks.rados.rados.0.smithi017.stdout:3775: left oid 24 (ObjNum 1427 snap 472 seq_num 1427) 2024-09-21T09:54:00.323 INFO:tasks.rados.rados.0.smithi017.stdout:3776: finishing rollback tid 2 to smithi01785587-4 2024-09-21T09:54:00.323 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 4 v 2056 (ObjNum 1268 snap 424 seq_num 1268) dirty exists 2024-09-21T09:54:00.324 INFO:tasks.rados.rados.0.smithi017.stdout:3777: finishing write tid 1 to smithi01785587-12 2024-09-21T09:54:00.324 INFO:tasks.rados.rados.0.smithi017.stdout:3779: seq_num 1430 ranges {434010=680434,1580074=442155} 2024-09-21T09:54:00.329 INFO:tasks.rados.rados.0.smithi017.stdout:3779: writing smithi01785587-26 from 434010 to 1114444 tid 1 2024-09-21T09:54:00.334 INFO:tasks.rados.rados.0.smithi017.stdout:3779: writing smithi01785587-26 from 1580074 to 2022229 tid 2 2024-09-21T09:54:00.334 INFO:tasks.rados.rados.0.smithi017.stdout:3775: done (4 left) 2024-09-21T09:54:00.334 INFO:tasks.rados.rados.0.smithi017.stdout:3776: done (3 left) 2024-09-21T09:54:00.334 INFO:tasks.rados.rados.0.smithi017.stdout:3780: read oid 25 snap -1 2024-09-21T09:54:00.334 INFO:tasks.rados.rados.0.smithi017.stdout:3780: expect (ObjNum 1304 snap 440 seq_num 1304) 2024-09-21T09:54:00.334 INFO:tasks.rados.rados.0.smithi017.stdout:3781: write oid 47 current snap is 472 2024-09-21T09:54:00.335 INFO:tasks.rados.rados.0.smithi017.stdout:3777: finishing write tid 2 to smithi01785587-12 2024-09-21T09:54:00.335 INFO:tasks.rados.rados.0.smithi017.stdout:3777: finishing write tid 3 to smithi01785587-12 2024-09-21T09:54:00.335 INFO:tasks.rados.rados.0.smithi017.stdout:3777: finishing write tid 4 to smithi01785587-12 2024-09-21T09:54:00.335 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 12 v 1304 (ObjNum 1428 snap 472 seq_num 1428) dirty exists 2024-09-21T09:54:00.335 INFO:tasks.rados.rados.0.smithi017.stdout:3777: left oid 12 (ObjNum 1428 snap 472 seq_num 1428) 2024-09-21T09:54:00.335 INFO:tasks.rados.rados.0.smithi017.stdout:3778: finishing write tid 1 to smithi01785587-29 2024-09-21T09:54:00.335 INFO:tasks.rados.rados.0.smithi017.stdout:3781: seq_num 1431 ranges {450817=755683,1638556=412261} 2024-09-21T09:54:00.340 INFO:tasks.rados.rados.0.smithi017.stdout:3781: writing smithi01785587-47 from 450817 to 1206500 tid 1 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:3781: writing smithi01785587-47 from 1638556 to 2050817 tid 2 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:3777: done (4 left) 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:3782: snap_create 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:3778: finishing write tid 2 to smithi01785587-29 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:3778: finishing write tid 3 to smithi01785587-29 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:3778: finishing write tid 4 to smithi01785587-29 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 1100 (ObjNum 1429 snap 472 seq_num 1429) dirty exists 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:3778: left oid 29 (ObjNum 1429 snap 472 seq_num 1429) 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:3779: finishing write tid 1 to smithi01785587-26 2024-09-21T09:54:00.346 INFO:tasks.rados.rados.0.smithi017.stdout:3779: finishing write tid 2 to smithi01785587-26 2024-09-21T09:54:00.347 INFO:tasks.rados.rados.0.smithi017.stdout:3779: finishing write tid 3 to smithi01785587-26 2024-09-21T09:54:00.347 INFO:tasks.rados.rados.0.smithi017.stdout:3779: finishing write tid 4 to smithi01785587-26 2024-09-21T09:54:00.347 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 2060 (ObjNum 1430 snap 472 seq_num 1430) dirty exists 2024-09-21T09:54:00.347 INFO:tasks.rados.rados.0.smithi017.stdout:3779: left oid 26 (ObjNum 1430 snap 472 seq_num 1430) 2024-09-21T09:54:00.347 INFO:tasks.rados.rados.0.smithi017.stdout:3780: expect (ObjNum 1304 snap 440 seq_num 1304) 2024-09-21T09:54:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:00 smithi134 ceph-mon[90085]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-21T09:54:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:54:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:54:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:00.655 INFO:tasks.rados.rados.0.smithi017.stdout:3781: finishing write tid 1 to smithi01785587-47 2024-09-21T09:54:00.655 INFO:tasks.rados.rados.0.smithi017.stdout:3781: finishing write tid 2 to smithi01785587-47 2024-09-21T09:54:00.655 INFO:tasks.rados.rados.0.smithi017.stdout:3781: finishing write tid 3 to smithi01785587-47 2024-09-21T09:54:00.655 INFO:tasks.rados.rados.0.smithi017.stdout:3781: finishing write tid 4 to smithi01785587-47 2024-09-21T09:54:00.655 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 1308 (ObjNum 1431 snap 472 seq_num 1431) dirty exists 2024-09-21T09:54:00.655 INFO:tasks.rados.rados.0.smithi017.stdout:3781: left oid 47 (ObjNum 1431 snap 472 seq_num 1431) 2024-09-21T09:54:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:00 smithi017 ceph-mon[102913]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-21T09:54:00.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:00 smithi017 ceph-mon[105151]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-21T09:54:01.283 INFO:tasks.rados.rados.0.smithi017.stdout:3778: done (4 left) 2024-09-21T09:54:01.283 INFO:tasks.rados.rados.0.smithi017.stdout:3779: done (3 left) 2024-09-21T09:54:01.283 INFO:tasks.rados.rados.0.smithi017.stdout:3780: done (2 left) 2024-09-21T09:54:01.283 INFO:tasks.rados.rados.0.smithi017.stdout:3781: done (1 left) 2024-09-21T09:54:01.283 INFO:tasks.rados.rados.0.smithi017.stdout:3782: done (0 left) 2024-09-21T09:54:01.284 INFO:tasks.rados.rados.0.smithi017.stdout:3783: read oid 33 snap -1 2024-09-21T09:54:01.284 INFO:tasks.rados.rados.0.smithi017.stdout:3783: expect (ObjNum 1329 snap 446 seq_num 1329) 2024-09-21T09:54:01.284 INFO:tasks.rados.rados.0.smithi017.stdout:3784: snap_remove snap 469 2024-09-21T09:54:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:01 smithi017 ceph-mon[102913]: pgmap v1101: 89 pgs: 89 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:54:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:01 smithi017 ceph-mon[102913]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-21T09:54:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:01 smithi017 ceph-mon[105151]: pgmap v1101: 89 pgs: 89 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:54:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:01 smithi017 ceph-mon[105151]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-21T09:54:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:01 smithi134 ceph-mon[90085]: pgmap v1101: 89 pgs: 89 active+clean; 162 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:54:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:01 smithi134 ceph-mon[90085]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-21T09:54:02.293 INFO:tasks.rados.rados.0.smithi017.stdout:3784: done (1 left) 2024-09-21T09:54:02.294 INFO:tasks.rados.rados.0.smithi017.stdout:3785: snap_remove snap 467 2024-09-21T09:54:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:02 smithi017 ceph-mon[102913]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-21T09:54:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:02 smithi017 ceph-mon[105151]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-21T09:54:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:02 smithi134 ceph-mon[90085]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-21T09:54:03.235 INFO:tasks.rados.rados.0.smithi017.stdout:3785: done (1 left) 2024-09-21T09:54:03.235 INFO:tasks.rados.rados.0.smithi017.stdout:3786: read oid 4 snap -1 2024-09-21T09:54:03.235 INFO:tasks.rados.rados.0.smithi017.stdout:3786: expect (ObjNum 1268 snap 424 seq_num 1268) 2024-09-21T09:54:03.235 INFO:tasks.rados.rados.0.smithi017.stdout:3787: write oid 1 current snap is 473 2024-09-21T09:54:03.235 INFO:tasks.rados.rados.0.smithi017.stdout:3787: seq_num 1432 ranges {467624=430932,1697038=370586} 2024-09-21T09:54:03.244 INFO:tasks.rados.rados.0.smithi017.stdout:3787: writing smithi01785587-1 from 467624 to 898556 tid 1 2024-09-21T09:54:03.247 INFO:tasks.rados.rados.0.smithi017.stdout:3787: writing smithi01785587-1 from 1697038 to 2067624 tid 2 2024-09-21T09:54:03.247 INFO:tasks.rados.rados.0.smithi017.stdout:3788: rollback oid 44 current snap is 473 2024-09-21T09:54:03.247 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 44 to 471 2024-09-21T09:54:03.247 INFO:tasks.rados.rados.0.smithi017.stdout:3783: expect (ObjNum 1329 snap 446 seq_num 1329) 2024-09-21T09:54:03.330 INFO:tasks.rados.rados.0.smithi017.stdout:3786: expect (ObjNum 1268 snap 424 seq_num 1268) 2024-09-21T09:54:03.495 INFO:tasks.rados.rados.0.smithi017.stdout:3787: finishing write tid 1 to smithi01785587-1 2024-09-21T09:54:03.495 INFO:tasks.rados.rados.0.smithi017.stdout:3787: finishing write tid 2 to smithi01785587-1 2024-09-21T09:54:03.495 INFO:tasks.rados.rados.0.smithi017.stdout:3787: finishing write tid 3 to smithi01785587-1 2024-09-21T09:54:03.495 INFO:tasks.rados.rados.0.smithi017.stdout:3787: finishing write tid 4 to smithi01785587-1 2024-09-21T09:54:03.495 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 1106 (ObjNum 1432 snap 473 seq_num 1432) dirty exists 2024-09-21T09:54:03.495 INFO:tasks.rados.rados.0.smithi017.stdout:3787: left oid 1 (ObjNum 1432 snap 473 seq_num 1432) 2024-09-21T09:54:03.496 INFO:tasks.rados.rados.0.smithi017.stdout:3783: done (3 left) 2024-09-21T09:54:03.496 INFO:tasks.rados.rados.0.smithi017.stdout:3786: done (2 left) 2024-09-21T09:54:03.496 INFO:tasks.rados.rados.0.smithi017.stdout:3787: done (1 left) 2024-09-21T09:54:03.496 INFO:tasks.rados.rados.0.smithi017.stdout:3789: write oid 1 current snap is 473 2024-09-21T09:54:03.496 INFO:tasks.rados.rados.0.smithi017.stdout:3788: finishing rollback tid 0 to smithi01785587-44 2024-09-21T09:54:03.496 INFO:tasks.rados.rados.0.smithi017.stdout:3789: seq_num 1433 ranges {484431=506181,1639167=445264} 2024-09-21T09:54:03.499 INFO:tasks.rados.rados.0.smithi017.stdout:3789: writing smithi01785587-1 from 484431 to 990612 tid 1 2024-09-21T09:54:03.505 INFO:tasks.rados.rados.0.smithi017.stdout:3789: writing smithi01785587-1 from 1639167 to 2084431 tid 2 2024-09-21T09:54:03.505 INFO:tasks.rados.rados.0.smithi017.stdout:3790: write oid 26 current snap is 473 2024-09-21T09:54:03.506 INFO:tasks.rados.rados.0.smithi017.stdout:3790: seq_num 1434 ranges {501238=581430,1697649=403589} 2024-09-21T09:54:03.511 INFO:tasks.rados.rados.0.smithi017.stdout:3790: writing smithi01785587-26 from 501238 to 1082668 tid 1 2024-09-21T09:54:03.518 INFO:tasks.rados.rados.0.smithi017.stdout:3790: writing smithi01785587-26 from 1697649 to 2101238 tid 2 2024-09-21T09:54:03.518 INFO:tasks.rados.rados.0.smithi017.stdout:3791: snap_create 2024-09-21T09:54:03.518 INFO:tasks.rados.rados.0.smithi017.stdout:3788: finishing rollback tid 1 to smithi01785587-44 2024-09-21T09:54:03.518 INFO:tasks.rados.rados.0.smithi017.stdout:3788: finishing rollback tid 2 to smithi01785587-44 2024-09-21T09:54:03.518 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 2206 (ObjNum 1407 snap 464 seq_num 1407) dirty exists 2024-09-21T09:54:03.518 INFO:tasks.rados.rados.0.smithi017.stdout:3789: finishing write tid 1 to smithi01785587-1 2024-09-21T09:54:03.518 INFO:tasks.rados.rados.0.smithi017.stdout:3789: finishing write tid 2 to smithi01785587-1 2024-09-21T09:54:03.519 INFO:tasks.rados.rados.0.smithi017.stdout:3789: finishing write tid 3 to smithi01785587-1 2024-09-21T09:54:03.519 INFO:tasks.rados.rados.0.smithi017.stdout:3789: finishing write tid 4 to smithi01785587-1 2024-09-21T09:54:03.519 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 1109 (ObjNum 1433 snap 473 seq_num 1433) dirty exists 2024-09-21T09:54:03.519 INFO:tasks.rados.rados.0.smithi017.stdout:3789: left oid 1 (ObjNum 1433 snap 473 seq_num 1433) 2024-09-21T09:54:03.519 INFO:tasks.rados.rados.0.smithi017.stdout:3790: finishing write tid 1 to smithi01785587-26 2024-09-21T09:54:03.522 INFO:tasks.rados.rados.0.smithi017.stdout:3790: finishing write tid 2 to smithi01785587-26 2024-09-21T09:54:03.522 INFO:tasks.rados.rados.0.smithi017.stdout:3790: finishing write tid 3 to smithi01785587-26 2024-09-21T09:54:03.522 INFO:tasks.rados.rados.0.smithi017.stdout:3790: finishing write tid 4 to smithi01785587-26 2024-09-21T09:54:03.522 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 2070 (ObjNum 1434 snap 473 seq_num 1434) dirty exists 2024-09-21T09:54:03.522 INFO:tasks.rados.rados.0.smithi017.stdout:3790: left oid 26 (ObjNum 1434 snap 473 seq_num 1434) 2024-09-21T09:54:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:03 smithi017 ceph-mon[102913]: pgmap v1104: 89 pgs: 89 active+clean; 178 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.3 MiB/s wr, 11 op/s 2024-09-21T09:54:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:03 smithi017 ceph-mon[102913]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-21T09:54:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:03 smithi017 ceph-mon[102913]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-21T09:54:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:03 smithi017 ceph-mon[105151]: pgmap v1104: 89 pgs: 89 active+clean; 178 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.3 MiB/s wr, 11 op/s 2024-09-21T09:54:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:03 smithi017 ceph-mon[105151]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-21T09:54:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:03 smithi017 ceph-mon[105151]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-21T09:54:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:03 smithi134 ceph-mon[90085]: pgmap v1104: 89 pgs: 89 active+clean; 178 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.3 MiB/s wr, 11 op/s 2024-09-21T09:54:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:03 smithi134 ceph-mon[90085]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-21T09:54:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:03 smithi134 ceph-mon[90085]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-21T09:54:04.300 INFO:tasks.rados.rados.0.smithi017.stdout:3788: done (3 left) 2024-09-21T09:54:04.300 INFO:tasks.rados.rados.0.smithi017.stdout:3789: done (2 left) 2024-09-21T09:54:04.300 INFO:tasks.rados.rados.0.smithi017.stdout:3790: done (1 left) 2024-09-21T09:54:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3791: done (0 left) 2024-09-21T09:54:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3792: read oid 30 snap 473 2024-09-21T09:54:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3792: expect (ObjNum 1425 snap 471 seq_num 1425) 2024-09-21T09:54:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3793: write oid 29 current snap is 474 2024-09-21T09:54:04.301 INFO:tasks.rados.rados.0.smithi017.stdout:3793: seq_num 1435 ranges {518045=656679,1756131=361914} 2024-09-21T09:54:04.306 INFO:tasks.rados.rados.0.smithi017.stdout:3793: writing smithi01785587-29 from 518045 to 1174724 tid 1 2024-09-21T09:54:04.311 INFO:tasks.rados.rados.0.smithi017.stdout:3793: writing smithi01785587-29 from 1756131 to 2118045 tid 2 2024-09-21T09:54:04.312 INFO:tasks.rados.rados.0.smithi017.stdout:3794: snap_remove snap 472 2024-09-21T09:54:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:54:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:54:04] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.51.0" 2024-09-21T09:54:05.305 INFO:tasks.rados.rados.0.smithi017.stdout:3793: finishing write tid 1 to smithi01785587-29 2024-09-21T09:54:05.306 INFO:tasks.rados.rados.0.smithi017.stdout:3793: finishing write tid 2 to smithi01785587-29 2024-09-21T09:54:05.306 INFO:tasks.rados.rados.0.smithi017.stdout:3793: finishing write tid 3 to smithi01785587-29 2024-09-21T09:54:05.306 INFO:tasks.rados.rados.0.smithi017.stdout:3793: finishing write tid 4 to smithi01785587-29 2024-09-21T09:54:05.306 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 1104 (ObjNum 1435 snap 474 seq_num 1435) dirty exists 2024-09-21T09:54:05.306 INFO:tasks.rados.rados.0.smithi017.stdout:3793: left oid 29 (ObjNum 1435 snap 474 seq_num 1435) 2024-09-21T09:54:05.306 INFO:tasks.rados.rados.0.smithi017.stdout:3792: expect (ObjNum 1425 snap 471 seq_num 1425) 2024-09-21T09:54:05.527 INFO:tasks.rados.rados.0.smithi017.stdout:3792: done (2 left) 2024-09-21T09:54:05.528 INFO:tasks.rados.rados.0.smithi017.stdout:3793: done (1 left) 2024-09-21T09:54:05.528 INFO:tasks.rados.rados.0.smithi017.stdout:3794: done (0 left) 2024-09-21T09:54:05.528 INFO:tasks.rados.rados.0.smithi017.stdout:3795: write oid 8 current snap is 474 2024-09-21T09:54:05.528 INFO:tasks.rados.rados.0.smithi017.stdout:3795: seq_num 1436 ranges {534852=731928,1814613=320239} 2024-09-21T09:54:05.534 INFO:tasks.rados.rados.0.smithi017.stdout:3795: writing smithi01785587-8 from 534852 to 1266780 tid 1 2024-09-21T09:54:05.539 INFO:tasks.rados.rados.0.smithi017.stdout:3795: writing smithi01785587-8 from 1814613 to 2134852 tid 2 2024-09-21T09:54:05.539 INFO:tasks.rados.rados.0.smithi017.stdout:3796: write oid 5 current snap is 474 2024-09-21T09:54:05.540 INFO:tasks.rados.rados.0.smithi017.stdout:3795: finishing write tid 1 to smithi01785587-8 2024-09-21T09:54:05.540 INFO:tasks.rados.rados.0.smithi017.stdout:3796: seq_num 1437 ranges {551659=523530,1873095=278564} 2024-09-21T09:54:05.545 INFO:tasks.rados.rados.0.smithi017.stdout:3796: writing smithi01785587-5 from 551659 to 1075189 tid 1 2024-09-21T09:54:05.551 INFO:tasks.rados.rados.0.smithi017.stdout:3796: writing smithi01785587-5 from 1873095 to 2151659 tid 2 2024-09-21T09:54:05.552 INFO:tasks.rados.rados.0.smithi017.stdout:3797: write oid 48 current snap is 474 2024-09-21T09:54:05.552 INFO:tasks.rados.rados.0.smithi017.stdout:3797: seq_num 1438 ranges {568466=598779,1931577=236889} 2024-09-21T09:54:05.558 INFO:tasks.rados.rados.0.smithi017.stdout:3797: writing smithi01785587-48 from 568466 to 1167245 tid 1 2024-09-21T09:54:05.564 INFO:tasks.rados.rados.0.smithi017.stdout:3797: writing smithi01785587-48 from 1931577 to 2168466 tid 2 2024-09-21T09:54:05.564 INFO:tasks.rados.rados.0.smithi017.stdout:3798: rollback oid 37 current snap is 474 2024-09-21T09:54:05.564 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 37 to 471 2024-09-21T09:54:05.564 INFO:tasks.rados.rados.0.smithi017.stdout:3795: finishing write tid 2 to smithi01785587-8 2024-09-21T09:54:05.564 INFO:tasks.rados.rados.0.smithi017.stdout:3799: delete oid 27 current snap is 474 2024-09-21T09:54:05.564 INFO:tasks.rados.rados.0.smithi017.stdout:3795: finishing write tid 3 to smithi01785587-8 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:3795: finishing write tid 4 to smithi01785587-8 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 1976 (ObjNum 1436 snap 474 seq_num 1436) dirty exists 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:3795: left oid 8 (ObjNum 1436 snap 474 seq_num 1436) 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:3796: finishing write tid 1 to smithi01785587-5 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:3796: finishing write tid 2 to smithi01785587-5 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:3796: finishing write tid 3 to smithi01785587-5 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:3796: finishing write tid 4 to smithi01785587-5 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 2212 (ObjNum 1437 snap 474 seq_num 1437) dirty exists 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:3796: left oid 5 (ObjNum 1437 snap 474 seq_num 1437) 2024-09-21T09:54:05.565 INFO:tasks.rados.rados.0.smithi017.stdout:3797: finishing write tid 1 to smithi01785587-48 2024-09-21T09:54:05.566 INFO:tasks.rados.rados.0.smithi017.stdout:3795: done (4 left) 2024-09-21T09:54:05.566 INFO:tasks.rados.rados.0.smithi017.stdout:3796: done (3 left) 2024-09-21T09:54:05.567 INFO:tasks.rados.rados.0.smithi017.stdout:3799: done (2 left) 2024-09-21T09:54:05.567 INFO:tasks.rados.rados.0.smithi017.stdout:3800: write oid 22 current snap is 474 2024-09-21T09:54:05.567 INFO:tasks.rados.rados.0.smithi017.stdout:3800: seq_num 1440 ranges {602080=749277,2048541=153539} 2024-09-21T09:54:05.574 INFO:tasks.rados.rados.0.smithi017.stdout:3800: writing smithi01785587-22 from 602080 to 1351357 tid 1 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:3800: writing smithi01785587-22 from 2048541 to 2202080 tid 2 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:3801: rollback oid 50 current snap is 474 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 50 to 471 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:3798: finishing rollback tid 0 to smithi01785587-37 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:3797: finishing write tid 2 to smithi01785587-48 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:3797: finishing write tid 3 to smithi01785587-48 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:3797: finishing write tid 4 to smithi01785587-48 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 48 v 1610 (ObjNum 1438 snap 474 seq_num 1438) dirty exists 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:3797: left oid 48 (ObjNum 1438 snap 474 seq_num 1438) 2024-09-21T09:54:05.580 INFO:tasks.rados.rados.0.smithi017.stdout:3798: finishing rollback tid 1 to smithi01785587-37 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3798: finishing rollback tid 2 to smithi01785587-37 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1968 (ObjNum 1285 snap 432 seq_num 1285) dirty exists 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3800: finishing write tid 1 to smithi01785587-22 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3797: done (3 left) 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3798: done (2 left) 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3802: read oid 11 snap 473 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3802: expect (ObjNum 1392 snap 462 seq_num 1392) 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3803: read oid 8 snap -1 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3803: expect (ObjNum 1436 snap 474 seq_num 1436) 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3804: rollback oid 31 current snap is 474 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 31 to 471 2024-09-21T09:54:05.581 INFO:tasks.rados.rados.0.smithi017.stdout:3805: read oid 9 snap 473 2024-09-21T09:54:05.582 INFO:tasks.rados.rados.0.smithi017.stdout:3805: expect (ObjNum 1414 snap 468 seq_num 1414) 2024-09-21T09:54:05.582 INFO:tasks.rados.rados.0.smithi017.stdout:3806: snap_create 2024-09-21T09:54:05.584 INFO:tasks.rados.rados.0.smithi017.stdout:3802: expect (ObjNum 1392 snap 462 seq_num 1392) 2024-09-21T09:54:05.700 INFO:tasks.rados.rados.0.smithi017.stdout:3800: finishing write tid 2 to smithi01785587-22 2024-09-21T09:54:05.701 INFO:tasks.rados.rados.0.smithi017.stdout:3800: finishing write tid 3 to smithi01785587-22 2024-09-21T09:54:05.701 INFO:tasks.rados.rados.0.smithi017.stdout:3800: finishing write tid 4 to smithi01785587-22 2024-09-21T09:54:05.701 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 2076 (ObjNum 1440 snap 474 seq_num 1440) dirty exists 2024-09-21T09:54:05.701 INFO:tasks.rados.rados.0.smithi017.stdout:3800: left oid 22 (ObjNum 1440 snap 474 seq_num 1440) 2024-09-21T09:54:05.702 INFO:tasks.rados.rados.0.smithi017.stdout:3804: finishing rollback tid 0 to smithi01785587-31 2024-09-21T09:54:05.702 INFO:tasks.rados.rados.0.smithi017.stdout:3804: finishing rollback tid 1 to smithi01785587-31 2024-09-21T09:54:05.703 INFO:tasks.rados.rados.0.smithi017.stdout:3804: finishing rollback tid 2 to smithi01785587-31 2024-09-21T09:54:05.703 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 2216 (ObjNum 1401 snap 462 seq_num 1401) dirty exists 2024-09-21T09:54:05.703 INFO:tasks.rados.rados.0.smithi017.stdout:3805: expect (ObjNum 1414 snap 468 seq_num 1414) 2024-09-21T09:54:05.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:05 smithi017 ceph-mon[105151]: pgmap v1107: 89 pgs: 89 active+clean; 180 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-09-21T09:54:05.706 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:05 smithi017 ceph-mon[105151]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-21T09:54:05.706 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:05 smithi017 ceph-mon[102913]: pgmap v1107: 89 pgs: 89 active+clean; 180 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-09-21T09:54:05.706 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:05 smithi017 ceph-mon[102913]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-21T09:54:05.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:05 smithi134 ceph-mon[90085]: pgmap v1107: 89 pgs: 89 active+clean; 180 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-09-21T09:54:05.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:05 smithi134 ceph-mon[90085]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-21T09:54:05.919 INFO:tasks.rados.rados.0.smithi017.stdout:3801: finishing rollback tid 0 to smithi01785587-50 2024-09-21T09:54:05.919 INFO:tasks.rados.rados.0.smithi017.stdout:3801: finishing rollback tid 1 to smithi01785587-50 2024-09-21T09:54:05.919 INFO:tasks.rados.rados.0.smithi017.stdout:3801: finishing rollback tid 2 to smithi01785587-50 2024-09-21T09:54:05.920 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 1179 (ObjNum 1093 snap 363 seq_num 1093) dirty exists 2024-09-21T09:54:05.920 INFO:tasks.rados.rados.0.smithi017.stdout:3803: expect (ObjNum 1436 snap 474 seq_num 1436) 2024-09-21T09:54:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3800: done (6 left) 2024-09-21T09:54:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3801: done (5 left) 2024-09-21T09:54:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3802: done (4 left) 2024-09-21T09:54:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3803: done (3 left) 2024-09-21T09:54:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3804: done (2 left) 2024-09-21T09:54:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3805: done (1 left) 2024-09-21T09:54:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3806: done (0 left) 2024-09-21T09:54:06.309 INFO:tasks.rados.rados.0.smithi017.stdout:3807: snap_create 2024-09-21T09:54:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:06 smithi017 ceph-mon[102913]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-21T09:54:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:06 smithi017 ceph-mon[105151]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-21T09:54:06.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:06 smithi134 ceph-mon[90085]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-21T09:54:07.315 INFO:tasks.rados.rados.0.smithi017.stdout:3807: done (0 left) 2024-09-21T09:54:07.315 INFO:tasks.rados.rados.0.smithi017.stdout:3808: delete oid 32 current snap is 476 2024-09-21T09:54:07.318 INFO:tasks.rados.rados.0.smithi017.stdout:3808: done (0 left) 2024-09-21T09:54:07.319 INFO:tasks.rados.rados.0.smithi017.stdout:3809: write oid 40 current snap is 476 2024-09-21T09:54:07.319 INFO:tasks.rados.rados.0.smithi017.stdout:3809: seq_num 1442 ranges {635694=499775,1649152=464169} 2024-09-21T09:54:07.325 INFO:tasks.rados.rados.0.smithi017.stdout:3809: writing smithi01785587-40 from 635694 to 1135469 tid 1 2024-09-21T09:54:07.331 INFO:tasks.rados.rados.0.smithi017.stdout:3809: writing smithi01785587-40 from 1649152 to 2113321 tid 2 2024-09-21T09:54:07.331 INFO:tasks.rados.rados.0.smithi017.stdout:3810: snap_create 2024-09-21T09:54:07.331 INFO:tasks.rados.rados.0.smithi017.stdout:3809: finishing write tid 1 to smithi01785587-40 2024-09-21T09:54:07.334 INFO:tasks.rados.rados.0.smithi017.stdout:3809: finishing write tid 2 to smithi01785587-40 2024-09-21T09:54:07.334 INFO:tasks.rados.rados.0.smithi017.stdout:3809: finishing write tid 3 to smithi01785587-40 2024-09-21T09:54:07.334 INFO:tasks.rados.rados.0.smithi017.stdout:3809: finishing write tid 4 to smithi01785587-40 2024-09-21T09:54:07.334 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 1971 (ObjNum 1442 snap 476 seq_num 1442) dirty exists 2024-09-21T09:54:07.335 INFO:tasks.rados.rados.0.smithi017.stdout:3809: left oid 40 (ObjNum 1442 snap 476 seq_num 1442) 2024-09-21T09:54:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:07 smithi017 ceph-mon[102913]: pgmap v1110: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:54:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:07 smithi017 ceph-mon[102913]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-21T09:54:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:07 smithi017 ceph-mon[105151]: pgmap v1110: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:54:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:07 smithi017 ceph-mon[105151]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-21T09:54:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:07 smithi134 ceph-mon[90085]: pgmap v1110: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:54:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:07 smithi134 ceph-mon[90085]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-21T09:54:08.236 INFO:tasks.rados.rados.0.smithi017.stdout:3809: done (1 left) 2024-09-21T09:54:08.236 INFO:tasks.rados.rados.0.smithi017.stdout:3810: done (0 left) 2024-09-21T09:54:08.236 INFO:tasks.rados.rados.0.smithi017.stdout:3811: delete oid 17 current snap is 477 2024-09-21T09:54:08.240 INFO:tasks.rados.rados.0.smithi017.stdout:3811: done (0 left) 2024-09-21T09:54:08.240 INFO:tasks.rados.rados.0.smithi017.stdout:3812: delete oid 17 current snap is 477 2024-09-21T09:54:08.243 INFO:tasks.rados.rados.0.smithi017.stdout:3812: done (0 left) 2024-09-21T09:54:08.243 INFO:tasks.rados.rados.0.smithi017.stdout:3813: read oid 32 snap -1 2024-09-21T09:54:08.243 INFO:tasks.rados.rados.0.smithi017.stdout:3813: expect deleted 2024-09-21T09:54:08.243 INFO:tasks.rados.rados.0.smithi017.stdout:3814: rollback oid 27 current snap is 477 2024-09-21T09:54:08.243 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 27 to 476 2024-09-21T09:54:08.243 INFO:tasks.rados.rados.0.smithi017.stdout:3815: write oid 28 current snap is 477 2024-09-21T09:54:08.243 INFO:tasks.rados.rados.0.smithi017.stdout:3815: seq_num 1445 ranges {686115=441875,1824598=461517} 2024-09-21T09:54:08.249 INFO:tasks.rados.rados.0.smithi017.stdout:3815: writing smithi01785587-28 from 686115 to 1127990 tid 1 2024-09-21T09:54:08.256 INFO:tasks.rados.rados.0.smithi017.stdout:3815: writing smithi01785587-28 from 1824598 to 2286115 tid 2 2024-09-21T09:54:08.256 INFO:tasks.rados.rados.0.smithi017.stdout:3816: read oid 38 snap -1 2024-09-21T09:54:08.256 INFO:tasks.rados.rados.0.smithi017.stdout:3816: expect deleted 2024-09-21T09:54:08.256 INFO:tasks.rados.rados.0.smithi017.stdout:3817: snap_remove snap 473 2024-09-21T09:54:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:08 smithi017 ceph-mon[102913]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-21T09:54:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:08 smithi017 ceph-mon[102913]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-21T09:54:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:08 smithi017 ceph-mon[105151]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-21T09:54:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:08 smithi017 ceph-mon[105151]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-21T09:54:08.759 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:08 smithi134 ceph-mon[90085]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-21T09:54:08.760 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:08 smithi134 ceph-mon[90085]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-21T09:54:09.240 INFO:tasks.rados.rados.0.smithi017.stdout:3817: done (4 left) 2024-09-21T09:54:09.240 INFO:tasks.rados.rados.0.smithi017.stdout:3818: read oid 34 snap -1 2024-09-21T09:54:09.240 INFO:tasks.rados.rados.0.smithi017.stdout:3818: expect deleted 2024-09-21T09:54:09.240 INFO:tasks.rados.rados.0.smithi017.stdout:3819: read oid 44 snap 453 2024-09-21T09:54:09.240 INFO:tasks.rados.rados.0.smithi017.stdout:3819: expect (ObjNum 1358 snap 451 seq_num 1358) 2024-09-21T09:54:09.240 INFO:tasks.rados.rados.0.smithi017.stdout:3820: write oid 17 current snap is 477 2024-09-21T09:54:09.240 INFO:tasks.rados.rados.0.smithi017.stdout:3820: seq_num 1446 ranges {702922=517124,1883080=419842} 2024-09-21T09:54:09.247 INFO:tasks.rados.rados.0.smithi017.stdout:3820: writing smithi01785587-17 from 702922 to 1220046 tid 1 2024-09-21T09:54:09.254 INFO:tasks.rados.rados.0.smithi017.stdout:3820: writing smithi01785587-17 from 1883080 to 2302922 tid 2 2024-09-21T09:54:09.254 INFO:tasks.rados.rados.0.smithi017.stdout:3821: write oid 41 current snap is 477 2024-09-21T09:54:09.254 INFO:tasks.rados.rados.0.smithi017.stdout:3821: seq_num 1447 ranges {719729=592373,1941562=378167} 2024-09-21T09:54:09.261 INFO:tasks.rados.rados.0.smithi017.stdout:3821: writing smithi01785587-41 from 719729 to 1312102 tid 1 2024-09-21T09:54:09.267 INFO:tasks.rados.rados.0.smithi017.stdout:3821: writing smithi01785587-41 from 1941562 to 2319729 tid 2 2024-09-21T09:54:09.267 INFO:tasks.rados.rados.0.smithi017.stdout:3822: delete oid 14 current snap is 477 2024-09-21T09:54:09.267 INFO:tasks.rados.rados.0.smithi017.stdout:3814: finishing rollback tid 1 to smithi01785587-27 2024-09-21T09:54:09.267 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 1320 (ObjNum 305 snap 0 seq_num 1) dirty dne 2024-09-21T09:54:09.267 INFO:tasks.rados.rados.0.smithi017.stdout:3815: finishing write tid 1 to smithi01785587-28 2024-09-21T09:54:09.267 INFO:tasks.rados.rados.0.smithi017.stdout:3815: finishing write tid 2 to smithi01785587-28 2024-09-21T09:54:09.267 INFO:tasks.rados.rados.0.smithi017.stdout:3815: finishing write tid 3 to smithi01785587-28 2024-09-21T09:54:09.268 INFO:tasks.rados.rados.0.smithi017.stdout:3815: finishing write tid 4 to smithi01785587-28 2024-09-21T09:54:09.268 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 28 v 2080 (ObjNum 1445 snap 477 seq_num 1445) dirty exists 2024-09-21T09:54:09.268 INFO:tasks.rados.rados.0.smithi017.stdout:3815: left oid 28 (ObjNum 1445 snap 477 seq_num 1445) 2024-09-21T09:54:09.268 INFO:tasks.rados.rados.0.smithi017.stdout:3819: expect (ObjNum 1358 snap 451 seq_num 1358) 2024-09-21T09:54:09.515 INFO:tasks.rados.rados.0.smithi017.stdout:3820: finishing write tid 1 to smithi01785587-17 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3813: done (8 left) 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3814: done (7 left) 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3815: done (6 left) 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3816: done (5 left) 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3818: done (4 left) 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3819: done (3 left) 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3822: done (2 left) 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3823: delete oid 8 current snap is 477 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3820: finishing write tid 2 to smithi01785587-17 2024-09-21T09:54:09.517 INFO:tasks.rados.rados.0.smithi017.stdout:3820: finishing write tid 3 to smithi01785587-17 2024-09-21T09:54:09.518 INFO:tasks.rados.rados.0.smithi017.stdout:3820: finishing write tid 4 to smithi01785587-17 2024-09-21T09:54:09.518 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 17 v 1331 (ObjNum 1446 snap 477 seq_num 1446) dirty exists 2024-09-21T09:54:09.518 INFO:tasks.rados.rados.0.smithi017.stdout:3820: left oid 17 (ObjNum 1446 snap 477 seq_num 1446) 2024-09-21T09:54:09.518 INFO:tasks.rados.rados.0.smithi017.stdout:3821: finishing write tid 1 to smithi01785587-41 2024-09-21T09:54:09.518 INFO:tasks.rados.rados.0.smithi017.stdout:3821: finishing write tid 2 to smithi01785587-41 2024-09-21T09:54:09.518 INFO:tasks.rados.rados.0.smithi017.stdout:3821: finishing write tid 3 to smithi01785587-41 2024-09-21T09:54:09.518 INFO:tasks.rados.rados.0.smithi017.stdout:3821: finishing write tid 4 to smithi01785587-41 2024-09-21T09:54:09.518 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 586 (ObjNum 1447 snap 477 seq_num 1447) dirty exists 2024-09-21T09:54:09.518 INFO:tasks.rados.rados.0.smithi017.stdout:3821: left oid 41 (ObjNum 1447 snap 477 seq_num 1447) 2024-09-21T09:54:09.519 INFO:tasks.rados.rados.0.smithi017.stdout:3820: done (2 left) 2024-09-21T09:54:09.519 INFO:tasks.rados.rados.0.smithi017.stdout:3821: done (1 left) 2024-09-21T09:54:09.519 INFO:tasks.rados.rados.0.smithi017.stdout:3823: done (0 left) 2024-09-21T09:54:09.519 INFO:tasks.rados.rados.0.smithi017.stdout:3824: snap_create 2024-09-21T09:54:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:09 smithi017 ceph-mon[102913]: pgmap v1113: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 21 op/s 2024-09-21T09:54:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:09 smithi017 ceph-mon[102913]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-21T09:54:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:09 smithi017 ceph-mon[105151]: pgmap v1113: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 21 op/s 2024-09-21T09:54:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:09 smithi017 ceph-mon[105151]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-21T09:54:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:09 smithi134 ceph-mon[90085]: pgmap v1113: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 21 op/s 2024-09-21T09:54:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:09 smithi134 ceph-mon[90085]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-21T09:54:10.329 INFO:tasks.rados.rados.0.smithi017.stdout:3824: done (0 left) 2024-09-21T09:54:10.330 INFO:tasks.rados.rados.0.smithi017.stdout:3825: snap_remove snap 475 2024-09-21T09:54:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:10.497 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:54:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:54:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:10.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:11.333 INFO:tasks.rados.rados.0.smithi017.stdout:3825: done (0 left) 2024-09-21T09:54:11.333 INFO:tasks.rados.rados.0.smithi017.stdout:3826: rollback oid 34 current snap is 478 2024-09-21T09:54:11.333 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 34 to 477 2024-09-21T09:54:11.333 INFO:tasks.rados.rados.0.smithi017.stdout:3827: delete oid 9 current snap is 478 2024-09-21T09:54:11.333 INFO:tasks.rados.rados.0.smithi017.stdout:3826: finishing rollback tid 1 to smithi01785587-34 2024-09-21T09:54:11.334 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:54:11.343 INFO:tasks.rados.rados.0.smithi017.stdout:3826: done (1 left) 2024-09-21T09:54:11.343 INFO:tasks.rados.rados.0.smithi017.stdout:3827: done (0 left) 2024-09-21T09:54:11.343 INFO:tasks.rados.rados.0.smithi017.stdout:3828: write oid 15 current snap is 478 2024-09-21T09:54:11.343 INFO:tasks.rados.rados.0.smithi017.stdout:3828: seq_num 1451 ranges {786957=493369,2059137=327820} 2024-09-21T09:54:11.350 INFO:tasks.rados.rados.0.smithi017.stdout:3828: writing smithi01785587-15 from 786957 to 1280326 tid 1 2024-09-21T09:54:11.356 INFO:tasks.rados.rados.0.smithi017.stdout:3828: writing smithi01785587-15 from 2059137 to 2386957 tid 2 2024-09-21T09:54:11.356 INFO:tasks.rados.rados.0.smithi017.stdout:3829: delete oid 24 current snap is 478 2024-09-21T09:54:11.356 INFO:tasks.rados.rados.0.smithi017.stdout:3828: finishing write tid 1 to smithi01785587-15 2024-09-21T09:54:11.358 INFO:tasks.rados.rados.0.smithi017.stdout:3829: done (1 left) 2024-09-21T09:54:11.359 INFO:tasks.rados.rados.0.smithi017.stdout:3830: delete oid 3 current snap is 478 2024-09-21T09:54:11.359 INFO:tasks.rados.rados.0.smithi017.stdout:3828: finishing write tid 2 to smithi01785587-15 2024-09-21T09:54:11.359 INFO:tasks.rados.rados.0.smithi017.stdout:3828: finishing write tid 3 to smithi01785587-15 2024-09-21T09:54:11.359 INFO:tasks.rados.rados.0.smithi017.stdout:3828: finishing write tid 4 to smithi01785587-15 2024-09-21T09:54:11.359 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 590 (ObjNum 1451 snap 478 seq_num 1451) dirty exists 2024-09-21T09:54:11.360 INFO:tasks.rados.rados.0.smithi017.stdout:3828: left oid 15 (ObjNum 1451 snap 478 seq_num 1451) 2024-09-21T09:54:11.362 INFO:tasks.rados.rados.0.smithi017.stdout:3828: done (1 left) 2024-09-21T09:54:11.362 INFO:tasks.rados.rados.0.smithi017.stdout:3830: done (0 left) 2024-09-21T09:54:11.362 INFO:tasks.rados.rados.0.smithi017.stdout:3831: read oid 3 snap -1 2024-09-21T09:54:11.362 INFO:tasks.rados.rados.0.smithi017.stdout:3831: expect deleted 2024-09-21T09:54:11.362 INFO:tasks.rados.rados.0.smithi017.stdout:3832: read oid 8 snap 471 2024-09-21T09:54:11.362 INFO:tasks.rados.rados.0.smithi017.stdout:3832: expect (ObjNum 1404 snap 464 seq_num 1404) 2024-09-21T09:54:11.362 INFO:tasks.rados.rados.0.smithi017.stdout:3833: read oid 33 snap -1 2024-09-21T09:54:11.362 INFO:tasks.rados.rados.0.smithi017.stdout:3833: expect (ObjNum 1329 snap 446 seq_num 1329) 2024-09-21T09:54:11.362 INFO:tasks.rados.rados.0.smithi017.stdout:3834: write oid 19 current snap is 478 2024-09-21T09:54:11.363 INFO:tasks.rados.rados.0.smithi017.stdout:3834: seq_num 1454 ranges {437378=435469,1434583=769830} 2024-09-21T09:54:11.367 INFO:tasks.rados.rados.0.smithi017.stdout:3834: writing smithi01785587-19 from 437378 to 872847 tid 1 2024-09-21T09:54:11.375 INFO:tasks.rados.rados.0.smithi017.stdout:3834: writing smithi01785587-19 from 1434583 to 2204413 tid 2 2024-09-21T09:54:11.375 INFO:tasks.rados.rados.0.smithi017.stdout:3835: snap_create 2024-09-21T09:54:11.375 INFO:tasks.rados.rados.0.smithi017.stdout:3832: expect (ObjNum 1404 snap 464 seq_num 1404) 2024-09-21T09:54:11.496 INFO:tasks.rados.rados.0.smithi017.stdout:3833: expect (ObjNum 1329 snap 446 seq_num 1329) 2024-09-21T09:54:11.576 INFO:tasks.rados.rados.0.smithi017.stdout:3834: finishing write tid 1 to smithi01785587-19 2024-09-21T09:54:11.576 INFO:tasks.rados.rados.0.smithi017.stdout:3834: finishing write tid 2 to smithi01785587-19 2024-09-21T09:54:11.576 INFO:tasks.rados.rados.0.smithi017.stdout:3834: finishing write tid 3 to smithi01785587-19 2024-09-21T09:54:11.576 INFO:tasks.rados.rados.0.smithi017.stdout:3834: finishing write tid 4 to smithi01785587-19 2024-09-21T09:54:11.577 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 19 v 1613 (ObjNum 1454 snap 478 seq_num 1454) dirty exists 2024-09-21T09:54:11.577 INFO:tasks.rados.rados.0.smithi017.stdout:3834: left oid 19 (ObjNum 1454 snap 478 seq_num 1454) 2024-09-21T09:54:11.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:11 smithi017 ceph-mon[102913]: pgmap v1116: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 493 KiB/s wr, 9 op/s 2024-09-21T09:54:11.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:11 smithi017 ceph-mon[102913]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-21T09:54:11.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:11 smithi017 ceph-mon[105151]: pgmap v1116: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 493 KiB/s wr, 9 op/s 2024-09-21T09:54:11.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:11 smithi017 ceph-mon[105151]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-21T09:54:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:11 smithi134 ceph-mon[90085]: pgmap v1116: 89 pgs: 89 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 493 KiB/s wr, 9 op/s 2024-09-21T09:54:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:11 smithi134 ceph-mon[90085]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-21T09:54:12.346 INFO:tasks.rados.rados.0.smithi017.stdout:3831: done (4 left) 2024-09-21T09:54:12.346 INFO:tasks.rados.rados.0.smithi017.stdout:3832: done (3 left) 2024-09-21T09:54:12.346 INFO:tasks.rados.rados.0.smithi017.stdout:3833: done (2 left) 2024-09-21T09:54:12.346 INFO:tasks.rados.rados.0.smithi017.stdout:3834: done (1 left) 2024-09-21T09:54:12.346 INFO:tasks.rados.rados.0.smithi017.stdout:3835: done (0 left) 2024-09-21T09:54:12.346 INFO:tasks.rados.rados.0.smithi017.stdout:3836: write oid 25 current snap is 479 2024-09-21T09:54:12.346 INFO:tasks.rados.rados.0.smithi017.stdout:3836: seq_num 1455 ranges {454185=510718,1493065=629841} 2024-09-21T09:54:12.351 INFO:tasks.rados.rados.0.smithi017.stdout:3836: writing smithi01785587-25 from 454185 to 964903 tid 1 2024-09-21T09:54:12.357 INFO:tasks.rados.rados.0.smithi017.stdout:3836: writing smithi01785587-25 from 1493065 to 2122906 tid 2 2024-09-21T09:54:12.357 INFO:tasks.rados.rados.0.smithi017.stdout:3837: read oid 43 snap -1 2024-09-21T09:54:12.357 INFO:tasks.rados.rados.0.smithi017.stdout:3837: expect deleted 2024-09-21T09:54:12.357 INFO:tasks.rados.rados.0.smithi017.stdout:3838: snap_remove snap 476 2024-09-21T09:54:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:12 smithi017 ceph-mon[102913]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-21T09:54:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:12 smithi017 ceph-mon[105151]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-21T09:54:12.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:12 smithi134 ceph-mon[90085]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-21T09:54:13.238 INFO:tasks.rados.rados.0.smithi017.stdout:3838: done (2 left) 2024-09-21T09:54:13.238 INFO:tasks.rados.rados.0.smithi017.stdout:3839: read oid 15 snap -1 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:3839: expect (ObjNum 1451 snap 478 seq_num 1451) 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:3840: read oid 11 snap -1 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:3840: expect (ObjNum 1392 snap 462 seq_num 1392) 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:3841: delete oid 1 current snap is 479 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:3836: finishing write tid 1 to smithi01785587-25 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:3836: finishing write tid 2 to smithi01785587-25 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:3836: finishing write tid 3 to smithi01785587-25 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:3836: finishing write tid 4 to smithi01785587-25 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 1112 (ObjNum 1455 snap 479 seq_num 1455) dirty exists 2024-09-21T09:54:13.239 INFO:tasks.rados.rados.0.smithi017.stdout:3836: left oid 25 (ObjNum 1455 snap 479 seq_num 1455) 2024-09-21T09:54:13.249 INFO:tasks.rados.rados.0.smithi017.stdout:3836: done (4 left) 2024-09-21T09:54:13.249 INFO:tasks.rados.rados.0.smithi017.stdout:3837: done (3 left) 2024-09-21T09:54:13.249 INFO:tasks.rados.rados.0.smithi017.stdout:3841: done (2 left) 2024-09-21T09:54:13.249 INFO:tasks.rados.rados.0.smithi017.stdout:3842: write oid 13 current snap is 479 2024-09-21T09:54:13.249 INFO:tasks.rados.rados.0.smithi017.stdout:3842: seq_num 1457 ranges {487799=661216,1893676=594123} 2024-09-21T09:54:13.251 INFO:tasks.rados.rados.0.smithi017.stdout:3842: writing smithi01785587-13 from 487799 to 1149015 tid 1 2024-09-21T09:54:13.258 INFO:tasks.rados.rados.0.smithi017.stdout:3842: writing smithi01785587-13 from 1893676 to 2487799 tid 2 2024-09-21T09:54:13.259 INFO:tasks.rados.rados.0.smithi017.stdout:3843: snap_remove snap 471 2024-09-21T09:54:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:13 smithi017 ceph-mon[105151]: pgmap v1119: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 890 KiB/s wr, 8 op/s 2024-09-21T09:54:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:13 smithi017 ceph-mon[105151]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-21T09:54:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:13 smithi017 ceph-mon[105151]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-21T09:54:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:13 smithi017 ceph-mon[102913]: pgmap v1119: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 890 KiB/s wr, 8 op/s 2024-09-21T09:54:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:13 smithi017 ceph-mon[102913]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-21T09:54:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:13 smithi017 ceph-mon[102913]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-21T09:54:13.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:13 smithi134 ceph-mon[90085]: pgmap v1119: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 890 KiB/s wr, 8 op/s 2024-09-21T09:54:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:13 smithi134 ceph-mon[90085]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-21T09:54:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:13 smithi134 ceph-mon[90085]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-21T09:54:14.240 INFO:tasks.rados.rados.0.smithi017.stdout:3839: expect (ObjNum 1451 snap 478 seq_num 1451) 2024-09-21T09:54:14.357 INFO:tasks.rados.rados.0.smithi017.stdout:3840: expect (ObjNum 1392 snap 462 seq_num 1392) 2024-09-21T09:54:14.473 INFO:tasks.rados.rados.0.smithi017.stdout:3842: finishing write tid 1 to smithi01785587-13 2024-09-21T09:54:14.473 INFO:tasks.rados.rados.0.smithi017.stdout:3842: finishing write tid 2 to smithi01785587-13 2024-09-21T09:54:14.473 INFO:tasks.rados.rados.0.smithi017.stdout:3839: done (3 left) 2024-09-21T09:54:14.473 INFO:tasks.rados.rados.0.smithi017.stdout:3840: done (2 left) 2024-09-21T09:54:14.474 INFO:tasks.rados.rados.0.smithi017.stdout:3843: done (1 left) 2024-09-21T09:54:14.474 INFO:tasks.rados.rados.0.smithi017.stdout:3844: read oid 44 snap 478 2024-09-21T09:54:14.474 INFO:tasks.rados.rados.0.smithi017.stdout:3844: expect (ObjNum 1407 snap 464 seq_num 1407) 2024-09-21T09:54:14.474 INFO:tasks.rados.rados.0.smithi017.stdout:3845: snap_create 2024-09-21T09:54:14.474 INFO:tasks.rados.rados.0.smithi017.stdout:3842: finishing write tid 3 to smithi01785587-13 2024-09-21T09:54:14.474 INFO:tasks.rados.rados.0.smithi017.stdout:3842: finishing write tid 4 to smithi01785587-13 2024-09-21T09:54:14.474 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1617 (ObjNum 1457 snap 479 seq_num 1457) dirty exists 2024-09-21T09:54:14.474 INFO:tasks.rados.rados.0.smithi017.stdout:3842: left oid 13 (ObjNum 1457 snap 479 seq_num 1457) 2024-09-21T09:54:14.477 INFO:tasks.rados.rados.0.smithi017.stdout:3844: expect (ObjNum 1407 snap 464 seq_num 1407) 2024-09-21T09:54:15.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:54:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:54:14] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.51.0" 2024-09-21T09:54:15.348 INFO:tasks.rados.rados.0.smithi017.stdout:3842: done (2 left) 2024-09-21T09:54:15.348 INFO:tasks.rados.rados.0.smithi017.stdout:3844: done (1 left) 2024-09-21T09:54:15.348 INFO:tasks.rados.rados.0.smithi017.stdout:3845: done (0 left) 2024-09-21T09:54:15.349 INFO:tasks.rados.rados.0.smithi017.stdout:3846: read oid 18 snap -1 2024-09-21T09:54:15.349 INFO:tasks.rados.rados.0.smithi017.stdout:3846: expect (ObjNum 1384 snap 460 seq_num 1384) 2024-09-21T09:54:15.349 INFO:tasks.rados.rados.0.smithi017.stdout:3847: snap_create 2024-09-21T09:54:15.364 INFO:tasks.rados.rados.0.smithi017.stdout:3846: expect (ObjNum 1384 snap 460 seq_num 1384) 2024-09-21T09:54:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:15 smithi134 ceph-mon[90085]: pgmap v1122: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 883 KiB/s wr, 8 op/s 2024-09-21T09:54:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:15 smithi134 ceph-mon[90085]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-21T09:54:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:15 smithi017 ceph-mon[105151]: pgmap v1122: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 883 KiB/s wr, 8 op/s 2024-09-21T09:54:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:15 smithi017 ceph-mon[105151]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-21T09:54:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:15 smithi017 ceph-mon[102913]: pgmap v1122: 89 pgs: 89 active+clean; 185 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 883 KiB/s wr, 8 op/s 2024-09-21T09:54:15.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:15 smithi017 ceph-mon[102913]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-21T09:54:16.353 INFO:tasks.rados.rados.0.smithi017.stdout:3846: done (1 left) 2024-09-21T09:54:16.353 INFO:tasks.rados.rados.0.smithi017.stdout:3847: done (0 left) 2024-09-21T09:54:16.354 INFO:tasks.rados.rados.0.smithi017.stdout:3848: write oid 20 current snap is 481 2024-09-21T09:54:16.354 INFO:tasks.rados.rados.0.smithi017.stdout:3848: seq_num 1458 ranges {504606=736465,1952158=552448} 2024-09-21T09:54:16.360 INFO:tasks.rados.rados.0.smithi017.stdout:3848: writing smithi01785587-20 from 504606 to 1241071 tid 1 2024-09-21T09:54:16.367 INFO:tasks.rados.rados.0.smithi017.stdout:3848: writing smithi01785587-20 from 1952158 to 2504606 tid 2 2024-09-21T09:54:16.367 INFO:tasks.rados.rados.0.smithi017.stdout:3849: snap_create 2024-09-21T09:54:16.367 INFO:tasks.rados.rados.0.smithi017.stdout:3848: finishing write tid 1 to smithi01785587-20 2024-09-21T09:54:16.371 INFO:tasks.rados.rados.0.smithi017.stdout:3848: finishing write tid 2 to smithi01785587-20 2024-09-21T09:54:16.371 INFO:tasks.rados.rados.0.smithi017.stdout:3848: finishing write tid 3 to smithi01785587-20 2024-09-21T09:54:16.371 INFO:tasks.rados.rados.0.smithi017.stdout:3848: finishing write tid 4 to smithi01785587-20 2024-09-21T09:54:16.371 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 1120 (ObjNum 1458 snap 481 seq_num 1458) dirty exists 2024-09-21T09:54:16.371 INFO:tasks.rados.rados.0.smithi017.stdout:3848: left oid 20 (ObjNum 1458 snap 481 seq_num 1458) 2024-09-21T09:54:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:16 smithi017 ceph-mon[105151]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-21T09:54:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:16 smithi017 ceph-mon[102913]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-21T09:54:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:16 smithi134 ceph-mon[90085]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-21T09:54:17.356 INFO:tasks.rados.rados.0.smithi017.stdout:3848: done (1 left) 2024-09-21T09:54:17.356 INFO:tasks.rados.rados.0.smithi017.stdout:3849: done (0 left) 2024-09-21T09:54:17.356 INFO:tasks.rados.rados.0.smithi017.stdout:3850: write oid 43 current snap is 482 2024-09-21T09:54:17.356 INFO:tasks.rados.rados.0.smithi017.stdout:3850: seq_num 1459 ranges {521413=411714,1610640=637179} 2024-09-21T09:54:17.361 INFO:tasks.rados.rados.0.smithi017.stdout:3850: writing smithi01785587-43 from 521413 to 933127 tid 1 2024-09-21T09:54:17.368 INFO:tasks.rados.rados.0.smithi017.stdout:3850: writing smithi01785587-43 from 1610640 to 2247819 tid 2 2024-09-21T09:54:17.369 INFO:tasks.rados.rados.0.smithi017.stdout:3851: rollback oid 16 current snap is 482 2024-09-21T09:54:17.369 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 16 to 479 2024-09-21T09:54:17.369 INFO:tasks.rados.rados.0.smithi017.stdout:3850: finishing write tid 1 to smithi01785587-43 2024-09-21T09:54:17.369 INFO:tasks.rados.rados.0.smithi017.stdout:3852: rollback oid 31 current snap is 482 2024-09-21T09:54:17.369 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 31 to 480 2024-09-21T09:54:17.369 INFO:tasks.rados.rados.0.smithi017.stdout:3853: read oid 4 snap -1 2024-09-21T09:54:17.369 INFO:tasks.rados.rados.0.smithi017.stdout:3853: expect (ObjNum 1268 snap 424 seq_num 1268) 2024-09-21T09:54:17.369 INFO:tasks.rados.rados.0.smithi017.stdout:3854: write oid 29 current snap is 482 2024-09-21T09:54:17.369 INFO:tasks.rados.rados.0.smithi017.stdout:3854: seq_num 1460 ranges {538220=603316,1785475=752745} 2024-09-21T09:54:17.375 INFO:tasks.rados.rados.0.smithi017.stdout:3854: writing smithi01785587-29 from 538220 to 1141536 tid 1 2024-09-21T09:54:17.383 INFO:tasks.rados.rados.0.smithi017.stdout:3854: writing smithi01785587-29 from 1785475 to 2538220 tid 2 2024-09-21T09:54:17.383 INFO:tasks.rados.rados.0.smithi017.stdout:3855: write oid 44 current snap is 482 2024-09-21T09:54:17.383 INFO:tasks.rados.rados.0.smithi017.stdout:3855: seq_num 1461 ranges {555027=678565,1727604=640848} 2024-09-21T09:54:17.389 INFO:tasks.rados.rados.0.smithi017.stdout:3855: writing smithi01785587-44 from 555027 to 1233592 tid 1 2024-09-21T09:54:17.396 INFO:tasks.rados.rados.0.smithi017.stdout:3855: writing smithi01785587-44 from 1727604 to 2368452 tid 2 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3856: read oid 33 snap -1 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3856: expect (ObjNum 1329 snap 446 seq_num 1329) 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3857: snap_create 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3850: finishing write tid 2 to smithi01785587-43 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3850: finishing write tid 3 to smithi01785587-43 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3850: finishing write tid 4 to smithi01785587-43 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 1118 (ObjNum 1459 snap 482 seq_num 1459) dirty exists 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3850: left oid 43 (ObjNum 1459 snap 482 seq_num 1459) 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3852: finishing rollback tid 0 to smithi01785587-31 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3852: finishing rollback tid 1 to smithi01785587-31 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3851: finishing rollback tid 0 to smithi01785587-16 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:3852: finishing rollback tid 2 to smithi01785587-31 2024-09-21T09:54:17.397 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 31 v 2237 (ObjNum 1401 snap 462 seq_num 1401) dirty exists 2024-09-21T09:54:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3851: finishing rollback tid 1 to smithi01785587-16 2024-09-21T09:54:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3851: finishing rollback tid 2 to smithi01785587-16 2024-09-21T09:54:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 2098 (ObjNum 1389 snap 461 seq_num 1389) dirty exists 2024-09-21T09:54:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3854: finishing write tid 1 to smithi01785587-29 2024-09-21T09:54:17.398 INFO:tasks.rados.rados.0.smithi017.stdout:3853: expect (ObjNum 1268 snap 424 seq_num 1268) 2024-09-21T09:54:17.561 INFO:tasks.rados.rados.0.smithi017.stdout:3854: finishing write tid 2 to smithi01785587-29 2024-09-21T09:54:17.561 INFO:tasks.rados.rados.0.smithi017.stdout:3854: finishing write tid 3 to smithi01785587-29 2024-09-21T09:54:17.561 INFO:tasks.rados.rados.0.smithi017.stdout:3854: finishing write tid 4 to smithi01785587-29 2024-09-21T09:54:17.562 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 1124 (ObjNum 1460 snap 482 seq_num 1460) dirty exists 2024-09-21T09:54:17.562 INFO:tasks.rados.rados.0.smithi017.stdout:3854: left oid 29 (ObjNum 1460 snap 482 seq_num 1460) 2024-09-21T09:54:17.562 INFO:tasks.rados.rados.0.smithi017.stdout:3855: finishing write tid 1 to smithi01785587-44 2024-09-21T09:54:17.562 INFO:tasks.rados.rados.0.smithi017.stdout:3855: finishing write tid 2 to smithi01785587-44 2024-09-21T09:54:17.562 INFO:tasks.rados.rados.0.smithi017.stdout:3855: finishing write tid 3 to smithi01785587-44 2024-09-21T09:54:17.562 INFO:tasks.rados.rados.0.smithi017.stdout:3855: finishing write tid 4 to smithi01785587-44 2024-09-21T09:54:17.562 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 2241 (ObjNum 1461 snap 482 seq_num 1461) dirty exists 2024-09-21T09:54:17.562 INFO:tasks.rados.rados.0.smithi017.stdout:3855: left oid 44 (ObjNum 1461 snap 482 seq_num 1461) 2024-09-21T09:54:17.562 INFO:tasks.rados.rados.0.smithi017.stdout:3856: expect (ObjNum 1329 snap 446 seq_num 1329) 2024-09-21T09:54:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:17 smithi017 ceph-mon[102913]: pgmap v1125: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-21T09:54:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:17 smithi017 ceph-mon[102913]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-21T09:54:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:17 smithi017 ceph-mon[105151]: pgmap v1125: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-21T09:54:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:17 smithi017 ceph-mon[105151]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-21T09:54:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:17 smithi134 ceph-mon[90085]: pgmap v1125: 89 pgs: 89 active+clean; 184 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-21T09:54:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:17 smithi134 ceph-mon[90085]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-21T09:54:18.241 INFO:tasks.rados.rados.0.smithi017.stdout:3850: done (7 left) 2024-09-21T09:54:18.241 INFO:tasks.rados.rados.0.smithi017.stdout:3851: done (6 left) 2024-09-21T09:54:18.241 INFO:tasks.rados.rados.0.smithi017.stdout:3852: done (5 left) 2024-09-21T09:54:18.241 INFO:tasks.rados.rados.0.smithi017.stdout:3853: done (4 left) 2024-09-21T09:54:18.241 INFO:tasks.rados.rados.0.smithi017.stdout:3854: done (3 left) 2024-09-21T09:54:18.241 INFO:tasks.rados.rados.0.smithi017.stdout:3855: done (2 left) 2024-09-21T09:54:18.241 INFO:tasks.rados.rados.0.smithi017.stdout:3856: done (1 left) 2024-09-21T09:54:18.242 INFO:tasks.rados.rados.0.smithi017.stdout:3857: done (0 left) 2024-09-21T09:54:18.242 INFO:tasks.rados.rados.0.smithi017.stdout:3858: write oid 44 current snap is 483 2024-09-21T09:54:18.242 INFO:tasks.rados.rados.0.smithi017.stdout:3858: seq_num 1462 ranges {571834=753814,1786086=784506} 2024-09-21T09:54:18.248 INFO:tasks.rados.rados.0.smithi017.stdout:3858: writing smithi01785587-44 from 571834 to 1325648 tid 1 2024-09-21T09:54:18.256 INFO:tasks.rados.rados.0.smithi017.stdout:3858: writing smithi01785587-44 from 1786086 to 2570592 tid 2 2024-09-21T09:54:18.256 INFO:tasks.rados.rados.0.smithi017.stdout:3859: read oid 10 snap 477 2024-09-21T09:54:18.256 INFO:tasks.rados.rados.0.smithi017.stdout:3859: expect deleted 2024-09-21T09:54:18.256 INFO:tasks.rados.rados.0.smithi017.stdout:3860: write oid 41 current snap is 483 2024-09-21T09:54:18.256 INFO:tasks.rados.rados.0.smithi017.stdout:3860: seq_num 1463 ranges {588641=429063,1444568=644517,2492662=95979} 2024-09-21T09:54:18.261 INFO:tasks.rados.rados.0.smithi017.stdout:3860: writing smithi01785587-41 from 588641 to 1017704 tid 1 2024-09-21T09:54:18.268 INFO:tasks.rados.rados.0.smithi017.stdout:3860: writing smithi01785587-41 from 1444568 to 2089085 tid 2 2024-09-21T09:54:18.271 INFO:tasks.rados.rados.0.smithi017.stdout:3860: writing smithi01785587-41 from 2492662 to 2588641 tid 3 2024-09-21T09:54:18.271 INFO:tasks.rados.rados.0.smithi017.stdout:3861: delete oid 37 current snap is 483 2024-09-21T09:54:18.271 INFO:tasks.rados.rados.0.smithi017.stdout:3858: finishing write tid 1 to smithi01785587-44 2024-09-21T09:54:18.272 INFO:tasks.rados.rados.0.smithi017.stdout:3858: finishing write tid 2 to smithi01785587-44 2024-09-21T09:54:18.272 INFO:tasks.rados.rados.0.smithi017.stdout:3858: finishing write tid 3 to smithi01785587-44 2024-09-21T09:54:18.272 INFO:tasks.rados.rados.0.smithi017.stdout:3858: finishing write tid 4 to smithi01785587-44 2024-09-21T09:54:18.272 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 2245 (ObjNum 1462 snap 483 seq_num 1462) dirty exists 2024-09-21T09:54:18.272 INFO:tasks.rados.rados.0.smithi017.stdout:3858: left oid 44 (ObjNum 1462 snap 483 seq_num 1462) 2024-09-21T09:54:18.272 INFO:tasks.rados.rados.0.smithi017.stdout:3860: finishing write tid 1 to smithi01785587-41 2024-09-21T09:54:18.273 INFO:tasks.rados.rados.0.smithi017.stdout:3860: finishing write tid 2 to smithi01785587-41 2024-09-21T09:54:18.273 INFO:tasks.rados.rados.0.smithi017.stdout:3860: finishing write tid 3 to smithi01785587-41 2024-09-21T09:54:18.273 INFO:tasks.rados.rados.0.smithi017.stdout:3860: finishing write tid 4 to smithi01785587-41 2024-09-21T09:54:18.273 INFO:tasks.rados.rados.0.smithi017.stdout:3860: finishing write tid 5 to smithi01785587-41 2024-09-21T09:54:18.274 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 41 v 599 (ObjNum 1463 snap 483 seq_num 1463) dirty exists 2024-09-21T09:54:18.274 INFO:tasks.rados.rados.0.smithi017.stdout:3860: left oid 41 (ObjNum 1463 snap 483 seq_num 1463) 2024-09-21T09:54:18.274 INFO:tasks.rados.rados.0.smithi017.stdout:3858: done (3 left) 2024-09-21T09:54:18.274 INFO:tasks.rados.rados.0.smithi017.stdout:3859: done (2 left) 2024-09-21T09:54:18.274 INFO:tasks.rados.rados.0.smithi017.stdout:3860: done (1 left) 2024-09-21T09:54:18.274 INFO:tasks.rados.rados.0.smithi017.stdout:3861: done (0 left) 2024-09-21T09:54:18.274 INFO:tasks.rados.rados.0.smithi017.stdout:3862: delete oid 18 current snap is 483 2024-09-21T09:54:18.277 INFO:tasks.rados.rados.0.smithi017.stdout:3862: done (0 left) 2024-09-21T09:54:18.277 INFO:tasks.rados.rados.0.smithi017.stdout:3863: delete oid 47 current snap is 483 2024-09-21T09:54:18.280 INFO:tasks.rados.rados.0.smithi017.stdout:3863: done (0 left) 2024-09-21T09:54:18.280 INFO:tasks.rados.rados.0.smithi017.stdout:3864: snap_remove snap 474 2024-09-21T09:54:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:18 smithi017 ceph-mon[102913]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-21T09:54:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:18 smithi017 ceph-mon[102913]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-21T09:54:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:18 smithi017 ceph-mon[105151]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-21T09:54:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:18 smithi017 ceph-mon[105151]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-21T09:54:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:18 smithi134 ceph-mon[90085]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-21T09:54:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:18 smithi134 ceph-mon[90085]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-21T09:54:19.242 INFO:tasks.rados.rados.0.smithi017.stdout:3864: done (0 left) 2024-09-21T09:54:19.243 INFO:tasks.rados.rados.0.smithi017.stdout:3865: write oid 30 current snap is 483 2024-09-21T09:54:19.243 INFO:tasks.rados.rados.0.smithi017.stdout:3865: seq_num 1467 ranges {655869=730059,1962143=535502} 2024-09-21T09:54:19.251 INFO:tasks.rados.rados.0.smithi017.stdout:3865: writing smithi01785587-30 from 655869 to 1385928 tid 1 2024-09-21T09:54:19.258 INFO:tasks.rados.rados.0.smithi017.stdout:3865: writing smithi01785587-30 from 1962143 to 2497645 tid 2 2024-09-21T09:54:19.258 INFO:tasks.rados.rados.0.smithi017.stdout:3866: snap_remove snap 478 2024-09-21T09:54:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:19 smithi017 ceph-mon[102913]: pgmap v1128: 89 pgs: 89 active+clean; 180 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.1 MiB/s wr, 19 op/s 2024-09-21T09:54:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:19 smithi017 ceph-mon[102913]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-21T09:54:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:19 smithi017 ceph-mon[105151]: pgmap v1128: 89 pgs: 89 active+clean; 180 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.1 MiB/s wr, 19 op/s 2024-09-21T09:54:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:19 smithi017 ceph-mon[105151]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-21T09:54:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:19 smithi134 ceph-mon[90085]: pgmap v1128: 89 pgs: 89 active+clean; 180 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.1 MiB/s wr, 19 op/s 2024-09-21T09:54:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:19 smithi134 ceph-mon[90085]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-21T09:54:20.244 INFO:tasks.rados.rados.0.smithi017.stdout:3866: done (1 left) 2024-09-21T09:54:20.244 INFO:tasks.rados.rados.0.smithi017.stdout:3867: rollback oid 43 current snap is 483 2024-09-21T09:54:20.244 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 479 2024-09-21T09:54:20.244 INFO:tasks.rados.rados.0.smithi017.stdout:3865: finishing write tid 1 to smithi01785587-30 2024-09-21T09:54:20.244 INFO:tasks.rados.rados.0.smithi017.stdout:3865: finishing write tid 2 to smithi01785587-30 2024-09-21T09:54:20.244 INFO:tasks.rados.rados.0.smithi017.stdout:3865: finishing write tid 3 to smithi01785587-30 2024-09-21T09:54:20.245 INFO:tasks.rados.rados.0.smithi017.stdout:3865: finishing write tid 4 to smithi01785587-30 2024-09-21T09:54:20.245 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1625 (ObjNum 1467 snap 483 seq_num 1467) dirty exists 2024-09-21T09:54:20.245 INFO:tasks.rados.rados.0.smithi017.stdout:3865: left oid 30 (ObjNum 1467 snap 483 seq_num 1467) 2024-09-21T09:54:20.245 INFO:tasks.rados.rados.0.smithi017.stdout:3865: done (1 left) 2024-09-21T09:54:20.245 INFO:tasks.rados.rados.0.smithi017.stdout:3868: snap_create 2024-09-21T09:54:20.249 INFO:tasks.rados.rados.0.smithi017.stdout:3867: finishing rollback tid 0 to smithi01785587-43 2024-09-21T09:54:20.250 INFO:tasks.rados.rados.0.smithi017.stdout:3867: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:54:20.250 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 1123 (ObjNum 33 snap 0 seq_num 2400796560) dirty dne 2024-09-21T09:54:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:54:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:54:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:21.244 INFO:tasks.rados.rados.0.smithi017.stdout:3867: done (1 left) 2024-09-21T09:54:21.244 INFO:tasks.rados.rados.0.smithi017.stdout:3868: done (0 left) 2024-09-21T09:54:21.244 INFO:tasks.rados.rados.0.smithi017.stdout:3869: snap_create 2024-09-21T09:54:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:21 smithi017 ceph-mon[102913]: pgmap v1131: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 0 B/s wr, 4 op/s 2024-09-21T09:54:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:21 smithi017 ceph-mon[102913]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-21T09:54:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:21 smithi017 ceph-mon[105151]: pgmap v1131: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 0 B/s wr, 4 op/s 2024-09-21T09:54:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:21 smithi017 ceph-mon[105151]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-21T09:54:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:21 smithi134 ceph-mon[90085]: pgmap v1131: 89 pgs: 89 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 0 B/s wr, 4 op/s 2024-09-21T09:54:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:21 smithi134 ceph-mon[90085]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-21T09:54:22.254 INFO:tasks.rados.rados.0.smithi017.stdout:3869: done (0 left) 2024-09-21T09:54:22.255 INFO:tasks.rados.rados.0.smithi017.stdout:3870: write oid 3 current snap is 485 2024-09-21T09:54:22.255 INFO:tasks.rados.rados.0.smithi017.stdout:3870: seq_num 1468 ranges {672676=521661,1736978=795513} 2024-09-21T09:54:22.261 INFO:tasks.rados.rados.0.smithi017.stdout:3870: writing smithi01785587-3 from 672676 to 1194337 tid 1 2024-09-21T09:54:22.269 INFO:tasks.rados.rados.0.smithi017.stdout:3870: writing smithi01785587-3 from 1736978 to 2532491 tid 2 2024-09-21T09:54:22.269 INFO:tasks.rados.rados.0.smithi017.stdout:3871: snap_remove snap 480 2024-09-21T09:54:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:22 smithi017 ceph-mon[102913]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-21T09:54:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:22 smithi017 ceph-mon[105151]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-21T09:54:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:22 smithi134 ceph-mon[90085]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-21T09:54:23.242 INFO:tasks.rados.rados.0.smithi017.stdout:3871: done (1 left) 2024-09-21T09:54:23.242 INFO:tasks.rados.rados.0.smithi017.stdout:3872: snap_remove snap 477 2024-09-21T09:54:23.242 INFO:tasks.rados.rados.0.smithi017.stdout:3870: finishing write tid 1 to smithi01785587-3 2024-09-21T09:54:23.242 INFO:tasks.rados.rados.0.smithi017.stdout:3870: finishing write tid 2 to smithi01785587-3 2024-09-21T09:54:23.242 INFO:tasks.rados.rados.0.smithi017.stdout:3870: finishing write tid 3 to smithi01785587-3 2024-09-21T09:54:23.242 INFO:tasks.rados.rados.0.smithi017.stdout:3870: finishing write tid 4 to smithi01785587-3 2024-09-21T09:54:23.242 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 2258 (ObjNum 1468 snap 485 seq_num 1468) dirty exists 2024-09-21T09:54:23.243 INFO:tasks.rados.rados.0.smithi017.stdout:3870: left oid 3 (ObjNum 1468 snap 485 seq_num 1468) 2024-09-21T09:54:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:23 smithi017 ceph-mon[105151]: pgmap v1134: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:54:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:23 smithi017 ceph-mon[105151]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-21T09:54:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:23 smithi017 ceph-mon[105151]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-21T09:54:23.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:23 smithi017 ceph-mon[102913]: pgmap v1134: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:54:23.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:23 smithi017 ceph-mon[102913]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-21T09:54:23.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:23 smithi017 ceph-mon[102913]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-21T09:54:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:23 smithi134 ceph-mon[90085]: pgmap v1134: 89 pgs: 89 active+clean; 182 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:54:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:23 smithi134 ceph-mon[90085]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-21T09:54:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:23 smithi134 ceph-mon[90085]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-21T09:54:24.243 INFO:tasks.rados.rados.0.smithi017.stdout:3870: done (1 left) 2024-09-21T09:54:24.243 INFO:tasks.rados.rados.0.smithi017.stdout:3872: done (0 left) 2024-09-21T09:54:24.244 INFO:tasks.rados.rados.0.smithi017.stdout:3873: delete oid 24 current snap is 485 2024-09-21T09:54:24.252 INFO:tasks.rados.rados.0.smithi017.stdout:3873: done (0 left) 2024-09-21T09:54:24.253 INFO:tasks.rados.rados.0.smithi017.stdout:3874: rollback oid 39 current snap is 485 2024-09-21T09:54:24.253 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 39 to 482 2024-09-21T09:54:24.253 INFO:tasks.rados.rados.0.smithi017.stdout:3875: write oid 50 current snap is 485 2024-09-21T09:54:24.253 INFO:tasks.rados.rados.0.smithi017.stdout:3875: seq_num 1470 ranges {706290=672159,2137589=568701} 2024-09-21T09:54:24.260 INFO:tasks.rados.rados.0.smithi017.stdout:3875: writing smithi01785587-50 from 706290 to 1378449 tid 1 2024-09-21T09:54:24.268 INFO:tasks.rados.rados.0.smithi017.stdout:3875: writing smithi01785587-50 from 2137589 to 2706290 tid 2 2024-09-21T09:54:24.268 INFO:tasks.rados.rados.0.smithi017.stdout:3876: write oid 37 current snap is 485 2024-09-21T09:54:24.268 INFO:tasks.rados.rados.0.smithi017.stdout:3876: seq_num 1471 ranges {723097=747408,2196071=527026} 2024-09-21T09:54:24.276 INFO:tasks.rados.rados.0.smithi017.stdout:3876: writing smithi01785587-37 from 723097 to 1470505 tid 1 2024-09-21T09:54:24.283 INFO:tasks.rados.rados.0.smithi017.stdout:3876: writing smithi01785587-37 from 2196071 to 2723097 tid 2 2024-09-21T09:54:24.283 INFO:tasks.rados.rados.0.smithi017.stdout:3877: snap_remove snap 479 2024-09-21T09:54:25.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:54:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:54:24] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.51.0" 2024-09-21T09:54:25.260 INFO:tasks.rados.rados.0.smithi017.stdout:3874: finishing rollback tid 0 to smithi01785587-39 2024-09-21T09:54:25.260 INFO:tasks.rados.rados.0.smithi017.stdout:3874: finishing rollback tid 1 to smithi01785587-39 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3874: finishing rollback tid 2 to smithi01785587-39 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 39 v 1989 (ObjNum 1424 snap 471 seq_num 1424) dirty exists 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3875: finishing write tid 1 to smithi01785587-50 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3875: finishing write tid 2 to smithi01785587-50 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3875: finishing write tid 3 to smithi01785587-50 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3875: finishing write tid 4 to smithi01785587-50 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 50 v 1196 (ObjNum 1470 snap 485 seq_num 1470) dirty exists 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3875: left oid 50 (ObjNum 1470 snap 485 seq_num 1470) 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3876: finishing write tid 1 to smithi01785587-37 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3876: finishing write tid 2 to smithi01785587-37 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3876: finishing write tid 3 to smithi01785587-37 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3876: finishing write tid 4 to smithi01785587-37 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 37 v 1992 (ObjNum 1471 snap 485 seq_num 1471) dirty exists 2024-09-21T09:54:25.261 INFO:tasks.rados.rados.0.smithi017.stdout:3876: left oid 37 (ObjNum 1471 snap 485 seq_num 1471) 2024-09-21T09:54:25.262 INFO:tasks.rados.rados.0.smithi017.stdout:3874: done (3 left) 2024-09-21T09:54:25.262 INFO:tasks.rados.rados.0.smithi017.stdout:3875: done (2 left) 2024-09-21T09:54:25.262 INFO:tasks.rados.rados.0.smithi017.stdout:3876: done (1 left) 2024-09-21T09:54:25.262 INFO:tasks.rados.rados.0.smithi017.stdout:3877: done (0 left) 2024-09-21T09:54:25.262 INFO:tasks.rados.rados.0.smithi017.stdout:3878: snap_create 2024-09-21T09:54:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:25 smithi134 ceph-mon[90085]: pgmap v1137: 89 pgs: 89 active+clean; 180 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:54:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:25 smithi134 ceph-mon[90085]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-21T09:54:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:25 smithi017 ceph-mon[105151]: pgmap v1137: 89 pgs: 89 active+clean; 180 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:54:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:25 smithi017 ceph-mon[105151]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-21T09:54:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:25 smithi017 ceph-mon[102913]: pgmap v1137: 89 pgs: 89 active+clean; 180 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:54:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:25 smithi017 ceph-mon[102913]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-21T09:54:26.261 INFO:tasks.rados.rados.0.smithi017.stdout:3878: done (0 left) 2024-09-21T09:54:26.261 INFO:tasks.rados.rados.0.smithi017.stdout:3879: snap_create 2024-09-21T09:54:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:26 smithi134 ceph-mon[90085]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-21T09:54:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:26 smithi017 ceph-mon[102913]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-21T09:54:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:26 smithi017 ceph-mon[105151]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-21T09:54:27.263 INFO:tasks.rados.rados.0.smithi017.stdout:3879: done (0 left) 2024-09-21T09:54:27.264 INFO:tasks.rados.rados.0.smithi017.stdout:3880: snap_create 2024-09-21T09:54:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:27 smithi017 ceph-mon[102913]: pgmap v1140: 89 pgs: 89 active+clean; 173 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 322 KiB/s wr, 1 op/s 2024-09-21T09:54:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:27 smithi017 ceph-mon[102913]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-21T09:54:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:27 smithi017 ceph-mon[105151]: pgmap v1140: 89 pgs: 89 active+clean; 173 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 322 KiB/s wr, 1 op/s 2024-09-21T09:54:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:27 smithi017 ceph-mon[105151]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-21T09:54:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:27 smithi134 ceph-mon[90085]: pgmap v1140: 89 pgs: 89 active+clean; 173 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 322 KiB/s wr, 1 op/s 2024-09-21T09:54:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:27 smithi134 ceph-mon[90085]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-21T09:54:28.242 INFO:tasks.rados.rados.0.smithi017.stdout:3880: done (0 left) 2024-09-21T09:54:28.242 INFO:tasks.rados.rados.0.smithi017.stdout:3881: write oid 5 current snap is 488 2024-09-21T09:54:28.242 INFO:tasks.rados.rados.0.smithi017.stdout:3881: seq_num 1472 ranges {739904=422657,1854553=402851} 2024-09-21T09:54:28.248 INFO:tasks.rados.rados.0.smithi017.stdout:3881: writing smithi01785587-5 from 739904 to 1162561 tid 1 2024-09-21T09:54:28.255 INFO:tasks.rados.rados.0.smithi017.stdout:3881: writing smithi01785587-5 from 1854553 to 2257404 tid 2 2024-09-21T09:54:28.255 INFO:tasks.rados.rados.0.smithi017.stdout:3882: write oid 16 current snap is 488 2024-09-21T09:54:28.255 INFO:tasks.rados.rados.0.smithi017.stdout:3881: finishing write tid 1 to smithi01785587-5 2024-09-21T09:54:28.256 INFO:tasks.rados.rados.0.smithi017.stdout:3882: seq_num 1473 ranges {756711=497906,1913035=546509} 2024-09-21T09:54:28.262 INFO:tasks.rados.rados.0.smithi017.stdout:3882: writing smithi01785587-16 from 756711 to 1254617 tid 1 2024-09-21T09:54:28.271 INFO:tasks.rados.rados.0.smithi017.stdout:3882: writing smithi01785587-16 from 1913035 to 2459544 tid 2 2024-09-21T09:54:28.271 INFO:tasks.rados.rados.0.smithi017.stdout:3883: write oid 46 current snap is 488 2024-09-21T09:54:28.271 INFO:tasks.rados.rados.0.smithi017.stdout:3881: finishing write tid 2 to smithi01785587-5 2024-09-21T09:54:28.271 INFO:tasks.rados.rados.0.smithi017.stdout:3881: finishing write tid 3 to smithi01785587-5 2024-09-21T09:54:28.271 INFO:tasks.rados.rados.0.smithi017.stdout:3881: finishing write tid 4 to smithi01785587-5 2024-09-21T09:54:28.271 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 2276 (ObjNum 1472 snap 488 seq_num 1472) dirty exists 2024-09-21T09:54:28.271 INFO:tasks.rados.rados.0.smithi017.stdout:3881: left oid 5 (ObjNum 1472 snap 488 seq_num 1472) 2024-09-21T09:54:28.271 INFO:tasks.rados.rados.0.smithi017.stdout:3882: finishing write tid 1 to smithi01785587-16 2024-09-21T09:54:28.271 INFO:tasks.rados.rados.0.smithi017.stdout:3883: seq_num 1474 ranges {773518=573155,1855164=406520} 2024-09-21T09:54:28.278 INFO:tasks.rados.rados.0.smithi017.stdout:3883: writing smithi01785587-46 from 773518 to 1346673 tid 1 2024-09-21T09:54:28.284 INFO:tasks.rados.rados.0.smithi017.stdout:3883: writing smithi01785587-46 from 1855164 to 2261684 tid 2 2024-09-21T09:54:28.284 INFO:tasks.rados.rados.0.smithi017.stdout:3881: done (2 left) 2024-09-21T09:54:28.284 INFO:tasks.rados.rados.0.smithi017.stdout:3884: read oid 37 snap -1 2024-09-21T09:54:28.284 INFO:tasks.rados.rados.0.smithi017.stdout:3884: expect (ObjNum 1471 snap 485 seq_num 1471) 2024-09-21T09:54:28.284 INFO:tasks.rados.rados.0.smithi017.stdout:3885: delete oid 6 current snap is 488 2024-09-21T09:54:28.284 INFO:tasks.rados.rados.0.smithi017.stdout:3882: finishing write tid 2 to smithi01785587-16 2024-09-21T09:54:28.284 INFO:tasks.rados.rados.0.smithi017.stdout:3882: finishing write tid 3 to smithi01785587-16 2024-09-21T09:54:28.284 INFO:tasks.rados.rados.0.smithi017.stdout:3882: finishing write tid 4 to smithi01785587-16 2024-09-21T09:54:28.284 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 16 v 2114 (ObjNum 1473 snap 488 seq_num 1473) dirty exists 2024-09-21T09:54:28.285 INFO:tasks.rados.rados.0.smithi017.stdout:3882: left oid 16 (ObjNum 1473 snap 488 seq_num 1473) 2024-09-21T09:54:28.287 INFO:tasks.rados.rados.0.smithi017.stdout:3882: done (3 left) 2024-09-21T09:54:28.287 INFO:tasks.rados.rados.0.smithi017.stdout:3885: done (2 left) 2024-09-21T09:54:28.287 INFO:tasks.rados.rados.0.smithi017.stdout:3886: read oid 3 snap -1 2024-09-21T09:54:28.287 INFO:tasks.rados.rados.0.smithi017.stdout:3886: expect (ObjNum 1468 snap 485 seq_num 1468) 2024-09-21T09:54:28.287 INFO:tasks.rados.rados.0.smithi017.stdout:3887: write oid 15 current snap is 488 2024-09-21T09:54:28.287 INFO:tasks.rados.rados.0.smithi017.stdout:3887: seq_num 1476 ranges {407132=440006,1288481=693836,2616455=190677} 2024-09-21T09:54:28.291 INFO:tasks.rados.rados.0.smithi017.stdout:3887: writing smithi01785587-15 from 407132 to 847138 tid 1 2024-09-21T09:54:28.298 INFO:tasks.rados.rados.0.smithi017.stdout:3887: writing smithi01785587-15 from 1288481 to 1982317 tid 2 2024-09-21T09:54:28.303 INFO:tasks.rados.rados.0.smithi017.stdout:3887: writing smithi01785587-15 from 2616455 to 2807132 tid 3 2024-09-21T09:54:28.303 INFO:tasks.rados.rados.0.smithi017.stdout:3888: delete oid 5 current snap is 488 2024-09-21T09:54:28.303 INFO:tasks.rados.rados.0.smithi017.stdout:3883: finishing write tid 1 to smithi01785587-46 2024-09-21T09:54:28.303 INFO:tasks.rados.rados.0.smithi017.stdout:3883: finishing write tid 2 to smithi01785587-46 2024-09-21T09:54:28.303 INFO:tasks.rados.rados.0.smithi017.stdout:3883: finishing write tid 3 to smithi01785587-46 2024-09-21T09:54:28.303 INFO:tasks.rados.rados.0.smithi017.stdout:3883: finishing write tid 4 to smithi01785587-46 2024-09-21T09:54:28.303 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 1993 (ObjNum 1474 snap 488 seq_num 1474) dirty exists 2024-09-21T09:54:28.304 INFO:tasks.rados.rados.0.smithi017.stdout:3883: left oid 46 (ObjNum 1474 snap 488 seq_num 1474) 2024-09-21T09:54:28.304 INFO:tasks.rados.rados.0.smithi017.stdout:3884: expect (ObjNum 1471 snap 485 seq_num 1471) 2024-09-21T09:54:28.394 INFO:tasks.rados.rados.0.smithi017.stdout:3886: expect (ObjNum 1468 snap 485 seq_num 1468) 2024-09-21T09:54:28.481 INFO:tasks.rados.rados.0.smithi017.stdout:3887: finishing write tid 1 to smithi01785587-15 2024-09-21T09:54:28.482 INFO:tasks.rados.rados.0.smithi017.stdout:3887: finishing write tid 2 to smithi01785587-15 2024-09-21T09:54:28.482 INFO:tasks.rados.rados.0.smithi017.stdout:3887: finishing write tid 3 to smithi01785587-15 2024-09-21T09:54:28.482 INFO:tasks.rados.rados.0.smithi017.stdout:3887: finishing write tid 4 to smithi01785587-15 2024-09-21T09:54:28.482 INFO:tasks.rados.rados.0.smithi017.stdout:3887: finishing write tid 5 to smithi01785587-15 2024-09-21T09:54:28.482 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 15 v 616 (ObjNum 1476 snap 488 seq_num 1476) dirty exists 2024-09-21T09:54:28.482 INFO:tasks.rados.rados.0.smithi017.stdout:3887: left oid 15 (ObjNum 1476 snap 488 seq_num 1476) 2024-09-21T09:54:28.482 INFO:tasks.rados.rados.0.smithi017.stdout:3883: done (4 left) 2024-09-21T09:54:28.483 INFO:tasks.rados.rados.0.smithi017.stdout:3884: done (3 left) 2024-09-21T09:54:28.483 INFO:tasks.rados.rados.0.smithi017.stdout:3886: done (2 left) 2024-09-21T09:54:28.483 INFO:tasks.rados.rados.0.smithi017.stdout:3887: done (1 left) 2024-09-21T09:54:28.483 INFO:tasks.rados.rados.0.smithi017.stdout:3888: done (0 left) 2024-09-21T09:54:28.483 INFO:tasks.rados.rados.0.smithi017.stdout:3889: write oid 47 current snap is 488 2024-09-21T09:54:28.483 INFO:tasks.rados.rados.0.smithi017.stdout:3889: seq_num 1478 ranges {440746=590504,1689092=697505} 2024-09-21T09:54:28.486 INFO:tasks.rados.rados.0.smithi017.stdout:3889: writing smithi01785587-47 from 440746 to 1031250 tid 1 2024-09-21T09:54:28.494 INFO:tasks.rados.rados.0.smithi017.stdout:3889: writing smithi01785587-47 from 1689092 to 2386597 tid 2 2024-09-21T09:54:28.494 INFO:tasks.rados.rados.0.smithi017.stdout:3890: read oid 49 snap -1 2024-09-21T09:54:28.494 INFO:tasks.rados.rados.0.smithi017.stdout:3890: expect (ObjNum 1393 snap 462 seq_num 1393) 2024-09-21T09:54:28.494 INFO:tasks.rados.rados.0.smithi017.stdout:3891: write oid 23 current snap is 488 2024-09-21T09:54:28.495 INFO:tasks.rados.rados.0.smithi017.stdout:3889: finishing write tid 1 to smithi01785587-47 2024-09-21T09:54:28.495 INFO:tasks.rados.rados.0.smithi017.stdout:3891: seq_num 1479 ranges {457553=665753,1747574=557516} 2024-09-21T09:54:28.500 INFO:tasks.rados.rados.0.smithi017.stdout:3891: writing smithi01785587-23 from 457553 to 1123306 tid 1 2024-09-21T09:54:28.507 INFO:tasks.rados.rados.0.smithi017.stdout:3891: writing smithi01785587-23 from 1747574 to 2305090 tid 2 2024-09-21T09:54:28.508 INFO:tasks.rados.rados.0.smithi017.stdout:3892: write oid 44 current snap is 488 2024-09-21T09:54:28.508 INFO:tasks.rados.rados.0.smithi017.stdout:3892: seq_num 1480 ranges {474360=741002,1806056=701174} 2024-09-21T09:54:28.513 INFO:tasks.rados.rados.0.smithi017.stdout:3892: writing smithi01785587-44 from 474360 to 1215362 tid 1 2024-09-21T09:54:28.521 INFO:tasks.rados.rados.0.smithi017.stdout:3892: writing smithi01785587-44 from 1806056 to 2507230 tid 2 2024-09-21T09:54:28.521 INFO:tasks.rados.rados.0.smithi017.stdout:3893: snap_create 2024-09-21T09:54:28.521 INFO:tasks.rados.rados.0.smithi017.stdout:3889: finishing write tid 2 to smithi01785587-47 2024-09-21T09:54:28.521 INFO:tasks.rados.rados.0.smithi017.stdout:3890: expect (ObjNum 1393 snap 462 seq_num 1393) 2024-09-21T09:54:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:28 smithi017 ceph-mon[102913]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-21T09:54:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:28 smithi017 ceph-mon[102913]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-21T09:54:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:28 smithi017 ceph-mon[105151]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-21T09:54:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:28 smithi017 ceph-mon[105151]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-21T09:54:28.729 INFO:tasks.rados.rados.0.smithi017.stdout:3889: finishing write tid 3 to smithi01785587-47 2024-09-21T09:54:28.729 INFO:tasks.rados.rados.0.smithi017.stdout:3889: finishing write tid 4 to smithi01785587-47 2024-09-21T09:54:28.729 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 47 v 1337 (ObjNum 1478 snap 488 seq_num 1478) dirty exists 2024-09-21T09:54:28.729 INFO:tasks.rados.rados.0.smithi017.stdout:3889: left oid 47 (ObjNum 1478 snap 488 seq_num 1478) 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3891: finishing write tid 1 to smithi01785587-23 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3891: finishing write tid 2 to smithi01785587-23 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3891: finishing write tid 3 to smithi01785587-23 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3891: finishing write tid 4 to smithi01785587-23 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 23 v 1997 (ObjNum 1479 snap 488 seq_num 1479) dirty exists 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3891: left oid 23 (ObjNum 1479 snap 488 seq_num 1479) 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3892: finishing write tid 1 to smithi01785587-44 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3892: finishing write tid 2 to smithi01785587-44 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3892: finishing write tid 3 to smithi01785587-44 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3892: finishing write tid 4 to smithi01785587-44 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 2281 (ObjNum 1480 snap 488 seq_num 1480) dirty exists 2024-09-21T09:54:28.730 INFO:tasks.rados.rados.0.smithi017.stdout:3892: left oid 44 (ObjNum 1480 snap 488 seq_num 1480) 2024-09-21T09:54:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:28 smithi134 ceph-mon[90085]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-21T09:54:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:28 smithi134 ceph-mon[90085]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-21T09:54:29.281 INFO:tasks.rados.rados.0.smithi017.stdout:3889: done (4 left) 2024-09-21T09:54:29.281 INFO:tasks.rados.rados.0.smithi017.stdout:3890: done (3 left) 2024-09-21T09:54:29.281 INFO:tasks.rados.rados.0.smithi017.stdout:3891: done (2 left) 2024-09-21T09:54:29.281 INFO:tasks.rados.rados.0.smithi017.stdout:3892: done (1 left) 2024-09-21T09:54:29.281 INFO:tasks.rados.rados.0.smithi017.stdout:3893: done (0 left) 2024-09-21T09:54:29.282 INFO:tasks.rados.rados.0.smithi017.stdout:3894: read oid 50 snap -1 2024-09-21T09:54:29.282 INFO:tasks.rados.rados.0.smithi017.stdout:3894: expect (ObjNum 1470 snap 485 seq_num 1470) 2024-09-21T09:54:29.282 INFO:tasks.rados.rados.0.smithi017.stdout:3895: write oid 13 current snap is 489 2024-09-21T09:54:29.282 INFO:tasks.rados.rados.0.smithi017.stdout:3895: seq_num 1481 ranges {491167=416251,1464538=561185,2753570=137597} 2024-09-21T09:54:29.286 INFO:tasks.rados.rados.0.smithi017.stdout:3895: writing smithi01785587-13 from 491167 to 907418 tid 1 2024-09-21T09:54:29.292 INFO:tasks.rados.rados.0.smithi017.stdout:3895: writing smithi01785587-13 from 1464538 to 2025723 tid 2 2024-09-21T09:54:29.298 INFO:tasks.rados.rados.0.smithi017.stdout:3895: writing smithi01785587-13 from 2753570 to 2891167 tid 3 2024-09-21T09:54:29.298 INFO:tasks.rados.rados.0.smithi017.stdout:3896: read oid 30 snap 453 2024-09-21T09:54:29.298 INFO:tasks.rados.rados.0.smithi017.stdout:3896: expect deleted 2024-09-21T09:54:29.298 INFO:tasks.rados.rados.0.smithi017.stdout:3894: expect (ObjNum 1470 snap 485 seq_num 1470) 2024-09-21T09:54:29.417 INFO:tasks.rados.rados.0.smithi017.stdout:3895: finishing write tid 1 to smithi01785587-13 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3895: finishing write tid 2 to smithi01785587-13 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3895: finishing write tid 3 to smithi01785587-13 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3895: finishing write tid 4 to smithi01785587-13 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3895: finishing write tid 5 to smithi01785587-13 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 13 v 1642 (ObjNum 1481 snap 489 seq_num 1481) dirty exists 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3895: left oid 13 (ObjNum 1481 snap 489 seq_num 1481) 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3897: read oid 5 snap -1 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3897: expect deleted 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3894: done (3 left) 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3895: done (2 left) 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3896: done (1 left) 2024-09-21T09:54:29.418 INFO:tasks.rados.rados.0.smithi017.stdout:3898: delete oid 30 current snap is 489 2024-09-21T09:54:29.419 INFO:tasks.rados.rados.0.smithi017.stdout:3897: done (1 left) 2024-09-21T09:54:29.419 INFO:tasks.rados.rados.0.smithi017.stdout:3898: done (0 left) 2024-09-21T09:54:29.419 INFO:tasks.rados.rados.0.smithi017.stdout:3899: snap_create 2024-09-21T09:54:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:29 smithi017 ceph-mon[102913]: pgmap v1143: 89 pgs: 89 active+clean; 167 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 625 KiB/s wr, 2 op/s 2024-09-21T09:54:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:29 smithi017 ceph-mon[105151]: pgmap v1143: 89 pgs: 89 active+clean; 167 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 625 KiB/s wr, 2 op/s 2024-09-21T09:54:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:29 smithi134 ceph-mon[90085]: pgmap v1143: 89 pgs: 89 active+clean; 167 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 625 KiB/s wr, 2 op/s 2024-09-21T09:54:30.288 INFO:tasks.rados.rados.0.smithi017.stdout:3899: done (0 left) 2024-09-21T09:54:30.288 INFO:tasks.rados.rados.0.smithi017.stdout:3900: read oid 3 snap -1 2024-09-21T09:54:30.288 INFO:tasks.rados.rados.0.smithi017.stdout:3900: expect (ObjNum 1468 snap 485 seq_num 1468) 2024-09-21T09:54:30.288 INFO:tasks.rados.rados.0.smithi017.stdout:3901: delete oid 34 current snap is 490 2024-09-21T09:54:30.294 INFO:tasks.rados.rados.0.smithi017.stdout:3900: expect (ObjNum 1468 snap 485 seq_num 1468) 2024-09-21T09:54:30.382 INFO:tasks.rados.rados.0.smithi017.stdout:3900: done (1 left) 2024-09-21T09:54:30.382 INFO:tasks.rados.rados.0.smithi017.stdout:3901: done (0 left) 2024-09-21T09:54:30.382 INFO:tasks.rados.rados.0.smithi017.stdout:3902: rollback oid 14 current snap is 490 2024-09-21T09:54:30.382 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 14 to 483 2024-09-21T09:54:30.382 INFO:tasks.rados.rados.0.smithi017.stdout:3903: write oid 26 current snap is 490 2024-09-21T09:54:30.382 INFO:tasks.rados.rados.0.smithi017.stdout:3903: seq_num 1484 ranges {541588=758351,2039984=708512} 2024-09-21T09:54:30.389 INFO:tasks.rados.rados.0.smithi017.stdout:3903: writing smithi01785587-26 from 541588 to 1299939 tid 1 2024-09-21T09:54:30.397 INFO:tasks.rados.rados.0.smithi017.stdout:3903: writing smithi01785587-26 from 2039984 to 2748496 tid 2 2024-09-21T09:54:30.397 INFO:tasks.rados.rados.0.smithi017.stdout:3904: read oid 38 snap -1 2024-09-21T09:54:30.397 INFO:tasks.rados.rados.0.smithi017.stdout:3904: expect deleted 2024-09-21T09:54:30.397 INFO:tasks.rados.rados.0.smithi017.stdout:3905: delete oid 21 current snap is 490 2024-09-21T09:54:30.397 INFO:tasks.rados.rados.0.smithi017.stdout:3902: finishing rollback tid 1 to smithi01785587-14 2024-09-21T09:54:30.397 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 14 v 2095 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:54:30.398 INFO:tasks.rados.rados.0.smithi017.stdout:3903: finishing write tid 1 to smithi01785587-26 2024-09-21T09:54:30.400 INFO:tasks.rados.rados.0.smithi017.stdout:3902: done (3 left) 2024-09-21T09:54:30.400 INFO:tasks.rados.rados.0.smithi017.stdout:3904: done (2 left) 2024-09-21T09:54:30.400 INFO:tasks.rados.rados.0.smithi017.stdout:3905: done (1 left) 2024-09-21T09:54:30.400 INFO:tasks.rados.rados.0.smithi017.stdout:3906: write oid 49 current snap is 490 2024-09-21T09:54:30.401 INFO:tasks.rados.rados.0.smithi017.stdout:3906: seq_num 1486 ranges {575202=508849,1640595=712181} 2024-09-21T09:54:30.406 INFO:tasks.rados.rados.0.smithi017.stdout:3906: writing smithi01785587-49 from 575202 to 1084051 tid 1 2024-09-21T09:54:30.413 INFO:tasks.rados.rados.0.smithi017.stdout:3906: writing smithi01785587-49 from 1640595 to 2352776 tid 2 2024-09-21T09:54:30.414 INFO:tasks.rados.rados.0.smithi017.stdout:3907: delete oid 46 current snap is 490 2024-09-21T09:54:30.414 INFO:tasks.rados.rados.0.smithi017.stdout:3903: finishing write tid 2 to smithi01785587-26 2024-09-21T09:54:30.414 INFO:tasks.rados.rados.0.smithi017.stdout:3903: finishing write tid 3 to smithi01785587-26 2024-09-21T09:54:30.414 INFO:tasks.rados.rados.0.smithi017.stdout:3903: finishing write tid 4 to smithi01785587-26 2024-09-21T09:54:30.414 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 2118 (ObjNum 1484 snap 490 seq_num 1484) dirty exists 2024-09-21T09:54:30.414 INFO:tasks.rados.rados.0.smithi017.stdout:3903: left oid 26 (ObjNum 1484 snap 490 seq_num 1484) 2024-09-21T09:54:30.414 INFO:tasks.rados.rados.0.smithi017.stdout:3906: finishing write tid 1 to smithi01785587-49 2024-09-21T09:54:30.416 INFO:tasks.rados.rados.0.smithi017.stdout:3903: done (2 left) 2024-09-21T09:54:30.416 INFO:tasks.rados.rados.0.smithi017.stdout:3907: done (1 left) 2024-09-21T09:54:30.417 INFO:tasks.rados.rados.0.smithi017.stdout:3908: write oid 27 current snap is 490 2024-09-21T09:54:30.417 INFO:tasks.rados.rados.0.smithi017.stdout:3908: seq_num 1488 ranges {608816=659347,1757559=715850} 2024-09-21T09:54:30.423 INFO:tasks.rados.rados.0.smithi017.stdout:3908: writing smithi01785587-27 from 608816 to 1268163 tid 1 2024-09-21T09:54:30.430 INFO:tasks.rados.rados.0.smithi017.stdout:3908: writing smithi01785587-27 from 1757559 to 2473409 tid 2 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:3909: read oid 38 snap -1 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:3909: expect deleted 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:3910: write oid 24 current snap is 490 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:3906: finishing write tid 2 to smithi01785587-49 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:3906: finishing write tid 3 to smithi01785587-49 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:3906: finishing write tid 4 to smithi01785587-49 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 49 v 1844 (ObjNum 1486 snap 490 seq_num 1486) dirty exists 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:3906: left oid 49 (ObjNum 1486 snap 490 seq_num 1486) 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:3908: finishing write tid 1 to smithi01785587-27 2024-09-21T09:54:30.431 INFO:tasks.rados.rados.0.smithi017.stdout:3910: seq_num 1489 ranges {625623=734596,1816041=459508,2740248=285375} 2024-09-21T09:54:30.438 INFO:tasks.rados.rados.0.smithi017.stdout:3910: writing smithi01785587-24 from 625623 to 1360219 tid 1 2024-09-21T09:54:30.443 INFO:tasks.rados.rados.0.smithi017.stdout:3910: writing smithi01785587-24 from 1816041 to 2275549 tid 2 2024-09-21T09:54:30.447 INFO:tasks.rados.rados.0.smithi017.stdout:3910: writing smithi01785587-24 from 2740248 to 3025623 tid 3 2024-09-21T09:54:30.447 INFO:tasks.rados.rados.0.smithi017.stdout:3906: done (3 left) 2024-09-21T09:54:30.448 INFO:tasks.rados.rados.0.smithi017.stdout:3911: delete oid 31 current snap is 490 2024-09-21T09:54:30.448 INFO:tasks.rados.rados.0.smithi017.stdout:3908: finishing write tid 2 to smithi01785587-27 2024-09-21T09:54:30.448 INFO:tasks.rados.rados.0.smithi017.stdout:3908: finishing write tid 3 to smithi01785587-27 2024-09-21T09:54:30.448 INFO:tasks.rados.rados.0.smithi017.stdout:3908: finishing write tid 4 to smithi01785587-27 2024-09-21T09:54:30.448 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 27 v 1344 (ObjNum 1488 snap 490 seq_num 1488) dirty exists 2024-09-21T09:54:30.448 INFO:tasks.rados.rados.0.smithi017.stdout:3908: left oid 27 (ObjNum 1488 snap 490 seq_num 1488) 2024-09-21T09:54:30.448 INFO:tasks.rados.rados.0.smithi017.stdout:3910: finishing write tid 1 to smithi01785587-24 2024-09-21T09:54:30.448 INFO:tasks.rados.rados.0.smithi017.stdout:3910: finishing write tid 2 to smithi01785587-24 2024-09-21T09:54:30.450 INFO:tasks.rados.rados.0.smithi017.stdout:3908: done (3 left) 2024-09-21T09:54:30.450 INFO:tasks.rados.rados.0.smithi017.stdout:3909: done (2 left) 2024-09-21T09:54:30.450 INFO:tasks.rados.rados.0.smithi017.stdout:3911: done (1 left) 2024-09-21T09:54:30.450 INFO:tasks.rados.rados.0.smithi017.stdout:3912: snap_create 2024-09-21T09:54:30.450 INFO:tasks.rados.rados.0.smithi017.stdout:3910: finishing write tid 3 to smithi01785587-24 2024-09-21T09:54:30.450 INFO:tasks.rados.rados.0.smithi017.stdout:3910: finishing write tid 4 to smithi01785587-24 2024-09-21T09:54:30.450 INFO:tasks.rados.rados.0.smithi017.stdout:3910: finishing write tid 5 to smithi01785587-24 2024-09-21T09:54:30.451 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 24 v 1202 (ObjNum 1489 snap 490 seq_num 1489) dirty exists 2024-09-21T09:54:30.451 INFO:tasks.rados.rados.0.smithi017.stdout:3910: left oid 24 (ObjNum 1489 snap 490 seq_num 1489) 2024-09-21T09:54:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:30 smithi134 ceph-mon[90085]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-21T09:54:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:54:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:54:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:30 smithi017 ceph-mon[102913]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-21T09:54:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:30 smithi017 ceph-mon[105151]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-21T09:54:31.290 INFO:tasks.rados.rados.0.smithi017.stdout:3910: done (1 left) 2024-09-21T09:54:31.290 INFO:tasks.rados.rados.0.smithi017.stdout:3912: done (0 left) 2024-09-21T09:54:31.291 INFO:tasks.rados.rados.0.smithi017.stdout:3913: read oid 41 snap 485 2024-09-21T09:54:31.291 INFO:tasks.rados.rados.0.smithi017.stdout:3913: expect (ObjNum 1463 snap 483 seq_num 1463) 2024-09-21T09:54:31.291 INFO:tasks.rados.rados.0.smithi017.stdout:3914: write oid 25 current snap is 491 2024-09-21T09:54:31.291 INFO:tasks.rados.rados.0.smithi017.stdout:3914: seq_num 1491 ranges {659237=601447,1933005=463177} 2024-09-21T09:54:31.296 INFO:tasks.rados.rados.0.smithi017.stdout:3914: writing smithi01785587-25 from 659237 to 1260684 tid 1 2024-09-21T09:54:31.303 INFO:tasks.rados.rados.0.smithi017.stdout:3914: writing smithi01785587-25 from 1933005 to 2396182 tid 2 2024-09-21T09:54:31.303 INFO:tasks.rados.rados.0.smithi017.stdout:3915: snap_remove snap 486 2024-09-21T09:54:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:31 smithi017 ceph-mon[102913]: pgmap v1146: 89 pgs: 89 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 614 KiB/s wr, 2 op/s 2024-09-21T09:54:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:31 smithi017 ceph-mon[102913]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-21T09:54:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:31 smithi017 ceph-mon[105151]: pgmap v1146: 89 pgs: 89 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 614 KiB/s wr, 2 op/s 2024-09-21T09:54:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:31 smithi017 ceph-mon[105151]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-21T09:54:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:31 smithi134 ceph-mon[90085]: pgmap v1146: 89 pgs: 89 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 614 KiB/s wr, 2 op/s 2024-09-21T09:54:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:31 smithi134 ceph-mon[90085]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-21T09:54:32.299 INFO:tasks.rados.rados.0.smithi017.stdout:3915: done (2 left) 2024-09-21T09:54:32.299 INFO:tasks.rados.rados.0.smithi017.stdout:3916: write oid 2 current snap is 491 2024-09-21T09:54:32.299 INFO:tasks.rados.rados.0.smithi017.stdout:3913: expect (ObjNum 1463 snap 483 seq_num 1463) 2024-09-21T09:54:32.418 INFO:tasks.rados.rados.0.smithi017.stdout:3914: finishing write tid 1 to smithi01785587-25 2024-09-21T09:54:32.418 INFO:tasks.rados.rados.0.smithi017.stdout:3914: finishing write tid 2 to smithi01785587-25 2024-09-21T09:54:32.418 INFO:tasks.rados.rados.0.smithi017.stdout:3914: finishing write tid 3 to smithi01785587-25 2024-09-21T09:54:32.418 INFO:tasks.rados.rados.0.smithi017.stdout:3914: finishing write tid 4 to smithi01785587-25 2024-09-21T09:54:32.418 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 25 v 1154 (ObjNum 1491 snap 491 seq_num 1491) dirty exists 2024-09-21T09:54:32.418 INFO:tasks.rados.rados.0.smithi017.stdout:3914: left oid 25 (ObjNum 1491 snap 491 seq_num 1491) 2024-09-21T09:54:32.418 INFO:tasks.rados.rados.0.smithi017.stdout:3916: seq_num 1492 ranges {676044=676696,1991487=723188} 2024-09-21T09:54:32.425 INFO:tasks.rados.rados.0.smithi017.stdout:3916: writing smithi01785587-2 from 676044 to 1352740 tid 1 2024-09-21T09:54:32.434 INFO:tasks.rados.rados.0.smithi017.stdout:3916: writing smithi01785587-2 from 1991487 to 2714675 tid 2 2024-09-21T09:54:32.434 INFO:tasks.rados.rados.0.smithi017.stdout:3913: done (2 left) 2024-09-21T09:54:32.434 INFO:tasks.rados.rados.0.smithi017.stdout:3914: done (1 left) 2024-09-21T09:54:32.434 INFO:tasks.rados.rados.0.smithi017.stdout:3917: read oid 18 snap -1 2024-09-21T09:54:32.434 INFO:tasks.rados.rados.0.smithi017.stdout:3917: expect deleted 2024-09-21T09:54:32.434 INFO:tasks.rados.rados.0.smithi017.stdout:3918: snap_remove snap 453 2024-09-21T09:54:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:32 smithi017 ceph-mon[102913]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-21T09:54:32.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:54:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:32 smithi017 ceph-mon[105151]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-21T09:54:32.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:54:32.658 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:32 smithi134 ceph-mon[90085]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-21T09:54:32.658 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:54:33.246 INFO:tasks.rados.rados.0.smithi017.stdout:3918: done (2 left) 2024-09-21T09:54:33.246 INFO:tasks.rados.rados.0.smithi017.stdout:3919: read oid 6 snap 490 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3919: expect deleted 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3920: read oid 48 snap -1 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3920: expect (ObjNum 1438 snap 474 seq_num 1438) 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3921: write oid 21 current snap is 491 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3916: finishing write tid 1 to smithi01785587-2 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3916: finishing write tid 2 to smithi01785587-2 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3916: finishing write tid 3 to smithi01785587-2 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3916: finishing write tid 4 to smithi01785587-2 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 759 (ObjNum 1492 snap 491 seq_num 1492) dirty exists 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3916: left oid 2 (ObjNum 1492 snap 491 seq_num 1492) 2024-09-21T09:54:33.247 INFO:tasks.rados.rados.0.smithi017.stdout:3921: seq_num 1493 ranges {692851=751945,2049969=466846} 2024-09-21T09:54:33.253 INFO:tasks.rados.rados.0.smithi017.stdout:3921: writing smithi01785587-21 from 692851 to 1444796 tid 1 2024-09-21T09:54:33.261 INFO:tasks.rados.rados.0.smithi017.stdout:3921: writing smithi01785587-21 from 2049969 to 2516815 tid 2 2024-09-21T09:54:33.261 INFO:tasks.rados.rados.0.smithi017.stdout:3916: done (4 left) 2024-09-21T09:54:33.261 INFO:tasks.rados.rados.0.smithi017.stdout:3917: done (3 left) 2024-09-21T09:54:33.261 INFO:tasks.rados.rados.0.smithi017.stdout:3919: done (2 left) 2024-09-21T09:54:33.261 INFO:tasks.rados.rados.0.smithi017.stdout:3922: delete oid 38 current snap is 491 2024-09-21T09:54:33.261 INFO:tasks.rados.rados.0.smithi017.stdout:3920: expect (ObjNum 1438 snap 474 seq_num 1438) 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:3921: finishing write tid 1 to smithi01785587-21 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:3921: finishing write tid 2 to smithi01785587-21 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:3921: finishing write tid 3 to smithi01785587-21 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:3921: finishing write tid 4 to smithi01785587-21 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1853 (ObjNum 1493 snap 491 seq_num 1493) dirty exists 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:3921: left oid 21 (ObjNum 1493 snap 491 seq_num 1493) 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:3920: done (2 left) 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:3921: done (1 left) 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:3922: done (0 left) 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:3923: rollback oid 35 current snap is 491 2024-09-21T09:54:33.340 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 35 to 481 2024-09-21T09:54:33.341 INFO:tasks.rados.rados.0.smithi017.stdout:3924: read oid 23 snap -1 2024-09-21T09:54:33.341 INFO:tasks.rados.rados.0.smithi017.stdout:3924: expect (ObjNum 1479 snap 488 seq_num 1479) 2024-09-21T09:54:33.341 INFO:tasks.rados.rados.0.smithi017.stdout:3925: snap_remove snap 489 2024-09-21T09:54:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:33 smithi017 ceph-mon[102913]: pgmap v1149: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.5 MiB/s wr, 24 op/s 2024-09-21T09:54:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:33 smithi017 ceph-mon[102913]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-21T09:54:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:33 smithi017 ceph-mon[102913]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-21T09:54:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:33 smithi017 ceph-mon[105151]: pgmap v1149: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.5 MiB/s wr, 24 op/s 2024-09-21T09:54:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:33 smithi017 ceph-mon[105151]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-21T09:54:33.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:33 smithi017 ceph-mon[105151]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-21T09:54:33.680 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:33 smithi134 ceph-mon[90085]: pgmap v1149: 89 pgs: 89 active+clean; 186 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.5 MiB/s wr, 24 op/s 2024-09-21T09:54:33.680 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:33 smithi134 ceph-mon[90085]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-21T09:54:33.680 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:33 smithi134 ceph-mon[90085]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-21T09:54:34.303 INFO:tasks.rados.rados.0.smithi017.stdout:3925: done (2 left) 2024-09-21T09:54:34.303 INFO:tasks.rados.rados.0.smithi017.stdout:3926: snap_remove snap 483 2024-09-21T09:54:34.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:54:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:54:34] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-09-21T09:54:35.309 INFO:tasks.rados.rados.0.smithi017.stdout:3926: done (2 left) 2024-09-21T09:54:35.310 INFO:tasks.rados.rados.0.smithi017.stdout:3927: snap_remove snap 488 2024-09-21T09:54:35.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:35 smithi134 ceph-mon[90085]: pgmap v1152: 89 pgs: 89 active+clean; 189 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 3.5 MiB/s wr, 25 op/s 2024-09-21T09:54:35.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:35 smithi134 ceph-mon[90085]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-21T09:54:35.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:35 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:35.567 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:35 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:35 smithi017 ceph-mon[105151]: pgmap v1152: 89 pgs: 89 active+clean; 189 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 3.5 MiB/s wr, 25 op/s 2024-09-21T09:54:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:35 smithi017 ceph-mon[105151]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-21T09:54:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:35 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:35 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:35 smithi017 ceph-mon[102913]: pgmap v1152: 89 pgs: 89 active+clean; 189 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 3.5 MiB/s wr, 25 op/s 2024-09-21T09:54:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:35 smithi017 ceph-mon[102913]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-21T09:54:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:35 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:35.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:35 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:36.314 INFO:tasks.rados.rados.0.smithi017.stdout:3927: done (2 left) 2024-09-21T09:54:36.314 INFO:tasks.rados.rados.0.smithi017.stdout:3928: rollback oid 18 current snap is 491 2024-09-21T09:54:36.315 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 18 to 485 2024-09-21T09:54:36.315 INFO:tasks.rados.rados.0.smithi017.stdout:3929: snap_remove snap 490 2024-09-21T09:54:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[102913]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-21T09:54:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:54:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:54:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[105151]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-21T09:54:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:54:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:54:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:36 smithi134 ceph-mon[90085]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-21T09:54:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:54:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:54:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:54:37.323 INFO:tasks.rados.rados.0.smithi017.stdout:3929: done (3 left) 2024-09-21T09:54:37.323 INFO:tasks.rados.rados.0.smithi017.stdout:3930: read oid 29 snap 484 2024-09-21T09:54:37.323 INFO:tasks.rados.rados.0.smithi017.stdout:3930: expect (ObjNum 1460 snap 482 seq_num 1460) 2024-09-21T09:54:37.323 INFO:tasks.rados.rados.0.smithi017.stdout:3931: snap_create 2024-09-21T09:54:37.323 INFO:tasks.rados.rados.0.smithi017.stdout:3923: finishing rollback tid 0 to smithi01785587-35 2024-09-21T09:54:37.323 INFO:tasks.rados.rados.0.smithi017.stdout:3923: finishing rollback tid 1 to smithi01785587-35 2024-09-21T09:54:37.323 INFO:tasks.rados.rados.0.smithi017.stdout:3924: expect (ObjNum 1479 snap 488 seq_num 1479) 2024-09-21T09:54:37.501 INFO:tasks.rados.rados.0.smithi017.stdout:3923: finishing rollback tid 2 to smithi01785587-35 2024-09-21T09:54:37.501 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 35 v 406 (ObjNum 1391 snap 462 seq_num 1391) dirty exists 2024-09-21T09:54:37.501 INFO:tasks.rados.rados.0.smithi017.stdout:3928: finishing rollback tid 1 to smithi01785587-18 2024-09-21T09:54:37.502 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 415 (ObjNum 657 snap 0 seq_num 4181696352) dirty dne 2024-09-21T09:54:37.502 INFO:tasks.rados.rados.0.smithi017.stdout:3930: expect (ObjNum 1460 snap 482 seq_num 1460) 2024-09-21T09:54:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:37 smithi017 ceph-mon[102913]: pgmap v1155: 89 pgs: 89 active+clean; 156 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 596 KiB/s wr, 11 op/s 2024-09-21T09:54:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:37 smithi017 ceph-mon[102913]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-21T09:54:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:37 smithi017 ceph-mon[105151]: pgmap v1155: 89 pgs: 89 active+clean; 156 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 596 KiB/s wr, 11 op/s 2024-09-21T09:54:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:37 smithi017 ceph-mon[105151]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-21T09:54:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:37 smithi134 ceph-mon[90085]: pgmap v1155: 89 pgs: 89 active+clean; 156 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 596 KiB/s wr, 11 op/s 2024-09-21T09:54:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:37 smithi134 ceph-mon[90085]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-21T09:54:38.344 INFO:tasks.rados.rados.0.smithi017.stdout:3923: done (4 left) 2024-09-21T09:54:38.344 INFO:tasks.rados.rados.0.smithi017.stdout:3924: done (3 left) 2024-09-21T09:54:38.344 INFO:tasks.rados.rados.0.smithi017.stdout:3928: done (2 left) 2024-09-21T09:54:38.344 INFO:tasks.rados.rados.0.smithi017.stdout:3930: done (1 left) 2024-09-21T09:54:38.344 INFO:tasks.rados.rados.0.smithi017.stdout:3931: done (0 left) 2024-09-21T09:54:38.344 INFO:tasks.rados.rados.0.smithi017.stdout:3932: snap_remove snap 485 2024-09-21T09:54:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:38 smithi017 ceph-mon[102913]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-21T09:54:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:38 smithi017 ceph-mon[105151]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-21T09:54:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:38 smithi134 ceph-mon[90085]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-21T09:54:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3932: done (0 left) 2024-09-21T09:54:39.347 INFO:tasks.rados.rados.0.smithi017.stdout:3933: write oid 9 current snap is 492 2024-09-21T09:54:39.348 INFO:tasks.rados.rados.0.smithi017.stdout:3933: seq_num 1495 ranges {726465=502443,1650580=470515,2788433=338032} 2024-09-21T09:54:39.354 INFO:tasks.rados.rados.0.smithi017.stdout:3933: writing smithi01785587-9 from 726465 to 1228908 tid 1 2024-09-21T09:54:39.360 INFO:tasks.rados.rados.0.smithi017.stdout:3933: writing smithi01785587-9 from 1650580 to 2121095 tid 2 2024-09-21T09:54:39.366 INFO:tasks.rados.rados.0.smithi017.stdout:3933: writing smithi01785587-9 from 2788433 to 3126465 tid 3 2024-09-21T09:54:39.366 INFO:tasks.rados.rados.0.smithi017.stdout:3934: rollback oid 5 current snap is 492 2024-09-21T09:54:39.366 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 5 to 491 2024-09-21T09:54:39.366 INFO:tasks.rados.rados.0.smithi017.stdout:3933: finishing write tid 1 to smithi01785587-9 2024-09-21T09:54:39.366 INFO:tasks.rados.rados.0.smithi017.stdout:3933: finishing write tid 2 to smithi01785587-9 2024-09-21T09:54:39.366 INFO:tasks.rados.rados.0.smithi017.stdout:3935: delete oid 39 current snap is 492 2024-09-21T09:54:39.369 INFO:tasks.rados.rados.0.smithi017.stdout:3933: finishing write tid 3 to smithi01785587-9 2024-09-21T09:54:39.369 INFO:tasks.rados.rados.0.smithi017.stdout:3933: finishing write tid 4 to smithi01785587-9 2024-09-21T09:54:39.369 INFO:tasks.rados.rados.0.smithi017.stdout:3935: done (2 left) 2024-09-21T09:54:39.369 INFO:tasks.rados.rados.0.smithi017.stdout:3936: snap_remove snap 484 2024-09-21T09:54:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:39 smithi017 ceph-mon[105151]: pgmap v1158: 89 pgs: 89 active+clean; 141 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 900 KiB/s wr, 12 op/s 2024-09-21T09:54:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:39 smithi017 ceph-mon[105151]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-21T09:54:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:39 smithi017 ceph-mon[102913]: pgmap v1158: 89 pgs: 89 active+clean; 141 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 900 KiB/s wr, 12 op/s 2024-09-21T09:54:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:39 smithi017 ceph-mon[102913]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-21T09:54:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:39 smithi134 ceph-mon[90085]: pgmap v1158: 89 pgs: 89 active+clean; 141 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 900 KiB/s wr, 12 op/s 2024-09-21T09:54:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:39 smithi134 ceph-mon[90085]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-21T09:54:40.359 INFO:tasks.rados.rados.0.smithi017.stdout:3936: done (2 left) 2024-09-21T09:54:40.359 INFO:tasks.rados.rados.0.smithi017.stdout:3937: snap_create 2024-09-21T09:54:40.359 INFO:tasks.rados.rados.0.smithi017.stdout:3933: finishing write tid 5 to smithi01785587-9 2024-09-21T09:54:40.359 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 9 v 763 (ObjNum 1495 snap 492 seq_num 1495) dirty exists 2024-09-21T09:54:40.359 INFO:tasks.rados.rados.0.smithi017.stdout:3933: left oid 9 (ObjNum 1495 snap 492 seq_num 1495) 2024-09-21T09:54:40.359 INFO:tasks.rados.rados.0.smithi017.stdout:3934: finishing rollback tid 1 to smithi01785587-5 2024-09-21T09:54:40.360 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 5 v 2277 (ObjNum 225 snap 0 seq_num 2399889824) dirty dne 2024-09-21T09:54:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:40 smithi134 ceph-mon[90085]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-21T09:54:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:54:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:54:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:40 smithi017 ceph-mon[102913]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-21T09:54:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:40 smithi017 ceph-mon[105151]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-21T09:54:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:41.366 INFO:tasks.rados.rados.0.smithi017.stdout:3933: done (2 left) 2024-09-21T09:54:41.366 INFO:tasks.rados.rados.0.smithi017.stdout:3934: done (1 left) 2024-09-21T09:54:41.366 INFO:tasks.rados.rados.0.smithi017.stdout:3937: done (0 left) 2024-09-21T09:54:41.366 INFO:tasks.rados.rados.0.smithi017.stdout:3938: write oid 33 current snap is 493 2024-09-21T09:54:41.366 INFO:tasks.rados.rados.0.smithi017.stdout:3938: seq_num 1497 ranges {760079=652941,2167544=474184} 2024-09-21T09:54:41.373 INFO:tasks.rados.rados.0.smithi017.stdout:3938: writing smithi01785587-33 from 760079 to 1413020 tid 1 2024-09-21T09:54:41.381 INFO:tasks.rados.rados.0.smithi017.stdout:3938: writing smithi01785587-33 from 2167544 to 2641728 tid 2 2024-09-21T09:54:41.381 INFO:tasks.rados.rados.0.smithi017.stdout:3939: snap_create 2024-09-21T09:54:41.381 INFO:tasks.rados.rados.0.smithi017.stdout:3938: finishing write tid 1 to smithi01785587-33 2024-09-21T09:54:41.385 INFO:tasks.rados.rados.0.smithi017.stdout:3938: finishing write tid 2 to smithi01785587-33 2024-09-21T09:54:41.385 INFO:tasks.rados.rados.0.smithi017.stdout:3938: finishing write tid 3 to smithi01785587-33 2024-09-21T09:54:41.385 INFO:tasks.rados.rados.0.smithi017.stdout:3938: finishing write tid 4 to smithi01785587-33 2024-09-21T09:54:41.385 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 2150 (ObjNum 1497 snap 493 seq_num 1497) dirty exists 2024-09-21T09:54:41.385 INFO:tasks.rados.rados.0.smithi017.stdout:3938: left oid 33 (ObjNum 1497 snap 493 seq_num 1497) 2024-09-21T09:54:41.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:41 smithi017 ceph-mon[102913]: pgmap v1161: 89 pgs: 89 active+clean; 134 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 640 KiB/s wr, 4 op/s 2024-09-21T09:54:41.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:41 smithi017 ceph-mon[102913]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-21T09:54:41.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:41 smithi017 ceph-mon[105151]: pgmap v1161: 89 pgs: 89 active+clean; 134 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 640 KiB/s wr, 4 op/s 2024-09-21T09:54:41.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:41 smithi017 ceph-mon[105151]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-21T09:54:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:41 smithi134 ceph-mon[90085]: pgmap v1161: 89 pgs: 89 active+clean; 134 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 640 KiB/s wr, 4 op/s 2024-09-21T09:54:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:41 smithi134 ceph-mon[90085]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-21T09:54:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:3938: done (1 left) 2024-09-21T09:54:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:3939: done (0 left) 2024-09-21T09:54:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:3940: read oid 13 snap -1 2024-09-21T09:54:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:3940: expect (ObjNum 1481 snap 489 seq_num 1481) 2024-09-21T09:54:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:3941: write oid 45 current snap is 494 2024-09-21T09:54:42.371 INFO:tasks.rados.rados.0.smithi017.stdout:3941: seq_num 1498 ranges {776886=444543,1826026=734195,2987055=189831} 2024-09-21T09:54:42.377 INFO:tasks.rados.rados.0.smithi017.stdout:3941: writing smithi01785587-45 from 776886 to 1221429 tid 1 2024-09-21T09:54:42.385 INFO:tasks.rados.rados.0.smithi017.stdout:3941: writing smithi01785587-45 from 1826026 to 2560221 tid 2 2024-09-21T09:54:42.388 INFO:tasks.rados.rados.0.smithi017.stdout:3941: writing smithi01785587-45 from 2987055 to 3176886 tid 3 2024-09-21T09:54:42.388 INFO:tasks.rados.rados.0.smithi017.stdout:3942: snap_remove snap 491 2024-09-21T09:54:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:42 smithi017 ceph-mon[105151]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-21T09:54:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:42 smithi017 ceph-mon[102913]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-21T09:54:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:42 smithi134 ceph-mon[90085]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-21T09:54:43.262 INFO:tasks.rados.rados.0.smithi017.stdout:3942: done (2 left) 2024-09-21T09:54:43.262 INFO:tasks.rados.rados.0.smithi017.stdout:3943: write oid 22 current snap is 494 2024-09-21T09:54:43.262 INFO:tasks.rados.rados.0.smithi017.stdout:3943: seq_num 1499 ranges {793693=519792,1884508=477853,2898125=295568} 2024-09-21T09:54:43.267 INFO:tasks.rados.rados.0.smithi017.stdout:3943: writing smithi01785587-22 from 793693 to 1313485 tid 1 2024-09-21T09:54:43.273 INFO:tasks.rados.rados.0.smithi017.stdout:3943: writing smithi01785587-22 from 1884508 to 2362361 tid 2 2024-09-21T09:54:43.278 INFO:tasks.rados.rados.0.smithi017.stdout:3943: writing smithi01785587-22 from 2898125 to 3193693 tid 3 2024-09-21T09:54:43.278 INFO:tasks.rados.rados.0.smithi017.stdout:3944: rollback oid 30 current snap is 494 2024-09-21T09:54:43.278 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 30 to 482 2024-09-21T09:54:43.278 INFO:tasks.rados.rados.0.smithi017.stdout:3940: expect (ObjNum 1481 snap 489 seq_num 1481) 2024-09-21T09:54:43.485 INFO:tasks.rados.rados.0.smithi017.stdout:3941: finishing write tid 1 to smithi01785587-45 2024-09-21T09:54:43.485 INFO:tasks.rados.rados.0.smithi017.stdout:3941: finishing write tid 2 to smithi01785587-45 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3941: finishing write tid 3 to smithi01785587-45 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3941: finishing write tid 4 to smithi01785587-45 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3941: finishing write tid 5 to smithi01785587-45 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1353 (ObjNum 1498 snap 494 seq_num 1498) dirty exists 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3941: left oid 45 (ObjNum 1498 snap 494 seq_num 1498) 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3943: finishing write tid 1 to smithi01785587-22 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3943: finishing write tid 2 to smithi01785587-22 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3940: done (3 left) 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3941: done (2 left) 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3945: snap_create 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3943: finishing write tid 3 to smithi01785587-22 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3943: finishing write tid 4 to smithi01785587-22 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:3943: finishing write tid 5 to smithi01785587-22 2024-09-21T09:54:43.486 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 22 v 2157 (ObjNum 1499 snap 494 seq_num 1499) dirty exists 2024-09-21T09:54:43.487 INFO:tasks.rados.rados.0.smithi017.stdout:3943: left oid 22 (ObjNum 1499 snap 494 seq_num 1499) 2024-09-21T09:54:43.487 INFO:tasks.rados.rados.0.smithi017.stdout:3944: finishing rollback tid 1 to smithi01785587-30 2024-09-21T09:54:43.487 INFO:tasks.rados.rados.0.smithi017.stdout:3944: finishing rollback tid 2 to smithi01785587-30 2024-09-21T09:54:43.487 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 30 v 1670 (ObjNum 1425 snap 471 seq_num 1425) dirty exists 2024-09-21T09:54:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:43 smithi017 ceph-mon[102913]: pgmap v1164: 89 pgs: 89 active+clean; 133 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2 op/s 2024-09-21T09:54:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:43 smithi017 ceph-mon[102913]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-21T09:54:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:43 smithi017 ceph-mon[102913]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-21T09:54:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:43 smithi017 ceph-mon[105151]: pgmap v1164: 89 pgs: 89 active+clean; 133 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2 op/s 2024-09-21T09:54:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:43 smithi017 ceph-mon[105151]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-21T09:54:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:43 smithi017 ceph-mon[105151]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-21T09:54:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:43 smithi134 ceph-mon[90085]: pgmap v1164: 89 pgs: 89 active+clean; 133 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2 op/s 2024-09-21T09:54:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:43 smithi134 ceph-mon[90085]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-21T09:54:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:43 smithi134 ceph-mon[90085]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-21T09:54:44.376 INFO:tasks.rados.rados.0.smithi017.stdout:3943: done (2 left) 2024-09-21T09:54:44.377 INFO:tasks.rados.rados.0.smithi017.stdout:3944: done (1 left) 2024-09-21T09:54:44.377 INFO:tasks.rados.rados.0.smithi017.stdout:3945: done (0 left) 2024-09-21T09:54:44.377 INFO:tasks.rados.rados.0.smithi017.stdout:3946: write oid 20 current snap is 495 2024-09-21T09:54:44.377 INFO:tasks.rados.rados.0.smithi017.stdout:3946: seq_num 1500 ranges {410500=595041,1542990=621511,2925548=284952} 2024-09-21T09:54:44.382 INFO:tasks.rados.rados.0.smithi017.stdout:3946: writing smithi01785587-20 from 410500 to 1005541 tid 1 2024-09-21T09:54:44.388 INFO:tasks.rados.rados.0.smithi017.stdout:3946: writing smithi01785587-20 from 1542990 to 2164501 tid 2 2024-09-21T09:54:44.394 INFO:tasks.rados.rados.0.smithi017.stdout:3946: writing smithi01785587-20 from 2925548 to 3210500 tid 3 2024-09-21T09:54:44.394 INFO:tasks.rados.rados.0.smithi017.stdout:3947: write oid 34 current snap is 495 2024-09-21T09:54:44.394 INFO:tasks.rados.rados.0.smithi017.stdout:3947: seq_num 1501 ranges {427307=670290,1601472=481522,2552971=483056} 2024-09-21T09:54:44.400 INFO:tasks.rados.rados.0.smithi017.stdout:3947: writing smithi01785587-34 from 427307 to 1097597 tid 1 2024-09-21T09:54:44.406 INFO:tasks.rados.rados.0.smithi017.stdout:3947: writing smithi01785587-34 from 1601472 to 2082994 tid 2 2024-09-21T09:54:44.411 INFO:tasks.rados.rados.0.smithi017.stdout:3947: writing smithi01785587-34 from 2552971 to 3036027 tid 3 2024-09-21T09:54:44.411 INFO:tasks.rados.rados.0.smithi017.stdout:3948: read oid 46 snap -1 2024-09-21T09:54:44.411 INFO:tasks.rados.rados.0.smithi017.stdout:3948: expect deleted 2024-09-21T09:54:44.411 INFO:tasks.rados.rados.0.smithi017.stdout:3949: snap_create 2024-09-21T09:54:44.412 INFO:tasks.rados.rados.0.smithi017.stdout:3946: finishing write tid 1 to smithi01785587-20 2024-09-21T09:54:44.412 INFO:tasks.rados.rados.0.smithi017.stdout:3946: finishing write tid 2 to smithi01785587-20 2024-09-21T09:54:44.412 INFO:tasks.rados.rados.0.smithi017.stdout:3946: finishing write tid 3 to smithi01785587-20 2024-09-21T09:54:44.412 INFO:tasks.rados.rados.0.smithi017.stdout:3946: finishing write tid 4 to smithi01785587-20 2024-09-21T09:54:44.412 INFO:tasks.rados.rados.0.smithi017.stdout:3946: finishing write tid 5 to smithi01785587-20 2024-09-21T09:54:44.412 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 20 v 1177 (ObjNum 1500 snap 495 seq_num 1500) dirty exists 2024-09-21T09:54:44.412 INFO:tasks.rados.rados.0.smithi017.stdout:3946: left oid 20 (ObjNum 1500 snap 495 seq_num 1500) 2024-09-21T09:54:44.412 INFO:tasks.rados.rados.0.smithi017.stdout:3947: finishing write tid 1 to smithi01785587-34 2024-09-21T09:54:44.412 INFO:tasks.rados.rados.0.smithi017.stdout:3947: finishing write tid 2 to smithi01785587-34 2024-09-21T09:54:44.415 INFO:tasks.rados.rados.0.smithi017.stdout:3947: finishing write tid 3 to smithi01785587-34 2024-09-21T09:54:44.415 INFO:tasks.rados.rados.0.smithi017.stdout:3947: finishing write tid 4 to smithi01785587-34 2024-09-21T09:54:44.415 INFO:tasks.rados.rados.0.smithi017.stdout:3947: finishing write tid 5 to smithi01785587-34 2024-09-21T09:54:44.415 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 34 v 1867 (ObjNum 1501 snap 495 seq_num 1501) dirty exists 2024-09-21T09:54:44.415 INFO:tasks.rados.rados.0.smithi017.stdout:3947: left oid 34 (ObjNum 1501 snap 495 seq_num 1501) 2024-09-21T09:54:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:54:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:54:44] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.51.0" 2024-09-21T09:54:45.382 INFO:tasks.rados.rados.0.smithi017.stdout:3946: done (3 left) 2024-09-21T09:54:45.382 INFO:tasks.rados.rados.0.smithi017.stdout:3947: done (2 left) 2024-09-21T09:54:45.382 INFO:tasks.rados.rados.0.smithi017.stdout:3948: done (1 left) 2024-09-21T09:54:45.382 INFO:tasks.rados.rados.0.smithi017.stdout:3949: done (0 left) 2024-09-21T09:54:45.382 INFO:tasks.rados.rados.0.smithi017.stdout:3950: snap_create 2024-09-21T09:54:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:45 smithi017 ceph-mon[102913]: pgmap v1167: 89 pgs: 89 active+clean; 137 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 655 KiB/s wr, 5 op/s 2024-09-21T09:54:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:45 smithi017 ceph-mon[102913]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-21T09:54:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:45 smithi017 ceph-mon[105151]: pgmap v1167: 89 pgs: 89 active+clean; 137 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 655 KiB/s wr, 5 op/s 2024-09-21T09:54:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:45 smithi017 ceph-mon[105151]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-21T09:54:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:45 smithi134 ceph-mon[90085]: pgmap v1167: 89 pgs: 89 active+clean; 137 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 655 KiB/s wr, 5 op/s 2024-09-21T09:54:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:45 smithi134 ceph-mon[90085]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-21T09:54:46.388 INFO:tasks.rados.rados.0.smithi017.stdout:3950: done (0 left) 2024-09-21T09:54:46.388 INFO:tasks.rados.rados.0.smithi017.stdout:3951: snap_remove snap 496 2024-09-21T09:54:46.636 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:46 smithi134 ceph-mon[90085]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-21T09:54:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:46 smithi017 ceph-mon[102913]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-21T09:54:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:46 smithi017 ceph-mon[105151]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-21T09:54:47.389 INFO:tasks.rados.rados.0.smithi017.stdout:3951: done (0 left) 2024-09-21T09:54:47.389 INFO:tasks.rados.rados.0.smithi017.stdout:3952: write oid 6 current snap is 497 2024-09-21T09:54:47.389 INFO:tasks.rados.rados.0.smithi017.stdout:3952: seq_num 1502 ranges {444114=745539,1943601=625180} 2024-09-21T09:54:47.395 INFO:tasks.rados.rados.0.smithi017.stdout:3952: writing smithi01785587-6 from 444114 to 1189653 tid 1 2024-09-21T09:54:47.404 INFO:tasks.rados.rados.0.smithi017.stdout:3952: writing smithi01785587-6 from 1943601 to 2568781 tid 2 2024-09-21T09:54:47.404 INFO:tasks.rados.rados.0.smithi017.stdout:3953: write oid 2 current snap is 497 2024-09-21T09:54:47.404 INFO:tasks.rados.rados.0.smithi017.stdout:3952: finishing write tid 1 to smithi01785587-6 2024-09-21T09:54:47.404 INFO:tasks.rados.rados.0.smithi017.stdout:3953: seq_num 1503 ranges {460921=420788,1602083=485191,2491464=621953} 2024-09-21T09:54:47.408 INFO:tasks.rados.rados.0.smithi017.stdout:3953: writing smithi01785587-2 from 460921 to 881709 tid 1 2024-09-21T09:54:47.416 INFO:tasks.rados.rados.0.smithi017.stdout:3953: writing smithi01785587-2 from 1602083 to 2087274 tid 2 2024-09-21T09:54:47.422 INFO:tasks.rados.rados.0.smithi017.stdout:3953: writing smithi01785587-2 from 2491464 to 3113417 tid 3 2024-09-21T09:54:47.422 INFO:tasks.rados.rados.0.smithi017.stdout:3954: snap_remove snap 487 2024-09-21T09:54:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:47 smithi017 ceph-mon[105151]: pgmap v1170: 89 pgs: 89 active+clean; 146 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-21T09:54:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:47 smithi017 ceph-mon[105151]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-21T09:54:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:47 smithi017 ceph-mon[102913]: pgmap v1170: 89 pgs: 89 active+clean; 146 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-21T09:54:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:47 smithi017 ceph-mon[102913]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-21T09:54:47.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:47 smithi134 ceph-mon[90085]: pgmap v1170: 89 pgs: 89 active+clean; 146 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-09-21T09:54:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:47 smithi134 ceph-mon[90085]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-21T09:54:48.262 INFO:tasks.rados.rados.0.smithi017.stdout:3954: done (2 left) 2024-09-21T09:54:48.262 INFO:tasks.rados.rados.0.smithi017.stdout:3955: snap_remove snap 495 2024-09-21T09:54:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:48 smithi017 ceph-mon[102913]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-21T09:54:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:48 smithi017 ceph-mon[102913]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-21T09:54:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:48 smithi017 ceph-mon[105151]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-21T09:54:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:48 smithi017 ceph-mon[105151]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-21T09:54:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:48 smithi134 ceph-mon[90085]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-21T09:54:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:48 smithi134 ceph-mon[90085]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-21T09:54:49.263 INFO:tasks.rados.rados.0.smithi017.stdout:3955: done (2 left) 2024-09-21T09:54:49.263 INFO:tasks.rados.rados.0.smithi017.stdout:3956: snap_remove snap 481 2024-09-21T09:54:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:49 smithi017 ceph-mon[102913]: pgmap v1173: 89 pgs: 89 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-21T09:54:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:49 smithi017 ceph-mon[102913]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-21T09:54:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:49 smithi017 ceph-mon[105151]: pgmap v1173: 89 pgs: 89 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-21T09:54:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:49 smithi017 ceph-mon[105151]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-21T09:54:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:49 smithi134 ceph-mon[90085]: pgmap v1173: 89 pgs: 89 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-09-21T09:54:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:49 smithi134 ceph-mon[90085]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-21T09:54:50.264 INFO:tasks.rados.rados.0.smithi017.stdout:3956: done (2 left) 2024-09-21T09:54:50.264 INFO:tasks.rados.rados.0.smithi017.stdout:3957: delete oid 16 current snap is 497 2024-09-21T09:54:50.264 INFO:tasks.rados.rados.0.smithi017.stdout:3952: finishing write tid 2 to smithi01785587-6 2024-09-21T09:54:50.264 INFO:tasks.rados.rados.0.smithi017.stdout:3952: finishing write tid 3 to smithi01785587-6 2024-09-21T09:54:50.264 INFO:tasks.rados.rados.0.smithi017.stdout:3952: finishing write tid 4 to smithi01785587-6 2024-09-21T09:54:50.264 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 6 v 1144 (ObjNum 1502 snap 497 seq_num 1502) dirty exists 2024-09-21T09:54:50.265 INFO:tasks.rados.rados.0.smithi017.stdout:3952: left oid 6 (ObjNum 1502 snap 497 seq_num 1502) 2024-09-21T09:54:50.265 INFO:tasks.rados.rados.0.smithi017.stdout:3953: finishing write tid 1 to smithi01785587-2 2024-09-21T09:54:50.265 INFO:tasks.rados.rados.0.smithi017.stdout:3953: finishing write tid 2 to smithi01785587-2 2024-09-21T09:54:50.265 INFO:tasks.rados.rados.0.smithi017.stdout:3953: finishing write tid 3 to smithi01785587-2 2024-09-21T09:54:50.265 INFO:tasks.rados.rados.0.smithi017.stdout:3953: finishing write tid 4 to smithi01785587-2 2024-09-21T09:54:50.265 INFO:tasks.rados.rados.0.smithi017.stdout:3953: finishing write tid 5 to smithi01785587-2 2024-09-21T09:54:50.265 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 2 v 768 (ObjNum 1503 snap 497 seq_num 1503) dirty exists 2024-09-21T09:54:50.265 INFO:tasks.rados.rados.0.smithi017.stdout:3953: left oid 2 (ObjNum 1503 snap 497 seq_num 1503) 2024-09-21T09:54:50.275 INFO:tasks.rados.rados.0.smithi017.stdout:3952: done (2 left) 2024-09-21T09:54:50.275 INFO:tasks.rados.rados.0.smithi017.stdout:3953: done (1 left) 2024-09-21T09:54:50.275 INFO:tasks.rados.rados.0.smithi017.stdout:3957: done (0 left) 2024-09-21T09:54:50.275 INFO:tasks.rados.rados.0.smithi017.stdout:3958: write oid 40 current snap is 497 2024-09-21T09:54:50.275 INFO:tasks.rados.rados.0.smithi017.stdout:3958: seq_num 1505 ranges {494535=571286,1719047=488860,2946310=348225} 2024-09-21T09:54:50.281 INFO:tasks.rados.rados.0.smithi017.stdout:3958: writing smithi01785587-40 from 494535 to 1065821 tid 1 2024-09-21T09:54:50.287 INFO:tasks.rados.rados.0.smithi017.stdout:3958: writing smithi01785587-40 from 1719047 to 2207907 tid 2 2024-09-21T09:54:50.294 INFO:tasks.rados.rados.0.smithi017.stdout:3958: writing smithi01785587-40 from 2946310 to 3294535 tid 3 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3958: finishing write tid 1 to smithi01785587-40 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3958: finishing write tid 2 to smithi01785587-40 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3959: read oid 20 snap -1 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3959: expect (ObjNum 1500 snap 495 seq_num 1500) 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3960: read oid 23 snap -1 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3960: expect (ObjNum 1479 snap 488 seq_num 1479) 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3961: read oid 8 snap -1 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3961: expect deleted 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3962: read oid 24 snap -1 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3962: expect (ObjNum 1489 snap 490 seq_num 1489) 2024-09-21T09:54:50.295 INFO:tasks.rados.rados.0.smithi017.stdout:3963: snap_remove snap 492 2024-09-21T09:54:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:54:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:54:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:51.269 INFO:tasks.rados.rados.0.smithi017.stdout:3963: done (5 left) 2024-09-21T09:54:51.269 INFO:tasks.rados.rados.0.smithi017.stdout:3964: read oid 2 snap -1 2024-09-21T09:54:51.269 INFO:tasks.rados.rados.0.smithi017.stdout:3964: expect (ObjNum 1503 snap 497 seq_num 1503) 2024-09-21T09:54:51.269 INFO:tasks.rados.rados.0.smithi017.stdout:3958: finishing write tid 3 to smithi01785587-40 2024-09-21T09:54:51.270 INFO:tasks.rados.rados.0.smithi017.stdout:3958: finishing write tid 4 to smithi01785587-40 2024-09-21T09:54:51.270 INFO:tasks.rados.rados.0.smithi017.stdout:3958: finishing write tid 5 to smithi01785587-40 2024-09-21T09:54:51.270 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 40 v 2021 (ObjNum 1505 snap 497 seq_num 1505) dirty exists 2024-09-21T09:54:51.270 INFO:tasks.rados.rados.0.smithi017.stdout:3958: left oid 40 (ObjNum 1505 snap 497 seq_num 1505) 2024-09-21T09:54:51.270 INFO:tasks.rados.rados.0.smithi017.stdout:3960: expect (ObjNum 1479 snap 488 seq_num 1479) 2024-09-21T09:54:51.450 INFO:tasks.rados.rados.0.smithi017.stdout:3962: expect (ObjNum 1489 snap 490 seq_num 1489) 2024-09-21T09:54:51.549 INFO:tasks.rados.rados.0.smithi017.stdout:3959: expect (ObjNum 1500 snap 495 seq_num 1500) 2024-09-21T09:54:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:51 smithi017 ceph-mon[102913]: pgmap v1176: 89 pgs: 89 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 399 KiB/s wr, 1 op/s 2024-09-21T09:54:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:51 smithi017 ceph-mon[102913]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-21T09:54:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:51 smithi017 ceph-mon[105151]: pgmap v1176: 89 pgs: 89 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 399 KiB/s wr, 1 op/s 2024-09-21T09:54:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:51 smithi017 ceph-mon[105151]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-21T09:54:51.737 INFO:tasks.rados.rados.0.smithi017.stdout:3964: expect (ObjNum 1503 snap 497 seq_num 1503) 2024-09-21T09:54:51.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:51 smithi134 ceph-mon[90085]: pgmap v1176: 89 pgs: 89 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 399 KiB/s wr, 1 op/s 2024-09-21T09:54:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:51 smithi134 ceph-mon[90085]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-21T09:54:51.884 INFO:tasks.rados.rados.0.smithi017.stdout:3958: done (5 left) 2024-09-21T09:54:51.884 INFO:tasks.rados.rados.0.smithi017.stdout:3959: done (4 left) 2024-09-21T09:54:51.884 INFO:tasks.rados.rados.0.smithi017.stdout:3960: done (3 left) 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi017.stdout:3961: done (2 left) 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi017.stdout:3962: done (1 left) 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi017.stdout:3964: done (0 left) 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi017.stdout:3965: delete oid 32 current snap is 497 2024-09-21T09:54:51.887 INFO:tasks.rados.rados.0.smithi017.stdout:3965: done (0 left) 2024-09-21T09:54:51.887 INFO:tasks.rados.rados.0.smithi017.stdout:3966: rollback oid 43 current snap is 497 2024-09-21T09:54:51.887 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 43 to 482 2024-09-21T09:54:51.887 INFO:tasks.rados.rados.0.smithi017.stdout:3967: delete oid 7 current snap is 497 2024-09-21T09:54:51.890 INFO:tasks.rados.rados.0.smithi017.stdout:3967: done (1 left) 2024-09-21T09:54:51.890 INFO:tasks.rados.rados.0.smithi017.stdout:3968: read oid 19 snap -1 2024-09-21T09:54:51.890 INFO:tasks.rados.rados.0.smithi017.stdout:3968: expect (ObjNum 1454 snap 478 seq_num 1454) 2024-09-21T09:54:51.890 INFO:tasks.rados.rados.0.smithi017.stdout:3969: write oid 45 current snap is 497 2024-09-21T09:54:51.890 INFO:tasks.rados.rados.0.smithi017.stdout:3969: seq_num 1508 ranges {544956=513386,1494493=636187,2628579=594666} 2024-09-21T09:54:51.895 INFO:tasks.rados.rados.0.smithi017.stdout:3969: writing smithi01785587-45 from 544956 to 1058342 tid 1 2024-09-21T09:54:51.901 INFO:tasks.rados.rados.0.smithi017.stdout:3969: writing smithi01785587-45 from 1494493 to 2130680 tid 2 2024-09-21T09:54:51.908 INFO:tasks.rados.rados.0.smithi017.stdout:3969: writing smithi01785587-45 from 2628579 to 3223245 tid 3 2024-09-21T09:54:51.908 INFO:tasks.rados.rados.0.smithi017.stdout:3970: read oid 6 snap -1 2024-09-21T09:54:51.908 INFO:tasks.rados.rados.0.smithi017.stdout:3970: expect (ObjNum 1502 snap 497 seq_num 1502) 2024-09-21T09:54:51.908 INFO:tasks.rados.rados.0.smithi017.stdout:3971: snap_create 2024-09-21T09:54:51.908 INFO:tasks.rados.rados.0.smithi017.stdout:3966: finishing rollback tid 1 to smithi01785587-43 2024-09-21T09:54:51.908 INFO:tasks.rados.rados.0.smithi017.stdout:3966: finishing rollback tid 2 to smithi01785587-43 2024-09-21T09:54:51.908 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 1150 (ObjNum 1459 snap 482 seq_num 1459) dirty exists 2024-09-21T09:54:51.908 INFO:tasks.rados.rados.0.smithi017.stdout:3968: expect (ObjNum 1454 snap 478 seq_num 1454) 2024-09-21T09:54:51.989 INFO:tasks.rados.rados.0.smithi017.stdout:3969: finishing write tid 1 to smithi01785587-45 2024-09-21T09:54:51.989 INFO:tasks.rados.rados.0.smithi017.stdout:3969: finishing write tid 2 to smithi01785587-45 2024-09-21T09:54:51.989 INFO:tasks.rados.rados.0.smithi017.stdout:3969: finishing write tid 3 to smithi01785587-45 2024-09-21T09:54:51.989 INFO:tasks.rados.rados.0.smithi017.stdout:3969: finishing write tid 4 to smithi01785587-45 2024-09-21T09:54:51.989 INFO:tasks.rados.rados.0.smithi017.stdout:3969: finishing write tid 5 to smithi01785587-45 2024-09-21T09:54:51.989 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 45 v 1366 (ObjNum 1508 snap 497 seq_num 1508) dirty exists 2024-09-21T09:54:51.989 INFO:tasks.rados.rados.0.smithi017.stdout:3969: left oid 45 (ObjNum 1508 snap 497 seq_num 1508) 2024-09-21T09:54:51.989 INFO:tasks.rados.rados.0.smithi017.stdout:3970: expect (ObjNum 1502 snap 497 seq_num 1502) 2024-09-21T09:54:52.274 INFO:tasks.rados.rados.0.smithi017.stdout:3966: done (4 left) 2024-09-21T09:54:52.274 INFO:tasks.rados.rados.0.smithi017.stdout:3968: done (3 left) 2024-09-21T09:54:52.274 INFO:tasks.rados.rados.0.smithi017.stdout:3969: done (2 left) 2024-09-21T09:54:52.274 INFO:tasks.rados.rados.0.smithi017.stdout:3970: done (1 left) 2024-09-21T09:54:52.274 INFO:tasks.rados.rados.0.smithi017.stdout:3971: done (0 left) 2024-09-21T09:54:52.275 INFO:tasks.rados.rados.0.smithi017.stdout:3972: write oid 43 current snap is 498 2024-09-21T09:54:52.275 INFO:tasks.rados.rados.0.smithi017.stdout:3972: seq_num 1509 ranges {561763=588635,1552975=496198,2656002=405938} 2024-09-21T09:54:52.280 INFO:tasks.rados.rados.0.smithi017.stdout:3972: writing smithi01785587-43 from 561763 to 1150398 tid 1 2024-09-21T09:54:52.285 INFO:tasks.rados.rados.0.smithi017.stdout:3972: writing smithi01785587-43 from 1552975 to 2049173 tid 2 2024-09-21T09:54:52.291 INFO:tasks.rados.rados.0.smithi017.stdout:3972: writing smithi01785587-43 from 2656002 to 3061940 tid 3 2024-09-21T09:54:52.291 INFO:tasks.rados.rados.0.smithi017.stdout:3973: write oid 18 current snap is 498 2024-09-21T09:54:52.291 INFO:tasks.rados.rados.0.smithi017.stdout:3973: seq_num 1510 ranges {578570=663884,1895104=639856,2967072=411498} 2024-09-21T09:54:52.298 INFO:tasks.rados.rados.0.smithi017.stdout:3973: writing smithi01785587-18 from 578570 to 1242454 tid 1 2024-09-21T09:54:52.305 INFO:tasks.rados.rados.0.smithi017.stdout:3973: writing smithi01785587-18 from 1895104 to 2534960 tid 2 2024-09-21T09:54:52.310 INFO:tasks.rados.rados.0.smithi017.stdout:3973: writing smithi01785587-18 from 2967072 to 3378570 tid 3 2024-09-21T09:54:52.310 INFO:tasks.rados.rados.0.smithi017.stdout:3974: write oid 3 current snap is 498 2024-09-21T09:54:52.310 INFO:tasks.rados.rados.0.smithi017.stdout:3974: seq_num 1511 ranges {595377=739133,1953586=499867,3110848=284529} 2024-09-21T09:54:52.317 INFO:tasks.rados.rados.0.smithi017.stdout:3974: writing smithi01785587-3 from 595377 to 1334510 tid 1 2024-09-21T09:54:52.324 INFO:tasks.rados.rados.0.smithi017.stdout:3974: writing smithi01785587-3 from 1953586 to 2453453 tid 2 2024-09-21T09:54:52.329 INFO:tasks.rados.rados.0.smithi017.stdout:3974: writing smithi01785587-3 from 3110848 to 3395377 tid 3 2024-09-21T09:54:52.330 INFO:tasks.rados.rados.0.smithi017.stdout:3975: read oid 8 snap 497 2024-09-21T09:54:52.330 INFO:tasks.rados.rados.0.smithi017.stdout:3975: expect deleted 2024-09-21T09:54:52.330 INFO:tasks.rados.rados.0.smithi017.stdout:3976: read oid 9 snap 493 2024-09-21T09:54:52.330 INFO:tasks.rados.rados.0.smithi017.stdout:3976: expect (ObjNum 1495 snap 492 seq_num 1495) 2024-09-21T09:54:52.330 INFO:tasks.rados.rados.0.smithi017.stdout:3977: snap_remove snap 482 2024-09-21T09:54:52.330 INFO:tasks.rados.rados.0.smithi017.stdout:3972: finishing write tid 1 to smithi01785587-43 2024-09-21T09:54:52.330 INFO:tasks.rados.rados.0.smithi017.stdout:3972: finishing write tid 2 to smithi01785587-43 2024-09-21T09:54:52.330 INFO:tasks.rados.rados.0.smithi017.stdout:3972: finishing write tid 3 to smithi01785587-43 2024-09-21T09:54:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:52 smithi017 ceph-mon[102913]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-21T09:54:52.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:52 smithi017 ceph-mon[105151]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-21T09:54:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:52 smithi134 ceph-mon[90085]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-21T09:54:53.262 INFO:tasks.rados.rados.0.smithi017.stdout:3977: done (5 left) 2024-09-21T09:54:53.262 INFO:tasks.rados.rados.0.smithi017.stdout:3978: snap_create 2024-09-21T09:54:53.262 INFO:tasks.rados.rados.0.smithi017.stdout:3972: finishing write tid 4 to smithi01785587-43 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3972: finishing write tid 5 to smithi01785587-43 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 43 v 1155 (ObjNum 1509 snap 498 seq_num 1509) dirty exists 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3972: left oid 43 (ObjNum 1509 snap 498 seq_num 1509) 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3973: finishing write tid 1 to smithi01785587-18 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3973: finishing write tid 2 to smithi01785587-18 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3973: finishing write tid 3 to smithi01785587-18 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3973: finishing write tid 4 to smithi01785587-18 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3973: finishing write tid 5 to smithi01785587-18 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 18 v 434 (ObjNum 1510 snap 498 seq_num 1510) dirty exists 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3973: left oid 18 (ObjNum 1510 snap 498 seq_num 1510) 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3974: finishing write tid 1 to smithi01785587-3 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3974: finishing write tid 2 to smithi01785587-3 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3974: finishing write tid 3 to smithi01785587-3 2024-09-21T09:54:53.263 INFO:tasks.rados.rados.0.smithi017.stdout:3974: finishing write tid 4 to smithi01785587-3 2024-09-21T09:54:53.264 INFO:tasks.rados.rados.0.smithi017.stdout:3974: finishing write tid 5 to smithi01785587-3 2024-09-21T09:54:53.264 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 3 v 2334 (ObjNum 1511 snap 498 seq_num 1511) dirty exists 2024-09-21T09:54:53.264 INFO:tasks.rados.rados.0.smithi017.stdout:3974: left oid 3 (ObjNum 1511 snap 498 seq_num 1511) 2024-09-21T09:54:53.264 INFO:tasks.rados.rados.0.smithi017.stdout:3976: expect (ObjNum 1495 snap 492 seq_num 1495) 2024-09-21T09:54:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:53 smithi017 ceph-mon[105151]: pgmap v1179: 89 pgs: 89 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 335 KiB/s wr, 8 op/s 2024-09-21T09:54:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:53 smithi017 ceph-mon[105151]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-21T09:54:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:53 smithi017 ceph-mon[105151]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-21T09:54:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:53 smithi017 ceph-mon[102913]: pgmap v1179: 89 pgs: 89 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 335 KiB/s wr, 8 op/s 2024-09-21T09:54:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:53 smithi017 ceph-mon[102913]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-21T09:54:53.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:53 smithi017 ceph-mon[102913]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-21T09:54:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:53 smithi134 ceph-mon[90085]: pgmap v1179: 89 pgs: 89 active+clean; 149 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 335 KiB/s wr, 8 op/s 2024-09-21T09:54:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:53 smithi134 ceph-mon[90085]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-21T09:54:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:53 smithi134 ceph-mon[90085]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-21T09:54:54.265 INFO:tasks.rados.rados.0.smithi017.stdout:3972: done (5 left) 2024-09-21T09:54:54.265 INFO:tasks.rados.rados.0.smithi017.stdout:3973: done (4 left) 2024-09-21T09:54:54.265 INFO:tasks.rados.rados.0.smithi017.stdout:3974: done (3 left) 2024-09-21T09:54:54.265 INFO:tasks.rados.rados.0.smithi017.stdout:3975: done (2 left) 2024-09-21T09:54:54.265 INFO:tasks.rados.rados.0.smithi017.stdout:3976: done (1 left) 2024-09-21T09:54:54.265 INFO:tasks.rados.rados.0.smithi017.stdout:3978: done (0 left) 2024-09-21T09:54:54.265 INFO:tasks.rados.rados.0.smithi017.stdout:3979: write oid 44 current snap is 499 2024-09-21T09:54:54.265 INFO:tasks.rados.rados.0.smithi017.stdout:3979: seq_num 1512 ranges {612184=414382,1612068=643525,3021918=390266} 2024-09-21T09:54:54.270 INFO:tasks.rados.rados.0.smithi017.stdout:3979: writing smithi01785587-44 from 612184 to 1026566 tid 1 2024-09-21T09:54:54.278 INFO:tasks.rados.rados.0.smithi017.stdout:3979: writing smithi01785587-44 from 1612068 to 2255593 tid 2 2024-09-21T09:54:54.284 INFO:tasks.rados.rados.0.smithi017.stdout:3979: writing smithi01785587-44 from 3021918 to 3412184 tid 3 2024-09-21T09:54:54.284 INFO:tasks.rados.rados.0.smithi017.stdout:3980: rollback oid 33 current snap is 499 2024-09-21T09:54:54.285 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 33 to 498 2024-09-21T09:54:54.285 INFO:tasks.rados.rados.0.smithi017.stdout:3979: finishing write tid 1 to smithi01785587-44 2024-09-21T09:54:54.285 INFO:tasks.rados.rados.0.smithi017.stdout:3979: finishing write tid 2 to smithi01785587-44 2024-09-21T09:54:54.285 INFO:tasks.rados.rados.0.smithi017.stdout:3981: snap_remove snap 493 2024-09-21T09:54:55.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:54:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:54:54] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.51.0" 2024-09-21T09:54:55.279 INFO:tasks.rados.rados.0.smithi017.stdout:3981: done (2 left) 2024-09-21T09:54:55.279 INFO:tasks.rados.rados.0.smithi017.stdout:3982: read oid 5 snap -1 2024-09-21T09:54:55.279 INFO:tasks.rados.rados.0.smithi017.stdout:3982: expect deleted 2024-09-21T09:54:55.279 INFO:tasks.rados.rados.0.smithi017.stdout:3983: snap_create 2024-09-21T09:54:55.280 INFO:tasks.rados.rados.0.smithi017.stdout:3979: finishing write tid 3 to smithi01785587-44 2024-09-21T09:54:55.280 INFO:tasks.rados.rados.0.smithi017.stdout:3979: finishing write tid 4 to smithi01785587-44 2024-09-21T09:54:55.280 INFO:tasks.rados.rados.0.smithi017.stdout:3980: finishing rollback tid 0 to smithi01785587-33 2024-09-21T09:54:55.280 INFO:tasks.rados.rados.0.smithi017.stdout:3980: finishing rollback tid 1 to smithi01785587-33 2024-09-21T09:54:55.280 INFO:tasks.rados.rados.0.smithi017.stdout:3979: finishing write tid 5 to smithi01785587-44 2024-09-21T09:54:55.280 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 44 v 2345 (ObjNum 1512 snap 499 seq_num 1512) dirty exists 2024-09-21T09:54:55.280 INFO:tasks.rados.rados.0.smithi017.stdout:3979: left oid 44 (ObjNum 1512 snap 499 seq_num 1512) 2024-09-21T09:54:55.280 INFO:tasks.rados.rados.0.smithi017.stdout:3980: finishing rollback tid 2 to smithi01785587-33 2024-09-21T09:54:55.280 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 33 v 2193 (ObjNum 1497 snap 493 seq_num 1497) dirty exists 2024-09-21T09:54:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:55 smithi017 ceph-mon[105151]: pgmap v1182: 89 pgs: 89 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:54:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:55 smithi017 ceph-mon[105151]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-21T09:54:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:55 smithi017 ceph-mon[102913]: pgmap v1182: 89 pgs: 89 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:54:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:55 smithi017 ceph-mon[102913]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-21T09:54:55.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:55 smithi134 ceph-mon[90085]: pgmap v1182: 89 pgs: 89 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-09-21T09:54:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:55 smithi134 ceph-mon[90085]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-21T09:54:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:56.286 INFO:tasks.rados.rados.0.smithi017.stdout:3979: done (3 left) 2024-09-21T09:54:56.287 INFO:tasks.rados.rados.0.smithi017.stdout:3980: done (2 left) 2024-09-21T09:54:56.287 INFO:tasks.rados.rados.0.smithi017.stdout:3982: done (1 left) 2024-09-21T09:54:56.287 INFO:tasks.rados.rados.0.smithi017.stdout:3983: done (0 left) 2024-09-21T09:54:56.287 INFO:tasks.rados.rados.0.smithi017.stdout:3984: write oid 29 current snap is 500 2024-09-21T09:54:56.287 INFO:tasks.rados.rados.0.smithi017.stdout:3984: seq_num 1513 ranges {628991=605984,1786903=787183,3165694=263297} 2024-09-21T09:54:56.293 INFO:tasks.rados.rados.0.smithi017.stdout:3984: writing smithi01785587-29 from 628991 to 1234975 tid 1 2024-09-21T09:54:56.300 INFO:tasks.rados.rados.0.smithi017.stdout:3984: writing smithi01785587-29 from 1786903 to 2574086 tid 2 2024-09-21T09:54:56.305 INFO:tasks.rados.rados.0.smithi017.stdout:3984: writing smithi01785587-29 from 3165694 to 3428991 tid 3 2024-09-21T09:54:56.305 INFO:tasks.rados.rados.0.smithi017.stdout:3985: snap_create 2024-09-21T09:54:56.305 INFO:tasks.rados.rados.0.smithi017.stdout:3984: finishing write tid 1 to smithi01785587-29 2024-09-21T09:54:56.305 INFO:tasks.rados.rados.0.smithi017.stdout:3984: finishing write tid 2 to smithi01785587-29 2024-09-21T09:54:56.309 INFO:tasks.rados.rados.0.smithi017.stdout:3984: finishing write tid 3 to smithi01785587-29 2024-09-21T09:54:56.309 INFO:tasks.rados.rados.0.smithi017.stdout:3984: finishing write tid 4 to smithi01785587-29 2024-09-21T09:54:56.309 INFO:tasks.rados.rados.0.smithi017.stdout:3984: finishing write tid 5 to smithi01785587-29 2024-09-21T09:54:56.309 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 29 v 1200 (ObjNum 1513 snap 500 seq_num 1513) dirty exists 2024-09-21T09:54:56.309 INFO:tasks.rados.rados.0.smithi017.stdout:3984: left oid 29 (ObjNum 1513 snap 500 seq_num 1513) 2024-09-21T09:54:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:56 smithi017 ceph-mon[102913]: osdmap e1070: 8 total, 8 up, 8 in 2024-09-21T09:54:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:56 smithi017 ceph-mon[105151]: osdmap e1070: 8 total, 8 up, 8 in 2024-09-21T09:54:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:56 smithi134 ceph-mon[90085]: osdmap e1070: 8 total, 8 up, 8 in 2024-09-21T09:54:57.290 INFO:tasks.rados.rados.0.smithi017.stdout:3984: done (1 left) 2024-09-21T09:54:57.291 INFO:tasks.rados.rados.0.smithi017.stdout:3985: done (0 left) 2024-09-21T09:54:57.291 INFO:tasks.rados.rados.0.smithi017.stdout:3986: snap_create 2024-09-21T09:54:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:57 smithi017 ceph-mon[102913]: pgmap v1185: 89 pgs: 89 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-21T09:54:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:57 smithi017 ceph-mon[102913]: osdmap e1071: 8 total, 8 up, 8 in 2024-09-21T09:54:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:57 smithi017 ceph-mon[105151]: pgmap v1185: 89 pgs: 89 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-21T09:54:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:57 smithi017 ceph-mon[105151]: osdmap e1071: 8 total, 8 up, 8 in 2024-09-21T09:54:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:57 smithi134 ceph-mon[90085]: pgmap v1185: 89 pgs: 89 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-21T09:54:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:57 smithi134 ceph-mon[90085]: osdmap e1071: 8 total, 8 up, 8 in 2024-09-21T09:54:58.264 INFO:tasks.rados.rados.0.smithi017.stdout:3986: done (0 left) 2024-09-21T09:54:58.264 INFO:tasks.rados.rados.0.smithi017.stdout:3987: read oid 8 snap -1 2024-09-21T09:54:58.264 INFO:tasks.rados.rados.0.smithi017.stdout:3987: expect deleted 2024-09-21T09:54:58.264 INFO:tasks.rados.rados.0.smithi017.stdout:3988: read oid 38 snap -1 2024-09-21T09:54:58.264 INFO:tasks.rados.rados.0.smithi017.stdout:3988: expect deleted 2024-09-21T09:54:58.264 INFO:tasks.rados.rados.0.smithi017.stdout:3989: rollback oid 26 current snap is 502 2024-09-21T09:54:58.264 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 26 to 499 2024-09-21T09:54:58.264 INFO:tasks.rados.rados.0.smithi017.stdout:3990: snap_create 2024-09-21T09:54:58.271 INFO:tasks.rados.rados.0.smithi017.stdout:3989: finishing rollback tid 0 to smithi01785587-26 2024-09-21T09:54:58.271 INFO:tasks.rados.rados.0.smithi017.stdout:3989: finishing rollback tid 1 to smithi01785587-26 2024-09-21T09:54:58.272 INFO:tasks.rados.rados.0.smithi017.stdout:3989: finishing rollback tid 2 to smithi01785587-26 2024-09-21T09:54:58.272 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 2203 (ObjNum 1484 snap 490 seq_num 1484) dirty exists 2024-09-21T09:54:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:58 smithi017 ceph-mon[102913]: osdmap e1072: 8 total, 8 up, 8 in 2024-09-21T09:54:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:58 smithi017 ceph-mon[102913]: osdmap e1073: 8 total, 8 up, 8 in 2024-09-21T09:54:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:58 smithi017 ceph-mon[105151]: osdmap e1072: 8 total, 8 up, 8 in 2024-09-21T09:54:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:58 smithi017 ceph-mon[105151]: osdmap e1073: 8 total, 8 up, 8 in 2024-09-21T09:54:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:58 smithi134 ceph-mon[90085]: osdmap e1072: 8 total, 8 up, 8 in 2024-09-21T09:54:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:58 smithi134 ceph-mon[90085]: osdmap e1073: 8 total, 8 up, 8 in 2024-09-21T09:54:59.266 INFO:tasks.rados.rados.0.smithi017.stdout:3987: done (3 left) 2024-09-21T09:54:59.266 INFO:tasks.rados.rados.0.smithi017.stdout:3988: done (2 left) 2024-09-21T09:54:59.266 INFO:tasks.rados.rados.0.smithi017.stdout:3989: done (1 left) 2024-09-21T09:54:59.266 INFO:tasks.rados.rados.0.smithi017.stdout:3990: done (0 left) 2024-09-21T09:54:59.266 INFO:tasks.rados.rados.0.smithi017.stdout:3991: read oid 35 snap -1 2024-09-21T09:54:59.266 INFO:tasks.rados.rados.0.smithi017.stdout:3991: expect (ObjNum 1391 snap 462 seq_num 1391) 2024-09-21T09:54:59.267 INFO:tasks.rados.rados.0.smithi017.stdout:3992: read oid 28 snap -1 2024-09-21T09:54:59.267 INFO:tasks.rados.rados.0.smithi017.stdout:3992: expect (ObjNum 1445 snap 477 seq_num 1445) 2024-09-21T09:54:59.267 INFO:tasks.rados.rados.0.smithi017.stdout:3993: read oid 8 snap 494 2024-09-21T09:54:59.267 INFO:tasks.rados.rados.0.smithi017.stdout:3993: expect deleted 2024-09-21T09:54:59.267 INFO:tasks.rados.rados.0.smithi017.stdout:3994: write oid 36 current snap is 503 2024-09-21T09:54:59.267 INFO:tasks.rados.rados.0.smithi017.stdout:3994: seq_num 1514 ranges {645798=681233,1729032=647194,3076764=369034} 2024-09-21T09:54:59.274 INFO:tasks.rados.rados.0.smithi017.stdout:3994: writing smithi01785587-36 from 645798 to 1327031 tid 1 2024-09-21T09:54:59.281 INFO:tasks.rados.rados.0.smithi017.stdout:3994: writing smithi01785587-36 from 1729032 to 2376226 tid 2 2024-09-21T09:54:59.288 INFO:tasks.rados.rados.0.smithi017.stdout:3994: writing smithi01785587-36 from 3076764 to 3445798 tid 3 2024-09-21T09:54:59.288 INFO:tasks.rados.rados.0.smithi017.stdout:3995: write oid 21 current snap is 503 2024-09-21T09:54:59.288 INFO:tasks.rados.rados.0.smithi017.stdout:3991: expect (ObjNum 1391 snap 462 seq_num 1391) 2024-09-21T09:54:59.398 INFO:tasks.rados.rados.0.smithi017.stdout:3992: expect (ObjNum 1445 snap 477 seq_num 1445) 2024-09-21T09:54:59.486 INFO:tasks.rados.rados.0.smithi017.stdout:3994: finishing write tid 1 to smithi01785587-36 2024-09-21T09:54:59.486 INFO:tasks.rados.rados.0.smithi017.stdout:3994: finishing write tid 2 to smithi01785587-36 2024-09-21T09:54:59.486 INFO:tasks.rados.rados.0.smithi017.stdout:3995: seq_num 1515 ranges {662605=756482,2187514=790852} 2024-09-21T09:54:59.494 INFO:tasks.rados.rados.0.smithi017.stdout:3995: writing smithi01785587-21 from 662605 to 1419087 tid 1 2024-09-21T09:54:59.503 INFO:tasks.rados.rados.0.smithi017.stdout:3995: writing smithi01785587-21 from 2187514 to 2978366 tid 2 2024-09-21T09:54:59.503 INFO:tasks.rados.rados.0.smithi017.stdout:3991: done (4 left) 2024-09-21T09:54:59.503 INFO:tasks.rados.rados.0.smithi017.stdout:3992: done (3 left) 2024-09-21T09:54:59.503 INFO:tasks.rados.rados.0.smithi017.stdout:3993: done (2 left) 2024-09-21T09:54:59.503 INFO:tasks.rados.rados.0.smithi017.stdout:3996: rollback oid 8 current snap is 503 2024-09-21T09:54:59.503 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 8 to 497 2024-09-21T09:54:59.504 INFO:tasks.rados.rados.0.smithi017.stdout:3994: finishing write tid 3 to smithi01785587-36 2024-09-21T09:54:59.504 INFO:tasks.rados.rados.0.smithi017.stdout:3994: finishing write tid 4 to smithi01785587-36 2024-09-21T09:54:59.504 INFO:tasks.rados.rados.0.smithi017.stdout:3994: finishing write tid 5 to smithi01785587-36 2024-09-21T09:54:59.504 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 36 v 2036 (ObjNum 1514 snap 503 seq_num 1514) dirty exists 2024-09-21T09:54:59.504 INFO:tasks.rados.rados.0.smithi017.stdout:3994: left oid 36 (ObjNum 1514 snap 503 seq_num 1514) 2024-09-21T09:54:59.504 INFO:tasks.rados.rados.0.smithi017.stdout:3995: finishing write tid 1 to smithi01785587-21 2024-09-21T09:54:59.504 INFO:tasks.rados.rados.0.smithi017.stdout:3994: done (2 left) 2024-09-21T09:54:59.504 INFO:tasks.rados.rados.0.smithi017.stdout:3997: write oid 46 current snap is 503 2024-09-21T09:54:59.504 INFO:tasks.rados.rados.0.smithi017.stdout:3997: seq_num 1516 ranges {679412=431731,1845996=650863,3131610=347802} 2024-09-21T09:54:59.508 INFO:tasks.rados.rados.0.smithi017.stdout:3997: writing smithi01785587-46 from 679412 to 1111143 tid 1 2024-09-21T09:54:59.516 INFO:tasks.rados.rados.0.smithi017.stdout:3997: writing smithi01785587-46 from 1845996 to 2496859 tid 2 2024-09-21T09:54:59.522 INFO:tasks.rados.rados.0.smithi017.stdout:3997: writing smithi01785587-46 from 3131610 to 3479412 tid 3 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:3998: rollback oid 26 current snap is 503 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 26 to 502 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:3996: finishing rollback tid 1 to smithi01785587-8 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 8 v 0 (ObjNum 177 snap 0 seq_num 2399984944) dirty dne 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:3996: done (3 left) 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:3999: rollback oid 1 current snap is 503 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:rollback oid 1 to 502 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:3995: finishing write tid 2 to smithi01785587-21 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:3995: finishing write tid 3 to smithi01785587-21 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:3995: finishing write tid 4 to smithi01785587-21 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 21 v 1878 (ObjNum 1515 snap 503 seq_num 1515) dirty exists 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:3995: left oid 21 (ObjNum 1515 snap 503 seq_num 1515) 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi017.stdout:3997: finishing write tid 1 to smithi01785587-46 2024-09-21T09:54:59.524 INFO:tasks.rados.rados.0.smithi017.stdout:3997: finishing write tid 2 to smithi01785587-46 2024-09-21T09:54:59.524 INFO:tasks.rados.rados.0.smithi017.stdout:3995: done (3 left) 2024-09-21T09:54:59.524 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 3 2024-09-21T09:54:59.524 INFO:tasks.rados.rados.0.smithi017.stdout:3999: finishing rollback tid 1 to smithi01785587-1 2024-09-21T09:54:59.524 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 1 v 0 (ObjNum 129 snap 0 seq_num 2399560688) dirty dne 2024-09-21T09:54:59.524 INFO:tasks.rados.rados.0.smithi017.stdout:3999: done (2 left) 2024-09-21T09:54:59.524 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 2 2024-09-21T09:54:59.525 INFO:tasks.rados.rados.0.smithi017.stdout:3997: finishing write tid 3 to smithi01785587-46 2024-09-21T09:54:59.525 INFO:tasks.rados.rados.0.smithi017.stdout:3997: finishing write tid 4 to smithi01785587-46 2024-09-21T09:54:59.525 INFO:tasks.rados.rados.0.smithi017.stdout:3997: finishing write tid 5 to smithi01785587-46 2024-09-21T09:54:59.525 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 46 v 2042 (ObjNum 1516 snap 503 seq_num 1516) dirty exists 2024-09-21T09:54:59.525 INFO:tasks.rados.rados.0.smithi017.stdout:3997: left oid 46 (ObjNum 1516 snap 503 seq_num 1516) 2024-09-21T09:54:59.525 INFO:tasks.rados.rados.0.smithi017.stdout:3997: done (1 left) 2024-09-21T09:54:59.525 INFO:tasks.rados.rados.0.smithi017.stdout: waiting on 1 2024-09-21T09:54:59.526 INFO:tasks.rados.rados.0.smithi017.stdout:3998: finishing rollback tid 0 to smithi01785587-26 2024-09-21T09:54:59.526 INFO:tasks.rados.rados.0.smithi017.stdout:3998: finishing rollback tid 1 to smithi01785587-26 2024-09-21T09:54:59.526 INFO:tasks.rados.rados.0.smithi017.stdout:3998: finishing rollback tid 2 to smithi01785587-26 2024-09-21T09:54:59.526 INFO:tasks.rados.rados.0.smithi017.stdout:update_object_version oid 26 v 2207 (ObjNum 1484 snap 490 seq_num 1484) dirty exists 2024-09-21T09:54:59.526 INFO:tasks.rados.rados.0.smithi017.stdout:3998: done (0 left) 2024-09-21T09:54:59.528 INFO:tasks.rados.rados.0.smithi017.stderr:0 errors. 2024-09-21T09:54:59.528 INFO:tasks.rados.rados.0.smithi017.stderr: 2024-09-21T09:54:59.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:54:59.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:59 smithi017 ceph-mon[102913]: pgmap v1188: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-21T09:54:59.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:54:59 smithi017 ceph-mon[102913]: osdmap e1074: 8 total, 8 up, 8 in 2024-09-21T09:54:59.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:59 smithi017 ceph-mon[105151]: pgmap v1188: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-21T09:54:59.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:54:59 smithi017 ceph-mon[105151]: osdmap e1074: 8 total, 8 up, 8 in 2024-09-21T09:54:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:59 smithi134 ceph-mon[90085]: pgmap v1188: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.5 MiB/s wr, 12 op/s 2024-09-21T09:54:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:54:59 smithi134 ceph-mon[90085]: osdmap e1074: 8 total, 8 up, 8 in 2024-09-21T09:54:59.826 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:55:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:55:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:01.428 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:01 smithi017 ceph-mon[102913]: pgmap v1191: 89 pgs: 89 active+clean; 118 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2 op/s 2024-09-21T09:55:01.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:01 smithi017 ceph-mon[105151]: pgmap v1191: 89 pgs: 89 active+clean; 118 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2 op/s 2024-09-21T09:55:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:01 smithi134 ceph-mon[90085]: pgmap v1191: 89 pgs: 89 active+clean; 118 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2 op/s 2024-09-21T09:55:01.845 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:55:01.845 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":1074,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T09:54:59.259049+0000","last_up_change":"2024-09-21T09:46:46.096331+0000","last_in_change":"2024-09-21T09:34:35.544589+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:27.624125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:41.967182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:42.538769+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-09-21T09:37:55.280215+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1074","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":604,"snap_epoch":1074,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":466,"up_from":479,"up_thru":581,"down_at":467,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6803","nonce":2105074120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6805","nonce":2105074120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6809","nonce":2105074120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6807","nonce":2105074120}]},"public_addr":"172.21.15.17:6803/2105074120","cluster_addr":"172.21.15.17:6805/2105074120","heartbeat_back_addr":"172.21.15.17:6809/2105074120","heartbeat_front_addr":"172.21.15.17:6807/2105074120","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":500,"up_from":513,"up_thru":581,"down_at":501,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6811","nonce":1078446230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6813","nonce":1078446230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6817","nonce":1078446230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6815","nonce":1078446230}]},"public_addr":"172.21.15.17:6811/1078446230","cluster_addr":"172.21.15.17:6813/1078446230","heartbeat_back_addr":"172.21.15.17:6817/1078446230","heartbeat_front_addr":"172.21.15.17:6815/1078446230","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":534,"up_from":547,"up_thru":581,"down_at":535,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6819","nonce":323142954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6821","nonce":323142954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6825","nonce":323142954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6823","nonce":323142954}]},"public_addr":"172.21.15.17:6819/323142954","cluster_addr":"172.21.15.17:6821/323142954","heartbeat_back_addr":"172.21.15.17:6825/323142954","heartbeat_front_addr":"172.21.15.17:6823/323142954","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":568,"up_from":581,"up_thru":581,"down_at":569,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6827","nonce":3146888816}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6829","nonce":3146888816}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6833","nonce":3146888816}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6831","nonce":3146888816}]},"public_addr":"172.21.15.17:6827/3146888816","cluster_addr":"172.21.15.17:6829/3146888816","heartbeat_back_addr":"172.21.15.17:6833/3146888816","heartbeat_front_addr":"172.21.15.17:6831/3146888816","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":581,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6801","nonce":2226434249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6803","nonce":2226434249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6807","nonce":2226434249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6805","nonce":2226434249}]},"public_addr":"172.21.15.134:6801/2226434249","cluster_addr":"172.21.15.134:6803/2226434249","heartbeat_back_addr":"172.21.15.134:6807/2226434249","heartbeat_front_addr":"172.21.15.134:6805/2226434249","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":581,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":569,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":581,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":466},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":500},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":534},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":568},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:6800/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/381036987":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3226197858":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/2599173714":"2024-09-22T09:41:39.758476+0000","172.21.15.134:0/2350962326":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/2266250644":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6801/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.134:0/1515484230":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.134:6834/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3671582122":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/758458220":"2024-09-22T09:41:39.758476+0000","172.21.15.134:6835/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3206271349":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/3530815435":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6800/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/1454017192":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/877591523":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6801/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/3275227882":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6801/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/3084797810":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3512479442":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/2291861207":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/3086297034":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/2023103508":"2024-09-22T09:41:39.758476+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":584,"length":1},{"begin":594,"length":2}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:55:02.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1294603037' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:02.620 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1294603037' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:02.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:55:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1294603037' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:02.955 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:03.390 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:03 smithi017 ceph-mon[105151]: pgmap v1192: 89 pgs: 89 active+clean; 125 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 797 KiB/s wr, 8 op/s 2024-09-21T09:55:03.390 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:03 smithi017 ceph-mon[105151]: osdmap e1075: 8 total, 8 up, 8 in 2024-09-21T09:55:03.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:03 smithi017 ceph-mon[102913]: pgmap v1192: 89 pgs: 89 active+clean; 125 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 797 KiB/s wr, 8 op/s 2024-09-21T09:55:03.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:03 smithi017 ceph-mon[102913]: osdmap e1075: 8 total, 8 up, 8 in 2024-09-21T09:55:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:03 smithi134 ceph-mon[90085]: pgmap v1192: 89 pgs: 89 active+clean; 125 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 797 KiB/s wr, 8 op/s 2024-09-21T09:55:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:03 smithi134 ceph-mon[90085]: osdmap e1075: 8 total, 8 up, 8 in 2024-09-21T09:55:04.734 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:55:04.735 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:55:05.075 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:55:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:55:04] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-09-21T09:55:05.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:05 smithi017 ceph-mon[102913]: pgmap v1194: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-09-21T09:55:05.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:05 smithi017 ceph-mon[105151]: pgmap v1194: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-09-21T09:55:05.429 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1194,"stamp":"2024-09-21T09:55:03.965764+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":133186865,"num_objects":79,"num_object_clones":14,"num_object_copies":237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79,"num_whiteouts":1,"num_read":6945,"num_read_kb":7001565,"num_write":5849,"num_write_kb":1352645,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":44747907,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21295,"ondisk_log_size":21295,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1677344,"kb_used_data":295352,"kb_used_omap":122,"kb_used_meta":1380677,"kb_avail":748283872,"statfs":{"total":767960285184,"available":766242684928,"internally_reserved":0,"allocated":302440448,"data_stored":312001116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125317,"internal_metadata":1413813883},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":140808,"num_objects":2,"num_object_clones":2,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":-1,"num_read":20,"num_read_kb":16280,"num_write":23,"num_write_kb":6189,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001572"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1481,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510639+0000","last_change":"2024-09-21T09:46:47.114206+0000","last_active":"2024-09-21T09:54:59.510639+0000","last_peered":"2024-09-21T09:54:59.510639+0000","last_clean":"2024-09-21T09:54:59.510639+0000","last_became_active":"2024-09-21T09:46:47.113819+0000","last_became_peered":"2024-09-21T09:46:47.113819+0000","last_unstale":"2024-09-21T09:54:59.510639+0000","last_undegraded":"2024-09-21T09:54:59.510639+0000","last_fullsized":"2024-09-21T09:54:59.510639+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:07.629544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":999,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.721554+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T09:54:58.721554+0000","last_peered":"2024-09-21T09:54:58.721554+0000","last_clean":"2024-09-21T09:54:58.721554+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T09:54:58.721554+0000","last_undegraded":"2024-09-21T09:54:58.721554+0000","last_fullsized":"2024-09-21T09:54:58.721554+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1018,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261944+0000","last_change":"2024-09-21T09:46:47.115484+0000","last_active":"2024-09-21T09:54:58.261944+0000","last_peered":"2024-09-21T09:54:58.261944+0000","last_clean":"2024-09-21T09:54:58.261944+0000","last_became_active":"2024-09-21T09:46:47.115080+0000","last_became_peered":"2024-09-21T09:46:47.115080+0000","last_unstale":"2024-09-21T09:54:58.261944+0000","last_undegraded":"2024-09-21T09:54:58.261944+0000","last_fullsized":"2024-09-21T09:54:58.261944+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:56.851805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1565,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.276596+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T09:54:59.276596+0000","last_peered":"2024-09-21T09:54:59.276596+0000","last_clean":"2024-09-21T09:54:59.276596+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T09:54:59.276596+0000","last_undegraded":"2024-09-21T09:54:59.276596+0000","last_fullsized":"2024-09-21T09:54:59.276596+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1567,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.277485+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T09:54:59.277485+0000","last_peered":"2024-09-21T09:54:59.277485+0000","last_clean":"2024-09-21T09:54:59.277485+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T09:54:59.277485+0000","last_undegraded":"2024-09-21T09:54:59.277485+0000","last_fullsized":"2024-09-21T09:54:59.277485+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":980,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.721839+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:54:58.721839+0000","last_peered":"2024-09-21T09:54:58.721839+0000","last_clean":"2024-09-21T09:54:58.721839+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:54:58.721839+0000","last_undegraded":"2024-09-21T09:54:58.721839+0000","last_fullsized":"2024-09-21T09:54:58.721839+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1481,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510854+0000","last_change":"2024-09-21T09:46:47.117580+0000","last_active":"2024-09-21T09:54:59.510854+0000","last_peered":"2024-09-21T09:54:59.510854+0000","last_clean":"2024-09-21T09:54:59.510854+0000","last_became_active":"2024-09-21T09:46:47.113030+0000","last_became_peered":"2024-09-21T09:46:47.113030+0000","last_unstale":"2024-09-21T09:54:59.510854+0000","last_undegraded":"2024-09-21T09:54:59.510854+0000","last_fullsized":"2024-09-21T09:54:59.510854+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:28.220180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":1006,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.260757+0000","last_change":"2024-09-21T09:45:05.080014+0000","last_active":"2024-09-21T09:54:58.260757+0000","last_peered":"2024-09-21T09:54:58.260757+0000","last_clean":"2024-09-21T09:54:58.260757+0000","last_became_active":"2024-09-21T09:45:05.079618+0000","last_became_peered":"2024-09-21T09:45:05.079618+0000","last_unstale":"2024-09-21T09:54:58.260757+0000","last_undegraded":"2024-09-21T09:54:58.260757+0000","last_fullsized":"2024-09-21T09:54:58.260757+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:42.160292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"1068'1682","reported_seq":4737,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.721737+0000","last_change":"2024-09-21T09:54:55.623078+0000","last_active":"2024-09-21T09:54:58.721737+0000","last_peered":"2024-09-21T09:54:58.721737+0000","last_clean":"2024-09-21T09:54:58.721737+0000","last_became_active":"2024-09-21T09:37:56.583636+0000","last_became_peered":"2024-09-21T09:37:56.583636+0000","last_unstale":"2024-09-21T09:54:58.721737+0000","last_undegraded":"2024-09-21T09:54:58.721737+0000","last_fullsized":"2024-09-21T09:54:58.721737+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":1682,"log_dups_size":0,"ondisk_log_size":1682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:33.953149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079586800000000003,"stat_sum":{"num_bytes":11446986,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":518,"num_read_kb":501627,"num_write":448,"num_write_kb":109248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.f","version":"63'1","reported_seq":998,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.722321+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T09:54:58.722321+0000","last_peered":"2024-09-21T09:54:58.722321+0000","last_clean":"2024-09-21T09:54:58.722321+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T09:54:58.722321+0000","last_undegraded":"2024-09-21T09:54:58.722321+0000","last_fullsized":"2024-09-21T09:54:58.722321+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1002,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.262040+0000","last_change":"2024-09-21T09:36:43.389124+0000","last_active":"2024-09-21T09:54:58.262040+0000","last_peered":"2024-09-21T09:54:58.262040+0000","last_clean":"2024-09-21T09:54:58.262040+0000","last_became_active":"2024-09-21T09:36:43.388588+0000","last_became_peered":"2024-09-21T09:36:43.388588+0000","last_unstale":"2024-09-21T09:54:58.262040+0000","last_undegraded":"2024-09-21T09:54:58.262040+0000","last_fullsized":"2024-09-21T09:54:58.262040+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:57.562156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"1068'1217","reported_seq":5295,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.267179+0000","last_change":"2024-09-21T09:54:55.743683+0000","last_active":"2024-09-21T09:54:59.267179+0000","last_peered":"2024-09-21T09:54:59.267179+0000","last_clean":"2024-09-21T09:54:59.267179+0000","last_became_active":"2024-09-21T09:46:13.261846+0000","last_became_peered":"2024-09-21T09:46:13.261846+0000","last_unstale":"2024-09-21T09:54:59.267179+0000","last_undegraded":"2024-09-21T09:54:59.267179+0000","last_fullsized":"2024-09-21T09:54:59.267179+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":1217,"log_dups_size":0,"ondisk_log_size":1217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:42.178428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0022888990000000001,"stat_sum":{"num_bytes":5731913,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":316,"num_read_kb":332417,"num_write":339,"num_write_kb":84935,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":5251214,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.e","version":"63'2","reported_seq":999,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.722259+0000","last_change":"2024-09-21T09:45:39.143190+0000","last_active":"2024-09-21T09:54:58.722259+0000","last_peered":"2024-09-21T09:54:58.722259+0000","last_clean":"2024-09-21T09:54:58.722259+0000","last_became_active":"2024-09-21T09:45:39.142826+0000","last_became_peered":"2024-09-21T09:45:39.142826+0000","last_unstale":"2024-09-21T09:54:58.722259+0000","last_undegraded":"2024-09-21T09:54:58.722259+0000","last_fullsized":"2024-09-21T09:54:58.722259+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:45.070945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1032,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265321+0000","last_change":"2024-09-21T09:46:47.119247+0000","last_active":"2024-09-21T09:54:58.265321+0000","last_peered":"2024-09-21T09:54:58.265321+0000","last_clean":"2024-09-21T09:54:58.265321+0000","last_became_active":"2024-09-21T09:46:47.118980+0000","last_became_peered":"2024-09-21T09:46:47.118980+0000","last_unstale":"2024-09-21T09:54:58.265321+0000","last_undegraded":"2024-09-21T09:54:58.265321+0000","last_fullsized":"2024-09-21T09:54:58.265321+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.140253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"998'378","reported_seq":4476,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.267412+0000","last_change":"2024-09-21T09:54:55.744093+0000","last_active":"2024-09-21T09:54:59.267412+0000","last_peered":"2024-09-21T09:54:59.267412+0000","last_clean":"2024-09-21T09:54:59.267412+0000","last_became_active":"2024-09-21T09:46:13.271736+0000","last_became_peered":"2024-09-21T09:46:13.271736+0000","last_unstale":"2024-09-21T09:54:59.267412+0000","last_undegraded":"2024-09-21T09:54:59.267412+0000","last_fullsized":"2024-09-21T09:54:59.267412+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":378,"log_dups_size":0,"ondisk_log_size":378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:02.428960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0024878740000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":101,"num_read_kb":99789,"num_write":96,"num_write_kb":23024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.d","version":"63'1","reported_seq":981,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.721958+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:54:58.721958+0000","last_peered":"2024-09-21T09:54:58.721958+0000","last_clean":"2024-09-21T09:54:58.721958+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:54:58.721958+0000","last_undegraded":"2024-09-21T09:54:58.721958+0000","last_fullsized":"2024-09-21T09:54:58.721958+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1007,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.260810+0000","last_change":"2024-09-21T09:46:13.260744+0000","last_active":"2024-09-21T09:54:58.260810+0000","last_peered":"2024-09-21T09:54:58.260810+0000","last_clean":"2024-09-21T09:54:58.260810+0000","last_became_active":"2024-09-21T09:46:13.260443+0000","last_became_peered":"2024-09-21T09:46:13.260443+0000","last_unstale":"2024-09-21T09:54:58.260810+0000","last_undegraded":"2024-09-21T09:54:58.260810+0000","last_fullsized":"2024-09-21T09:54:58.260810+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:22.699110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"1074'2042","reported_seq":6048,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.523823+0000","last_change":"2024-09-21T09:54:55.745545+0000","last_active":"2024-09-21T09:54:59.523823+0000","last_peered":"2024-09-21T09:54:59.523823+0000","last_clean":"2024-09-21T09:54:59.523823+0000","last_became_active":"2024-09-21T09:46:47.115344+0000","last_became_peered":"2024-09-21T09:46:47.115344+0000","last_unstale":"2024-09-21T09:54:59.523823+0000","last_undegraded":"2024-09-21T09:54:59.523823+0000","last_fullsized":"2024-09-21T09:54:59.523823+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":2042,"log_dups_size":0,"ondisk_log_size":2042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:46.670512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0038271830000000001,"stat_sum":{"num_bytes":9732309,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":668,"num_read_kb":725930,"num_write":557,"num_write_kb":134619,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":8581695,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.c","version":"0'0","reported_seq":1020,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261918+0000","last_change":"2024-09-21T09:46:47.115343+0000","last_active":"2024-09-21T09:54:58.261918+0000","last_peered":"2024-09-21T09:54:58.261918+0000","last_clean":"2024-09-21T09:54:58.261918+0000","last_became_active":"2024-09-21T09:46:47.115017+0000","last_became_peered":"2024-09-21T09:46:47.115017+0000","last_unstale":"2024-09-21T09:54:58.261918+0000","last_undegraded":"2024-09-21T09:54:58.261918+0000","last_fullsized":"2024-09-21T09:54:58.261918+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.820662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1567,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.276540+0000","last_change":"2024-09-21T09:45:39.138289+0000","last_active":"2024-09-21T09:54:59.276540+0000","last_peered":"2024-09-21T09:54:59.276540+0000","last_clean":"2024-09-21T09:54:59.276540+0000","last_became_active":"2024-09-21T09:45:39.137902+0000","last_became_peered":"2024-09-21T09:45:39.137902+0000","last_unstale":"2024-09-21T09:54:59.276540+0000","last_undegraded":"2024-09-21T09:54:59.276540+0000","last_fullsized":"2024-09-21T09:54:59.276540+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:40:16.383906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"1074'1878","reported_seq":5894,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.505992+0000","last_change":"2024-09-21T09:54:55.745930+0000","last_active":"2024-09-21T09:54:59.505992+0000","last_peered":"2024-09-21T09:54:59.505992+0000","last_clean":"2024-09-21T09:54:59.505992+0000","last_became_active":"2024-09-21T09:46:13.259858+0000","last_became_peered":"2024-09-21T09:46:13.259858+0000","last_unstale":"2024-09-21T09:54:59.505992+0000","last_undegraded":"2024-09-21T09:54:59.505992+0000","last_fullsized":"2024-09-21T09:54:59.505992+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":1878,"log_dups_size":0,"ondisk_log_size":1878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:25:55.055433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0041391980000000002,"stat_sum":{"num_bytes":11212448,"num_objects":4,"num_object_clones":1,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":693,"num_read_kb":805498,"num_write":505,"num_write_kb":120979,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":5440065,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.b","version":"63'2","reported_seq":1491,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510293+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T09:54:59.510293+0000","last_peered":"2024-09-21T09:54:59.510293+0000","last_clean":"2024-09-21T09:54:59.510293+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T09:54:59.510293+0000","last_undegraded":"2024-09-21T09:54:59.510293+0000","last_fullsized":"2024-09-21T09:54:59.510293+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1015,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265425+0000","last_change":"2024-09-21T09:46:47.114293+0000","last_active":"2024-09-21T09:54:58.265425+0000","last_peered":"2024-09-21T09:54:58.265425+0000","last_clean":"2024-09-21T09:54:58.265425+0000","last_became_active":"2024-09-21T09:46:47.113993+0000","last_became_peered":"2024-09-21T09:46:47.113993+0000","last_unstale":"2024-09-21T09:54:58.265425+0000","last_undegraded":"2024-09-21T09:54:58.265425+0000","last_fullsized":"2024-09-21T09:54:58.265425+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:30.941931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"1070'774","reported_seq":4930,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.281991+0000","last_change":"2024-09-21T09:54:55.287757+0000","last_active":"2024-09-21T09:54:59.281991+0000","last_peered":"2024-09-21T09:54:59.281991+0000","last_clean":"2024-09-21T09:54:59.281991+0000","last_became_active":"2024-09-21T09:45:39.131965+0000","last_became_peered":"2024-09-21T09:45:39.131965+0000","last_unstale":"2024-09-21T09:54:59.281991+0000","last_undegraded":"2024-09-21T09:54:59.281991+0000","last_fullsized":"2024-09-21T09:54:59.281991+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":774,"log_dups_size":0,"ondisk_log_size":774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:31.984924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.003416246,"stat_sum":{"num_bytes":7877945,"num_objects":3,"num_object_clones":1,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":205,"num_read_kb":172357,"num_write":210,"num_write_kb":50100,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":1011600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.a","version":"0'0","reported_seq":998,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261202+0000","last_change":"2024-09-21T09:45:39.127186+0000","last_active":"2024-09-21T09:54:58.261202+0000","last_peered":"2024-09-21T09:54:58.261202+0000","last_clean":"2024-09-21T09:54:58.261202+0000","last_became_active":"2024-09-21T09:45:39.126940+0000","last_became_peered":"2024-09-21T09:45:39.126940+0000","last_unstale":"2024-09-21T09:54:58.261202+0000","last_undegraded":"2024-09-21T09:54:58.261202+0000","last_fullsized":"2024-09-21T09:54:58.261202+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:52.205703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1032,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265636+0000","last_change":"2024-09-21T09:46:13.259814+0000","last_active":"2024-09-21T09:54:58.265636+0000","last_peered":"2024-09-21T09:54:58.265636+0000","last_clean":"2024-09-21T09:54:58.265636+0000","last_became_active":"2024-09-21T09:46:13.259527+0000","last_became_peered":"2024-09-21T09:46:13.259527+0000","last_unstale":"2024-09-21T09:54:58.265636+0000","last_undegraded":"2024-09-21T09:54:58.265636+0000","last_fullsized":"2024-09-21T09:54:58.265636+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:00.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"1071'1200","reported_seq":4346,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.262202+0000","last_change":"2024-09-21T09:54:55.281492+0000","last_active":"2024-09-21T09:54:58.262202+0000","last_peered":"2024-09-21T09:54:58.262202+0000","last_clean":"2024-09-21T09:54:58.262202+0000","last_became_active":"2024-09-21T09:37:56.582757+0000","last_became_peered":"2024-09-21T09:37:56.582757+0000","last_unstale":"2024-09-21T09:54:58.262202+0000","last_undegraded":"2024-09-21T09:54:58.262202+0000","last_fullsized":"2024-09-21T09:54:58.262202+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":1200,"log_dups_size":0,"ondisk_log_size":1200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:30:43.634844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0030338850000000001,"stat_sum":{"num_bytes":12272581,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":413,"num_read_kb":428208,"num_write":320,"num_write_kb":79051,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.9","version":"0'0","reported_seq":1016,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265424+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T09:54:58.265424+0000","last_peered":"2024-09-21T09:54:58.265424+0000","last_clean":"2024-09-21T09:54:58.265424+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T09:54:58.265424+0000","last_undegraded":"2024-09-21T09:54:58.265424+0000","last_fullsized":"2024-09-21T09:54:58.265424+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1015,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265883+0000","last_change":"2024-09-21T09:45:39.143302+0000","last_active":"2024-09-21T09:54:58.265883+0000","last_peered":"2024-09-21T09:54:58.265883+0000","last_clean":"2024-09-21T09:54:58.265883+0000","last_became_active":"2024-09-21T09:45:39.142964+0000","last_became_peered":"2024-09-21T09:45:39.142964+0000","last_unstale":"2024-09-21T09:54:58.265883+0000","last_undegraded":"2024-09-21T09:54:58.265883+0000","last_fullsized":"2024-09-21T09:54:58.265883+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:54.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"1074'2207","reported_seq":5284,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.525513+0000","last_change":"2024-09-21T09:54:55.289911+0000","last_active":"2024-09-21T09:54:59.525513+0000","last_peered":"2024-09-21T09:54:59.525513+0000","last_clean":"2024-09-21T09:54:59.525513+0000","last_became_active":"2024-09-21T09:45:05.097993+0000","last_became_peered":"2024-09-21T09:45:05.097993+0000","last_unstale":"2024-09-21T09:54:59.525513+0000","last_undegraded":"2024-09-21T09:54:59.525513+0000","last_fullsized":"2024-09-21T09:54:59.525513+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":2207,"log_dups_size":0,"ondisk_log_size":2207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:50.537341+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.0075462150000000002,"stat_sum":{"num_bytes":17649462,"num_objects":10,"num_object_clones":4,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":1,"num_read":748,"num_read_kb":769804,"num_write":588,"num_write_kb":135508,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":1030273,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.8","version":"0'0","reported_seq":1482,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510355+0000","last_change":"2024-09-21T09:46:47.121643+0000","last_active":"2024-09-21T09:54:59.510355+0000","last_peered":"2024-09-21T09:54:59.510355+0000","last_clean":"2024-09-21T09:54:59.510355+0000","last_became_active":"2024-09-21T09:46:47.120528+0000","last_became_peered":"2024-09-21T09:46:47.120528+0000","last_unstale":"2024-09-21T09:54:59.510355+0000","last_undegraded":"2024-09-21T09:54:59.510355+0000","last_fullsized":"2024-09-21T09:54:59.510355+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:07.989377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1529,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.282295+0000","last_change":"2024-09-21T09:46:47.121159+0000","last_active":"2024-09-21T09:54:59.282295+0000","last_peered":"2024-09-21T09:54:59.282295+0000","last_clean":"2024-09-21T09:54:59.282295+0000","last_became_active":"2024-09-21T09:46:47.120829+0000","last_became_peered":"2024-09-21T09:46:47.120829+0000","last_unstale":"2024-09-21T09:54:59.282295+0000","last_undegraded":"2024-09-21T09:54:59.282295+0000","last_fullsized":"2024-09-21T09:54:59.282295+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:43.393840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"1068'424","reported_seq":3594,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.269342+0000","last_change":"2024-09-21T09:54:55.622877+0000","last_active":"2024-09-21T09:54:59.269342+0000","last_peered":"2024-09-21T09:54:59.269342+0000","last_clean":"2024-09-21T09:54:59.269342+0000","last_became_active":"2024-09-21T09:46:13.257560+0000","last_became_peered":"2024-09-21T09:46:13.257560+0000","last_unstale":"2024-09-21T09:54:59.269342+0000","last_undegraded":"2024-09-21T09:54:59.269342+0000","last_fullsized":"2024-09-21T09:54:59.269342+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":424,"log_dups_size":0,"ondisk_log_size":424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:09.330301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055506200000000005,"stat_sum":{"num_bytes":3378537,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":134,"num_read_kb":141034,"num_write":105,"num_write_kb":21581,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"1.0","version":"288'100","reported_seq":1114,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.721414+0000","last_change":"2024-09-21T09:45:05.087928+0000","last_active":"2024-09-21T09:54:58.721414+0000","last_peered":"2024-09-21T09:54:58.721414+0000","last_clean":"2024-09-21T09:54:58.721414+0000","last_became_active":"2024-09-21T09:45:05.087478+0000","last_became_peered":"2024-09-21T09:45:05.087478+0000","last_unstale":"2024-09-21T09:54:58.721414+0000","last_undegraded":"2024-09-21T09:54:58.721414+0000","last_fullsized":"2024-09-21T09:54:58.721414+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:48.651314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1033,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261848+0000","last_change":"2024-09-21T09:46:13.262337+0000","last_active":"2024-09-21T09:54:58.261848+0000","last_peered":"2024-09-21T09:54:58.261848+0000","last_clean":"2024-09-21T09:54:58.261848+0000","last_became_active":"2024-09-21T09:46:13.262005+0000","last_became_peered":"2024-09-21T09:46:13.262005+0000","last_unstale":"2024-09-21T09:54:58.261848+0000","last_undegraded":"2024-09-21T09:54:58.261848+0000","last_fullsized":"2024-09-21T09:54:58.261848+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:32.413210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1464,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510163+0000","last_change":"2024-09-21T09:46:47.116045+0000","last_active":"2024-09-21T09:54:59.510163+0000","last_peered":"2024-09-21T09:54:59.510163+0000","last_clean":"2024-09-21T09:54:59.510163+0000","last_became_active":"2024-09-21T09:46:47.115526+0000","last_became_peered":"2024-09-21T09:46:47.115526+0000","last_unstale":"2024-09-21T09:54:59.510163+0000","last_undegraded":"2024-09-21T09:54:59.510163+0000","last_fullsized":"2024-09-21T09:54:59.510163+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:43:35.293486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":980,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261037+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:54:58.261037+0000","last_peered":"2024-09-21T09:54:58.261037+0000","last_clean":"2024-09-21T09:54:58.261037+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:54:58.261037+0000","last_undegraded":"2024-09-21T09:54:58.261037+0000","last_fullsized":"2024-09-21T09:54:58.261037+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"1068'638","reported_seq":3844,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.262538+0000","last_change":"2024-09-21T09:54:55.279026+0000","last_active":"2024-09-21T09:54:58.262538+0000","last_peered":"2024-09-21T09:54:58.262538+0000","last_clean":"2024-09-21T09:54:58.262538+0000","last_became_active":"2024-09-21T09:45:39.130634+0000","last_became_peered":"2024-09-21T09:45:39.130634+0000","last_unstale":"2024-09-21T09:54:58.262538+0000","last_undegraded":"2024-09-21T09:54:58.262538+0000","last_fullsized":"2024-09-21T09:54:58.262538+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":638,"log_dups_size":0,"ondisk_log_size":638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:51:15.172181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038284099999999998,"stat_sum":{"num_bytes":5395773,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":192483,"num_write":192,"num_write_kb":45368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"2.0","version":"0'0","reported_seq":1026,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.721364+0000","last_change":"2024-09-21T09:45:39.129555+0000","last_active":"2024-09-21T09:54:58.721364+0000","last_peered":"2024-09-21T09:54:58.721364+0000","last_clean":"2024-09-21T09:54:58.721364+0000","last_became_active":"2024-09-21T09:45:39.129205+0000","last_became_peered":"2024-09-21T09:45:39.129205+0000","last_unstale":"2024-09-21T09:54:58.721364+0000","last_undegraded":"2024-09-21T09:54:58.721364+0000","last_fullsized":"2024-09-21T09:54:58.721364+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:07.229126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1565,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.275995+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T09:54:59.275995+0000","last_peered":"2024-09-21T09:54:59.275995+0000","last_clean":"2024-09-21T09:54:59.275995+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T09:54:59.275995+0000","last_undegraded":"2024-09-21T09:54:59.275995+0000","last_fullsized":"2024-09-21T09:54:59.275995+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1578,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.277041+0000","last_change":"2024-09-21T09:46:13.273573+0000","last_active":"2024-09-21T09:54:59.277041+0000","last_peered":"2024-09-21T09:54:59.277041+0000","last_clean":"2024-09-21T09:54:59.277041+0000","last_became_active":"2024-09-21T09:46:13.273297+0000","last_became_peered":"2024-09-21T09:46:13.273297+0000","last_unstale":"2024-09-21T09:54:59.277041+0000","last_undegraded":"2024-09-21T09:54:59.277041+0000","last_fullsized":"2024-09-21T09:54:59.277041+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:38.957745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"1068'1347","reported_seq":5352,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.267636+0000","last_change":"2024-09-21T09:54:55.747190+0000","last_active":"2024-09-21T09:54:59.267636+0000","last_peered":"2024-09-21T09:54:59.267636+0000","last_clean":"2024-09-21T09:54:59.267636+0000","last_became_active":"2024-09-21T09:46:13.257601+0000","last_became_peered":"2024-09-21T09:46:13.257601+0000","last_unstale":"2024-09-21T09:54:59.267636+0000","last_undegraded":"2024-09-21T09:54:59.267636+0000","last_fullsized":"2024-09-21T09:54:59.267636+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":1347,"log_dups_size":0,"ondisk_log_size":1347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:32.302640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0053256340000000001,"stat_sum":{"num_bytes":4841142,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":368,"num_read_kb":399464,"num_write":358,"num_write_kb":90442,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":2496542,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"2.1","version":"0'0","reported_seq":1540,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.267172+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T09:54:59.267172+0000","last_peered":"2024-09-21T09:54:59.267172+0000","last_clean":"2024-09-21T09:54:59.267172+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T09:54:59.267172+0000","last_undegraded":"2024-09-21T09:54:59.267172+0000","last_fullsized":"2024-09-21T09:54:59.267172+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1530,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.281160+0000","last_change":"2024-09-21T09:46:13.255342+0000","last_active":"2024-09-21T09:54:59.281160+0000","last_peered":"2024-09-21T09:54:59.281160+0000","last_clean":"2024-09-21T09:54:59.281160+0000","last_became_active":"2024-09-21T09:46:13.255014+0000","last_became_peered":"2024-09-21T09:46:13.255014+0000","last_unstale":"2024-09-21T09:54:59.281160+0000","last_undegraded":"2024-09-21T09:54:59.281160+0000","last_fullsized":"2024-09-21T09:54:59.281160+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:53.012266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1541,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.282154+0000","last_change":"2024-09-21T09:45:39.134465+0000","last_active":"2024-09-21T09:54:59.282154+0000","last_peered":"2024-09-21T09:54:59.282154+0000","last_clean":"2024-09-21T09:54:59.282154+0000","last_became_active":"2024-09-21T09:45:39.129692+0000","last_became_peered":"2024-09-21T09:45:39.129692+0000","last_unstale":"2024-09-21T09:54:59.282154+0000","last_undegraded":"2024-09-21T09:54:59.282154+0000","last_fullsized":"2024-09-21T09:54:59.282154+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:14.288180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"1074'2036","reported_seq":6080,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.290150+0000","last_change":"2024-09-21T09:54:55.288890+0000","last_active":"2024-09-21T09:54:59.290150+0000","last_peered":"2024-09-21T09:54:59.290150+0000","last_clean":"2024-09-21T09:54:59.290150+0000","last_became_active":"2024-09-21T09:45:39.138917+0000","last_became_peered":"2024-09-21T09:45:39.138917+0000","last_unstale":"2024-09-21T09:54:59.290150+0000","last_undegraded":"2024-09-21T09:54:59.290150+0000","last_fullsized":"2024-09-21T09:54:59.290150+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":2036,"log_dups_size":0,"ondisk_log_size":2036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:24:54.350466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0034302690000000001,"stat_sum":{"num_bytes":12523530,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":482,"num_read_kb":480798,"num_write":497,"num_write_kb":116690,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":8183513,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"2.2","version":"58'2","reported_seq":1034,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261947+0000","last_change":"2024-09-21T09:45:39.131285+0000","last_active":"2024-09-21T09:54:58.261947+0000","last_peered":"2024-09-21T09:54:58.261947+0000","last_clean":"2024-09-21T09:54:58.261947+0000","last_became_active":"2024-09-21T09:45:39.130505+0000","last_became_peered":"2024-09-21T09:45:39.130505+0000","last_unstale":"2024-09-21T09:54:58.261947+0000","last_undegraded":"2024-09-21T09:54:58.261947+0000","last_fullsized":"2024-09-21T09:54:58.261947+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:37.523277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1016,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265381+0000","last_change":"2024-09-21T09:45:05.097518+0000","last_active":"2024-09-21T09:54:58.265381+0000","last_peered":"2024-09-21T09:54:58.265381+0000","last_clean":"2024-09-21T09:54:58.265381+0000","last_became_active":"2024-09-21T09:45:05.096918+0000","last_became_peered":"2024-09-21T09:45:05.096918+0000","last_unstale":"2024-09-21T09:54:58.265381+0000","last_undegraded":"2024-09-21T09:54:58.265381+0000","last_fullsized":"2024-09-21T09:54:58.265381+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:13.781847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1540,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.281160+0000","last_change":"2024-09-21T09:46:47.113687+0000","last_active":"2024-09-21T09:54:59.281160+0000","last_peered":"2024-09-21T09:54:59.281160+0000","last_clean":"2024-09-21T09:54:59.281160+0000","last_became_active":"2024-09-21T09:46:47.113235+0000","last_became_peered":"2024-09-21T09:46:47.113235+0000","last_unstale":"2024-09-21T09:54:59.281160+0000","last_undegraded":"2024-09-21T09:54:59.281160+0000","last_fullsized":"2024-09-21T09:54:59.281160+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:50.851616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"1070'1370","reported_seq":5537,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.276979+0000","last_change":"2024-09-21T09:54:55.289180+0000","last_active":"2024-09-21T09:54:59.276979+0000","last_peered":"2024-09-21T09:54:59.276979+0000","last_clean":"2024-09-21T09:54:59.276979+0000","last_became_active":"2024-09-21T09:45:05.086021+0000","last_became_peered":"2024-09-21T09:45:05.086021+0000","last_unstale":"2024-09-21T09:54:59.276979+0000","last_undegraded":"2024-09-21T09:54:59.276979+0000","last_fullsized":"2024-09-21T09:54:59.276979+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":1370,"log_dups_size":0,"ondisk_log_size":1370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:43.815539+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.003613836,"stat_sum":{"num_bytes":10354574,"num_objects":4,"num_object_clones":1,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":436,"num_read_kb":445254,"num_write":407,"num_write_kb":100712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":1436775,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.5","version":"0'0","reported_seq":1036,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.262071+0000","last_change":"2024-09-21T09:46:47.115673+0000","last_active":"2024-09-21T09:54:58.262071+0000","last_peered":"2024-09-21T09:54:58.262071+0000","last_clean":"2024-09-21T09:54:58.262071+0000","last_became_active":"2024-09-21T09:46:47.115166+0000","last_became_peered":"2024-09-21T09:46:47.115166+0000","last_unstale":"2024-09-21T09:54:58.262071+0000","last_undegraded":"2024-09-21T09:54:58.262071+0000","last_fullsized":"2024-09-21T09:54:58.262071+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:16.660834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1554,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.281354+0000","last_change":"2024-09-21T09:45:39.132441+0000","last_active":"2024-09-21T09:54:59.281354+0000","last_peered":"2024-09-21T09:54:59.281354+0000","last_clean":"2024-09-21T09:54:59.281354+0000","last_became_active":"2024-09-21T09:45:39.129666+0000","last_became_peered":"2024-09-21T09:45:39.129666+0000","last_unstale":"2024-09-21T09:54:59.281354+0000","last_undegraded":"2024-09-21T09:54:59.281354+0000","last_fullsized":"2024-09-21T09:54:59.281354+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:34.450160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1493,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.267165+0000","last_change":"2024-09-21T09:46:13.258244+0000","last_active":"2024-09-21T09:54:59.267165+0000","last_peered":"2024-09-21T09:54:59.267165+0000","last_clean":"2024-09-21T09:54:59.267165+0000","last_became_active":"2024-09-21T09:46:13.257820+0000","last_became_peered":"2024-09-21T09:46:13.257820+0000","last_unstale":"2024-09-21T09:54:59.267165+0000","last_undegraded":"2024-09-21T09:54:59.267165+0000","last_fullsized":"2024-09-21T09:54:59.267165+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:04.618304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.2","version":"1068'436","reported_seq":3626,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261009+0000","last_change":"2024-09-21T09:54:55.277563+0000","last_active":"2024-09-21T09:54:58.261009+0000","last_peered":"2024-09-21T09:54:58.261009+0000","last_clean":"2024-09-21T09:54:58.261009+0000","last_became_active":"2024-09-21T09:45:05.085408+0000","last_became_peered":"2024-09-21T09:45:05.085408+0000","last_unstale":"2024-09-21T09:54:58.261009+0000","last_undegraded":"2024-09-21T09:54:58.261009+0000","last_fullsized":"2024-09-21T09:54:58.261009+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":436,"log_dups_size":0,"ondisk_log_size":436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:56.020319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064375700000000001,"stat_sum":{"num_bytes":3378570,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":193,"num_read_kb":191556,"num_write":113,"num_write_kb":27294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.4","version":"0'0","reported_seq":1530,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.281659+0000","last_change":"2024-09-21T09:46:13.260666+0000","last_active":"2024-09-21T09:54:59.281659+0000","last_peered":"2024-09-21T09:54:59.281659+0000","last_clean":"2024-09-21T09:54:59.281659+0000","last_became_active":"2024-09-21T09:46:13.260324+0000","last_became_peered":"2024-09-21T09:46:13.260324+0000","last_unstale":"2024-09-21T09:54:59.281659+0000","last_undegraded":"2024-09-21T09:54:59.281659+0000","last_fullsized":"2024-09-21T09:54:59.281659+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:25.140797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1009,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.722101+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T09:54:58.722101+0000","last_peered":"2024-09-21T09:54:58.722101+0000","last_clean":"2024-09-21T09:54:58.722101+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T09:54:58.722101+0000","last_undegraded":"2024-09-21T09:54:58.722101+0000","last_fullsized":"2024-09-21T09:54:58.722101+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1556,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.276664+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T09:54:59.276664+0000","last_peered":"2024-09-21T09:54:59.276664+0000","last_clean":"2024-09-21T09:54:59.276664+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T09:54:59.276664+0000","last_undegraded":"2024-09-21T09:54:59.276664+0000","last_fullsized":"2024-09-21T09:54:59.276664+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"1068'1159","reported_seq":4296,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.267006+0000","last_change":"2024-09-21T09:54:55.282334+0000","last_active":"2024-09-21T09:54:59.267006+0000","last_peered":"2024-09-21T09:54:59.267006+0000","last_clean":"2024-09-21T09:54:59.267006+0000","last_became_active":"2024-09-21T09:46:47.119127+0000","last_became_peered":"2024-09-21T09:46:47.119127+0000","last_unstale":"2024-09-21T09:54:59.267006+0000","last_undegraded":"2024-09-21T09:54:59.267006+0000","last_fullsized":"2024-09-21T09:54:59.267006+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":1159,"log_dups_size":0,"ondisk_log_size":1159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:14.367506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001146022,"stat_sum":{"num_bytes":7690710,"num_objects":3,"num_object_clones":1,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":389,"num_read_kb":415933,"num_write":298,"num_write_kb":68114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":2541554,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.7","version":"63'2","reported_seq":1491,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.511077+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T09:54:59.511077+0000","last_peered":"2024-09-21T09:54:59.511077+0000","last_clean":"2024-09-21T09:54:59.511077+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T09:54:59.511077+0000","last_undegraded":"2024-09-21T09:54:59.511077+0000","last_fullsized":"2024-09-21T09:54:59.511077+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1528,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.282229+0000","last_change":"2024-09-21T09:45:39.130168+0000","last_active":"2024-09-21T09:54:59.282229+0000","last_peered":"2024-09-21T09:54:59.282229+0000","last_clean":"2024-09-21T09:54:59.282229+0000","last_became_active":"2024-09-21T09:45:39.129498+0000","last_became_peered":"2024-09-21T09:45:39.129498+0000","last_unstale":"2024-09-21T09:54:59.282229+0000","last_undegraded":"2024-09-21T09:54:59.282229+0000","last_fullsized":"2024-09-21T09:54:59.282229+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:58.310876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1489,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510212+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T09:54:59.510212+0000","last_peered":"2024-09-21T09:54:59.510212+0000","last_clean":"2024-09-21T09:54:59.510212+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T09:54:59.510212+0000","last_undegraded":"2024-09-21T09:54:59.510212+0000","last_fullsized":"2024-09-21T09:54:59.510212+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"1070'2349","reported_seq":6257,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510040+0000","last_change":"2024-09-21T09:54:55.288311+0000","last_active":"2024-09-21T09:54:59.510040+0000","last_peered":"2024-09-21T09:54:59.510040+0000","last_clean":"2024-09-21T09:54:59.510040+0000","last_became_active":"2024-09-21T09:46:47.118851+0000","last_became_peered":"2024-09-21T09:46:47.118851+0000","last_unstale":"2024-09-21T09:54:59.510040+0000","last_undegraded":"2024-09-21T09:54:59.510040+0000","last_fullsized":"2024-09-21T09:54:59.510040+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:55.558507+0000","last_clean_scrub_stamp":"2024-09-21T09:37:55.558507+0000","objects_scrubbed":0,"log_size":2349,"log_dups_size":0,"ondisk_log_size":2349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:09:05.699647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0079884329999999996,"stat_sum":{"num_bytes":9104468,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":763,"num_read_kb":898654,"num_write":604,"num_write_kb":143321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":8184308,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"248","length":"1"},{"start":"252","length":"2"}]},{"pgid":"3.6","version":"63'1","reported_seq":1569,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.277155+0000","last_change":"2024-09-21T09:45:39.140762+0000","last_active":"2024-09-21T09:54:59.277155+0000","last_peered":"2024-09-21T09:54:59.277155+0000","last_clean":"2024-09-21T09:54:59.277155+0000","last_became_active":"2024-09-21T09:45:39.140402+0000","last_became_peered":"2024-09-21T09:45:39.140402+0000","last_unstale":"2024-09-21T09:54:59.277155+0000","last_undegraded":"2024-09-21T09:54:59.277155+0000","last_fullsized":"2024-09-21T09:54:59.277155+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:06.545428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1010,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261226+0000","last_change":"2024-09-21T09:46:13.260656+0000","last_active":"2024-09-21T09:54:58.261226+0000","last_peered":"2024-09-21T09:54:58.261226+0000","last_clean":"2024-09-21T09:54:58.261226+0000","last_became_active":"2024-09-21T09:46:13.260368+0000","last_became_peered":"2024-09-21T09:46:13.260368+0000","last_unstale":"2024-09-21T09:54:58.261226+0000","last_undegraded":"2024-09-21T09:54:58.261226+0000","last_fullsized":"2024-09-21T09:54:58.261226+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:22.059036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":998,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265236+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:54:58.265236+0000","last_peered":"2024-09-21T09:54:58.265236+0000","last_clean":"2024-09-21T09:54:58.265236+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:54:58.265236+0000","last_undegraded":"2024-09-21T09:54:58.265236+0000","last_fullsized":"2024-09-21T09:54:58.265236+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1015,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265252+0000","last_change":"2024-09-21T09:45:39.129538+0000","last_active":"2024-09-21T09:54:58.265252+0000","last_peered":"2024-09-21T09:54:58.265252+0000","last_clean":"2024-09-21T09:54:58.265252+0000","last_became_active":"2024-09-21T09:45:39.129198+0000","last_became_peered":"2024-09-21T09:45:39.129198+0000","last_unstale":"2024-09-21T09:54:58.265252+0000","last_undegraded":"2024-09-21T09:54:58.265252+0000","last_fullsized":"2024-09-21T09:54:58.265252+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.798277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1021,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265442+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:54:58.265442+0000","last_peered":"2024-09-21T09:54:58.265442+0000","last_clean":"2024-09-21T09:54:58.265442+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:54:58.265442+0000","last_undegraded":"2024-09-21T09:54:58.265442+0000","last_fullsized":"2024-09-21T09:54:58.265442+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1529,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.281399+0000","last_change":"2024-09-21T09:46:13.254014+0000","last_active":"2024-09-21T09:54:59.281399+0000","last_peered":"2024-09-21T09:54:59.281399+0000","last_clean":"2024-09-21T09:54:59.281399+0000","last_became_active":"2024-09-21T09:46:13.253673+0000","last_became_peered":"2024-09-21T09:46:13.253673+0000","last_unstale":"2024-09-21T09:54:59.281399+0000","last_undegraded":"2024-09-21T09:54:59.281399+0000","last_fullsized":"2024-09-21T09:54:59.281399+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:02.784613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1000,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.721779+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T09:54:58.721779+0000","last_peered":"2024-09-21T09:54:58.721779+0000","last_clean":"2024-09-21T09:54:58.721779+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T09:54:58.721779+0000","last_undegraded":"2024-09-21T09:54:58.721779+0000","last_fullsized":"2024-09-21T09:54:58.721779+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1020,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.262117+0000","last_change":"2024-09-21T09:45:39.130886+0000","last_active":"2024-09-21T09:54:58.262117+0000","last_peered":"2024-09-21T09:54:58.262117+0000","last_clean":"2024-09-21T09:54:58.262117+0000","last_became_active":"2024-09-21T09:45:39.130559+0000","last_became_peered":"2024-09-21T09:45:39.130559+0000","last_unstale":"2024-09-21T09:54:58.262117+0000","last_undegraded":"2024-09-21T09:54:58.262117+0000","last_fullsized":"2024-09-21T09:54:58.262117+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:39.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1464,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510013+0000","last_change":"2024-09-21T09:46:47.122643+0000","last_active":"2024-09-21T09:54:59.510013+0000","last_peered":"2024-09-21T09:54:59.510013+0000","last_clean":"2024-09-21T09:54:59.510013+0000","last_became_active":"2024-09-21T09:46:47.122364+0000","last_became_peered":"2024-09-21T09:46:47.122364+0000","last_unstale":"2024-09-21T09:54:59.510013+0000","last_undegraded":"2024-09-21T09:54:59.510013+0000","last_fullsized":"2024-09-21T09:54:59.510013+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:29.222333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1565,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.277005+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T09:54:59.277005+0000","last_peered":"2024-09-21T09:54:59.277005+0000","last_clean":"2024-09-21T09:54:59.277005+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T09:54:59.277005+0000","last_undegraded":"2024-09-21T09:54:59.277005+0000","last_fullsized":"2024-09-21T09:54:59.277005+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1481,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.509971+0000","last_change":"2024-09-21T09:46:47.121360+0000","last_active":"2024-09-21T09:54:59.509971+0000","last_peered":"2024-09-21T09:54:59.509971+0000","last_clean":"2024-09-21T09:54:59.509971+0000","last_became_active":"2024-09-21T09:46:47.120494+0000","last_became_peered":"2024-09-21T09:46:47.120494+0000","last_unstale":"2024-09-21T09:54:59.509971+0000","last_undegraded":"2024-09-21T09:54:59.509971+0000","last_fullsized":"2024-09-21T09:54:59.509971+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:45.862984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1007,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261129+0000","last_change":"2024-09-21T09:46:13.260037+0000","last_active":"2024-09-21T09:54:58.261129+0000","last_peered":"2024-09-21T09:54:58.261129+0000","last_clean":"2024-09-21T09:54:58.261129+0000","last_became_active":"2024-09-21T09:46:13.259690+0000","last_became_peered":"2024-09-21T09:46:13.259690+0000","last_unstale":"2024-09-21T09:54:58.261129+0000","last_undegraded":"2024-09-21T09:54:58.261129+0000","last_fullsized":"2024-09-21T09:54:58.261129+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:57.466762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1499,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510240+0000","last_change":"2024-09-21T09:46:47.120830+0000","last_active":"2024-09-21T09:54:59.510240+0000","last_peered":"2024-09-21T09:54:59.510240+0000","last_clean":"2024-09-21T09:54:59.510240+0000","last_became_active":"2024-09-21T09:46:47.118419+0000","last_became_peered":"2024-09-21T09:46:47.118419+0000","last_unstale":"2024-09-21T09:54:59.510240+0000","last_undegraded":"2024-09-21T09:54:59.510240+0000","last_fullsized":"2024-09-21T09:54:59.510240+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:13.789806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1568,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.276870+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T09:54:59.276870+0000","last_peered":"2024-09-21T09:54:59.276870+0000","last_clean":"2024-09-21T09:54:59.276870+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T09:54:59.276870+0000","last_undegraded":"2024-09-21T09:54:59.276870+0000","last_fullsized":"2024-09-21T09:54:59.276870+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1529,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.280910+0000","last_change":"2024-09-21T09:46:47.121724+0000","last_active":"2024-09-21T09:54:59.280910+0000","last_peered":"2024-09-21T09:54:59.280910+0000","last_clean":"2024-09-21T09:54:59.280910+0000","last_became_active":"2024-09-21T09:46:47.121406+0000","last_became_peered":"2024-09-21T09:46:47.121406+0000","last_unstale":"2024-09-21T09:54:59.280910+0000","last_undegraded":"2024-09-21T09:54:59.280910+0000","last_fullsized":"2024-09-21T09:54:59.280910+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:55.493704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1481,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510638+0000","last_change":"2024-09-21T09:46:47.113184+0000","last_active":"2024-09-21T09:54:59.510638+0000","last_peered":"2024-09-21T09:54:59.510638+0000","last_clean":"2024-09-21T09:54:59.510638+0000","last_became_active":"2024-09-21T09:46:47.112857+0000","last_became_peered":"2024-09-21T09:46:47.112857+0000","last_unstale":"2024-09-21T09:54:59.510638+0000","last_undegraded":"2024-09-21T09:54:59.510638+0000","last_fullsized":"2024-09-21T09:54:59.510638+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:46.783986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1034,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265238+0000","last_change":"2024-09-21T09:46:13.262555+0000","last_active":"2024-09-21T09:54:58.265238+0000","last_peered":"2024-09-21T09:54:58.265238+0000","last_clean":"2024-09-21T09:54:58.265238+0000","last_became_active":"2024-09-21T09:46:13.261474+0000","last_became_peered":"2024-09-21T09:46:13.261474+0000","last_unstale":"2024-09-21T09:54:58.265238+0000","last_undegraded":"2024-09-21T09:54:58.265238+0000","last_fullsized":"2024-09-21T09:54:58.265238+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:44.865988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1519,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.267432+0000","last_change":"2024-09-21T09:46:47.114406+0000","last_active":"2024-09-21T09:54:59.267432+0000","last_peered":"2024-09-21T09:54:59.267432+0000","last_clean":"2024-09-21T09:54:59.267432+0000","last_became_active":"2024-09-21T09:46:47.113986+0000","last_became_peered":"2024-09-21T09:46:47.113986+0000","last_unstale":"2024-09-21T09:54:59.267432+0000","last_undegraded":"2024-09-21T09:54:59.267432+0000","last_fullsized":"2024-09-21T09:54:59.267432+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:32.292201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1558,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.277342+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T09:54:59.277342+0000","last_peered":"2024-09-21T09:54:59.277342+0000","last_clean":"2024-09-21T09:54:59.277342+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T09:54:59.277342+0000","last_undegraded":"2024-09-21T09:54:59.277342+0000","last_fullsized":"2024-09-21T09:54:59.277342+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1032,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265158+0000","last_change":"2024-09-21T09:46:47.114188+0000","last_active":"2024-09-21T09:54:58.265158+0000","last_peered":"2024-09-21T09:54:58.265158+0000","last_clean":"2024-09-21T09:54:58.265158+0000","last_became_active":"2024-09-21T09:46:47.113859+0000","last_became_peered":"2024-09-21T09:46:47.113859+0000","last_unstale":"2024-09-21T09:54:58.265158+0000","last_undegraded":"2024-09-21T09:54:58.265158+0000","last_fullsized":"2024-09-21T09:54:58.265158+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:22.670368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":1022,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.261867+0000","last_change":"2024-09-21T09:45:39.131124+0000","last_active":"2024-09-21T09:54:58.261867+0000","last_peered":"2024-09-21T09:54:58.261867+0000","last_clean":"2024-09-21T09:54:58.261867+0000","last_became_active":"2024-09-21T09:45:39.130756+0000","last_became_peered":"2024-09-21T09:45:39.130756+0000","last_unstale":"2024-09-21T09:54:58.261867+0000","last_undegraded":"2024-09-21T09:54:58.261867+0000","last_fullsized":"2024-09-21T09:54:58.261867+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:49.130698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1032,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.265144+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T09:54:58.265144+0000","last_peered":"2024-09-21T09:54:58.265144+0000","last_clean":"2024-09-21T09:54:58.265144+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T09:54:58.265144+0000","last_undegraded":"2024-09-21T09:54:58.265144+0000","last_fullsized":"2024-09-21T09:54:58.265144+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":1004,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.262011+0000","last_change":"2024-09-21T09:36:43.115791+0000","last_active":"2024-09-21T09:54:58.262011+0000","last_peered":"2024-09-21T09:54:58.262011+0000","last_clean":"2024-09-21T09:54:58.262011+0000","last_became_active":"2024-09-21T09:36:43.115525+0000","last_became_peered":"2024-09-21T09:36:43.115525+0000","last_unstale":"2024-09-21T09:54:58.262011+0000","last_undegraded":"2024-09-21T09:54:58.262011+0000","last_fullsized":"2024-09-21T09:54:58.262011+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:28.396722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1491,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510337+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T09:54:59.510337+0000","last_peered":"2024-09-21T09:54:59.510337+0000","last_clean":"2024-09-21T09:54:59.510337+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T09:54:59.510337+0000","last_undegraded":"2024-09-21T09:54:59.510337+0000","last_fullsized":"2024-09-21T09:54:59.510337+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1477,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.510830+0000","last_change":"2024-09-21T09:46:47.115788+0000","last_active":"2024-09-21T09:54:59.510830+0000","last_peered":"2024-09-21T09:54:59.510830+0000","last_clean":"2024-09-21T09:54:59.510830+0000","last_became_active":"2024-09-21T09:46:47.114864+0000","last_became_peered":"2024-09-21T09:46:47.114864+0000","last_unstale":"2024-09-21T09:54:59.510830+0000","last_undegraded":"2024-09-21T09:54:59.510830+0000","last_fullsized":"2024-09-21T09:54:59.510830+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:00.034944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1015,"reported_epoch":1073,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.264823+0000","last_change":"2024-09-21T09:45:39.143064+0000","last_active":"2024-09-21T09:54:58.264823+0000","last_peered":"2024-09-21T09:54:58.264823+0000","last_clean":"2024-09-21T09:54:58.264823+0000","last_became_active":"2024-09-21T09:45:39.142699+0000","last_became_peered":"2024-09-21T09:45:39.142699+0000","last_unstale":"2024-09-21T09:54:58.264823+0000","last_undegraded":"2024-09-21T09:54:58.264823+0000","last_fullsized":"2024-09-21T09:54:58.264823+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:14.684721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1572,"reported_epoch":1074,"state":"active+clean","last_fresh":"2024-09-21T09:54:59.277072+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T09:54:59.277072+0000","last_peered":"2024-09-21T09:54:59.277072+0000","last_clean":"2024-09-21T09:54:59.277072+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T09:54:59.277072+0000","last_undegraded":"2024-09-21T09:54:59.277072+0000","last_fullsized":"2024-09-21T09:54:59.277072+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":132590948,"num_objects":53,"num_object_clones":14,"num_object_copies":159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53,"num_whiteouts":1,"num_read":6639,"num_read_kb":7000806,"num_write":5637,"num_write_kb":1350986,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":44157539,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":236605440,"data_stored":265462269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21137,"ondisk_log_size":21137,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397749,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":338484,"kb_used_data":28652,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93406668,"statfs":{"total":95995035648,"available":95648428032,"internally_reserved":0,"allocated":29339648,"data_stored":27416483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593978,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":336712,"kb_used_data":31036,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93408440,"statfs":{"total":95995035648,"available":95650242560,"internally_reserved":0,"allocated":31780864,"data_stored":31409948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790207,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357956,"kb_used_data":48112,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93387196,"statfs":{"total":95995035648,"available":95628488704,"internally_reserved":0,"allocated":49266688,"data_stored":54207259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986434,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":360908,"kb_used_data":59388,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93384244,"statfs":{"total":95995035648,"available":95625465856,"internally_reserved":0,"allocated":60813312,"data_stored":67714209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999076,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":62540,"kb_used_data":22036,"kb_used_omap":25,"kb_used_meta":40102,"kb_avail":93682612,"statfs":{"total":95995035648,"available":95930994688,"internally_reserved":0,"allocated":22564864,"data_stored":21221800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":41065415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111019,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":54776,"kb_used_data":26560,"kb_used_omap":31,"kb_used_meta":27808,"kb_avail":93690376,"statfs":{"total":95995035648,"available":95938945024,"internally_reserved":0,"allocated":27197440,"data_stored":24434741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":28475624},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318222962,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":61188,"kb_used_data":33284,"kb_used_omap":26,"kb_used_meta":27685,"kb_avail":93683964,"statfs":{"total":95995035648,"available":95932379136,"internally_reserved":0,"allocated":34082816,"data_stored":33863670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":28349452},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289334905,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":104780,"kb_used_data":46284,"kb_used_omap":31,"kb_used_meta":58272,"kb_avail":93640372,"statfs":{"total":95995035648,"available":95887740928,"internally_reserved":0,"allocated":47394816,"data_stored":51733006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":59670624},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":37457920,"data_stored":44550604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":24928256,"data_stored":27401853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":17854464,"data_stored":17842703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":14684160,"data_stored":16069280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":53727232,"data_stored":62720823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":42192896,"data_stored":49218104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24104960,"data_stored":25826251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":21655552,"data_stored":21832651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:55:05.434 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-21T09:55:05.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-21T09:55:05.694 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:05 smithi134 ceph-mon[90085]: pgmap v1194: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-09-21T09:55:06.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:06 smithi017 ceph-mon[102913]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:06 smithi017 ceph-mon[105151]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:06 smithi134 ceph-mon[90085]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:07.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:07 smithi017 ceph-mon[102913]: pgmap v1195: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 803 KiB/s wr, 5 op/s 2024-09-21T09:55:07.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:07 smithi017 ceph-mon[105151]: pgmap v1195: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 803 KiB/s wr, 5 op/s 2024-09-21T09:55:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:07 smithi134 ceph-mon[90085]: pgmap v1195: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 803 KiB/s wr, 5 op/s 2024-09-21T09:55:08.321 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_0' removed 2024-09-21T09:55:08.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:08 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2900298068' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:08.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:08 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:08.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:08 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2900298068' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:08.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:08 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:08 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2900298068' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:08 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:08.993 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T09:55:08.993 INFO:teuthology.task.print:**** done rados/snaps-few-objects 5-workload 2024-09-21T09:55:08.994 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:08.994 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:55:09.004 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-21T09:55:09.004 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T09:55:09.012 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:55:09.012 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:55:09.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:55:09.275 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:09 smithi017 ceph-mon[102913]: pgmap v1196: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 711 KiB/s wr, 4 op/s 2024-09-21T09:55:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:09 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:55:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:09 smithi017 ceph-mon[102913]: osdmap e1076: 8 total, 8 up, 8 in 2024-09-21T09:55:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:09 smithi017 ceph-mon[105151]: pgmap v1196: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 711 KiB/s wr, 4 op/s 2024-09-21T09:55:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:09 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:55:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:09 smithi017 ceph-mon[105151]: osdmap e1076: 8 total, 8 up, 8 in 2024-09-21T09:55:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:09 smithi134 ceph-mon[90085]: pgmap v1196: 89 pgs: 89 active+clean; 127 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 711 KiB/s wr, 4 op/s 2024-09-21T09:55:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:09 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:55:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:09 smithi134 ceph-mon[90085]: osdmap e1076: 8 total, 8 up, 8 in 2024-09-21T09:55:09.744 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 09:55:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=sqlstore.transactions t=2024-09-21T09:55:09.415103395Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:55:10.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:10.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:55:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:55:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:55:11.480 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:11 smithi017 ceph-mon[102913]: pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:11.480 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:11 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2898847521' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-21T09:55:11.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:11 smithi017 ceph-mon[105151]: pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:11.480 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:11 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2898847521' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-21T09:55:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:11 smithi134 ceph-mon[90085]: pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:11 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2898847521' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-21T09:55:11.837 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":40,"flags":0,"active_gid":25589,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6800","nonce":2478405364},{"type":"v1","addr":"172.21.15.17:6801","nonce":2478405364}]},"active_addr":"172.21.15.17:6801/2478405364","active_change":"2024-09-21T09:41:39.758785+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":25330,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.8","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.8","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":286,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":2641946511}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":2071741810}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":1644548562}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":4040999284}]}]} 2024-09-21T09:55:11.841 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:55:11.842 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:55:11.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:55:12.100 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:13.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:13 smithi017 ceph-mon[102913]: pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:13.554 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:13 smithi017 ceph-mon[105151]: pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:13 smithi134 ceph-mon[90085]: pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:55:14.043 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":1076,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T09:55:08.309073+0000","last_up_change":"2024-09-21T09:46:46.096331+0000","last_in_change":"2024-09-21T09:34:35.544589+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:27.624125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:41.967182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:42.538769+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":466,"up_from":479,"up_thru":581,"down_at":467,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6803","nonce":2105074120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6805","nonce":2105074120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6809","nonce":2105074120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6807","nonce":2105074120}]},"public_addr":"172.21.15.17:6803/2105074120","cluster_addr":"172.21.15.17:6805/2105074120","heartbeat_back_addr":"172.21.15.17:6809/2105074120","heartbeat_front_addr":"172.21.15.17:6807/2105074120","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":500,"up_from":513,"up_thru":581,"down_at":501,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6811","nonce":1078446230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6813","nonce":1078446230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6817","nonce":1078446230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6815","nonce":1078446230}]},"public_addr":"172.21.15.17:6811/1078446230","cluster_addr":"172.21.15.17:6813/1078446230","heartbeat_back_addr":"172.21.15.17:6817/1078446230","heartbeat_front_addr":"172.21.15.17:6815/1078446230","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":534,"up_from":547,"up_thru":581,"down_at":535,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6819","nonce":323142954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6821","nonce":323142954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6825","nonce":323142954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6823","nonce":323142954}]},"public_addr":"172.21.15.17:6819/323142954","cluster_addr":"172.21.15.17:6821/323142954","heartbeat_back_addr":"172.21.15.17:6825/323142954","heartbeat_front_addr":"172.21.15.17:6823/323142954","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":568,"up_from":581,"up_thru":581,"down_at":569,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6827","nonce":3146888816}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6829","nonce":3146888816}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6833","nonce":3146888816}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6831","nonce":3146888816}]},"public_addr":"172.21.15.17:6827/3146888816","cluster_addr":"172.21.15.17:6829/3146888816","heartbeat_back_addr":"172.21.15.17:6833/3146888816","heartbeat_front_addr":"172.21.15.17:6831/3146888816","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":581,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6801","nonce":2226434249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6803","nonce":2226434249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6807","nonce":2226434249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6805","nonce":2226434249}]},"public_addr":"172.21.15.134:6801/2226434249","cluster_addr":"172.21.15.134:6803/2226434249","heartbeat_back_addr":"172.21.15.134:6807/2226434249","heartbeat_front_addr":"172.21.15.134:6805/2226434249","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":581,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":569,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":581,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":466},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":500},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":534},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":568},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/2291861207":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/3530815435":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3512479442":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/3084797810":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/2599173714":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/381036987":"2024-09-22T09:41:39.758476+0000","172.21.15.134:6834/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3671582122":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6800/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.134:6835/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/758458220":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3206271349":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/3226197858":"2024-09-22T09:41:39.758476+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/2266250644":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/2350962326":"2024-09-22T09:40:59.077787+0000","172.21.15.134:0/3275227882":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6801/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/1454017192":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/877591523":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6801/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/1515484230":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6801/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3086297034":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/2023103508":"2024-09-22T09:41:39.758476+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:55:14.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2281137178' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:14.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2281137178' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:14.689 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:55:14.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:55:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2281137178' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:14.948 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:55:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:55:14] "GET /metrics HTTP/1.1" 200 34064 "" "Prometheus/2.51.0" 2024-09-21T09:55:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:15 smithi017 ceph-mon[102913]: pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:15 smithi017 ceph-mon[105151]: pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:15 smithi134 ceph-mon[90085]: pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:16.873 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:55:16.873 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":1076,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T09:55:08.309073+0000","last_up_change":"2024-09-21T09:46:46.096331+0000","last_in_change":"2024-09-21T09:34:35.544589+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:27.624125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:41.967182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:42.538769+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":466,"up_from":479,"up_thru":581,"down_at":467,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6803","nonce":2105074120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6805","nonce":2105074120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6809","nonce":2105074120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6807","nonce":2105074120}]},"public_addr":"172.21.15.17:6803/2105074120","cluster_addr":"172.21.15.17:6805/2105074120","heartbeat_back_addr":"172.21.15.17:6809/2105074120","heartbeat_front_addr":"172.21.15.17:6807/2105074120","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":500,"up_from":513,"up_thru":581,"down_at":501,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6811","nonce":1078446230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6813","nonce":1078446230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6817","nonce":1078446230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1078446230},{"type":"v1","addr":"172.21.15.17:6815","nonce":1078446230}]},"public_addr":"172.21.15.17:6811/1078446230","cluster_addr":"172.21.15.17:6813/1078446230","heartbeat_back_addr":"172.21.15.17:6817/1078446230","heartbeat_front_addr":"172.21.15.17:6815/1078446230","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":534,"up_from":547,"up_thru":581,"down_at":535,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6819","nonce":323142954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6821","nonce":323142954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6825","nonce":323142954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6823","nonce":323142954}]},"public_addr":"172.21.15.17:6819/323142954","cluster_addr":"172.21.15.17:6821/323142954","heartbeat_back_addr":"172.21.15.17:6825/323142954","heartbeat_front_addr":"172.21.15.17:6823/323142954","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":568,"up_from":581,"up_thru":581,"down_at":569,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6827","nonce":3146888816}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6829","nonce":3146888816}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6833","nonce":3146888816}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6831","nonce":3146888816}]},"public_addr":"172.21.15.17:6827/3146888816","cluster_addr":"172.21.15.17:6829/3146888816","heartbeat_back_addr":"172.21.15.17:6833/3146888816","heartbeat_front_addr":"172.21.15.17:6831/3146888816","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":581,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6801","nonce":2226434249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6803","nonce":2226434249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6807","nonce":2226434249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2226434249},{"type":"v1","addr":"172.21.15.134:6805","nonce":2226434249}]},"public_addr":"172.21.15.134:6801/2226434249","cluster_addr":"172.21.15.134:6803/2226434249","heartbeat_back_addr":"172.21.15.134:6807/2226434249","heartbeat_front_addr":"172.21.15.134:6805/2226434249","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":581,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":569,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":581,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":466},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":500},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":534},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":568},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:6800/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/381036987":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3226197858":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/2599173714":"2024-09-22T09:41:39.758476+0000","172.21.15.134:0/2350962326":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/2266250644":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6801/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.134:0/1515484230":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.134:6834/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3671582122":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/758458220":"2024-09-22T09:41:39.758476+0000","172.21.15.134:6835/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3206271349":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/3530815435":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6800/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/1454017192":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/877591523":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6801/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/3275227882":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6801/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/3084797810":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3512479442":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/2291861207":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/3086297034":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/2023103508":"2024-09-22T09:41:39.758476+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:55:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:17 smithi017 ceph-mon[105151]: pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:17.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2845442473' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:17.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:17 smithi017 ceph-mon[102913]: pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:17.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2845442473' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:17.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:55:17.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:55:17.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:55:17.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:55:17.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:55:17.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:55:17.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:55:17.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:55:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:17 smithi134 ceph-mon[90085]: pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2845442473' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:18.025 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:18.028 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:18.031 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:18.031 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:18.032 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:18.032 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:18.034 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:18.041 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:19.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:19 smithi017 ceph-mon[102913]: pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:19.606 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:19 smithi017 ceph-mon[105151]: pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:19 smithi134 ceph-mon[90085]: pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:55:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:55:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:21.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:21 smithi017 ceph-mon[102913]: pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:21.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:21 smithi017 ceph-mon[105151]: pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:21 smithi134 ceph-mon[90085]: pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:23 smithi017 ceph-mon[102913]: pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:23 smithi017 ceph-mon[105151]: pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:23 smithi134 ceph-mon[90085]: pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:24.883 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:55:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:55:24] "GET /metrics HTTP/1.1" 200 34064 "" "Prometheus/2.51.0" 2024-09-21T09:55:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:25 smithi017 ceph-mon[102913]: pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:25 smithi017 ceph-mon[105151]: pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:25 smithi134 ceph-mon[90085]: pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:25.947 INFO:teuthology.orchestra.run.smithi017.stdout:2495375999082 2024-09-21T09:55:25.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:55:26.860 INFO:teuthology.orchestra.run.smithi017.stdout:184683593984 2024-09-21T09:55:26.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:55:27.564 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:27 smithi017 ceph-mon[102913]: pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:27.564 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:27 smithi017 ceph-mon[105151]: pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:27 smithi134 ceph-mon[90085]: pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:29 smithi017 ceph-mon[102913]: pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:29 smithi017 ceph-mon[105151]: pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:29 smithi134 ceph-mon[90085]: pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:55:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:55:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:30.814 INFO:teuthology.orchestra.run.smithi017.stdout:2349347111025 2024-09-21T09:55:30.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:55:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:31 smithi017 ceph-mon[102913]: pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:31 smithi017 ceph-mon[105151]: pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:31 smithi134 ceph-mon[90085]: pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:32.593 INFO:teuthology.orchestra.run.smithi017.stdout:2203318222968 2024-09-21T09:55:32.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:55:32.650 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:33.065 INFO:teuthology.orchestra.run.smithi017.stdout:2057289334911 2024-09-21T09:55:33.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:55:33.510 INFO:teuthology.orchestra.run.smithi017.stdout:210453397756 2024-09-21T09:55:33.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:55:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:33 smithi017 ceph-mon[102913]: pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:33 smithi017 ceph-mon[105151]: pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:33 smithi134 ceph-mon[90085]: pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:34.620 INFO:teuthology.orchestra.run.smithi017.stdout:133143986441 2024-09-21T09:55:34.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:55:34.626 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:34.893 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:55:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:55:34] "GET /metrics HTTP/1.1" 200 34063 "" "Prometheus/2.51.0" 2024-09-21T09:55:35.115 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:35.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:35 smithi017 ceph-mon[102913]: pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:35.643 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:35 smithi017 ceph-mon[105151]: pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:35 smithi134 ceph-mon[90085]: pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:36.462 INFO:teuthology.orchestra.run.smithi017.stdout:158913790214 2024-09-21T09:55:36.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:55:36.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:36.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:36.662 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:37.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:37 smithi017 ceph-mon[105151]: pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:37.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:37 smithi017 ceph-mon[102913]: pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:37.666 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:37 smithi134 ceph-mon[90085]: pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:37.921 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:37.921 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:38.410 INFO:teuthology.orchestra.run.smithi017.stdout:184683593986 2024-09-21T09:55:39.012 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593984 got 184683593986 for osd.6 2024-09-21T09:55:39.012 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:39.403 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:39 smithi017 ceph-mon[102913]: pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:39.403 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:39 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2598007418' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T09:55:39.403 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:39 smithi017 ceph-mon[105151]: pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:39.403 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:39 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2598007418' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T09:55:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:39 smithi134 ceph-mon[90085]: pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:39 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2598007418' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T09:55:40.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:40.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:40.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:40.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:40.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:40.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:55:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:55:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:41.139 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:41.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:41 smithi017 ceph-mon[102913]: pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:41.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:41 smithi017 ceph-mon[105151]: pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:41 smithi134 ceph-mon[90085]: pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:42.239 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:42.770 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:43.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:43 smithi017 ceph-mon[102913]: pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:43.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:43 smithi017 ceph-mon[105151]: pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:43 smithi134 ceph-mon[90085]: pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:43.762 INFO:teuthology.orchestra.run.smithi017.stdout:2057289334913 2024-09-21T09:55:44.349 INFO:teuthology.orchestra.run.smithi017.stdout:2495375999085 2024-09-21T09:55:44.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2042830054' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T09:55:44.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3683051450' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T09:55:44.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2042830054' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T09:55:44.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3683051450' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T09:55:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2042830054' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T09:55:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3683051450' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T09:55:44.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:55:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:55:44] "GET /metrics HTTP/1.1" 200 34065 "" "Prometheus/2.51.0" 2024-09-21T09:55:45.551 INFO:teuthology.orchestra.run.smithi017.stdout:2349347111028 2024-09-21T09:55:45.552 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:45 smithi017 ceph-mon[102913]: pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:45.552 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:45 smithi017 ceph-mon[105151]: pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:45 smithi134 ceph-mon[90085]: pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:45.870 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2057289334911 got 2057289334913 for osd.0 2024-09-21T09:55:45.870 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/33403962' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T09:55:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/33403962' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T09:55:46.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/33403962' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T09:55:47.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:47 smithi017 ceph-mon[105151]: pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:47.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:47 smithi017 ceph-mon[102913]: pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:47 smithi134 ceph-mon[90085]: pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:48.529 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:48 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1882195691' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T09:55:48.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:48 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1882195691' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T09:55:48.554 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2495375999082 got 2495375999085 for osd.3 2024-09-21T09:55:48.554 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:48 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1882195691' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T09:55:48.831 INFO:teuthology.orchestra.run.smithi017.stdout:133143986444 2024-09-21T09:55:49.435 INFO:teuthology.orchestra.run.smithi017.stdout:2203318222972 2024-09-21T09:55:49.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:49 smithi017 ceph-mon[102913]: pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:49.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:49 smithi017 ceph-mon[105151]: pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:49.695 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986441 got 133143986444 for osd.4 2024-09-21T09:55:49.695 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:49 smithi134 ceph-mon[90085]: pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:50.395 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2203318222968 got 2203318222972 for osd.1 2024-09-21T09:55:50.396 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:50 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/900545662' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T09:55:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:55:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:55:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:50.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:50 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/900545662' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T09:55:50.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:50 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/900545662' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T09:55:50.765 INFO:teuthology.orchestra.run.smithi017.stdout:210453397759 2024-09-21T09:55:51.666 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:51 smithi017 ceph-mon[102913]: pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:51.666 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:51 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2306210386' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T09:55:51.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:51 smithi017 ceph-mon[105151]: pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:51.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:51 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2306210386' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T09:55:51.690 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2349347111025 got 2349347111028 for osd.2 2024-09-21T09:55:51.690 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:51 smithi134 ceph-mon[90085]: pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:51 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2306210386' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T09:55:52.069 INFO:teuthology.orchestra.run.smithi017.stdout:158913790217 2024-09-21T09:55:52.303 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397756 got 210453397759 for osd.7 2024-09-21T09:55:52.303 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:52.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:52 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1018967415' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T09:55:52.616 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:52 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1018967415' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T09:55:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:52 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1018967415' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T09:55:52.864 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790214 got 158913790217 for osd.5 2024-09-21T09:55:52.865 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:52.865 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:55:52.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:55:53.515 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:53 smithi017 ceph-mon[102913]: pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:53 smithi017 ceph-mon[105151]: pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:53 smithi134 ceph-mon[90085]: pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:54.997 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:55:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:55:54] "GET /metrics HTTP/1.1" 200 34065 "" "Prometheus/2.51.0" 2024-09-21T09:55:55.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:55 smithi017 ceph-mon[102913]: pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:55.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:55.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:55 smithi017 ceph-mon[105151]: pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:55.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:55 smithi134 ceph-mon[90085]: pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:56.658 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:55:56.658 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:55:57.392 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1221,"stamp":"2024-09-21T09:55:55.979219+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":212,"num_write_kb":1659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158,"ondisk_log_size":158,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1454064,"kb_used_data":64320,"kb_used_omap":122,"kb_used_meta":1388997,"kb_avail":748507152,"statfs":{"total":767960285184,"available":766471323648,"internally_reserved":0,"allocated":65863680,"data_stored":46944695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125317,"internal_metadata":1422333563},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003113"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911046+0000","last_change":"2024-09-21T09:46:47.114206+0000","last_active":"2024-09-21T09:55:08.911046+0000","last_peered":"2024-09-21T09:55:08.911046+0000","last_clean":"2024-09-21T09:55:08.911046+0000","last_became_active":"2024-09-21T09:46:47.113819+0000","last_became_peered":"2024-09-21T09:46:47.113819+0000","last_unstale":"2024-09-21T09:55:08.911046+0000","last_undegraded":"2024-09-21T09:55:08.911046+0000","last_fullsized":"2024-09-21T09:55:08.911046+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:07.629544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1001,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624302+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T09:55:03.624302+0000","last_peered":"2024-09-21T09:55:03.624302+0000","last_clean":"2024-09-21T09:55:03.624302+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T09:55:03.624302+0000","last_undegraded":"2024-09-21T09:55:03.624302+0000","last_fullsized":"2024-09-21T09:55:03.624302+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1020,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412012+0000","last_change":"2024-09-21T09:46:47.115484+0000","last_active":"2024-09-21T09:55:03.412012+0000","last_peered":"2024-09-21T09:55:03.412012+0000","last_clean":"2024-09-21T09:55:03.412012+0000","last_became_active":"2024-09-21T09:46:47.115080+0000","last_became_peered":"2024-09-21T09:46:47.115080+0000","last_unstale":"2024-09-21T09:55:03.412012+0000","last_undegraded":"2024-09-21T09:55:03.412012+0000","last_fullsized":"2024-09-21T09:55:03.412012+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:56.851805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1569,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912062+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T09:55:08.912062+0000","last_peered":"2024-09-21T09:55:08.912062+0000","last_clean":"2024-09-21T09:55:08.912062+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T09:55:08.912062+0000","last_undegraded":"2024-09-21T09:55:08.912062+0000","last_fullsized":"2024-09-21T09:55:08.912062+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1571,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912710+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T09:55:08.912710+0000","last_peered":"2024-09-21T09:55:08.912710+0000","last_clean":"2024-09-21T09:55:08.912710+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T09:55:08.912710+0000","last_undegraded":"2024-09-21T09:55:08.912710+0000","last_fullsized":"2024-09-21T09:55:08.912710+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":982,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624220+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:55:03.624220+0000","last_peered":"2024-09-21T09:55:03.624220+0000","last_clean":"2024-09-21T09:55:03.624220+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:55:03.624220+0000","last_undegraded":"2024-09-21T09:55:03.624220+0000","last_fullsized":"2024-09-21T09:55:03.624220+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912905+0000","last_change":"2024-09-21T09:46:47.117580+0000","last_active":"2024-09-21T09:55:08.912905+0000","last_peered":"2024-09-21T09:55:08.912905+0000","last_clean":"2024-09-21T09:55:08.912905+0000","last_became_active":"2024-09-21T09:46:47.113030+0000","last_became_peered":"2024-09-21T09:46:47.113030+0000","last_unstale":"2024-09-21T09:55:08.912905+0000","last_undegraded":"2024-09-21T09:55:08.912905+0000","last_fullsized":"2024-09-21T09:55:08.912905+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:28.220180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":1008,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049340+0000","last_change":"2024-09-21T09:45:05.080014+0000","last_active":"2024-09-21T09:55:04.049340+0000","last_peered":"2024-09-21T09:55:04.049340+0000","last_clean":"2024-09-21T09:55:04.049340+0000","last_became_active":"2024-09-21T09:45:05.079618+0000","last_became_peered":"2024-09-21T09:45:05.079618+0000","last_unstale":"2024-09-21T09:55:04.049340+0000","last_undegraded":"2024-09-21T09:55:04.049340+0000","last_fullsized":"2024-09-21T09:55:04.049340+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:42.160292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624817+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T09:55:03.624817+0000","last_peered":"2024-09-21T09:55:03.624817+0000","last_clean":"2024-09-21T09:55:03.624817+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T09:55:03.624817+0000","last_undegraded":"2024-09-21T09:55:03.624817+0000","last_fullsized":"2024-09-21T09:55:03.624817+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1004,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.411948+0000","last_change":"2024-09-21T09:36:43.389124+0000","last_active":"2024-09-21T09:55:03.411948+0000","last_peered":"2024-09-21T09:55:03.411948+0000","last_clean":"2024-09-21T09:55:03.411948+0000","last_became_active":"2024-09-21T09:36:43.388588+0000","last_became_peered":"2024-09-21T09:36:43.388588+0000","last_unstale":"2024-09-21T09:55:03.411948+0000","last_undegraded":"2024-09-21T09:55:03.411948+0000","last_fullsized":"2024-09-21T09:55:03.411948+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:57.562156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1001,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624842+0000","last_change":"2024-09-21T09:45:39.143190+0000","last_active":"2024-09-21T09:55:03.624842+0000","last_peered":"2024-09-21T09:55:03.624842+0000","last_clean":"2024-09-21T09:55:03.624842+0000","last_became_active":"2024-09-21T09:45:39.142826+0000","last_became_peered":"2024-09-21T09:45:39.142826+0000","last_unstale":"2024-09-21T09:55:03.624842+0000","last_undegraded":"2024-09-21T09:55:03.624842+0000","last_fullsized":"2024-09-21T09:55:03.624842+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:45.070945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300475+0000","last_change":"2024-09-21T09:46:47.119247+0000","last_active":"2024-09-21T09:55:03.300475+0000","last_peered":"2024-09-21T09:55:03.300475+0000","last_clean":"2024-09-21T09:55:03.300475+0000","last_became_active":"2024-09-21T09:46:47.118980+0000","last_became_peered":"2024-09-21T09:46:47.118980+0000","last_unstale":"2024-09-21T09:55:03.300475+0000","last_undegraded":"2024-09-21T09:55:03.300475+0000","last_fullsized":"2024-09-21T09:55:03.300475+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.140253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":983,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624823+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:55:03.624823+0000","last_peered":"2024-09-21T09:55:03.624823+0000","last_clean":"2024-09-21T09:55:03.624823+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:55:03.624823+0000","last_undegraded":"2024-09-21T09:55:03.624823+0000","last_fullsized":"2024-09-21T09:55:03.624823+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1009,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049323+0000","last_change":"2024-09-21T09:46:13.260744+0000","last_active":"2024-09-21T09:55:04.049323+0000","last_peered":"2024-09-21T09:55:04.049323+0000","last_clean":"2024-09-21T09:55:04.049323+0000","last_became_active":"2024-09-21T09:46:13.260443+0000","last_became_peered":"2024-09-21T09:46:13.260443+0000","last_unstale":"2024-09-21T09:55:04.049323+0000","last_undegraded":"2024-09-21T09:55:04.049323+0000","last_fullsized":"2024-09-21T09:55:04.049323+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:22.699110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1022,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412189+0000","last_change":"2024-09-21T09:46:47.115343+0000","last_active":"2024-09-21T09:55:03.412189+0000","last_peered":"2024-09-21T09:55:03.412189+0000","last_clean":"2024-09-21T09:55:03.412189+0000","last_became_active":"2024-09-21T09:46:47.115017+0000","last_became_peered":"2024-09-21T09:46:47.115017+0000","last_unstale":"2024-09-21T09:55:03.412189+0000","last_undegraded":"2024-09-21T09:55:03.412189+0000","last_fullsized":"2024-09-21T09:55:03.412189+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.820662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1571,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912938+0000","last_change":"2024-09-21T09:45:39.138289+0000","last_active":"2024-09-21T09:55:08.912938+0000","last_peered":"2024-09-21T09:55:08.912938+0000","last_clean":"2024-09-21T09:55:08.912938+0000","last_became_active":"2024-09-21T09:45:39.137902+0000","last_became_peered":"2024-09-21T09:45:39.137902+0000","last_unstale":"2024-09-21T09:55:08.912938+0000","last_undegraded":"2024-09-21T09:55:08.912938+0000","last_fullsized":"2024-09-21T09:55:08.912938+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:40:16.383906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1496,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911031+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T09:55:08.911031+0000","last_peered":"2024-09-21T09:55:08.911031+0000","last_clean":"2024-09-21T09:55:08.911031+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T09:55:08.911031+0000","last_undegraded":"2024-09-21T09:55:08.911031+0000","last_fullsized":"2024-09-21T09:55:08.911031+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300113+0000","last_change":"2024-09-21T09:46:47.114293+0000","last_active":"2024-09-21T09:55:03.300113+0000","last_peered":"2024-09-21T09:55:03.300113+0000","last_clean":"2024-09-21T09:55:03.300113+0000","last_became_active":"2024-09-21T09:46:47.113993+0000","last_became_peered":"2024-09-21T09:46:47.113993+0000","last_unstale":"2024-09-21T09:55:03.300113+0000","last_undegraded":"2024-09-21T09:55:03.300113+0000","last_fullsized":"2024-09-21T09:55:03.300113+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:30.941931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049541+0000","last_change":"2024-09-21T09:45:39.127186+0000","last_active":"2024-09-21T09:55:04.049541+0000","last_peered":"2024-09-21T09:55:04.049541+0000","last_clean":"2024-09-21T09:55:04.049541+0000","last_became_active":"2024-09-21T09:45:39.126940+0000","last_became_peered":"2024-09-21T09:45:39.126940+0000","last_unstale":"2024-09-21T09:55:04.049541+0000","last_undegraded":"2024-09-21T09:55:04.049541+0000","last_fullsized":"2024-09-21T09:55:04.049541+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:52.205703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300795+0000","last_change":"2024-09-21T09:46:13.259814+0000","last_active":"2024-09-21T09:55:03.300795+0000","last_peered":"2024-09-21T09:55:03.300795+0000","last_clean":"2024-09-21T09:55:03.300795+0000","last_became_active":"2024-09-21T09:46:13.259527+0000","last_became_peered":"2024-09-21T09:46:13.259527+0000","last_unstale":"2024-09-21T09:55:03.300795+0000","last_undegraded":"2024-09-21T09:55:03.300795+0000","last_fullsized":"2024-09-21T09:55:03.300795+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:00.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1018,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300593+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T09:55:03.300593+0000","last_peered":"2024-09-21T09:55:03.300593+0000","last_clean":"2024-09-21T09:55:03.300593+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T09:55:03.300593+0000","last_undegraded":"2024-09-21T09:55:03.300593+0000","last_fullsized":"2024-09-21T09:55:03.300593+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300762+0000","last_change":"2024-09-21T09:45:39.143302+0000","last_active":"2024-09-21T09:55:03.300762+0000","last_peered":"2024-09-21T09:55:03.300762+0000","last_clean":"2024-09-21T09:55:03.300762+0000","last_became_active":"2024-09-21T09:45:39.142964+0000","last_became_peered":"2024-09-21T09:45:39.142964+0000","last_unstale":"2024-09-21T09:55:03.300762+0000","last_undegraded":"2024-09-21T09:55:03.300762+0000","last_fullsized":"2024-09-21T09:55:03.300762+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:54.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1487,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913222+0000","last_change":"2024-09-21T09:46:47.121643+0000","last_active":"2024-09-21T09:55:08.913222+0000","last_peered":"2024-09-21T09:55:08.913222+0000","last_clean":"2024-09-21T09:55:08.913222+0000","last_became_active":"2024-09-21T09:46:47.120528+0000","last_became_peered":"2024-09-21T09:46:47.120528+0000","last_unstale":"2024-09-21T09:55:08.913222+0000","last_undegraded":"2024-09-21T09:55:08.913222+0000","last_fullsized":"2024-09-21T09:55:08.913222+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:07.989377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1533,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913431+0000","last_change":"2024-09-21T09:46:47.121159+0000","last_active":"2024-09-21T09:55:08.913431+0000","last_peered":"2024-09-21T09:55:08.913431+0000","last_clean":"2024-09-21T09:55:08.913431+0000","last_became_active":"2024-09-21T09:46:47.120829+0000","last_became_peered":"2024-09-21T09:46:47.120829+0000","last_unstale":"2024-09-21T09:55:08.913431+0000","last_undegraded":"2024-09-21T09:55:08.913431+0000","last_fullsized":"2024-09-21T09:55:08.913431+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:43.393840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1116,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624090+0000","last_change":"2024-09-21T09:45:05.087928+0000","last_active":"2024-09-21T09:55:03.624090+0000","last_peered":"2024-09-21T09:55:03.624090+0000","last_clean":"2024-09-21T09:55:03.624090+0000","last_became_active":"2024-09-21T09:45:05.087478+0000","last_became_peered":"2024-09-21T09:45:05.087478+0000","last_unstale":"2024-09-21T09:55:03.624090+0000","last_undegraded":"2024-09-21T09:55:03.624090+0000","last_fullsized":"2024-09-21T09:55:03.624090+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:48.651314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1035,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412049+0000","last_change":"2024-09-21T09:46:13.262337+0000","last_active":"2024-09-21T09:55:03.412049+0000","last_peered":"2024-09-21T09:55:03.412049+0000","last_clean":"2024-09-21T09:55:03.412049+0000","last_became_active":"2024-09-21T09:46:13.262005+0000","last_became_peered":"2024-09-21T09:46:13.262005+0000","last_unstale":"2024-09-21T09:55:03.412049+0000","last_undegraded":"2024-09-21T09:55:03.412049+0000","last_fullsized":"2024-09-21T09:55:03.412049+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:32.413210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1469,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912212+0000","last_change":"2024-09-21T09:46:47.116045+0000","last_active":"2024-09-21T09:55:08.912212+0000","last_peered":"2024-09-21T09:55:08.912212+0000","last_clean":"2024-09-21T09:55:08.912212+0000","last_became_active":"2024-09-21T09:46:47.115526+0000","last_became_peered":"2024-09-21T09:46:47.115526+0000","last_unstale":"2024-09-21T09:55:08.912212+0000","last_undegraded":"2024-09-21T09:55:08.912212+0000","last_fullsized":"2024-09-21T09:55:08.912212+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:43:35.293486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":982,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049787+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:55:04.049787+0000","last_peered":"2024-09-21T09:55:04.049787+0000","last_clean":"2024-09-21T09:55:04.049787+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:55:04.049787+0000","last_undegraded":"2024-09-21T09:55:04.049787+0000","last_fullsized":"2024-09-21T09:55:04.049787+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1028,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624013+0000","last_change":"2024-09-21T09:45:39.129555+0000","last_active":"2024-09-21T09:55:03.624013+0000","last_peered":"2024-09-21T09:55:03.624013+0000","last_clean":"2024-09-21T09:55:03.624013+0000","last_became_active":"2024-09-21T09:45:39.129205+0000","last_became_peered":"2024-09-21T09:45:39.129205+0000","last_unstale":"2024-09-21T09:55:03.624013+0000","last_undegraded":"2024-09-21T09:55:03.624013+0000","last_fullsized":"2024-09-21T09:55:03.624013+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:07.229126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1570,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913207+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T09:55:08.913207+0000","last_peered":"2024-09-21T09:55:08.913207+0000","last_clean":"2024-09-21T09:55:08.913207+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T09:55:08.913207+0000","last_undegraded":"2024-09-21T09:55:08.913207+0000","last_fullsized":"2024-09-21T09:55:08.913207+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1583,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912533+0000","last_change":"2024-09-21T09:46:13.273573+0000","last_active":"2024-09-21T09:55:08.912533+0000","last_peered":"2024-09-21T09:55:08.912533+0000","last_clean":"2024-09-21T09:55:08.912533+0000","last_became_active":"2024-09-21T09:46:13.273297+0000","last_became_peered":"2024-09-21T09:46:13.273297+0000","last_unstale":"2024-09-21T09:55:08.912533+0000","last_undegraded":"2024-09-21T09:55:08.912533+0000","last_fullsized":"2024-09-21T09:55:08.912533+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:38.957745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1544,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316664+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T09:55:08.316664+0000","last_peered":"2024-09-21T09:55:08.316664+0000","last_clean":"2024-09-21T09:55:08.316664+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T09:55:08.316664+0000","last_undegraded":"2024-09-21T09:55:08.316664+0000","last_fullsized":"2024-09-21T09:55:08.316664+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1535,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911392+0000","last_change":"2024-09-21T09:46:13.255342+0000","last_active":"2024-09-21T09:55:08.911392+0000","last_peered":"2024-09-21T09:55:08.911392+0000","last_clean":"2024-09-21T09:55:08.911392+0000","last_became_active":"2024-09-21T09:46:13.255014+0000","last_became_peered":"2024-09-21T09:46:13.255014+0000","last_unstale":"2024-09-21T09:55:08.911392+0000","last_undegraded":"2024-09-21T09:55:08.911392+0000","last_fullsized":"2024-09-21T09:55:08.911392+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:53.012266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1545,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913260+0000","last_change":"2024-09-21T09:45:39.134465+0000","last_active":"2024-09-21T09:55:08.913260+0000","last_peered":"2024-09-21T09:55:08.913260+0000","last_clean":"2024-09-21T09:55:08.913260+0000","last_became_active":"2024-09-21T09:45:39.129692+0000","last_became_peered":"2024-09-21T09:45:39.129692+0000","last_unstale":"2024-09-21T09:55:08.913260+0000","last_undegraded":"2024-09-21T09:55:08.913260+0000","last_fullsized":"2024-09-21T09:55:08.913260+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:14.288180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1036,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412260+0000","last_change":"2024-09-21T09:45:39.131285+0000","last_active":"2024-09-21T09:55:03.412260+0000","last_peered":"2024-09-21T09:55:03.412260+0000","last_clean":"2024-09-21T09:55:03.412260+0000","last_became_active":"2024-09-21T09:45:39.130505+0000","last_became_peered":"2024-09-21T09:45:39.130505+0000","last_unstale":"2024-09-21T09:55:03.412260+0000","last_undegraded":"2024-09-21T09:55:03.412260+0000","last_fullsized":"2024-09-21T09:55:03.412260+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:37.523277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1018,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300486+0000","last_change":"2024-09-21T09:45:05.097518+0000","last_active":"2024-09-21T09:55:03.300486+0000","last_peered":"2024-09-21T09:55:03.300486+0000","last_clean":"2024-09-21T09:55:03.300486+0000","last_became_active":"2024-09-21T09:45:05.096918+0000","last_became_peered":"2024-09-21T09:45:05.096918+0000","last_unstale":"2024-09-21T09:55:03.300486+0000","last_undegraded":"2024-09-21T09:55:03.300486+0000","last_fullsized":"2024-09-21T09:55:03.300486+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:13.781847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1544,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911019+0000","last_change":"2024-09-21T09:46:47.113687+0000","last_active":"2024-09-21T09:55:08.911019+0000","last_peered":"2024-09-21T09:55:08.911019+0000","last_clean":"2024-09-21T09:55:08.911019+0000","last_became_active":"2024-09-21T09:46:47.113235+0000","last_became_peered":"2024-09-21T09:46:47.113235+0000","last_unstale":"2024-09-21T09:55:08.911019+0000","last_undegraded":"2024-09-21T09:55:08.911019+0000","last_fullsized":"2024-09-21T09:55:08.911019+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:50.851616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1038,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412109+0000","last_change":"2024-09-21T09:46:47.115673+0000","last_active":"2024-09-21T09:55:03.412109+0000","last_peered":"2024-09-21T09:55:03.412109+0000","last_clean":"2024-09-21T09:55:03.412109+0000","last_became_active":"2024-09-21T09:46:47.115166+0000","last_became_peered":"2024-09-21T09:46:47.115166+0000","last_unstale":"2024-09-21T09:55:03.412109+0000","last_undegraded":"2024-09-21T09:55:03.412109+0000","last_fullsized":"2024-09-21T09:55:03.412109+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:16.660834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1558,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911131+0000","last_change":"2024-09-21T09:45:39.132441+0000","last_active":"2024-09-21T09:55:08.911131+0000","last_peered":"2024-09-21T09:55:08.911131+0000","last_clean":"2024-09-21T09:55:08.911131+0000","last_became_active":"2024-09-21T09:45:39.129666+0000","last_became_peered":"2024-09-21T09:45:39.129666+0000","last_unstale":"2024-09-21T09:55:08.911131+0000","last_undegraded":"2024-09-21T09:55:08.911131+0000","last_fullsized":"2024-09-21T09:55:08.911131+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:34.450160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1497,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316744+0000","last_change":"2024-09-21T09:46:13.258244+0000","last_active":"2024-09-21T09:55:08.316744+0000","last_peered":"2024-09-21T09:55:08.316744+0000","last_clean":"2024-09-21T09:55:08.316744+0000","last_became_active":"2024-09-21T09:46:13.257820+0000","last_became_peered":"2024-09-21T09:46:13.257820+0000","last_unstale":"2024-09-21T09:55:08.316744+0000","last_undegraded":"2024-09-21T09:55:08.316744+0000","last_fullsized":"2024-09-21T09:55:08.316744+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:04.618304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1534,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911366+0000","last_change":"2024-09-21T09:46:13.260666+0000","last_active":"2024-09-21T09:55:08.911366+0000","last_peered":"2024-09-21T09:55:08.911366+0000","last_clean":"2024-09-21T09:55:08.911366+0000","last_became_active":"2024-09-21T09:46:13.260324+0000","last_became_peered":"2024-09-21T09:46:13.260324+0000","last_unstale":"2024-09-21T09:55:08.911366+0000","last_undegraded":"2024-09-21T09:55:08.911366+0000","last_fullsized":"2024-09-21T09:55:08.911366+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:25.140797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1011,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624946+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T09:55:03.624946+0000","last_peered":"2024-09-21T09:55:03.624946+0000","last_clean":"2024-09-21T09:55:03.624946+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T09:55:03.624946+0000","last_undegraded":"2024-09-21T09:55:03.624946+0000","last_fullsized":"2024-09-21T09:55:03.624946+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1561,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914110+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T09:55:08.914110+0000","last_peered":"2024-09-21T09:55:08.914110+0000","last_clean":"2024-09-21T09:55:08.914110+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T09:55:08.914110+0000","last_undegraded":"2024-09-21T09:55:08.914110+0000","last_fullsized":"2024-09-21T09:55:08.914110+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1496,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913455+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T09:55:08.913455+0000","last_peered":"2024-09-21T09:55:08.913455+0000","last_clean":"2024-09-21T09:55:08.913455+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T09:55:08.913455+0000","last_undegraded":"2024-09-21T09:55:08.913455+0000","last_fullsized":"2024-09-21T09:55:08.913455+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1532,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912680+0000","last_change":"2024-09-21T09:45:39.130168+0000","last_active":"2024-09-21T09:55:08.912680+0000","last_peered":"2024-09-21T09:55:08.912680+0000","last_clean":"2024-09-21T09:55:08.912680+0000","last_became_active":"2024-09-21T09:45:39.129498+0000","last_became_peered":"2024-09-21T09:45:39.129498+0000","last_unstale":"2024-09-21T09:55:08.912680+0000","last_undegraded":"2024-09-21T09:55:08.912680+0000","last_fullsized":"2024-09-21T09:55:08.912680+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:58.310876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1493,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912070+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T09:55:08.912070+0000","last_peered":"2024-09-21T09:55:08.912070+0000","last_clean":"2024-09-21T09:55:08.912070+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T09:55:08.912070+0000","last_undegraded":"2024-09-21T09:55:08.912070+0000","last_fullsized":"2024-09-21T09:55:08.912070+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1574,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912871+0000","last_change":"2024-09-21T09:45:39.140762+0000","last_active":"2024-09-21T09:55:08.912871+0000","last_peered":"2024-09-21T09:55:08.912871+0000","last_clean":"2024-09-21T09:55:08.912871+0000","last_became_active":"2024-09-21T09:45:39.140402+0000","last_became_peered":"2024-09-21T09:45:39.140402+0000","last_unstale":"2024-09-21T09:55:08.912871+0000","last_undegraded":"2024-09-21T09:55:08.912871+0000","last_fullsized":"2024-09-21T09:55:08.912871+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:06.545428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1012,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049926+0000","last_change":"2024-09-21T09:46:13.260656+0000","last_active":"2024-09-21T09:55:04.049926+0000","last_peered":"2024-09-21T09:55:04.049926+0000","last_clean":"2024-09-21T09:55:04.049926+0000","last_became_active":"2024-09-21T09:46:13.260368+0000","last_became_peered":"2024-09-21T09:46:13.260368+0000","last_unstale":"2024-09-21T09:55:04.049926+0000","last_undegraded":"2024-09-21T09:55:04.049926+0000","last_fullsized":"2024-09-21T09:55:04.049926+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:22.059036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300041+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:55:03.300041+0000","last_peered":"2024-09-21T09:55:03.300041+0000","last_clean":"2024-09-21T09:55:03.300041+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:55:03.300041+0000","last_undegraded":"2024-09-21T09:55:03.300041+0000","last_fullsized":"2024-09-21T09:55:03.300041+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300296+0000","last_change":"2024-09-21T09:45:39.129538+0000","last_active":"2024-09-21T09:55:03.300296+0000","last_peered":"2024-09-21T09:55:03.300296+0000","last_clean":"2024-09-21T09:55:03.300296+0000","last_became_active":"2024-09-21T09:45:39.129198+0000","last_became_peered":"2024-09-21T09:45:39.129198+0000","last_unstale":"2024-09-21T09:55:03.300296+0000","last_undegraded":"2024-09-21T09:55:03.300296+0000","last_fullsized":"2024-09-21T09:55:03.300296+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.798277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1023,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300173+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:55:03.300173+0000","last_peered":"2024-09-21T09:55:03.300173+0000","last_clean":"2024-09-21T09:55:03.300173+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:55:03.300173+0000","last_undegraded":"2024-09-21T09:55:03.300173+0000","last_fullsized":"2024-09-21T09:55:03.300173+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1533,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.910728+0000","last_change":"2024-09-21T09:46:13.254014+0000","last_active":"2024-09-21T09:55:08.910728+0000","last_peered":"2024-09-21T09:55:08.910728+0000","last_clean":"2024-09-21T09:55:08.910728+0000","last_became_active":"2024-09-21T09:46:13.253673+0000","last_became_peered":"2024-09-21T09:46:13.253673+0000","last_unstale":"2024-09-21T09:55:08.910728+0000","last_undegraded":"2024-09-21T09:55:08.910728+0000","last_fullsized":"2024-09-21T09:55:08.910728+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:02.784613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1002,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624515+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T09:55:03.624515+0000","last_peered":"2024-09-21T09:55:03.624515+0000","last_clean":"2024-09-21T09:55:03.624515+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T09:55:03.624515+0000","last_undegraded":"2024-09-21T09:55:03.624515+0000","last_fullsized":"2024-09-21T09:55:03.624515+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1022,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412623+0000","last_change":"2024-09-21T09:45:39.130886+0000","last_active":"2024-09-21T09:55:03.412623+0000","last_peered":"2024-09-21T09:55:03.412623+0000","last_clean":"2024-09-21T09:55:03.412623+0000","last_became_active":"2024-09-21T09:45:39.130559+0000","last_became_peered":"2024-09-21T09:45:39.130559+0000","last_unstale":"2024-09-21T09:55:03.412623+0000","last_undegraded":"2024-09-21T09:55:03.412623+0000","last_fullsized":"2024-09-21T09:55:03.412623+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:39.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1468,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911482+0000","last_change":"2024-09-21T09:46:47.122643+0000","last_active":"2024-09-21T09:55:08.911482+0000","last_peered":"2024-09-21T09:55:08.911482+0000","last_clean":"2024-09-21T09:55:08.911482+0000","last_became_active":"2024-09-21T09:46:47.122364+0000","last_became_peered":"2024-09-21T09:46:47.122364+0000","last_unstale":"2024-09-21T09:55:08.911482+0000","last_undegraded":"2024-09-21T09:55:08.911482+0000","last_fullsized":"2024-09-21T09:55:08.911482+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:29.222333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1569,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914102+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T09:55:08.914102+0000","last_peered":"2024-09-21T09:55:08.914102+0000","last_clean":"2024-09-21T09:55:08.914102+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T09:55:08.914102+0000","last_undegraded":"2024-09-21T09:55:08.914102+0000","last_fullsized":"2024-09-21T09:55:08.914102+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911965+0000","last_change":"2024-09-21T09:46:47.121360+0000","last_active":"2024-09-21T09:55:08.911965+0000","last_peered":"2024-09-21T09:55:08.911965+0000","last_clean":"2024-09-21T09:55:08.911965+0000","last_became_active":"2024-09-21T09:46:47.120494+0000","last_became_peered":"2024-09-21T09:46:47.120494+0000","last_unstale":"2024-09-21T09:55:08.911965+0000","last_undegraded":"2024-09-21T09:55:08.911965+0000","last_fullsized":"2024-09-21T09:55:08.911965+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:45.862984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1009,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049924+0000","last_change":"2024-09-21T09:46:13.260037+0000","last_active":"2024-09-21T09:55:04.049924+0000","last_peered":"2024-09-21T09:55:04.049924+0000","last_clean":"2024-09-21T09:55:04.049924+0000","last_became_active":"2024-09-21T09:46:13.259690+0000","last_became_peered":"2024-09-21T09:46:13.259690+0000","last_unstale":"2024-09-21T09:55:04.049924+0000","last_undegraded":"2024-09-21T09:55:04.049924+0000","last_fullsized":"2024-09-21T09:55:04.049924+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:57.466762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1503,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912173+0000","last_change":"2024-09-21T09:46:47.120830+0000","last_active":"2024-09-21T09:55:08.912173+0000","last_peered":"2024-09-21T09:55:08.912173+0000","last_clean":"2024-09-21T09:55:08.912173+0000","last_became_active":"2024-09-21T09:46:47.118419+0000","last_became_peered":"2024-09-21T09:46:47.118419+0000","last_unstale":"2024-09-21T09:55:08.912173+0000","last_undegraded":"2024-09-21T09:55:08.912173+0000","last_fullsized":"2024-09-21T09:55:08.912173+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:13.789806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1572,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912600+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T09:55:08.912600+0000","last_peered":"2024-09-21T09:55:08.912600+0000","last_clean":"2024-09-21T09:55:08.912600+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T09:55:08.912600+0000","last_undegraded":"2024-09-21T09:55:08.912600+0000","last_fullsized":"2024-09-21T09:55:08.912600+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1534,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.910698+0000","last_change":"2024-09-21T09:46:47.121724+0000","last_active":"2024-09-21T09:55:08.910698+0000","last_peered":"2024-09-21T09:55:08.910698+0000","last_clean":"2024-09-21T09:55:08.910698+0000","last_became_active":"2024-09-21T09:46:47.121406+0000","last_became_peered":"2024-09-21T09:46:47.121406+0000","last_unstale":"2024-09-21T09:55:08.910698+0000","last_undegraded":"2024-09-21T09:55:08.910698+0000","last_fullsized":"2024-09-21T09:55:08.910698+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:55.493704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911362+0000","last_change":"2024-09-21T09:46:47.113184+0000","last_active":"2024-09-21T09:55:08.911362+0000","last_peered":"2024-09-21T09:55:08.911362+0000","last_clean":"2024-09-21T09:55:08.911362+0000","last_became_active":"2024-09-21T09:46:47.112857+0000","last_became_peered":"2024-09-21T09:46:47.112857+0000","last_unstale":"2024-09-21T09:55:08.911362+0000","last_undegraded":"2024-09-21T09:55:08.911362+0000","last_fullsized":"2024-09-21T09:55:08.911362+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:46.783986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1036,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300495+0000","last_change":"2024-09-21T09:46:13.262555+0000","last_active":"2024-09-21T09:55:03.300495+0000","last_peered":"2024-09-21T09:55:03.300495+0000","last_clean":"2024-09-21T09:55:03.300495+0000","last_became_active":"2024-09-21T09:46:13.261474+0000","last_became_peered":"2024-09-21T09:46:13.261474+0000","last_unstale":"2024-09-21T09:55:03.300495+0000","last_undegraded":"2024-09-21T09:55:03.300495+0000","last_fullsized":"2024-09-21T09:55:03.300495+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:44.865988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1523,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316853+0000","last_change":"2024-09-21T09:46:47.114406+0000","last_active":"2024-09-21T09:55:08.316853+0000","last_peered":"2024-09-21T09:55:08.316853+0000","last_clean":"2024-09-21T09:55:08.316853+0000","last_became_active":"2024-09-21T09:46:47.113986+0000","last_became_peered":"2024-09-21T09:46:47.113986+0000","last_unstale":"2024-09-21T09:55:08.316853+0000","last_undegraded":"2024-09-21T09:55:08.316853+0000","last_fullsized":"2024-09-21T09:55:08.316853+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:32.292201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1562,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914239+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T09:55:08.914239+0000","last_peered":"2024-09-21T09:55:08.914239+0000","last_clean":"2024-09-21T09:55:08.914239+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T09:55:08.914239+0000","last_undegraded":"2024-09-21T09:55:08.914239+0000","last_fullsized":"2024-09-21T09:55:08.914239+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300266+0000","last_change":"2024-09-21T09:46:47.114188+0000","last_active":"2024-09-21T09:55:03.300266+0000","last_peered":"2024-09-21T09:55:03.300266+0000","last_clean":"2024-09-21T09:55:03.300266+0000","last_became_active":"2024-09-21T09:46:47.113859+0000","last_became_peered":"2024-09-21T09:46:47.113859+0000","last_unstale":"2024-09-21T09:55:03.300266+0000","last_undegraded":"2024-09-21T09:55:03.300266+0000","last_fullsized":"2024-09-21T09:55:03.300266+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:22.670368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":1024,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412132+0000","last_change":"2024-09-21T09:45:39.131124+0000","last_active":"2024-09-21T09:55:03.412132+0000","last_peered":"2024-09-21T09:55:03.412132+0000","last_clean":"2024-09-21T09:55:03.412132+0000","last_became_active":"2024-09-21T09:45:39.130756+0000","last_became_peered":"2024-09-21T09:45:39.130756+0000","last_unstale":"2024-09-21T09:55:03.412132+0000","last_undegraded":"2024-09-21T09:55:03.412132+0000","last_fullsized":"2024-09-21T09:55:03.412132+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:49.130698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300392+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T09:55:03.300392+0000","last_peered":"2024-09-21T09:55:03.300392+0000","last_clean":"2024-09-21T09:55:03.300392+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T09:55:03.300392+0000","last_undegraded":"2024-09-21T09:55:03.300392+0000","last_fullsized":"2024-09-21T09:55:03.300392+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":1006,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412104+0000","last_change":"2024-09-21T09:36:43.115791+0000","last_active":"2024-09-21T09:55:03.412104+0000","last_peered":"2024-09-21T09:55:03.412104+0000","last_clean":"2024-09-21T09:55:03.412104+0000","last_became_active":"2024-09-21T09:36:43.115525+0000","last_became_peered":"2024-09-21T09:36:43.115525+0000","last_unstale":"2024-09-21T09:55:03.412104+0000","last_undegraded":"2024-09-21T09:55:03.412104+0000","last_fullsized":"2024-09-21T09:55:03.412104+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:28.396722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1495,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911833+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T09:55:08.911833+0000","last_peered":"2024-09-21T09:55:08.911833+0000","last_clean":"2024-09-21T09:55:08.911833+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T09:55:08.911833+0000","last_undegraded":"2024-09-21T09:55:08.911833+0000","last_fullsized":"2024-09-21T09:55:08.911833+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1481,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913152+0000","last_change":"2024-09-21T09:46:47.115788+0000","last_active":"2024-09-21T09:55:08.913152+0000","last_peered":"2024-09-21T09:55:08.913152+0000","last_clean":"2024-09-21T09:55:08.913152+0000","last_became_active":"2024-09-21T09:46:47.114864+0000","last_became_peered":"2024-09-21T09:46:47.114864+0000","last_unstale":"2024-09-21T09:55:08.913152+0000","last_undegraded":"2024-09-21T09:55:08.913152+0000","last_fullsized":"2024-09-21T09:55:08.913152+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:00.034944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.299925+0000","last_change":"2024-09-21T09:45:39.143064+0000","last_active":"2024-09-21T09:55:03.299925+0000","last_peered":"2024-09-21T09:55:03.299925+0000","last_clean":"2024-09-21T09:55:03.299925+0000","last_became_active":"2024-09-21T09:45:39.142699+0000","last_became_peered":"2024-09-21T09:45:39.142699+0000","last_unstale":"2024-09-21T09:55:03.299925+0000","last_undegraded":"2024-09-21T09:55:03.299925+0000","last_fullsized":"2024-09-21T09:55:03.299925+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:14.684721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1577,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914281+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T09:55:08.914281+0000","last_peered":"2024-09-21T09:55:08.914281+0000","last_clean":"2024-09-21T09:55:08.914281+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T09:55:08.914281+0000","last_undegraded":"2024-09-21T09:55:08.914281+0000","last_fullsized":"2024-09-21T09:55:08.914281+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397761,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317368,"kb_used_data":7532,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93427784,"statfs":{"total":95995035648,"available":95670050816,"internally_reserved":0,"allocated":7712768,"data_stored":5600360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593990,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313200,"kb_used_data":7524,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93431952,"statfs":{"total":95995035648,"available":95674318848,"internally_reserved":0,"allocated":7704576,"data_stored":5600225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790219,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316768,"kb_used_data":6936,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93428384,"statfs":{"total":95995035648,"available":95670665216,"internally_reserved":0,"allocated":7102464,"data_stored":5005683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986446,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308460,"kb_used_data":6948,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93436692,"statfs":{"total":95995035648,"available":95679172608,"internally_reserved":0,"allocated":7114752,"data_stored":5009914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999088,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":48744,"kb_used_data":8608,"kb_used_omap":25,"kb_used_meta":40102,"kb_avail":93696408,"statfs":{"total":95995035648,"available":95945121792,"internally_reserved":0,"allocated":8814592,"data_stored":6219336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":41065415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111030,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40984,"kb_used_data":8792,"kb_used_omap":31,"kb_used_meta":31968,"kb_avail":93704168,"statfs":{"total":95995035648,"available":95953068032,"internally_reserved":0,"allocated":9003008,"data_stored":6349770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":32735464},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318222973,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40672,"kb_used_data":8608,"kb_used_omap":26,"kb_used_meta":31845,"kb_avail":93704480,"statfs":{"total":95995035648,"available":95953387520,"internally_reserved":0,"allocated":8814592,"data_stored":6219273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":32609292},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289334916,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67868,"kb_used_data":9372,"kb_used_omap":31,"kb_used_meta":58272,"kb_avail":93677284,"statfs":{"total":95995035648,"available":95925538816,"internally_reserved":0,"allocated":9596928,"data_stored":6940134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":59670624},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:55:57.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:55:57.659 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:57 smithi017 ceph-mon[102913]: pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:57 smithi017 ceph-mon[105151]: pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:57 smithi134 ceph-mon[90085]: pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:58 smithi017 ceph-mon[102913]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:58 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:58 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:58 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:58 smithi017 ceph-mon[105151]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:58 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:58 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:58 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:58 smithi134 ceph-mon[90085]: from='client.35223 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:58 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:58 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:58 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:55:59.501 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:55:59.501 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:55:59.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:55:59 smithi017 ceph-mon[102913]: pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:59.813 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:55:59 smithi017 ceph-mon[105151]: pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:55:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:55:59 smithi134 ceph-mon[90085]: pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:00.132 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1222,"stamp":"2024-09-21T09:55:57.979707+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":212,"num_write_kb":1659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158,"ondisk_log_size":158,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1454064,"kb_used_data":64320,"kb_used_omap":122,"kb_used_meta":1388997,"kb_avail":748507152,"statfs":{"total":767960285184,"available":766471323648,"internally_reserved":0,"allocated":65863680,"data_stored":46944695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125317,"internal_metadata":1422333563},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003165"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911046+0000","last_change":"2024-09-21T09:46:47.114206+0000","last_active":"2024-09-21T09:55:08.911046+0000","last_peered":"2024-09-21T09:55:08.911046+0000","last_clean":"2024-09-21T09:55:08.911046+0000","last_became_active":"2024-09-21T09:46:47.113819+0000","last_became_peered":"2024-09-21T09:46:47.113819+0000","last_unstale":"2024-09-21T09:55:08.911046+0000","last_undegraded":"2024-09-21T09:55:08.911046+0000","last_fullsized":"2024-09-21T09:55:08.911046+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:07.629544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1001,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624302+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T09:55:03.624302+0000","last_peered":"2024-09-21T09:55:03.624302+0000","last_clean":"2024-09-21T09:55:03.624302+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T09:55:03.624302+0000","last_undegraded":"2024-09-21T09:55:03.624302+0000","last_fullsized":"2024-09-21T09:55:03.624302+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1020,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412012+0000","last_change":"2024-09-21T09:46:47.115484+0000","last_active":"2024-09-21T09:55:03.412012+0000","last_peered":"2024-09-21T09:55:03.412012+0000","last_clean":"2024-09-21T09:55:03.412012+0000","last_became_active":"2024-09-21T09:46:47.115080+0000","last_became_peered":"2024-09-21T09:46:47.115080+0000","last_unstale":"2024-09-21T09:55:03.412012+0000","last_undegraded":"2024-09-21T09:55:03.412012+0000","last_fullsized":"2024-09-21T09:55:03.412012+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:56.851805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1569,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912062+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T09:55:08.912062+0000","last_peered":"2024-09-21T09:55:08.912062+0000","last_clean":"2024-09-21T09:55:08.912062+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T09:55:08.912062+0000","last_undegraded":"2024-09-21T09:55:08.912062+0000","last_fullsized":"2024-09-21T09:55:08.912062+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1571,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912710+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T09:55:08.912710+0000","last_peered":"2024-09-21T09:55:08.912710+0000","last_clean":"2024-09-21T09:55:08.912710+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T09:55:08.912710+0000","last_undegraded":"2024-09-21T09:55:08.912710+0000","last_fullsized":"2024-09-21T09:55:08.912710+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":982,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624220+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:55:03.624220+0000","last_peered":"2024-09-21T09:55:03.624220+0000","last_clean":"2024-09-21T09:55:03.624220+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:55:03.624220+0000","last_undegraded":"2024-09-21T09:55:03.624220+0000","last_fullsized":"2024-09-21T09:55:03.624220+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912905+0000","last_change":"2024-09-21T09:46:47.117580+0000","last_active":"2024-09-21T09:55:08.912905+0000","last_peered":"2024-09-21T09:55:08.912905+0000","last_clean":"2024-09-21T09:55:08.912905+0000","last_became_active":"2024-09-21T09:46:47.113030+0000","last_became_peered":"2024-09-21T09:46:47.113030+0000","last_unstale":"2024-09-21T09:55:08.912905+0000","last_undegraded":"2024-09-21T09:55:08.912905+0000","last_fullsized":"2024-09-21T09:55:08.912905+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:28.220180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":1008,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049340+0000","last_change":"2024-09-21T09:45:05.080014+0000","last_active":"2024-09-21T09:55:04.049340+0000","last_peered":"2024-09-21T09:55:04.049340+0000","last_clean":"2024-09-21T09:55:04.049340+0000","last_became_active":"2024-09-21T09:45:05.079618+0000","last_became_peered":"2024-09-21T09:45:05.079618+0000","last_unstale":"2024-09-21T09:55:04.049340+0000","last_undegraded":"2024-09-21T09:55:04.049340+0000","last_fullsized":"2024-09-21T09:55:04.049340+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:42.160292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624817+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T09:55:03.624817+0000","last_peered":"2024-09-21T09:55:03.624817+0000","last_clean":"2024-09-21T09:55:03.624817+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T09:55:03.624817+0000","last_undegraded":"2024-09-21T09:55:03.624817+0000","last_fullsized":"2024-09-21T09:55:03.624817+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1004,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.411948+0000","last_change":"2024-09-21T09:36:43.389124+0000","last_active":"2024-09-21T09:55:03.411948+0000","last_peered":"2024-09-21T09:55:03.411948+0000","last_clean":"2024-09-21T09:55:03.411948+0000","last_became_active":"2024-09-21T09:36:43.388588+0000","last_became_peered":"2024-09-21T09:36:43.388588+0000","last_unstale":"2024-09-21T09:55:03.411948+0000","last_undegraded":"2024-09-21T09:55:03.411948+0000","last_fullsized":"2024-09-21T09:55:03.411948+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:57.562156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1001,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624842+0000","last_change":"2024-09-21T09:45:39.143190+0000","last_active":"2024-09-21T09:55:03.624842+0000","last_peered":"2024-09-21T09:55:03.624842+0000","last_clean":"2024-09-21T09:55:03.624842+0000","last_became_active":"2024-09-21T09:45:39.142826+0000","last_became_peered":"2024-09-21T09:45:39.142826+0000","last_unstale":"2024-09-21T09:55:03.624842+0000","last_undegraded":"2024-09-21T09:55:03.624842+0000","last_fullsized":"2024-09-21T09:55:03.624842+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:45.070945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300475+0000","last_change":"2024-09-21T09:46:47.119247+0000","last_active":"2024-09-21T09:55:03.300475+0000","last_peered":"2024-09-21T09:55:03.300475+0000","last_clean":"2024-09-21T09:55:03.300475+0000","last_became_active":"2024-09-21T09:46:47.118980+0000","last_became_peered":"2024-09-21T09:46:47.118980+0000","last_unstale":"2024-09-21T09:55:03.300475+0000","last_undegraded":"2024-09-21T09:55:03.300475+0000","last_fullsized":"2024-09-21T09:55:03.300475+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.140253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":983,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624823+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:55:03.624823+0000","last_peered":"2024-09-21T09:55:03.624823+0000","last_clean":"2024-09-21T09:55:03.624823+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:55:03.624823+0000","last_undegraded":"2024-09-21T09:55:03.624823+0000","last_fullsized":"2024-09-21T09:55:03.624823+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1009,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049323+0000","last_change":"2024-09-21T09:46:13.260744+0000","last_active":"2024-09-21T09:55:04.049323+0000","last_peered":"2024-09-21T09:55:04.049323+0000","last_clean":"2024-09-21T09:55:04.049323+0000","last_became_active":"2024-09-21T09:46:13.260443+0000","last_became_peered":"2024-09-21T09:46:13.260443+0000","last_unstale":"2024-09-21T09:55:04.049323+0000","last_undegraded":"2024-09-21T09:55:04.049323+0000","last_fullsized":"2024-09-21T09:55:04.049323+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:22.699110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1022,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412189+0000","last_change":"2024-09-21T09:46:47.115343+0000","last_active":"2024-09-21T09:55:03.412189+0000","last_peered":"2024-09-21T09:55:03.412189+0000","last_clean":"2024-09-21T09:55:03.412189+0000","last_became_active":"2024-09-21T09:46:47.115017+0000","last_became_peered":"2024-09-21T09:46:47.115017+0000","last_unstale":"2024-09-21T09:55:03.412189+0000","last_undegraded":"2024-09-21T09:55:03.412189+0000","last_fullsized":"2024-09-21T09:55:03.412189+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.820662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1571,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912938+0000","last_change":"2024-09-21T09:45:39.138289+0000","last_active":"2024-09-21T09:55:08.912938+0000","last_peered":"2024-09-21T09:55:08.912938+0000","last_clean":"2024-09-21T09:55:08.912938+0000","last_became_active":"2024-09-21T09:45:39.137902+0000","last_became_peered":"2024-09-21T09:45:39.137902+0000","last_unstale":"2024-09-21T09:55:08.912938+0000","last_undegraded":"2024-09-21T09:55:08.912938+0000","last_fullsized":"2024-09-21T09:55:08.912938+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:40:16.383906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1496,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911031+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T09:55:08.911031+0000","last_peered":"2024-09-21T09:55:08.911031+0000","last_clean":"2024-09-21T09:55:08.911031+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T09:55:08.911031+0000","last_undegraded":"2024-09-21T09:55:08.911031+0000","last_fullsized":"2024-09-21T09:55:08.911031+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300113+0000","last_change":"2024-09-21T09:46:47.114293+0000","last_active":"2024-09-21T09:55:03.300113+0000","last_peered":"2024-09-21T09:55:03.300113+0000","last_clean":"2024-09-21T09:55:03.300113+0000","last_became_active":"2024-09-21T09:46:47.113993+0000","last_became_peered":"2024-09-21T09:46:47.113993+0000","last_unstale":"2024-09-21T09:55:03.300113+0000","last_undegraded":"2024-09-21T09:55:03.300113+0000","last_fullsized":"2024-09-21T09:55:03.300113+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:30.941931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049541+0000","last_change":"2024-09-21T09:45:39.127186+0000","last_active":"2024-09-21T09:55:04.049541+0000","last_peered":"2024-09-21T09:55:04.049541+0000","last_clean":"2024-09-21T09:55:04.049541+0000","last_became_active":"2024-09-21T09:45:39.126940+0000","last_became_peered":"2024-09-21T09:45:39.126940+0000","last_unstale":"2024-09-21T09:55:04.049541+0000","last_undegraded":"2024-09-21T09:55:04.049541+0000","last_fullsized":"2024-09-21T09:55:04.049541+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:52.205703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300795+0000","last_change":"2024-09-21T09:46:13.259814+0000","last_active":"2024-09-21T09:55:03.300795+0000","last_peered":"2024-09-21T09:55:03.300795+0000","last_clean":"2024-09-21T09:55:03.300795+0000","last_became_active":"2024-09-21T09:46:13.259527+0000","last_became_peered":"2024-09-21T09:46:13.259527+0000","last_unstale":"2024-09-21T09:55:03.300795+0000","last_undegraded":"2024-09-21T09:55:03.300795+0000","last_fullsized":"2024-09-21T09:55:03.300795+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:00.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1018,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300593+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T09:55:03.300593+0000","last_peered":"2024-09-21T09:55:03.300593+0000","last_clean":"2024-09-21T09:55:03.300593+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T09:55:03.300593+0000","last_undegraded":"2024-09-21T09:55:03.300593+0000","last_fullsized":"2024-09-21T09:55:03.300593+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300762+0000","last_change":"2024-09-21T09:45:39.143302+0000","last_active":"2024-09-21T09:55:03.300762+0000","last_peered":"2024-09-21T09:55:03.300762+0000","last_clean":"2024-09-21T09:55:03.300762+0000","last_became_active":"2024-09-21T09:45:39.142964+0000","last_became_peered":"2024-09-21T09:45:39.142964+0000","last_unstale":"2024-09-21T09:55:03.300762+0000","last_undegraded":"2024-09-21T09:55:03.300762+0000","last_fullsized":"2024-09-21T09:55:03.300762+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:54.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1487,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913222+0000","last_change":"2024-09-21T09:46:47.121643+0000","last_active":"2024-09-21T09:55:08.913222+0000","last_peered":"2024-09-21T09:55:08.913222+0000","last_clean":"2024-09-21T09:55:08.913222+0000","last_became_active":"2024-09-21T09:46:47.120528+0000","last_became_peered":"2024-09-21T09:46:47.120528+0000","last_unstale":"2024-09-21T09:55:08.913222+0000","last_undegraded":"2024-09-21T09:55:08.913222+0000","last_fullsized":"2024-09-21T09:55:08.913222+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:07.989377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1533,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913431+0000","last_change":"2024-09-21T09:46:47.121159+0000","last_active":"2024-09-21T09:55:08.913431+0000","last_peered":"2024-09-21T09:55:08.913431+0000","last_clean":"2024-09-21T09:55:08.913431+0000","last_became_active":"2024-09-21T09:46:47.120829+0000","last_became_peered":"2024-09-21T09:46:47.120829+0000","last_unstale":"2024-09-21T09:55:08.913431+0000","last_undegraded":"2024-09-21T09:55:08.913431+0000","last_fullsized":"2024-09-21T09:55:08.913431+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:43.393840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1116,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624090+0000","last_change":"2024-09-21T09:45:05.087928+0000","last_active":"2024-09-21T09:55:03.624090+0000","last_peered":"2024-09-21T09:55:03.624090+0000","last_clean":"2024-09-21T09:55:03.624090+0000","last_became_active":"2024-09-21T09:45:05.087478+0000","last_became_peered":"2024-09-21T09:45:05.087478+0000","last_unstale":"2024-09-21T09:55:03.624090+0000","last_undegraded":"2024-09-21T09:55:03.624090+0000","last_fullsized":"2024-09-21T09:55:03.624090+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:48.651314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1035,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412049+0000","last_change":"2024-09-21T09:46:13.262337+0000","last_active":"2024-09-21T09:55:03.412049+0000","last_peered":"2024-09-21T09:55:03.412049+0000","last_clean":"2024-09-21T09:55:03.412049+0000","last_became_active":"2024-09-21T09:46:13.262005+0000","last_became_peered":"2024-09-21T09:46:13.262005+0000","last_unstale":"2024-09-21T09:55:03.412049+0000","last_undegraded":"2024-09-21T09:55:03.412049+0000","last_fullsized":"2024-09-21T09:55:03.412049+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:32.413210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1469,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912212+0000","last_change":"2024-09-21T09:46:47.116045+0000","last_active":"2024-09-21T09:55:08.912212+0000","last_peered":"2024-09-21T09:55:08.912212+0000","last_clean":"2024-09-21T09:55:08.912212+0000","last_became_active":"2024-09-21T09:46:47.115526+0000","last_became_peered":"2024-09-21T09:46:47.115526+0000","last_unstale":"2024-09-21T09:55:08.912212+0000","last_undegraded":"2024-09-21T09:55:08.912212+0000","last_fullsized":"2024-09-21T09:55:08.912212+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:43:35.293486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":982,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049787+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:55:04.049787+0000","last_peered":"2024-09-21T09:55:04.049787+0000","last_clean":"2024-09-21T09:55:04.049787+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:55:04.049787+0000","last_undegraded":"2024-09-21T09:55:04.049787+0000","last_fullsized":"2024-09-21T09:55:04.049787+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1028,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624013+0000","last_change":"2024-09-21T09:45:39.129555+0000","last_active":"2024-09-21T09:55:03.624013+0000","last_peered":"2024-09-21T09:55:03.624013+0000","last_clean":"2024-09-21T09:55:03.624013+0000","last_became_active":"2024-09-21T09:45:39.129205+0000","last_became_peered":"2024-09-21T09:45:39.129205+0000","last_unstale":"2024-09-21T09:55:03.624013+0000","last_undegraded":"2024-09-21T09:55:03.624013+0000","last_fullsized":"2024-09-21T09:55:03.624013+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:07.229126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1570,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913207+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T09:55:08.913207+0000","last_peered":"2024-09-21T09:55:08.913207+0000","last_clean":"2024-09-21T09:55:08.913207+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T09:55:08.913207+0000","last_undegraded":"2024-09-21T09:55:08.913207+0000","last_fullsized":"2024-09-21T09:55:08.913207+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1583,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912533+0000","last_change":"2024-09-21T09:46:13.273573+0000","last_active":"2024-09-21T09:55:08.912533+0000","last_peered":"2024-09-21T09:55:08.912533+0000","last_clean":"2024-09-21T09:55:08.912533+0000","last_became_active":"2024-09-21T09:46:13.273297+0000","last_became_peered":"2024-09-21T09:46:13.273297+0000","last_unstale":"2024-09-21T09:55:08.912533+0000","last_undegraded":"2024-09-21T09:55:08.912533+0000","last_fullsized":"2024-09-21T09:55:08.912533+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:38.957745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1544,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316664+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T09:55:08.316664+0000","last_peered":"2024-09-21T09:55:08.316664+0000","last_clean":"2024-09-21T09:55:08.316664+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T09:55:08.316664+0000","last_undegraded":"2024-09-21T09:55:08.316664+0000","last_fullsized":"2024-09-21T09:55:08.316664+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1535,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911392+0000","last_change":"2024-09-21T09:46:13.255342+0000","last_active":"2024-09-21T09:55:08.911392+0000","last_peered":"2024-09-21T09:55:08.911392+0000","last_clean":"2024-09-21T09:55:08.911392+0000","last_became_active":"2024-09-21T09:46:13.255014+0000","last_became_peered":"2024-09-21T09:46:13.255014+0000","last_unstale":"2024-09-21T09:55:08.911392+0000","last_undegraded":"2024-09-21T09:55:08.911392+0000","last_fullsized":"2024-09-21T09:55:08.911392+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:53.012266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1545,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913260+0000","last_change":"2024-09-21T09:45:39.134465+0000","last_active":"2024-09-21T09:55:08.913260+0000","last_peered":"2024-09-21T09:55:08.913260+0000","last_clean":"2024-09-21T09:55:08.913260+0000","last_became_active":"2024-09-21T09:45:39.129692+0000","last_became_peered":"2024-09-21T09:45:39.129692+0000","last_unstale":"2024-09-21T09:55:08.913260+0000","last_undegraded":"2024-09-21T09:55:08.913260+0000","last_fullsized":"2024-09-21T09:55:08.913260+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:14.288180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1036,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412260+0000","last_change":"2024-09-21T09:45:39.131285+0000","last_active":"2024-09-21T09:55:03.412260+0000","last_peered":"2024-09-21T09:55:03.412260+0000","last_clean":"2024-09-21T09:55:03.412260+0000","last_became_active":"2024-09-21T09:45:39.130505+0000","last_became_peered":"2024-09-21T09:45:39.130505+0000","last_unstale":"2024-09-21T09:55:03.412260+0000","last_undegraded":"2024-09-21T09:55:03.412260+0000","last_fullsized":"2024-09-21T09:55:03.412260+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:37.523277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1018,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300486+0000","last_change":"2024-09-21T09:45:05.097518+0000","last_active":"2024-09-21T09:55:03.300486+0000","last_peered":"2024-09-21T09:55:03.300486+0000","last_clean":"2024-09-21T09:55:03.300486+0000","last_became_active":"2024-09-21T09:45:05.096918+0000","last_became_peered":"2024-09-21T09:45:05.096918+0000","last_unstale":"2024-09-21T09:55:03.300486+0000","last_undegraded":"2024-09-21T09:55:03.300486+0000","last_fullsized":"2024-09-21T09:55:03.300486+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:13.781847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1544,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911019+0000","last_change":"2024-09-21T09:46:47.113687+0000","last_active":"2024-09-21T09:55:08.911019+0000","last_peered":"2024-09-21T09:55:08.911019+0000","last_clean":"2024-09-21T09:55:08.911019+0000","last_became_active":"2024-09-21T09:46:47.113235+0000","last_became_peered":"2024-09-21T09:46:47.113235+0000","last_unstale":"2024-09-21T09:55:08.911019+0000","last_undegraded":"2024-09-21T09:55:08.911019+0000","last_fullsized":"2024-09-21T09:55:08.911019+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:50.851616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1038,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412109+0000","last_change":"2024-09-21T09:46:47.115673+0000","last_active":"2024-09-21T09:55:03.412109+0000","last_peered":"2024-09-21T09:55:03.412109+0000","last_clean":"2024-09-21T09:55:03.412109+0000","last_became_active":"2024-09-21T09:46:47.115166+0000","last_became_peered":"2024-09-21T09:46:47.115166+0000","last_unstale":"2024-09-21T09:55:03.412109+0000","last_undegraded":"2024-09-21T09:55:03.412109+0000","last_fullsized":"2024-09-21T09:55:03.412109+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:16.660834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1558,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911131+0000","last_change":"2024-09-21T09:45:39.132441+0000","last_active":"2024-09-21T09:55:08.911131+0000","last_peered":"2024-09-21T09:55:08.911131+0000","last_clean":"2024-09-21T09:55:08.911131+0000","last_became_active":"2024-09-21T09:45:39.129666+0000","last_became_peered":"2024-09-21T09:45:39.129666+0000","last_unstale":"2024-09-21T09:55:08.911131+0000","last_undegraded":"2024-09-21T09:55:08.911131+0000","last_fullsized":"2024-09-21T09:55:08.911131+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:34.450160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1497,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316744+0000","last_change":"2024-09-21T09:46:13.258244+0000","last_active":"2024-09-21T09:55:08.316744+0000","last_peered":"2024-09-21T09:55:08.316744+0000","last_clean":"2024-09-21T09:55:08.316744+0000","last_became_active":"2024-09-21T09:46:13.257820+0000","last_became_peered":"2024-09-21T09:46:13.257820+0000","last_unstale":"2024-09-21T09:55:08.316744+0000","last_undegraded":"2024-09-21T09:55:08.316744+0000","last_fullsized":"2024-09-21T09:55:08.316744+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:04.618304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1534,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911366+0000","last_change":"2024-09-21T09:46:13.260666+0000","last_active":"2024-09-21T09:55:08.911366+0000","last_peered":"2024-09-21T09:55:08.911366+0000","last_clean":"2024-09-21T09:55:08.911366+0000","last_became_active":"2024-09-21T09:46:13.260324+0000","last_became_peered":"2024-09-21T09:46:13.260324+0000","last_unstale":"2024-09-21T09:55:08.911366+0000","last_undegraded":"2024-09-21T09:55:08.911366+0000","last_fullsized":"2024-09-21T09:55:08.911366+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:25.140797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1011,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624946+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T09:55:03.624946+0000","last_peered":"2024-09-21T09:55:03.624946+0000","last_clean":"2024-09-21T09:55:03.624946+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T09:55:03.624946+0000","last_undegraded":"2024-09-21T09:55:03.624946+0000","last_fullsized":"2024-09-21T09:55:03.624946+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1561,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914110+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T09:55:08.914110+0000","last_peered":"2024-09-21T09:55:08.914110+0000","last_clean":"2024-09-21T09:55:08.914110+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T09:55:08.914110+0000","last_undegraded":"2024-09-21T09:55:08.914110+0000","last_fullsized":"2024-09-21T09:55:08.914110+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1496,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913455+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T09:55:08.913455+0000","last_peered":"2024-09-21T09:55:08.913455+0000","last_clean":"2024-09-21T09:55:08.913455+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T09:55:08.913455+0000","last_undegraded":"2024-09-21T09:55:08.913455+0000","last_fullsized":"2024-09-21T09:55:08.913455+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1532,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912680+0000","last_change":"2024-09-21T09:45:39.130168+0000","last_active":"2024-09-21T09:55:08.912680+0000","last_peered":"2024-09-21T09:55:08.912680+0000","last_clean":"2024-09-21T09:55:08.912680+0000","last_became_active":"2024-09-21T09:45:39.129498+0000","last_became_peered":"2024-09-21T09:45:39.129498+0000","last_unstale":"2024-09-21T09:55:08.912680+0000","last_undegraded":"2024-09-21T09:55:08.912680+0000","last_fullsized":"2024-09-21T09:55:08.912680+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:58.310876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1493,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912070+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T09:55:08.912070+0000","last_peered":"2024-09-21T09:55:08.912070+0000","last_clean":"2024-09-21T09:55:08.912070+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T09:55:08.912070+0000","last_undegraded":"2024-09-21T09:55:08.912070+0000","last_fullsized":"2024-09-21T09:55:08.912070+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1574,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912871+0000","last_change":"2024-09-21T09:45:39.140762+0000","last_active":"2024-09-21T09:55:08.912871+0000","last_peered":"2024-09-21T09:55:08.912871+0000","last_clean":"2024-09-21T09:55:08.912871+0000","last_became_active":"2024-09-21T09:45:39.140402+0000","last_became_peered":"2024-09-21T09:45:39.140402+0000","last_unstale":"2024-09-21T09:55:08.912871+0000","last_undegraded":"2024-09-21T09:55:08.912871+0000","last_fullsized":"2024-09-21T09:55:08.912871+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:06.545428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1012,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049926+0000","last_change":"2024-09-21T09:46:13.260656+0000","last_active":"2024-09-21T09:55:04.049926+0000","last_peered":"2024-09-21T09:55:04.049926+0000","last_clean":"2024-09-21T09:55:04.049926+0000","last_became_active":"2024-09-21T09:46:13.260368+0000","last_became_peered":"2024-09-21T09:46:13.260368+0000","last_unstale":"2024-09-21T09:55:04.049926+0000","last_undegraded":"2024-09-21T09:55:04.049926+0000","last_fullsized":"2024-09-21T09:55:04.049926+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:22.059036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300041+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:55:03.300041+0000","last_peered":"2024-09-21T09:55:03.300041+0000","last_clean":"2024-09-21T09:55:03.300041+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:55:03.300041+0000","last_undegraded":"2024-09-21T09:55:03.300041+0000","last_fullsized":"2024-09-21T09:55:03.300041+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300296+0000","last_change":"2024-09-21T09:45:39.129538+0000","last_active":"2024-09-21T09:55:03.300296+0000","last_peered":"2024-09-21T09:55:03.300296+0000","last_clean":"2024-09-21T09:55:03.300296+0000","last_became_active":"2024-09-21T09:45:39.129198+0000","last_became_peered":"2024-09-21T09:45:39.129198+0000","last_unstale":"2024-09-21T09:55:03.300296+0000","last_undegraded":"2024-09-21T09:55:03.300296+0000","last_fullsized":"2024-09-21T09:55:03.300296+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.798277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1023,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300173+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:55:03.300173+0000","last_peered":"2024-09-21T09:55:03.300173+0000","last_clean":"2024-09-21T09:55:03.300173+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:55:03.300173+0000","last_undegraded":"2024-09-21T09:55:03.300173+0000","last_fullsized":"2024-09-21T09:55:03.300173+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1533,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.910728+0000","last_change":"2024-09-21T09:46:13.254014+0000","last_active":"2024-09-21T09:55:08.910728+0000","last_peered":"2024-09-21T09:55:08.910728+0000","last_clean":"2024-09-21T09:55:08.910728+0000","last_became_active":"2024-09-21T09:46:13.253673+0000","last_became_peered":"2024-09-21T09:46:13.253673+0000","last_unstale":"2024-09-21T09:55:08.910728+0000","last_undegraded":"2024-09-21T09:55:08.910728+0000","last_fullsized":"2024-09-21T09:55:08.910728+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:02.784613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1002,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624515+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T09:55:03.624515+0000","last_peered":"2024-09-21T09:55:03.624515+0000","last_clean":"2024-09-21T09:55:03.624515+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T09:55:03.624515+0000","last_undegraded":"2024-09-21T09:55:03.624515+0000","last_fullsized":"2024-09-21T09:55:03.624515+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1022,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412623+0000","last_change":"2024-09-21T09:45:39.130886+0000","last_active":"2024-09-21T09:55:03.412623+0000","last_peered":"2024-09-21T09:55:03.412623+0000","last_clean":"2024-09-21T09:55:03.412623+0000","last_became_active":"2024-09-21T09:45:39.130559+0000","last_became_peered":"2024-09-21T09:45:39.130559+0000","last_unstale":"2024-09-21T09:55:03.412623+0000","last_undegraded":"2024-09-21T09:55:03.412623+0000","last_fullsized":"2024-09-21T09:55:03.412623+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:39.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1468,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911482+0000","last_change":"2024-09-21T09:46:47.122643+0000","last_active":"2024-09-21T09:55:08.911482+0000","last_peered":"2024-09-21T09:55:08.911482+0000","last_clean":"2024-09-21T09:55:08.911482+0000","last_became_active":"2024-09-21T09:46:47.122364+0000","last_became_peered":"2024-09-21T09:46:47.122364+0000","last_unstale":"2024-09-21T09:55:08.911482+0000","last_undegraded":"2024-09-21T09:55:08.911482+0000","last_fullsized":"2024-09-21T09:55:08.911482+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:29.222333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1569,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914102+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T09:55:08.914102+0000","last_peered":"2024-09-21T09:55:08.914102+0000","last_clean":"2024-09-21T09:55:08.914102+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T09:55:08.914102+0000","last_undegraded":"2024-09-21T09:55:08.914102+0000","last_fullsized":"2024-09-21T09:55:08.914102+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911965+0000","last_change":"2024-09-21T09:46:47.121360+0000","last_active":"2024-09-21T09:55:08.911965+0000","last_peered":"2024-09-21T09:55:08.911965+0000","last_clean":"2024-09-21T09:55:08.911965+0000","last_became_active":"2024-09-21T09:46:47.120494+0000","last_became_peered":"2024-09-21T09:46:47.120494+0000","last_unstale":"2024-09-21T09:55:08.911965+0000","last_undegraded":"2024-09-21T09:55:08.911965+0000","last_fullsized":"2024-09-21T09:55:08.911965+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:45.862984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1009,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049924+0000","last_change":"2024-09-21T09:46:13.260037+0000","last_active":"2024-09-21T09:55:04.049924+0000","last_peered":"2024-09-21T09:55:04.049924+0000","last_clean":"2024-09-21T09:55:04.049924+0000","last_became_active":"2024-09-21T09:46:13.259690+0000","last_became_peered":"2024-09-21T09:46:13.259690+0000","last_unstale":"2024-09-21T09:55:04.049924+0000","last_undegraded":"2024-09-21T09:55:04.049924+0000","last_fullsized":"2024-09-21T09:55:04.049924+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:57.466762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1503,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912173+0000","last_change":"2024-09-21T09:46:47.120830+0000","last_active":"2024-09-21T09:55:08.912173+0000","last_peered":"2024-09-21T09:55:08.912173+0000","last_clean":"2024-09-21T09:55:08.912173+0000","last_became_active":"2024-09-21T09:46:47.118419+0000","last_became_peered":"2024-09-21T09:46:47.118419+0000","last_unstale":"2024-09-21T09:55:08.912173+0000","last_undegraded":"2024-09-21T09:55:08.912173+0000","last_fullsized":"2024-09-21T09:55:08.912173+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:13.789806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1572,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912600+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T09:55:08.912600+0000","last_peered":"2024-09-21T09:55:08.912600+0000","last_clean":"2024-09-21T09:55:08.912600+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T09:55:08.912600+0000","last_undegraded":"2024-09-21T09:55:08.912600+0000","last_fullsized":"2024-09-21T09:55:08.912600+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1534,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.910698+0000","last_change":"2024-09-21T09:46:47.121724+0000","last_active":"2024-09-21T09:55:08.910698+0000","last_peered":"2024-09-21T09:55:08.910698+0000","last_clean":"2024-09-21T09:55:08.910698+0000","last_became_active":"2024-09-21T09:46:47.121406+0000","last_became_peered":"2024-09-21T09:46:47.121406+0000","last_unstale":"2024-09-21T09:55:08.910698+0000","last_undegraded":"2024-09-21T09:55:08.910698+0000","last_fullsized":"2024-09-21T09:55:08.910698+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:55.493704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911362+0000","last_change":"2024-09-21T09:46:47.113184+0000","last_active":"2024-09-21T09:55:08.911362+0000","last_peered":"2024-09-21T09:55:08.911362+0000","last_clean":"2024-09-21T09:55:08.911362+0000","last_became_active":"2024-09-21T09:46:47.112857+0000","last_became_peered":"2024-09-21T09:46:47.112857+0000","last_unstale":"2024-09-21T09:55:08.911362+0000","last_undegraded":"2024-09-21T09:55:08.911362+0000","last_fullsized":"2024-09-21T09:55:08.911362+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:46.783986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1036,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300495+0000","last_change":"2024-09-21T09:46:13.262555+0000","last_active":"2024-09-21T09:55:03.300495+0000","last_peered":"2024-09-21T09:55:03.300495+0000","last_clean":"2024-09-21T09:55:03.300495+0000","last_became_active":"2024-09-21T09:46:13.261474+0000","last_became_peered":"2024-09-21T09:46:13.261474+0000","last_unstale":"2024-09-21T09:55:03.300495+0000","last_undegraded":"2024-09-21T09:55:03.300495+0000","last_fullsized":"2024-09-21T09:55:03.300495+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:44.865988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1523,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316853+0000","last_change":"2024-09-21T09:46:47.114406+0000","last_active":"2024-09-21T09:55:08.316853+0000","last_peered":"2024-09-21T09:55:08.316853+0000","last_clean":"2024-09-21T09:55:08.316853+0000","last_became_active":"2024-09-21T09:46:47.113986+0000","last_became_peered":"2024-09-21T09:46:47.113986+0000","last_unstale":"2024-09-21T09:55:08.316853+0000","last_undegraded":"2024-09-21T09:55:08.316853+0000","last_fullsized":"2024-09-21T09:55:08.316853+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:32.292201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1562,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914239+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T09:55:08.914239+0000","last_peered":"2024-09-21T09:55:08.914239+0000","last_clean":"2024-09-21T09:55:08.914239+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T09:55:08.914239+0000","last_undegraded":"2024-09-21T09:55:08.914239+0000","last_fullsized":"2024-09-21T09:55:08.914239+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300266+0000","last_change":"2024-09-21T09:46:47.114188+0000","last_active":"2024-09-21T09:55:03.300266+0000","last_peered":"2024-09-21T09:55:03.300266+0000","last_clean":"2024-09-21T09:55:03.300266+0000","last_became_active":"2024-09-21T09:46:47.113859+0000","last_became_peered":"2024-09-21T09:46:47.113859+0000","last_unstale":"2024-09-21T09:55:03.300266+0000","last_undegraded":"2024-09-21T09:55:03.300266+0000","last_fullsized":"2024-09-21T09:55:03.300266+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:22.670368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":1024,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412132+0000","last_change":"2024-09-21T09:45:39.131124+0000","last_active":"2024-09-21T09:55:03.412132+0000","last_peered":"2024-09-21T09:55:03.412132+0000","last_clean":"2024-09-21T09:55:03.412132+0000","last_became_active":"2024-09-21T09:45:39.130756+0000","last_became_peered":"2024-09-21T09:45:39.130756+0000","last_unstale":"2024-09-21T09:55:03.412132+0000","last_undegraded":"2024-09-21T09:55:03.412132+0000","last_fullsized":"2024-09-21T09:55:03.412132+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:49.130698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300392+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T09:55:03.300392+0000","last_peered":"2024-09-21T09:55:03.300392+0000","last_clean":"2024-09-21T09:55:03.300392+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T09:55:03.300392+0000","last_undegraded":"2024-09-21T09:55:03.300392+0000","last_fullsized":"2024-09-21T09:55:03.300392+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":1006,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412104+0000","last_change":"2024-09-21T09:36:43.115791+0000","last_active":"2024-09-21T09:55:03.412104+0000","last_peered":"2024-09-21T09:55:03.412104+0000","last_clean":"2024-09-21T09:55:03.412104+0000","last_became_active":"2024-09-21T09:36:43.115525+0000","last_became_peered":"2024-09-21T09:36:43.115525+0000","last_unstale":"2024-09-21T09:55:03.412104+0000","last_undegraded":"2024-09-21T09:55:03.412104+0000","last_fullsized":"2024-09-21T09:55:03.412104+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:28.396722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1495,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911833+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T09:55:08.911833+0000","last_peered":"2024-09-21T09:55:08.911833+0000","last_clean":"2024-09-21T09:55:08.911833+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T09:55:08.911833+0000","last_undegraded":"2024-09-21T09:55:08.911833+0000","last_fullsized":"2024-09-21T09:55:08.911833+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1481,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913152+0000","last_change":"2024-09-21T09:46:47.115788+0000","last_active":"2024-09-21T09:55:08.913152+0000","last_peered":"2024-09-21T09:55:08.913152+0000","last_clean":"2024-09-21T09:55:08.913152+0000","last_became_active":"2024-09-21T09:46:47.114864+0000","last_became_peered":"2024-09-21T09:46:47.114864+0000","last_unstale":"2024-09-21T09:55:08.913152+0000","last_undegraded":"2024-09-21T09:55:08.913152+0000","last_fullsized":"2024-09-21T09:55:08.913152+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:00.034944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.299925+0000","last_change":"2024-09-21T09:45:39.143064+0000","last_active":"2024-09-21T09:55:03.299925+0000","last_peered":"2024-09-21T09:55:03.299925+0000","last_clean":"2024-09-21T09:55:03.299925+0000","last_became_active":"2024-09-21T09:45:39.142699+0000","last_became_peered":"2024-09-21T09:45:39.142699+0000","last_unstale":"2024-09-21T09:55:03.299925+0000","last_undegraded":"2024-09-21T09:55:03.299925+0000","last_fullsized":"2024-09-21T09:55:03.299925+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:14.684721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1577,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914281+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T09:55:08.914281+0000","last_peered":"2024-09-21T09:55:08.914281+0000","last_clean":"2024-09-21T09:55:08.914281+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T09:55:08.914281+0000","last_undegraded":"2024-09-21T09:55:08.914281+0000","last_fullsized":"2024-09-21T09:55:08.914281+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397761,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317368,"kb_used_data":7532,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93427784,"statfs":{"total":95995035648,"available":95670050816,"internally_reserved":0,"allocated":7712768,"data_stored":5600360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593990,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313200,"kb_used_data":7524,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93431952,"statfs":{"total":95995035648,"available":95674318848,"internally_reserved":0,"allocated":7704576,"data_stored":5600225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790219,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316768,"kb_used_data":6936,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93428384,"statfs":{"total":95995035648,"available":95670665216,"internally_reserved":0,"allocated":7102464,"data_stored":5005683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986446,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308460,"kb_used_data":6948,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93436692,"statfs":{"total":95995035648,"available":95679172608,"internally_reserved":0,"allocated":7114752,"data_stored":5009914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999088,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":48744,"kb_used_data":8608,"kb_used_omap":25,"kb_used_meta":40102,"kb_avail":93696408,"statfs":{"total":95995035648,"available":95945121792,"internally_reserved":0,"allocated":8814592,"data_stored":6219336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":41065415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111031,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40984,"kb_used_data":8792,"kb_used_omap":31,"kb_used_meta":31968,"kb_avail":93704168,"statfs":{"total":95995035648,"available":95953068032,"internally_reserved":0,"allocated":9003008,"data_stored":6349770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":32735464},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318222974,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40672,"kb_used_data":8608,"kb_used_omap":26,"kb_used_meta":31845,"kb_avail":93704480,"statfs":{"total":95995035648,"available":95953387520,"internally_reserved":0,"allocated":8814592,"data_stored":6219273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":32609292},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289334916,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67868,"kb_used_data":9372,"kb_used_omap":31,"kb_used_meta":58272,"kb_avail":93677284,"statfs":{"total":95995035648,"available":95925538816,"internally_reserved":0,"allocated":9596928,"data_stored":6940134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":59670624},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:56:00.135 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:56:00.136 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:56:00.136 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:56:00.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:56:00.392 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:56:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:56:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:00.786 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:00 smithi017 ceph-mon[102913]: from='client.35229 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:00.786 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:00 smithi017 ceph-mon[105151]: from='client.35229 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:00 smithi134 ceph-mon[90085]: from='client.35229 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:01.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:01 smithi017 ceph-mon[102913]: pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:01.830 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:01 smithi017 ceph-mon[105151]: pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:01 smithi134 ceph-mon[90085]: pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:02.262 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:56:02.262 INFO:teuthology.orchestra.run.smithi017.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:56:02.815 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:56:02.815 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:56:02.830 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-09-21T09:56:02.830 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T09:56:02.838 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T09:56:02.839 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:56:02.839 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-09-21T09:56:02.841 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-21T09:56:02.841 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:56:02.841 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-21T09:56:02.841 INFO:tasks.thrashosds:Beginning thrashosds... 2024-09-21T09:56:02.841 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:56:02.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:56:02.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4069549427' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-21T09:56:02.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4069549427' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-21T09:56:02.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4069549427' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-21T09:56:03.096 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:03 smithi017 ceph-mon[102913]: pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:03 smithi017 ceph-mon[105151]: pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:03 smithi134 ceph-mon[90085]: pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:04.880 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:56:04.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:56:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:56:04] "GET /metrics HTTP/1.1" 200 34065 "" "Prometheus/2.51.0" 2024-09-21T09:56:04.882 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:56:05.563 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1225,"stamp":"2024-09-21T09:56:03.981057+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":212,"num_write_kb":1659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158,"ondisk_log_size":158,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1454064,"kb_used_data":64320,"kb_used_omap":122,"kb_used_meta":1388997,"kb_avail":748507152,"statfs":{"total":767960285184,"available":766471323648,"internally_reserved":0,"allocated":65863680,"data_stored":46944695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125317,"internal_metadata":1422333563},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003053"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911046+0000","last_change":"2024-09-21T09:46:47.114206+0000","last_active":"2024-09-21T09:55:08.911046+0000","last_peered":"2024-09-21T09:55:08.911046+0000","last_clean":"2024-09-21T09:55:08.911046+0000","last_became_active":"2024-09-21T09:46:47.113819+0000","last_became_peered":"2024-09-21T09:46:47.113819+0000","last_unstale":"2024-09-21T09:55:08.911046+0000","last_undegraded":"2024-09-21T09:55:08.911046+0000","last_fullsized":"2024-09-21T09:55:08.911046+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:07.629544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1001,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624302+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T09:55:03.624302+0000","last_peered":"2024-09-21T09:55:03.624302+0000","last_clean":"2024-09-21T09:55:03.624302+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T09:55:03.624302+0000","last_undegraded":"2024-09-21T09:55:03.624302+0000","last_fullsized":"2024-09-21T09:55:03.624302+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1020,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412012+0000","last_change":"2024-09-21T09:46:47.115484+0000","last_active":"2024-09-21T09:55:03.412012+0000","last_peered":"2024-09-21T09:55:03.412012+0000","last_clean":"2024-09-21T09:55:03.412012+0000","last_became_active":"2024-09-21T09:46:47.115080+0000","last_became_peered":"2024-09-21T09:46:47.115080+0000","last_unstale":"2024-09-21T09:55:03.412012+0000","last_undegraded":"2024-09-21T09:55:03.412012+0000","last_fullsized":"2024-09-21T09:55:03.412012+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:56.851805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1569,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912062+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T09:55:08.912062+0000","last_peered":"2024-09-21T09:55:08.912062+0000","last_clean":"2024-09-21T09:55:08.912062+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T09:55:08.912062+0000","last_undegraded":"2024-09-21T09:55:08.912062+0000","last_fullsized":"2024-09-21T09:55:08.912062+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1571,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912710+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T09:55:08.912710+0000","last_peered":"2024-09-21T09:55:08.912710+0000","last_clean":"2024-09-21T09:55:08.912710+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T09:55:08.912710+0000","last_undegraded":"2024-09-21T09:55:08.912710+0000","last_fullsized":"2024-09-21T09:55:08.912710+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":982,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624220+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:55:03.624220+0000","last_peered":"2024-09-21T09:55:03.624220+0000","last_clean":"2024-09-21T09:55:03.624220+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:55:03.624220+0000","last_undegraded":"2024-09-21T09:55:03.624220+0000","last_fullsized":"2024-09-21T09:55:03.624220+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912905+0000","last_change":"2024-09-21T09:46:47.117580+0000","last_active":"2024-09-21T09:55:08.912905+0000","last_peered":"2024-09-21T09:55:08.912905+0000","last_clean":"2024-09-21T09:55:08.912905+0000","last_became_active":"2024-09-21T09:46:47.113030+0000","last_became_peered":"2024-09-21T09:46:47.113030+0000","last_unstale":"2024-09-21T09:55:08.912905+0000","last_undegraded":"2024-09-21T09:55:08.912905+0000","last_fullsized":"2024-09-21T09:55:08.912905+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:28.220180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":1008,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049340+0000","last_change":"2024-09-21T09:45:05.080014+0000","last_active":"2024-09-21T09:55:04.049340+0000","last_peered":"2024-09-21T09:55:04.049340+0000","last_clean":"2024-09-21T09:55:04.049340+0000","last_became_active":"2024-09-21T09:45:05.079618+0000","last_became_peered":"2024-09-21T09:45:05.079618+0000","last_unstale":"2024-09-21T09:55:04.049340+0000","last_undegraded":"2024-09-21T09:55:04.049340+0000","last_fullsized":"2024-09-21T09:55:04.049340+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:42.160292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624817+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T09:55:03.624817+0000","last_peered":"2024-09-21T09:55:03.624817+0000","last_clean":"2024-09-21T09:55:03.624817+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T09:55:03.624817+0000","last_undegraded":"2024-09-21T09:55:03.624817+0000","last_fullsized":"2024-09-21T09:55:03.624817+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1004,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.411948+0000","last_change":"2024-09-21T09:36:43.389124+0000","last_active":"2024-09-21T09:55:03.411948+0000","last_peered":"2024-09-21T09:55:03.411948+0000","last_clean":"2024-09-21T09:55:03.411948+0000","last_became_active":"2024-09-21T09:36:43.388588+0000","last_became_peered":"2024-09-21T09:36:43.388588+0000","last_unstale":"2024-09-21T09:55:03.411948+0000","last_undegraded":"2024-09-21T09:55:03.411948+0000","last_fullsized":"2024-09-21T09:55:03.411948+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:57.562156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1001,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624842+0000","last_change":"2024-09-21T09:45:39.143190+0000","last_active":"2024-09-21T09:55:03.624842+0000","last_peered":"2024-09-21T09:55:03.624842+0000","last_clean":"2024-09-21T09:55:03.624842+0000","last_became_active":"2024-09-21T09:45:39.142826+0000","last_became_peered":"2024-09-21T09:45:39.142826+0000","last_unstale":"2024-09-21T09:55:03.624842+0000","last_undegraded":"2024-09-21T09:55:03.624842+0000","last_fullsized":"2024-09-21T09:55:03.624842+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:45.070945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300475+0000","last_change":"2024-09-21T09:46:47.119247+0000","last_active":"2024-09-21T09:55:03.300475+0000","last_peered":"2024-09-21T09:55:03.300475+0000","last_clean":"2024-09-21T09:55:03.300475+0000","last_became_active":"2024-09-21T09:46:47.118980+0000","last_became_peered":"2024-09-21T09:46:47.118980+0000","last_unstale":"2024-09-21T09:55:03.300475+0000","last_undegraded":"2024-09-21T09:55:03.300475+0000","last_fullsized":"2024-09-21T09:55:03.300475+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.140253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":983,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624823+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:55:03.624823+0000","last_peered":"2024-09-21T09:55:03.624823+0000","last_clean":"2024-09-21T09:55:03.624823+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:55:03.624823+0000","last_undegraded":"2024-09-21T09:55:03.624823+0000","last_fullsized":"2024-09-21T09:55:03.624823+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1009,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049323+0000","last_change":"2024-09-21T09:46:13.260744+0000","last_active":"2024-09-21T09:55:04.049323+0000","last_peered":"2024-09-21T09:55:04.049323+0000","last_clean":"2024-09-21T09:55:04.049323+0000","last_became_active":"2024-09-21T09:46:13.260443+0000","last_became_peered":"2024-09-21T09:46:13.260443+0000","last_unstale":"2024-09-21T09:55:04.049323+0000","last_undegraded":"2024-09-21T09:55:04.049323+0000","last_fullsized":"2024-09-21T09:55:04.049323+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:22.699110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1022,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412189+0000","last_change":"2024-09-21T09:46:47.115343+0000","last_active":"2024-09-21T09:55:03.412189+0000","last_peered":"2024-09-21T09:55:03.412189+0000","last_clean":"2024-09-21T09:55:03.412189+0000","last_became_active":"2024-09-21T09:46:47.115017+0000","last_became_peered":"2024-09-21T09:46:47.115017+0000","last_unstale":"2024-09-21T09:55:03.412189+0000","last_undegraded":"2024-09-21T09:55:03.412189+0000","last_fullsized":"2024-09-21T09:55:03.412189+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.820662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1571,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912938+0000","last_change":"2024-09-21T09:45:39.138289+0000","last_active":"2024-09-21T09:55:08.912938+0000","last_peered":"2024-09-21T09:55:08.912938+0000","last_clean":"2024-09-21T09:55:08.912938+0000","last_became_active":"2024-09-21T09:45:39.137902+0000","last_became_peered":"2024-09-21T09:45:39.137902+0000","last_unstale":"2024-09-21T09:55:08.912938+0000","last_undegraded":"2024-09-21T09:55:08.912938+0000","last_fullsized":"2024-09-21T09:55:08.912938+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:40:16.383906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1496,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911031+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T09:55:08.911031+0000","last_peered":"2024-09-21T09:55:08.911031+0000","last_clean":"2024-09-21T09:55:08.911031+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T09:55:08.911031+0000","last_undegraded":"2024-09-21T09:55:08.911031+0000","last_fullsized":"2024-09-21T09:55:08.911031+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300113+0000","last_change":"2024-09-21T09:46:47.114293+0000","last_active":"2024-09-21T09:55:03.300113+0000","last_peered":"2024-09-21T09:55:03.300113+0000","last_clean":"2024-09-21T09:55:03.300113+0000","last_became_active":"2024-09-21T09:46:47.113993+0000","last_became_peered":"2024-09-21T09:46:47.113993+0000","last_unstale":"2024-09-21T09:55:03.300113+0000","last_undegraded":"2024-09-21T09:55:03.300113+0000","last_fullsized":"2024-09-21T09:55:03.300113+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:30.941931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049541+0000","last_change":"2024-09-21T09:45:39.127186+0000","last_active":"2024-09-21T09:55:04.049541+0000","last_peered":"2024-09-21T09:55:04.049541+0000","last_clean":"2024-09-21T09:55:04.049541+0000","last_became_active":"2024-09-21T09:45:39.126940+0000","last_became_peered":"2024-09-21T09:45:39.126940+0000","last_unstale":"2024-09-21T09:55:04.049541+0000","last_undegraded":"2024-09-21T09:55:04.049541+0000","last_fullsized":"2024-09-21T09:55:04.049541+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:52.205703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300795+0000","last_change":"2024-09-21T09:46:13.259814+0000","last_active":"2024-09-21T09:55:03.300795+0000","last_peered":"2024-09-21T09:55:03.300795+0000","last_clean":"2024-09-21T09:55:03.300795+0000","last_became_active":"2024-09-21T09:46:13.259527+0000","last_became_peered":"2024-09-21T09:46:13.259527+0000","last_unstale":"2024-09-21T09:55:03.300795+0000","last_undegraded":"2024-09-21T09:55:03.300795+0000","last_fullsized":"2024-09-21T09:55:03.300795+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:00.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1018,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300593+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T09:55:03.300593+0000","last_peered":"2024-09-21T09:55:03.300593+0000","last_clean":"2024-09-21T09:55:03.300593+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T09:55:03.300593+0000","last_undegraded":"2024-09-21T09:55:03.300593+0000","last_fullsized":"2024-09-21T09:55:03.300593+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300762+0000","last_change":"2024-09-21T09:45:39.143302+0000","last_active":"2024-09-21T09:55:03.300762+0000","last_peered":"2024-09-21T09:55:03.300762+0000","last_clean":"2024-09-21T09:55:03.300762+0000","last_became_active":"2024-09-21T09:45:39.142964+0000","last_became_peered":"2024-09-21T09:45:39.142964+0000","last_unstale":"2024-09-21T09:55:03.300762+0000","last_undegraded":"2024-09-21T09:55:03.300762+0000","last_fullsized":"2024-09-21T09:55:03.300762+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:54.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1487,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913222+0000","last_change":"2024-09-21T09:46:47.121643+0000","last_active":"2024-09-21T09:55:08.913222+0000","last_peered":"2024-09-21T09:55:08.913222+0000","last_clean":"2024-09-21T09:55:08.913222+0000","last_became_active":"2024-09-21T09:46:47.120528+0000","last_became_peered":"2024-09-21T09:46:47.120528+0000","last_unstale":"2024-09-21T09:55:08.913222+0000","last_undegraded":"2024-09-21T09:55:08.913222+0000","last_fullsized":"2024-09-21T09:55:08.913222+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:07.989377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1533,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913431+0000","last_change":"2024-09-21T09:46:47.121159+0000","last_active":"2024-09-21T09:55:08.913431+0000","last_peered":"2024-09-21T09:55:08.913431+0000","last_clean":"2024-09-21T09:55:08.913431+0000","last_became_active":"2024-09-21T09:46:47.120829+0000","last_became_peered":"2024-09-21T09:46:47.120829+0000","last_unstale":"2024-09-21T09:55:08.913431+0000","last_undegraded":"2024-09-21T09:55:08.913431+0000","last_fullsized":"2024-09-21T09:55:08.913431+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:43.393840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1116,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624090+0000","last_change":"2024-09-21T09:45:05.087928+0000","last_active":"2024-09-21T09:55:03.624090+0000","last_peered":"2024-09-21T09:55:03.624090+0000","last_clean":"2024-09-21T09:55:03.624090+0000","last_became_active":"2024-09-21T09:45:05.087478+0000","last_became_peered":"2024-09-21T09:45:05.087478+0000","last_unstale":"2024-09-21T09:55:03.624090+0000","last_undegraded":"2024-09-21T09:55:03.624090+0000","last_fullsized":"2024-09-21T09:55:03.624090+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:48.651314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1035,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412049+0000","last_change":"2024-09-21T09:46:13.262337+0000","last_active":"2024-09-21T09:55:03.412049+0000","last_peered":"2024-09-21T09:55:03.412049+0000","last_clean":"2024-09-21T09:55:03.412049+0000","last_became_active":"2024-09-21T09:46:13.262005+0000","last_became_peered":"2024-09-21T09:46:13.262005+0000","last_unstale":"2024-09-21T09:55:03.412049+0000","last_undegraded":"2024-09-21T09:55:03.412049+0000","last_fullsized":"2024-09-21T09:55:03.412049+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:32.413210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1469,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912212+0000","last_change":"2024-09-21T09:46:47.116045+0000","last_active":"2024-09-21T09:55:08.912212+0000","last_peered":"2024-09-21T09:55:08.912212+0000","last_clean":"2024-09-21T09:55:08.912212+0000","last_became_active":"2024-09-21T09:46:47.115526+0000","last_became_peered":"2024-09-21T09:46:47.115526+0000","last_unstale":"2024-09-21T09:55:08.912212+0000","last_undegraded":"2024-09-21T09:55:08.912212+0000","last_fullsized":"2024-09-21T09:55:08.912212+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:43:35.293486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":982,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049787+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:55:04.049787+0000","last_peered":"2024-09-21T09:55:04.049787+0000","last_clean":"2024-09-21T09:55:04.049787+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:55:04.049787+0000","last_undegraded":"2024-09-21T09:55:04.049787+0000","last_fullsized":"2024-09-21T09:55:04.049787+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1028,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624013+0000","last_change":"2024-09-21T09:45:39.129555+0000","last_active":"2024-09-21T09:55:03.624013+0000","last_peered":"2024-09-21T09:55:03.624013+0000","last_clean":"2024-09-21T09:55:03.624013+0000","last_became_active":"2024-09-21T09:45:39.129205+0000","last_became_peered":"2024-09-21T09:45:39.129205+0000","last_unstale":"2024-09-21T09:55:03.624013+0000","last_undegraded":"2024-09-21T09:55:03.624013+0000","last_fullsized":"2024-09-21T09:55:03.624013+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:07.229126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1570,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913207+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T09:55:08.913207+0000","last_peered":"2024-09-21T09:55:08.913207+0000","last_clean":"2024-09-21T09:55:08.913207+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T09:55:08.913207+0000","last_undegraded":"2024-09-21T09:55:08.913207+0000","last_fullsized":"2024-09-21T09:55:08.913207+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1583,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912533+0000","last_change":"2024-09-21T09:46:13.273573+0000","last_active":"2024-09-21T09:55:08.912533+0000","last_peered":"2024-09-21T09:55:08.912533+0000","last_clean":"2024-09-21T09:55:08.912533+0000","last_became_active":"2024-09-21T09:46:13.273297+0000","last_became_peered":"2024-09-21T09:46:13.273297+0000","last_unstale":"2024-09-21T09:55:08.912533+0000","last_undegraded":"2024-09-21T09:55:08.912533+0000","last_fullsized":"2024-09-21T09:55:08.912533+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:38.957745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1544,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316664+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T09:55:08.316664+0000","last_peered":"2024-09-21T09:55:08.316664+0000","last_clean":"2024-09-21T09:55:08.316664+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T09:55:08.316664+0000","last_undegraded":"2024-09-21T09:55:08.316664+0000","last_fullsized":"2024-09-21T09:55:08.316664+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1535,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911392+0000","last_change":"2024-09-21T09:46:13.255342+0000","last_active":"2024-09-21T09:55:08.911392+0000","last_peered":"2024-09-21T09:55:08.911392+0000","last_clean":"2024-09-21T09:55:08.911392+0000","last_became_active":"2024-09-21T09:46:13.255014+0000","last_became_peered":"2024-09-21T09:46:13.255014+0000","last_unstale":"2024-09-21T09:55:08.911392+0000","last_undegraded":"2024-09-21T09:55:08.911392+0000","last_fullsized":"2024-09-21T09:55:08.911392+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:53.012266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1545,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913260+0000","last_change":"2024-09-21T09:45:39.134465+0000","last_active":"2024-09-21T09:55:08.913260+0000","last_peered":"2024-09-21T09:55:08.913260+0000","last_clean":"2024-09-21T09:55:08.913260+0000","last_became_active":"2024-09-21T09:45:39.129692+0000","last_became_peered":"2024-09-21T09:45:39.129692+0000","last_unstale":"2024-09-21T09:55:08.913260+0000","last_undegraded":"2024-09-21T09:55:08.913260+0000","last_fullsized":"2024-09-21T09:55:08.913260+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:14.288180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1036,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412260+0000","last_change":"2024-09-21T09:45:39.131285+0000","last_active":"2024-09-21T09:55:03.412260+0000","last_peered":"2024-09-21T09:55:03.412260+0000","last_clean":"2024-09-21T09:55:03.412260+0000","last_became_active":"2024-09-21T09:45:39.130505+0000","last_became_peered":"2024-09-21T09:45:39.130505+0000","last_unstale":"2024-09-21T09:55:03.412260+0000","last_undegraded":"2024-09-21T09:55:03.412260+0000","last_fullsized":"2024-09-21T09:55:03.412260+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:37.523277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1018,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300486+0000","last_change":"2024-09-21T09:45:05.097518+0000","last_active":"2024-09-21T09:55:03.300486+0000","last_peered":"2024-09-21T09:55:03.300486+0000","last_clean":"2024-09-21T09:55:03.300486+0000","last_became_active":"2024-09-21T09:45:05.096918+0000","last_became_peered":"2024-09-21T09:45:05.096918+0000","last_unstale":"2024-09-21T09:55:03.300486+0000","last_undegraded":"2024-09-21T09:55:03.300486+0000","last_fullsized":"2024-09-21T09:55:03.300486+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:13.781847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1545,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911071+0000","last_change":"2024-09-21T09:46:47.113687+0000","last_active":"2024-09-21T09:55:08.911071+0000","last_peered":"2024-09-21T09:55:08.911071+0000","last_clean":"2024-09-21T09:55:08.911071+0000","last_became_active":"2024-09-21T09:46:47.113235+0000","last_became_peered":"2024-09-21T09:46:47.113235+0000","last_unstale":"2024-09-21T09:55:08.911071+0000","last_undegraded":"2024-09-21T09:55:08.911071+0000","last_fullsized":"2024-09-21T09:55:08.911071+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:50.851616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1038,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412109+0000","last_change":"2024-09-21T09:46:47.115673+0000","last_active":"2024-09-21T09:55:03.412109+0000","last_peered":"2024-09-21T09:55:03.412109+0000","last_clean":"2024-09-21T09:55:03.412109+0000","last_became_active":"2024-09-21T09:46:47.115166+0000","last_became_peered":"2024-09-21T09:46:47.115166+0000","last_unstale":"2024-09-21T09:55:03.412109+0000","last_undegraded":"2024-09-21T09:55:03.412109+0000","last_fullsized":"2024-09-21T09:55:03.412109+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:16.660834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1559,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911179+0000","last_change":"2024-09-21T09:45:39.132441+0000","last_active":"2024-09-21T09:55:08.911179+0000","last_peered":"2024-09-21T09:55:08.911179+0000","last_clean":"2024-09-21T09:55:08.911179+0000","last_became_active":"2024-09-21T09:45:39.129666+0000","last_became_peered":"2024-09-21T09:45:39.129666+0000","last_unstale":"2024-09-21T09:55:08.911179+0000","last_undegraded":"2024-09-21T09:55:08.911179+0000","last_fullsized":"2024-09-21T09:55:08.911179+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:34.450160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1497,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316744+0000","last_change":"2024-09-21T09:46:13.258244+0000","last_active":"2024-09-21T09:55:08.316744+0000","last_peered":"2024-09-21T09:55:08.316744+0000","last_clean":"2024-09-21T09:55:08.316744+0000","last_became_active":"2024-09-21T09:46:13.257820+0000","last_became_peered":"2024-09-21T09:46:13.257820+0000","last_unstale":"2024-09-21T09:55:08.316744+0000","last_undegraded":"2024-09-21T09:55:08.316744+0000","last_fullsized":"2024-09-21T09:55:08.316744+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:04.618304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1535,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911419+0000","last_change":"2024-09-21T09:46:13.260666+0000","last_active":"2024-09-21T09:55:08.911419+0000","last_peered":"2024-09-21T09:55:08.911419+0000","last_clean":"2024-09-21T09:55:08.911419+0000","last_became_active":"2024-09-21T09:46:13.260324+0000","last_became_peered":"2024-09-21T09:46:13.260324+0000","last_unstale":"2024-09-21T09:55:08.911419+0000","last_undegraded":"2024-09-21T09:55:08.911419+0000","last_fullsized":"2024-09-21T09:55:08.911419+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:25.140797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1011,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624946+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T09:55:03.624946+0000","last_peered":"2024-09-21T09:55:03.624946+0000","last_clean":"2024-09-21T09:55:03.624946+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T09:55:03.624946+0000","last_undegraded":"2024-09-21T09:55:03.624946+0000","last_fullsized":"2024-09-21T09:55:03.624946+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1561,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914110+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T09:55:08.914110+0000","last_peered":"2024-09-21T09:55:08.914110+0000","last_clean":"2024-09-21T09:55:08.914110+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T09:55:08.914110+0000","last_undegraded":"2024-09-21T09:55:08.914110+0000","last_fullsized":"2024-09-21T09:55:08.914110+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1496,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913455+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T09:55:08.913455+0000","last_peered":"2024-09-21T09:55:08.913455+0000","last_clean":"2024-09-21T09:55:08.913455+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T09:55:08.913455+0000","last_undegraded":"2024-09-21T09:55:08.913455+0000","last_fullsized":"2024-09-21T09:55:08.913455+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1532,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912680+0000","last_change":"2024-09-21T09:45:39.130168+0000","last_active":"2024-09-21T09:55:08.912680+0000","last_peered":"2024-09-21T09:55:08.912680+0000","last_clean":"2024-09-21T09:55:08.912680+0000","last_became_active":"2024-09-21T09:45:39.129498+0000","last_became_peered":"2024-09-21T09:45:39.129498+0000","last_unstale":"2024-09-21T09:55:08.912680+0000","last_undegraded":"2024-09-21T09:55:08.912680+0000","last_fullsized":"2024-09-21T09:55:08.912680+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:58.310876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1493,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912070+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T09:55:08.912070+0000","last_peered":"2024-09-21T09:55:08.912070+0000","last_clean":"2024-09-21T09:55:08.912070+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T09:55:08.912070+0000","last_undegraded":"2024-09-21T09:55:08.912070+0000","last_fullsized":"2024-09-21T09:55:08.912070+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1574,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912871+0000","last_change":"2024-09-21T09:45:39.140762+0000","last_active":"2024-09-21T09:55:08.912871+0000","last_peered":"2024-09-21T09:55:08.912871+0000","last_clean":"2024-09-21T09:55:08.912871+0000","last_became_active":"2024-09-21T09:45:39.140402+0000","last_became_peered":"2024-09-21T09:45:39.140402+0000","last_unstale":"2024-09-21T09:55:08.912871+0000","last_undegraded":"2024-09-21T09:55:08.912871+0000","last_fullsized":"2024-09-21T09:55:08.912871+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:06.545428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1012,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049926+0000","last_change":"2024-09-21T09:46:13.260656+0000","last_active":"2024-09-21T09:55:04.049926+0000","last_peered":"2024-09-21T09:55:04.049926+0000","last_clean":"2024-09-21T09:55:04.049926+0000","last_became_active":"2024-09-21T09:46:13.260368+0000","last_became_peered":"2024-09-21T09:46:13.260368+0000","last_unstale":"2024-09-21T09:55:04.049926+0000","last_undegraded":"2024-09-21T09:55:04.049926+0000","last_fullsized":"2024-09-21T09:55:04.049926+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:22.059036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300041+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:55:03.300041+0000","last_peered":"2024-09-21T09:55:03.300041+0000","last_clean":"2024-09-21T09:55:03.300041+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:55:03.300041+0000","last_undegraded":"2024-09-21T09:55:03.300041+0000","last_fullsized":"2024-09-21T09:55:03.300041+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300296+0000","last_change":"2024-09-21T09:45:39.129538+0000","last_active":"2024-09-21T09:55:03.300296+0000","last_peered":"2024-09-21T09:55:03.300296+0000","last_clean":"2024-09-21T09:55:03.300296+0000","last_became_active":"2024-09-21T09:45:39.129198+0000","last_became_peered":"2024-09-21T09:45:39.129198+0000","last_unstale":"2024-09-21T09:55:03.300296+0000","last_undegraded":"2024-09-21T09:55:03.300296+0000","last_fullsized":"2024-09-21T09:55:03.300296+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.798277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1023,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300173+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:55:03.300173+0000","last_peered":"2024-09-21T09:55:03.300173+0000","last_clean":"2024-09-21T09:55:03.300173+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:55:03.300173+0000","last_undegraded":"2024-09-21T09:55:03.300173+0000","last_fullsized":"2024-09-21T09:55:03.300173+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1533,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.910728+0000","last_change":"2024-09-21T09:46:13.254014+0000","last_active":"2024-09-21T09:55:08.910728+0000","last_peered":"2024-09-21T09:55:08.910728+0000","last_clean":"2024-09-21T09:55:08.910728+0000","last_became_active":"2024-09-21T09:46:13.253673+0000","last_became_peered":"2024-09-21T09:46:13.253673+0000","last_unstale":"2024-09-21T09:55:08.910728+0000","last_undegraded":"2024-09-21T09:55:08.910728+0000","last_fullsized":"2024-09-21T09:55:08.910728+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:02.784613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1002,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624515+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T09:55:03.624515+0000","last_peered":"2024-09-21T09:55:03.624515+0000","last_clean":"2024-09-21T09:55:03.624515+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T09:55:03.624515+0000","last_undegraded":"2024-09-21T09:55:03.624515+0000","last_fullsized":"2024-09-21T09:55:03.624515+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1022,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412623+0000","last_change":"2024-09-21T09:45:39.130886+0000","last_active":"2024-09-21T09:55:03.412623+0000","last_peered":"2024-09-21T09:55:03.412623+0000","last_clean":"2024-09-21T09:55:03.412623+0000","last_became_active":"2024-09-21T09:45:39.130559+0000","last_became_peered":"2024-09-21T09:45:39.130559+0000","last_unstale":"2024-09-21T09:55:03.412623+0000","last_undegraded":"2024-09-21T09:55:03.412623+0000","last_fullsized":"2024-09-21T09:55:03.412623+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:39.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1468,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911482+0000","last_change":"2024-09-21T09:46:47.122643+0000","last_active":"2024-09-21T09:55:08.911482+0000","last_peered":"2024-09-21T09:55:08.911482+0000","last_clean":"2024-09-21T09:55:08.911482+0000","last_became_active":"2024-09-21T09:46:47.122364+0000","last_became_peered":"2024-09-21T09:46:47.122364+0000","last_unstale":"2024-09-21T09:55:08.911482+0000","last_undegraded":"2024-09-21T09:55:08.911482+0000","last_fullsized":"2024-09-21T09:55:08.911482+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:29.222333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1569,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914102+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T09:55:08.914102+0000","last_peered":"2024-09-21T09:55:08.914102+0000","last_clean":"2024-09-21T09:55:08.914102+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T09:55:08.914102+0000","last_undegraded":"2024-09-21T09:55:08.914102+0000","last_fullsized":"2024-09-21T09:55:08.914102+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911965+0000","last_change":"2024-09-21T09:46:47.121360+0000","last_active":"2024-09-21T09:55:08.911965+0000","last_peered":"2024-09-21T09:55:08.911965+0000","last_clean":"2024-09-21T09:55:08.911965+0000","last_became_active":"2024-09-21T09:46:47.120494+0000","last_became_peered":"2024-09-21T09:46:47.120494+0000","last_unstale":"2024-09-21T09:55:08.911965+0000","last_undegraded":"2024-09-21T09:55:08.911965+0000","last_fullsized":"2024-09-21T09:55:08.911965+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:45.862984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1009,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049924+0000","last_change":"2024-09-21T09:46:13.260037+0000","last_active":"2024-09-21T09:55:04.049924+0000","last_peered":"2024-09-21T09:55:04.049924+0000","last_clean":"2024-09-21T09:55:04.049924+0000","last_became_active":"2024-09-21T09:46:13.259690+0000","last_became_peered":"2024-09-21T09:46:13.259690+0000","last_unstale":"2024-09-21T09:55:04.049924+0000","last_undegraded":"2024-09-21T09:55:04.049924+0000","last_fullsized":"2024-09-21T09:55:04.049924+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:57.466762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1503,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912173+0000","last_change":"2024-09-21T09:46:47.120830+0000","last_active":"2024-09-21T09:55:08.912173+0000","last_peered":"2024-09-21T09:55:08.912173+0000","last_clean":"2024-09-21T09:55:08.912173+0000","last_became_active":"2024-09-21T09:46:47.118419+0000","last_became_peered":"2024-09-21T09:46:47.118419+0000","last_unstale":"2024-09-21T09:55:08.912173+0000","last_undegraded":"2024-09-21T09:55:08.912173+0000","last_fullsized":"2024-09-21T09:55:08.912173+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:13.789806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1572,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912600+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T09:55:08.912600+0000","last_peered":"2024-09-21T09:55:08.912600+0000","last_clean":"2024-09-21T09:55:08.912600+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T09:55:08.912600+0000","last_undegraded":"2024-09-21T09:55:08.912600+0000","last_fullsized":"2024-09-21T09:55:08.912600+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1534,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.910698+0000","last_change":"2024-09-21T09:46:47.121724+0000","last_active":"2024-09-21T09:55:08.910698+0000","last_peered":"2024-09-21T09:55:08.910698+0000","last_clean":"2024-09-21T09:55:08.910698+0000","last_became_active":"2024-09-21T09:46:47.121406+0000","last_became_peered":"2024-09-21T09:46:47.121406+0000","last_unstale":"2024-09-21T09:55:08.910698+0000","last_undegraded":"2024-09-21T09:55:08.910698+0000","last_fullsized":"2024-09-21T09:55:08.910698+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:55.493704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911362+0000","last_change":"2024-09-21T09:46:47.113184+0000","last_active":"2024-09-21T09:55:08.911362+0000","last_peered":"2024-09-21T09:55:08.911362+0000","last_clean":"2024-09-21T09:55:08.911362+0000","last_became_active":"2024-09-21T09:46:47.112857+0000","last_became_peered":"2024-09-21T09:46:47.112857+0000","last_unstale":"2024-09-21T09:55:08.911362+0000","last_undegraded":"2024-09-21T09:55:08.911362+0000","last_fullsized":"2024-09-21T09:55:08.911362+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:46.783986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1036,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300495+0000","last_change":"2024-09-21T09:46:13.262555+0000","last_active":"2024-09-21T09:55:03.300495+0000","last_peered":"2024-09-21T09:55:03.300495+0000","last_clean":"2024-09-21T09:55:03.300495+0000","last_became_active":"2024-09-21T09:46:13.261474+0000","last_became_peered":"2024-09-21T09:46:13.261474+0000","last_unstale":"2024-09-21T09:55:03.300495+0000","last_undegraded":"2024-09-21T09:55:03.300495+0000","last_fullsized":"2024-09-21T09:55:03.300495+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:44.865988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1523,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316853+0000","last_change":"2024-09-21T09:46:47.114406+0000","last_active":"2024-09-21T09:55:08.316853+0000","last_peered":"2024-09-21T09:55:08.316853+0000","last_clean":"2024-09-21T09:55:08.316853+0000","last_became_active":"2024-09-21T09:46:47.113986+0000","last_became_peered":"2024-09-21T09:46:47.113986+0000","last_unstale":"2024-09-21T09:55:08.316853+0000","last_undegraded":"2024-09-21T09:55:08.316853+0000","last_fullsized":"2024-09-21T09:55:08.316853+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:32.292201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1562,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914239+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T09:55:08.914239+0000","last_peered":"2024-09-21T09:55:08.914239+0000","last_clean":"2024-09-21T09:55:08.914239+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T09:55:08.914239+0000","last_undegraded":"2024-09-21T09:55:08.914239+0000","last_fullsized":"2024-09-21T09:55:08.914239+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300266+0000","last_change":"2024-09-21T09:46:47.114188+0000","last_active":"2024-09-21T09:55:03.300266+0000","last_peered":"2024-09-21T09:55:03.300266+0000","last_clean":"2024-09-21T09:55:03.300266+0000","last_became_active":"2024-09-21T09:46:47.113859+0000","last_became_peered":"2024-09-21T09:46:47.113859+0000","last_unstale":"2024-09-21T09:55:03.300266+0000","last_undegraded":"2024-09-21T09:55:03.300266+0000","last_fullsized":"2024-09-21T09:55:03.300266+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:22.670368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":1024,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412132+0000","last_change":"2024-09-21T09:45:39.131124+0000","last_active":"2024-09-21T09:55:03.412132+0000","last_peered":"2024-09-21T09:55:03.412132+0000","last_clean":"2024-09-21T09:55:03.412132+0000","last_became_active":"2024-09-21T09:45:39.130756+0000","last_became_peered":"2024-09-21T09:45:39.130756+0000","last_unstale":"2024-09-21T09:55:03.412132+0000","last_undegraded":"2024-09-21T09:55:03.412132+0000","last_fullsized":"2024-09-21T09:55:03.412132+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:49.130698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300392+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T09:55:03.300392+0000","last_peered":"2024-09-21T09:55:03.300392+0000","last_clean":"2024-09-21T09:55:03.300392+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T09:55:03.300392+0000","last_undegraded":"2024-09-21T09:55:03.300392+0000","last_fullsized":"2024-09-21T09:55:03.300392+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":1006,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412104+0000","last_change":"2024-09-21T09:36:43.115791+0000","last_active":"2024-09-21T09:55:03.412104+0000","last_peered":"2024-09-21T09:55:03.412104+0000","last_clean":"2024-09-21T09:55:03.412104+0000","last_became_active":"2024-09-21T09:36:43.115525+0000","last_became_peered":"2024-09-21T09:36:43.115525+0000","last_unstale":"2024-09-21T09:55:03.412104+0000","last_undegraded":"2024-09-21T09:55:03.412104+0000","last_fullsized":"2024-09-21T09:55:03.412104+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:28.396722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1495,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911833+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T09:55:08.911833+0000","last_peered":"2024-09-21T09:55:08.911833+0000","last_clean":"2024-09-21T09:55:08.911833+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T09:55:08.911833+0000","last_undegraded":"2024-09-21T09:55:08.911833+0000","last_fullsized":"2024-09-21T09:55:08.911833+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1481,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913152+0000","last_change":"2024-09-21T09:46:47.115788+0000","last_active":"2024-09-21T09:55:08.913152+0000","last_peered":"2024-09-21T09:55:08.913152+0000","last_clean":"2024-09-21T09:55:08.913152+0000","last_became_active":"2024-09-21T09:46:47.114864+0000","last_became_peered":"2024-09-21T09:46:47.114864+0000","last_unstale":"2024-09-21T09:55:08.913152+0000","last_undegraded":"2024-09-21T09:55:08.913152+0000","last_fullsized":"2024-09-21T09:55:08.913152+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:00.034944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.299925+0000","last_change":"2024-09-21T09:45:39.143064+0000","last_active":"2024-09-21T09:55:03.299925+0000","last_peered":"2024-09-21T09:55:03.299925+0000","last_clean":"2024-09-21T09:55:03.299925+0000","last_became_active":"2024-09-21T09:45:39.142699+0000","last_became_peered":"2024-09-21T09:45:39.142699+0000","last_unstale":"2024-09-21T09:55:03.299925+0000","last_undegraded":"2024-09-21T09:55:03.299925+0000","last_fullsized":"2024-09-21T09:55:03.299925+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:14.684721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1577,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914281+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T09:55:08.914281+0000","last_peered":"2024-09-21T09:55:08.914281+0000","last_clean":"2024-09-21T09:55:08.914281+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T09:55:08.914281+0000","last_undegraded":"2024-09-21T09:55:08.914281+0000","last_fullsized":"2024-09-21T09:55:08.914281+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397762,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317368,"kb_used_data":7532,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93427784,"statfs":{"total":95995035648,"available":95670050816,"internally_reserved":0,"allocated":7712768,"data_stored":5600360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593991,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313200,"kb_used_data":7524,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93431952,"statfs":{"total":95995035648,"available":95674318848,"internally_reserved":0,"allocated":7704576,"data_stored":5600225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790220,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316768,"kb_used_data":6936,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93428384,"statfs":{"total":95995035648,"available":95670665216,"internally_reserved":0,"allocated":7102464,"data_stored":5005683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986447,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308460,"kb_used_data":6948,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93436692,"statfs":{"total":95995035648,"available":95679172608,"internally_reserved":0,"allocated":7114752,"data_stored":5009914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999089,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":48744,"kb_used_data":8608,"kb_used_omap":25,"kb_used_meta":40102,"kb_avail":93696408,"statfs":{"total":95995035648,"available":95945121792,"internally_reserved":0,"allocated":8814592,"data_stored":6219336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":41065415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111032,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40984,"kb_used_data":8792,"kb_used_omap":31,"kb_used_meta":31968,"kb_avail":93704168,"statfs":{"total":95995035648,"available":95953068032,"internally_reserved":0,"allocated":9003008,"data_stored":6349770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":32735464},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318222975,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40672,"kb_used_data":8608,"kb_used_omap":26,"kb_used_meta":31845,"kb_avail":93704480,"statfs":{"total":95995035648,"available":95953387520,"internally_reserved":0,"allocated":8814592,"data_stored":6219273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":32609292},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289334918,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67868,"kb_used_data":9372,"kb_used_omap":31,"kb_used_meta":58272,"kb_avail":93677284,"statfs":{"total":95995035648,"available":95925538816,"internally_reserved":0,"allocated":9596928,"data_stored":6940134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":59670624},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:56:05.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:56:05.859 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:05 smithi017 ceph-mon[102913]: pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:05.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:05 smithi017 ceph-mon[105151]: pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:05.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:05 smithi134 ceph-mon[90085]: pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:06 smithi017 ceph-mon[102913]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:06.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:06 smithi017 ceph-mon[105151]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:06.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:06 smithi134 ceph-mon[90085]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:07.643 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:56:07.644 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:56:07.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:07 smithi017 ceph-mon[102913]: pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:07.918 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:07 smithi017 ceph-mon[105151]: pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:07 smithi134 ceph-mon[90085]: pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:08.262 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1226,"stamp":"2024-09-21T09:56:05.981543+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":212,"num_write_kb":1659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158,"ondisk_log_size":158,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1454064,"kb_used_data":64320,"kb_used_omap":122,"kb_used_meta":1388997,"kb_avail":748507152,"statfs":{"total":767960285184,"available":766471323648,"internally_reserved":0,"allocated":65863680,"data_stored":46944695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125317,"internal_metadata":1422333563},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002900"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1486,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911094+0000","last_change":"2024-09-21T09:46:47.114206+0000","last_active":"2024-09-21T09:55:08.911094+0000","last_peered":"2024-09-21T09:55:08.911094+0000","last_clean":"2024-09-21T09:55:08.911094+0000","last_became_active":"2024-09-21T09:46:47.113819+0000","last_became_peered":"2024-09-21T09:46:47.113819+0000","last_unstale":"2024-09-21T09:55:08.911094+0000","last_undegraded":"2024-09-21T09:55:08.911094+0000","last_fullsized":"2024-09-21T09:55:08.911094+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:07.629544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1001,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624302+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T09:55:03.624302+0000","last_peered":"2024-09-21T09:55:03.624302+0000","last_clean":"2024-09-21T09:55:03.624302+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T09:55:03.624302+0000","last_undegraded":"2024-09-21T09:55:03.624302+0000","last_fullsized":"2024-09-21T09:55:03.624302+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1020,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412012+0000","last_change":"2024-09-21T09:46:47.115484+0000","last_active":"2024-09-21T09:55:03.412012+0000","last_peered":"2024-09-21T09:55:03.412012+0000","last_clean":"2024-09-21T09:55:03.412012+0000","last_became_active":"2024-09-21T09:46:47.115080+0000","last_became_peered":"2024-09-21T09:46:47.115080+0000","last_unstale":"2024-09-21T09:55:03.412012+0000","last_undegraded":"2024-09-21T09:55:03.412012+0000","last_fullsized":"2024-09-21T09:55:03.412012+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:56.851805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1569,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912062+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T09:55:08.912062+0000","last_peered":"2024-09-21T09:55:08.912062+0000","last_clean":"2024-09-21T09:55:08.912062+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T09:55:08.912062+0000","last_undegraded":"2024-09-21T09:55:08.912062+0000","last_fullsized":"2024-09-21T09:55:08.912062+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1571,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912710+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T09:55:08.912710+0000","last_peered":"2024-09-21T09:55:08.912710+0000","last_clean":"2024-09-21T09:55:08.912710+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T09:55:08.912710+0000","last_undegraded":"2024-09-21T09:55:08.912710+0000","last_fullsized":"2024-09-21T09:55:08.912710+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":982,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624220+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:55:03.624220+0000","last_peered":"2024-09-21T09:55:03.624220+0000","last_clean":"2024-09-21T09:55:03.624220+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:55:03.624220+0000","last_undegraded":"2024-09-21T09:55:03.624220+0000","last_fullsized":"2024-09-21T09:55:03.624220+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912905+0000","last_change":"2024-09-21T09:46:47.117580+0000","last_active":"2024-09-21T09:55:08.912905+0000","last_peered":"2024-09-21T09:55:08.912905+0000","last_clean":"2024-09-21T09:55:08.912905+0000","last_became_active":"2024-09-21T09:46:47.113030+0000","last_became_peered":"2024-09-21T09:46:47.113030+0000","last_unstale":"2024-09-21T09:55:08.912905+0000","last_undegraded":"2024-09-21T09:55:08.912905+0000","last_fullsized":"2024-09-21T09:55:08.912905+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:28.220180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":1008,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049340+0000","last_change":"2024-09-21T09:45:05.080014+0000","last_active":"2024-09-21T09:55:04.049340+0000","last_peered":"2024-09-21T09:55:04.049340+0000","last_clean":"2024-09-21T09:55:04.049340+0000","last_became_active":"2024-09-21T09:45:05.079618+0000","last_became_peered":"2024-09-21T09:45:05.079618+0000","last_unstale":"2024-09-21T09:55:04.049340+0000","last_undegraded":"2024-09-21T09:55:04.049340+0000","last_fullsized":"2024-09-21T09:55:04.049340+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:42.160292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624817+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T09:55:03.624817+0000","last_peered":"2024-09-21T09:55:03.624817+0000","last_clean":"2024-09-21T09:55:03.624817+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T09:55:03.624817+0000","last_undegraded":"2024-09-21T09:55:03.624817+0000","last_fullsized":"2024-09-21T09:55:03.624817+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1004,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.411948+0000","last_change":"2024-09-21T09:36:43.389124+0000","last_active":"2024-09-21T09:55:03.411948+0000","last_peered":"2024-09-21T09:55:03.411948+0000","last_clean":"2024-09-21T09:55:03.411948+0000","last_became_active":"2024-09-21T09:36:43.388588+0000","last_became_peered":"2024-09-21T09:36:43.388588+0000","last_unstale":"2024-09-21T09:55:03.411948+0000","last_undegraded":"2024-09-21T09:55:03.411948+0000","last_fullsized":"2024-09-21T09:55:03.411948+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:57.562156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1001,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624842+0000","last_change":"2024-09-21T09:45:39.143190+0000","last_active":"2024-09-21T09:55:03.624842+0000","last_peered":"2024-09-21T09:55:03.624842+0000","last_clean":"2024-09-21T09:55:03.624842+0000","last_became_active":"2024-09-21T09:45:39.142826+0000","last_became_peered":"2024-09-21T09:45:39.142826+0000","last_unstale":"2024-09-21T09:55:03.624842+0000","last_undegraded":"2024-09-21T09:55:03.624842+0000","last_fullsized":"2024-09-21T09:55:03.624842+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:45.070945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300475+0000","last_change":"2024-09-21T09:46:47.119247+0000","last_active":"2024-09-21T09:55:03.300475+0000","last_peered":"2024-09-21T09:55:03.300475+0000","last_clean":"2024-09-21T09:55:03.300475+0000","last_became_active":"2024-09-21T09:46:47.118980+0000","last_became_peered":"2024-09-21T09:46:47.118980+0000","last_unstale":"2024-09-21T09:55:03.300475+0000","last_undegraded":"2024-09-21T09:55:03.300475+0000","last_fullsized":"2024-09-21T09:55:03.300475+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.140253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":983,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624823+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:55:03.624823+0000","last_peered":"2024-09-21T09:55:03.624823+0000","last_clean":"2024-09-21T09:55:03.624823+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:55:03.624823+0000","last_undegraded":"2024-09-21T09:55:03.624823+0000","last_fullsized":"2024-09-21T09:55:03.624823+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1009,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049323+0000","last_change":"2024-09-21T09:46:13.260744+0000","last_active":"2024-09-21T09:55:04.049323+0000","last_peered":"2024-09-21T09:55:04.049323+0000","last_clean":"2024-09-21T09:55:04.049323+0000","last_became_active":"2024-09-21T09:46:13.260443+0000","last_became_peered":"2024-09-21T09:46:13.260443+0000","last_unstale":"2024-09-21T09:55:04.049323+0000","last_undegraded":"2024-09-21T09:55:04.049323+0000","last_fullsized":"2024-09-21T09:55:04.049323+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:22.699110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1022,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412189+0000","last_change":"2024-09-21T09:46:47.115343+0000","last_active":"2024-09-21T09:55:03.412189+0000","last_peered":"2024-09-21T09:55:03.412189+0000","last_clean":"2024-09-21T09:55:03.412189+0000","last_became_active":"2024-09-21T09:46:47.115017+0000","last_became_peered":"2024-09-21T09:46:47.115017+0000","last_unstale":"2024-09-21T09:55:03.412189+0000","last_undegraded":"2024-09-21T09:55:03.412189+0000","last_fullsized":"2024-09-21T09:55:03.412189+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.820662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1571,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912938+0000","last_change":"2024-09-21T09:45:39.138289+0000","last_active":"2024-09-21T09:55:08.912938+0000","last_peered":"2024-09-21T09:55:08.912938+0000","last_clean":"2024-09-21T09:55:08.912938+0000","last_became_active":"2024-09-21T09:45:39.137902+0000","last_became_peered":"2024-09-21T09:45:39.137902+0000","last_unstale":"2024-09-21T09:55:08.912938+0000","last_undegraded":"2024-09-21T09:55:08.912938+0000","last_fullsized":"2024-09-21T09:55:08.912938+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:40:16.383906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1496,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911031+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T09:55:08.911031+0000","last_peered":"2024-09-21T09:55:08.911031+0000","last_clean":"2024-09-21T09:55:08.911031+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T09:55:08.911031+0000","last_undegraded":"2024-09-21T09:55:08.911031+0000","last_fullsized":"2024-09-21T09:55:08.911031+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300113+0000","last_change":"2024-09-21T09:46:47.114293+0000","last_active":"2024-09-21T09:55:03.300113+0000","last_peered":"2024-09-21T09:55:03.300113+0000","last_clean":"2024-09-21T09:55:03.300113+0000","last_became_active":"2024-09-21T09:46:47.113993+0000","last_became_peered":"2024-09-21T09:46:47.113993+0000","last_unstale":"2024-09-21T09:55:03.300113+0000","last_undegraded":"2024-09-21T09:55:03.300113+0000","last_fullsized":"2024-09-21T09:55:03.300113+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:30.941931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049541+0000","last_change":"2024-09-21T09:45:39.127186+0000","last_active":"2024-09-21T09:55:04.049541+0000","last_peered":"2024-09-21T09:55:04.049541+0000","last_clean":"2024-09-21T09:55:04.049541+0000","last_became_active":"2024-09-21T09:45:39.126940+0000","last_became_peered":"2024-09-21T09:45:39.126940+0000","last_unstale":"2024-09-21T09:55:04.049541+0000","last_undegraded":"2024-09-21T09:55:04.049541+0000","last_fullsized":"2024-09-21T09:55:04.049541+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:52.205703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300795+0000","last_change":"2024-09-21T09:46:13.259814+0000","last_active":"2024-09-21T09:55:03.300795+0000","last_peered":"2024-09-21T09:55:03.300795+0000","last_clean":"2024-09-21T09:55:03.300795+0000","last_became_active":"2024-09-21T09:46:13.259527+0000","last_became_peered":"2024-09-21T09:46:13.259527+0000","last_unstale":"2024-09-21T09:55:03.300795+0000","last_undegraded":"2024-09-21T09:55:03.300795+0000","last_fullsized":"2024-09-21T09:55:03.300795+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:00.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1018,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300593+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T09:55:03.300593+0000","last_peered":"2024-09-21T09:55:03.300593+0000","last_clean":"2024-09-21T09:55:03.300593+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T09:55:03.300593+0000","last_undegraded":"2024-09-21T09:55:03.300593+0000","last_fullsized":"2024-09-21T09:55:03.300593+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300762+0000","last_change":"2024-09-21T09:45:39.143302+0000","last_active":"2024-09-21T09:55:03.300762+0000","last_peered":"2024-09-21T09:55:03.300762+0000","last_clean":"2024-09-21T09:55:03.300762+0000","last_became_active":"2024-09-21T09:45:39.142964+0000","last_became_peered":"2024-09-21T09:45:39.142964+0000","last_unstale":"2024-09-21T09:55:03.300762+0000","last_undegraded":"2024-09-21T09:55:03.300762+0000","last_fullsized":"2024-09-21T09:55:03.300762+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:54.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1487,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913222+0000","last_change":"2024-09-21T09:46:47.121643+0000","last_active":"2024-09-21T09:55:08.913222+0000","last_peered":"2024-09-21T09:55:08.913222+0000","last_clean":"2024-09-21T09:55:08.913222+0000","last_became_active":"2024-09-21T09:46:47.120528+0000","last_became_peered":"2024-09-21T09:46:47.120528+0000","last_unstale":"2024-09-21T09:55:08.913222+0000","last_undegraded":"2024-09-21T09:55:08.913222+0000","last_fullsized":"2024-09-21T09:55:08.913222+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:07.989377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1533,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913431+0000","last_change":"2024-09-21T09:46:47.121159+0000","last_active":"2024-09-21T09:55:08.913431+0000","last_peered":"2024-09-21T09:55:08.913431+0000","last_clean":"2024-09-21T09:55:08.913431+0000","last_became_active":"2024-09-21T09:46:47.120829+0000","last_became_peered":"2024-09-21T09:46:47.120829+0000","last_unstale":"2024-09-21T09:55:08.913431+0000","last_undegraded":"2024-09-21T09:55:08.913431+0000","last_fullsized":"2024-09-21T09:55:08.913431+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:43.393840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1116,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624090+0000","last_change":"2024-09-21T09:45:05.087928+0000","last_active":"2024-09-21T09:55:03.624090+0000","last_peered":"2024-09-21T09:55:03.624090+0000","last_clean":"2024-09-21T09:55:03.624090+0000","last_became_active":"2024-09-21T09:45:05.087478+0000","last_became_peered":"2024-09-21T09:45:05.087478+0000","last_unstale":"2024-09-21T09:55:03.624090+0000","last_undegraded":"2024-09-21T09:55:03.624090+0000","last_fullsized":"2024-09-21T09:55:03.624090+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:48.651314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1035,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412049+0000","last_change":"2024-09-21T09:46:13.262337+0000","last_active":"2024-09-21T09:55:03.412049+0000","last_peered":"2024-09-21T09:55:03.412049+0000","last_clean":"2024-09-21T09:55:03.412049+0000","last_became_active":"2024-09-21T09:46:13.262005+0000","last_became_peered":"2024-09-21T09:46:13.262005+0000","last_unstale":"2024-09-21T09:55:03.412049+0000","last_undegraded":"2024-09-21T09:55:03.412049+0000","last_fullsized":"2024-09-21T09:55:03.412049+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:32.413210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1469,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912212+0000","last_change":"2024-09-21T09:46:47.116045+0000","last_active":"2024-09-21T09:55:08.912212+0000","last_peered":"2024-09-21T09:55:08.912212+0000","last_clean":"2024-09-21T09:55:08.912212+0000","last_became_active":"2024-09-21T09:46:47.115526+0000","last_became_peered":"2024-09-21T09:46:47.115526+0000","last_unstale":"2024-09-21T09:55:08.912212+0000","last_undegraded":"2024-09-21T09:55:08.912212+0000","last_fullsized":"2024-09-21T09:55:08.912212+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:43:35.293486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":982,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049787+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:55:04.049787+0000","last_peered":"2024-09-21T09:55:04.049787+0000","last_clean":"2024-09-21T09:55:04.049787+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:55:04.049787+0000","last_undegraded":"2024-09-21T09:55:04.049787+0000","last_fullsized":"2024-09-21T09:55:04.049787+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1028,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624013+0000","last_change":"2024-09-21T09:45:39.129555+0000","last_active":"2024-09-21T09:55:03.624013+0000","last_peered":"2024-09-21T09:55:03.624013+0000","last_clean":"2024-09-21T09:55:03.624013+0000","last_became_active":"2024-09-21T09:45:39.129205+0000","last_became_peered":"2024-09-21T09:45:39.129205+0000","last_unstale":"2024-09-21T09:55:03.624013+0000","last_undegraded":"2024-09-21T09:55:03.624013+0000","last_fullsized":"2024-09-21T09:55:03.624013+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:07.229126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1570,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913207+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T09:55:08.913207+0000","last_peered":"2024-09-21T09:55:08.913207+0000","last_clean":"2024-09-21T09:55:08.913207+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T09:55:08.913207+0000","last_undegraded":"2024-09-21T09:55:08.913207+0000","last_fullsized":"2024-09-21T09:55:08.913207+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1583,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912533+0000","last_change":"2024-09-21T09:46:13.273573+0000","last_active":"2024-09-21T09:55:08.912533+0000","last_peered":"2024-09-21T09:55:08.912533+0000","last_clean":"2024-09-21T09:55:08.912533+0000","last_became_active":"2024-09-21T09:46:13.273297+0000","last_became_peered":"2024-09-21T09:46:13.273297+0000","last_unstale":"2024-09-21T09:55:08.912533+0000","last_undegraded":"2024-09-21T09:55:08.912533+0000","last_fullsized":"2024-09-21T09:55:08.912533+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:38.957745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1544,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316664+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T09:55:08.316664+0000","last_peered":"2024-09-21T09:55:08.316664+0000","last_clean":"2024-09-21T09:55:08.316664+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T09:55:08.316664+0000","last_undegraded":"2024-09-21T09:55:08.316664+0000","last_fullsized":"2024-09-21T09:55:08.316664+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1535,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911392+0000","last_change":"2024-09-21T09:46:13.255342+0000","last_active":"2024-09-21T09:55:08.911392+0000","last_peered":"2024-09-21T09:55:08.911392+0000","last_clean":"2024-09-21T09:55:08.911392+0000","last_became_active":"2024-09-21T09:46:13.255014+0000","last_became_peered":"2024-09-21T09:46:13.255014+0000","last_unstale":"2024-09-21T09:55:08.911392+0000","last_undegraded":"2024-09-21T09:55:08.911392+0000","last_fullsized":"2024-09-21T09:55:08.911392+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:53.012266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1545,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913260+0000","last_change":"2024-09-21T09:45:39.134465+0000","last_active":"2024-09-21T09:55:08.913260+0000","last_peered":"2024-09-21T09:55:08.913260+0000","last_clean":"2024-09-21T09:55:08.913260+0000","last_became_active":"2024-09-21T09:45:39.129692+0000","last_became_peered":"2024-09-21T09:45:39.129692+0000","last_unstale":"2024-09-21T09:55:08.913260+0000","last_undegraded":"2024-09-21T09:55:08.913260+0000","last_fullsized":"2024-09-21T09:55:08.913260+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:14.288180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1036,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412260+0000","last_change":"2024-09-21T09:45:39.131285+0000","last_active":"2024-09-21T09:55:03.412260+0000","last_peered":"2024-09-21T09:55:03.412260+0000","last_clean":"2024-09-21T09:55:03.412260+0000","last_became_active":"2024-09-21T09:45:39.130505+0000","last_became_peered":"2024-09-21T09:45:39.130505+0000","last_unstale":"2024-09-21T09:55:03.412260+0000","last_undegraded":"2024-09-21T09:55:03.412260+0000","last_fullsized":"2024-09-21T09:55:03.412260+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:37.523277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1018,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300486+0000","last_change":"2024-09-21T09:45:05.097518+0000","last_active":"2024-09-21T09:55:03.300486+0000","last_peered":"2024-09-21T09:55:03.300486+0000","last_clean":"2024-09-21T09:55:03.300486+0000","last_became_active":"2024-09-21T09:45:05.096918+0000","last_became_peered":"2024-09-21T09:45:05.096918+0000","last_unstale":"2024-09-21T09:55:03.300486+0000","last_undegraded":"2024-09-21T09:55:03.300486+0000","last_fullsized":"2024-09-21T09:55:03.300486+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:13.781847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1545,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911071+0000","last_change":"2024-09-21T09:46:47.113687+0000","last_active":"2024-09-21T09:55:08.911071+0000","last_peered":"2024-09-21T09:55:08.911071+0000","last_clean":"2024-09-21T09:55:08.911071+0000","last_became_active":"2024-09-21T09:46:47.113235+0000","last_became_peered":"2024-09-21T09:46:47.113235+0000","last_unstale":"2024-09-21T09:55:08.911071+0000","last_undegraded":"2024-09-21T09:55:08.911071+0000","last_fullsized":"2024-09-21T09:55:08.911071+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:50.851616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1038,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412109+0000","last_change":"2024-09-21T09:46:47.115673+0000","last_active":"2024-09-21T09:55:03.412109+0000","last_peered":"2024-09-21T09:55:03.412109+0000","last_clean":"2024-09-21T09:55:03.412109+0000","last_became_active":"2024-09-21T09:46:47.115166+0000","last_became_peered":"2024-09-21T09:46:47.115166+0000","last_unstale":"2024-09-21T09:55:03.412109+0000","last_undegraded":"2024-09-21T09:55:03.412109+0000","last_fullsized":"2024-09-21T09:55:03.412109+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:16.660834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1559,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911179+0000","last_change":"2024-09-21T09:45:39.132441+0000","last_active":"2024-09-21T09:55:08.911179+0000","last_peered":"2024-09-21T09:55:08.911179+0000","last_clean":"2024-09-21T09:55:08.911179+0000","last_became_active":"2024-09-21T09:45:39.129666+0000","last_became_peered":"2024-09-21T09:45:39.129666+0000","last_unstale":"2024-09-21T09:55:08.911179+0000","last_undegraded":"2024-09-21T09:55:08.911179+0000","last_fullsized":"2024-09-21T09:55:08.911179+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:34.450160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1497,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316744+0000","last_change":"2024-09-21T09:46:13.258244+0000","last_active":"2024-09-21T09:55:08.316744+0000","last_peered":"2024-09-21T09:55:08.316744+0000","last_clean":"2024-09-21T09:55:08.316744+0000","last_became_active":"2024-09-21T09:46:13.257820+0000","last_became_peered":"2024-09-21T09:46:13.257820+0000","last_unstale":"2024-09-21T09:55:08.316744+0000","last_undegraded":"2024-09-21T09:55:08.316744+0000","last_fullsized":"2024-09-21T09:55:08.316744+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:04.618304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1535,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911419+0000","last_change":"2024-09-21T09:46:13.260666+0000","last_active":"2024-09-21T09:55:08.911419+0000","last_peered":"2024-09-21T09:55:08.911419+0000","last_clean":"2024-09-21T09:55:08.911419+0000","last_became_active":"2024-09-21T09:46:13.260324+0000","last_became_peered":"2024-09-21T09:46:13.260324+0000","last_unstale":"2024-09-21T09:55:08.911419+0000","last_undegraded":"2024-09-21T09:55:08.911419+0000","last_fullsized":"2024-09-21T09:55:08.911419+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:25.140797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1011,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624946+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T09:55:03.624946+0000","last_peered":"2024-09-21T09:55:03.624946+0000","last_clean":"2024-09-21T09:55:03.624946+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T09:55:03.624946+0000","last_undegraded":"2024-09-21T09:55:03.624946+0000","last_fullsized":"2024-09-21T09:55:03.624946+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1561,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914110+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T09:55:08.914110+0000","last_peered":"2024-09-21T09:55:08.914110+0000","last_clean":"2024-09-21T09:55:08.914110+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T09:55:08.914110+0000","last_undegraded":"2024-09-21T09:55:08.914110+0000","last_fullsized":"2024-09-21T09:55:08.914110+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1496,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913455+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T09:55:08.913455+0000","last_peered":"2024-09-21T09:55:08.913455+0000","last_clean":"2024-09-21T09:55:08.913455+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T09:55:08.913455+0000","last_undegraded":"2024-09-21T09:55:08.913455+0000","last_fullsized":"2024-09-21T09:55:08.913455+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1532,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912680+0000","last_change":"2024-09-21T09:45:39.130168+0000","last_active":"2024-09-21T09:55:08.912680+0000","last_peered":"2024-09-21T09:55:08.912680+0000","last_clean":"2024-09-21T09:55:08.912680+0000","last_became_active":"2024-09-21T09:45:39.129498+0000","last_became_peered":"2024-09-21T09:45:39.129498+0000","last_unstale":"2024-09-21T09:55:08.912680+0000","last_undegraded":"2024-09-21T09:55:08.912680+0000","last_fullsized":"2024-09-21T09:55:08.912680+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:58.310876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1493,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912070+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T09:55:08.912070+0000","last_peered":"2024-09-21T09:55:08.912070+0000","last_clean":"2024-09-21T09:55:08.912070+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T09:55:08.912070+0000","last_undegraded":"2024-09-21T09:55:08.912070+0000","last_fullsized":"2024-09-21T09:55:08.912070+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1574,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912871+0000","last_change":"2024-09-21T09:45:39.140762+0000","last_active":"2024-09-21T09:55:08.912871+0000","last_peered":"2024-09-21T09:55:08.912871+0000","last_clean":"2024-09-21T09:55:08.912871+0000","last_became_active":"2024-09-21T09:45:39.140402+0000","last_became_peered":"2024-09-21T09:45:39.140402+0000","last_unstale":"2024-09-21T09:55:08.912871+0000","last_undegraded":"2024-09-21T09:55:08.912871+0000","last_fullsized":"2024-09-21T09:55:08.912871+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:06.545428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1012,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049926+0000","last_change":"2024-09-21T09:46:13.260656+0000","last_active":"2024-09-21T09:55:04.049926+0000","last_peered":"2024-09-21T09:55:04.049926+0000","last_clean":"2024-09-21T09:55:04.049926+0000","last_became_active":"2024-09-21T09:46:13.260368+0000","last_became_peered":"2024-09-21T09:46:13.260368+0000","last_unstale":"2024-09-21T09:55:04.049926+0000","last_undegraded":"2024-09-21T09:55:04.049926+0000","last_fullsized":"2024-09-21T09:55:04.049926+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:22.059036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1000,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300041+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:55:03.300041+0000","last_peered":"2024-09-21T09:55:03.300041+0000","last_clean":"2024-09-21T09:55:03.300041+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:55:03.300041+0000","last_undegraded":"2024-09-21T09:55:03.300041+0000","last_fullsized":"2024-09-21T09:55:03.300041+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300296+0000","last_change":"2024-09-21T09:45:39.129538+0000","last_active":"2024-09-21T09:55:03.300296+0000","last_peered":"2024-09-21T09:55:03.300296+0000","last_clean":"2024-09-21T09:55:03.300296+0000","last_became_active":"2024-09-21T09:45:39.129198+0000","last_became_peered":"2024-09-21T09:45:39.129198+0000","last_unstale":"2024-09-21T09:55:03.300296+0000","last_undegraded":"2024-09-21T09:55:03.300296+0000","last_fullsized":"2024-09-21T09:55:03.300296+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.798277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1023,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300173+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:55:03.300173+0000","last_peered":"2024-09-21T09:55:03.300173+0000","last_clean":"2024-09-21T09:55:03.300173+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:55:03.300173+0000","last_undegraded":"2024-09-21T09:55:03.300173+0000","last_fullsized":"2024-09-21T09:55:03.300173+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1533,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.910728+0000","last_change":"2024-09-21T09:46:13.254014+0000","last_active":"2024-09-21T09:55:08.910728+0000","last_peered":"2024-09-21T09:55:08.910728+0000","last_clean":"2024-09-21T09:55:08.910728+0000","last_became_active":"2024-09-21T09:46:13.253673+0000","last_became_peered":"2024-09-21T09:46:13.253673+0000","last_unstale":"2024-09-21T09:55:08.910728+0000","last_undegraded":"2024-09-21T09:55:08.910728+0000","last_fullsized":"2024-09-21T09:55:08.910728+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:02.784613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1002,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.624515+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T09:55:03.624515+0000","last_peered":"2024-09-21T09:55:03.624515+0000","last_clean":"2024-09-21T09:55:03.624515+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T09:55:03.624515+0000","last_undegraded":"2024-09-21T09:55:03.624515+0000","last_fullsized":"2024-09-21T09:55:03.624515+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1022,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412623+0000","last_change":"2024-09-21T09:45:39.130886+0000","last_active":"2024-09-21T09:55:03.412623+0000","last_peered":"2024-09-21T09:55:03.412623+0000","last_clean":"2024-09-21T09:55:03.412623+0000","last_became_active":"2024-09-21T09:45:39.130559+0000","last_became_peered":"2024-09-21T09:45:39.130559+0000","last_unstale":"2024-09-21T09:55:03.412623+0000","last_undegraded":"2024-09-21T09:55:03.412623+0000","last_fullsized":"2024-09-21T09:55:03.412623+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:39.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1469,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911522+0000","last_change":"2024-09-21T09:46:47.122643+0000","last_active":"2024-09-21T09:55:08.911522+0000","last_peered":"2024-09-21T09:55:08.911522+0000","last_clean":"2024-09-21T09:55:08.911522+0000","last_became_active":"2024-09-21T09:46:47.122364+0000","last_became_peered":"2024-09-21T09:46:47.122364+0000","last_unstale":"2024-09-21T09:55:08.911522+0000","last_undegraded":"2024-09-21T09:55:08.911522+0000","last_fullsized":"2024-09-21T09:55:08.911522+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:29.222333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1569,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914102+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T09:55:08.914102+0000","last_peered":"2024-09-21T09:55:08.914102+0000","last_clean":"2024-09-21T09:55:08.914102+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T09:55:08.914102+0000","last_undegraded":"2024-09-21T09:55:08.914102+0000","last_fullsized":"2024-09-21T09:55:08.914102+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1486,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912010+0000","last_change":"2024-09-21T09:46:47.121360+0000","last_active":"2024-09-21T09:55:08.912010+0000","last_peered":"2024-09-21T09:55:08.912010+0000","last_clean":"2024-09-21T09:55:08.912010+0000","last_became_active":"2024-09-21T09:46:47.120494+0000","last_became_peered":"2024-09-21T09:46:47.120494+0000","last_unstale":"2024-09-21T09:55:08.912010+0000","last_undegraded":"2024-09-21T09:55:08.912010+0000","last_fullsized":"2024-09-21T09:55:08.912010+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:45.862984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1009,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.049924+0000","last_change":"2024-09-21T09:46:13.260037+0000","last_active":"2024-09-21T09:55:04.049924+0000","last_peered":"2024-09-21T09:55:04.049924+0000","last_clean":"2024-09-21T09:55:04.049924+0000","last_became_active":"2024-09-21T09:46:13.259690+0000","last_became_peered":"2024-09-21T09:46:13.259690+0000","last_unstale":"2024-09-21T09:55:04.049924+0000","last_undegraded":"2024-09-21T09:55:04.049924+0000","last_fullsized":"2024-09-21T09:55:04.049924+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:57.466762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1503,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912173+0000","last_change":"2024-09-21T09:46:47.120830+0000","last_active":"2024-09-21T09:55:08.912173+0000","last_peered":"2024-09-21T09:55:08.912173+0000","last_clean":"2024-09-21T09:55:08.912173+0000","last_became_active":"2024-09-21T09:46:47.118419+0000","last_became_peered":"2024-09-21T09:46:47.118419+0000","last_unstale":"2024-09-21T09:55:08.912173+0000","last_undegraded":"2024-09-21T09:55:08.912173+0000","last_fullsized":"2024-09-21T09:55:08.912173+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:13.789806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1572,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.912600+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T09:55:08.912600+0000","last_peered":"2024-09-21T09:55:08.912600+0000","last_clean":"2024-09-21T09:55:08.912600+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T09:55:08.912600+0000","last_undegraded":"2024-09-21T09:55:08.912600+0000","last_fullsized":"2024-09-21T09:55:08.912600+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1534,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.910698+0000","last_change":"2024-09-21T09:46:47.121724+0000","last_active":"2024-09-21T09:55:08.910698+0000","last_peered":"2024-09-21T09:55:08.910698+0000","last_clean":"2024-09-21T09:55:08.910698+0000","last_became_active":"2024-09-21T09:46:47.121406+0000","last_became_peered":"2024-09-21T09:46:47.121406+0000","last_unstale":"2024-09-21T09:55:08.910698+0000","last_undegraded":"2024-09-21T09:55:08.910698+0000","last_fullsized":"2024-09-21T09:55:08.910698+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:55.493704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1485,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911362+0000","last_change":"2024-09-21T09:46:47.113184+0000","last_active":"2024-09-21T09:55:08.911362+0000","last_peered":"2024-09-21T09:55:08.911362+0000","last_clean":"2024-09-21T09:55:08.911362+0000","last_became_active":"2024-09-21T09:46:47.112857+0000","last_became_peered":"2024-09-21T09:46:47.112857+0000","last_unstale":"2024-09-21T09:55:08.911362+0000","last_undegraded":"2024-09-21T09:55:08.911362+0000","last_fullsized":"2024-09-21T09:55:08.911362+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:46.783986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1036,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300495+0000","last_change":"2024-09-21T09:46:13.262555+0000","last_active":"2024-09-21T09:55:03.300495+0000","last_peered":"2024-09-21T09:55:03.300495+0000","last_clean":"2024-09-21T09:55:03.300495+0000","last_became_active":"2024-09-21T09:46:13.261474+0000","last_became_peered":"2024-09-21T09:46:13.261474+0000","last_unstale":"2024-09-21T09:55:03.300495+0000","last_undegraded":"2024-09-21T09:55:03.300495+0000","last_fullsized":"2024-09-21T09:55:03.300495+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:44.865988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1523,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.316853+0000","last_change":"2024-09-21T09:46:47.114406+0000","last_active":"2024-09-21T09:55:08.316853+0000","last_peered":"2024-09-21T09:55:08.316853+0000","last_clean":"2024-09-21T09:55:08.316853+0000","last_became_active":"2024-09-21T09:46:47.113986+0000","last_became_peered":"2024-09-21T09:46:47.113986+0000","last_unstale":"2024-09-21T09:55:08.316853+0000","last_undegraded":"2024-09-21T09:55:08.316853+0000","last_fullsized":"2024-09-21T09:55:08.316853+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:32.292201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1562,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914239+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T09:55:08.914239+0000","last_peered":"2024-09-21T09:55:08.914239+0000","last_clean":"2024-09-21T09:55:08.914239+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T09:55:08.914239+0000","last_undegraded":"2024-09-21T09:55:08.914239+0000","last_fullsized":"2024-09-21T09:55:08.914239+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300266+0000","last_change":"2024-09-21T09:46:47.114188+0000","last_active":"2024-09-21T09:55:03.300266+0000","last_peered":"2024-09-21T09:55:03.300266+0000","last_clean":"2024-09-21T09:55:03.300266+0000","last_became_active":"2024-09-21T09:46:47.113859+0000","last_became_peered":"2024-09-21T09:46:47.113859+0000","last_unstale":"2024-09-21T09:55:03.300266+0000","last_undegraded":"2024-09-21T09:55:03.300266+0000","last_fullsized":"2024-09-21T09:55:03.300266+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:22.670368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":1024,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412132+0000","last_change":"2024-09-21T09:45:39.131124+0000","last_active":"2024-09-21T09:55:03.412132+0000","last_peered":"2024-09-21T09:55:03.412132+0000","last_clean":"2024-09-21T09:55:03.412132+0000","last_became_active":"2024-09-21T09:45:39.130756+0000","last_became_peered":"2024-09-21T09:45:39.130756+0000","last_unstale":"2024-09-21T09:55:03.412132+0000","last_undegraded":"2024-09-21T09:55:03.412132+0000","last_fullsized":"2024-09-21T09:55:03.412132+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:49.130698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1034,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.300392+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T09:55:03.300392+0000","last_peered":"2024-09-21T09:55:03.300392+0000","last_clean":"2024-09-21T09:55:03.300392+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T09:55:03.300392+0000","last_undegraded":"2024-09-21T09:55:03.300392+0000","last_fullsized":"2024-09-21T09:55:03.300392+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":1006,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.412104+0000","last_change":"2024-09-21T09:36:43.115791+0000","last_active":"2024-09-21T09:55:03.412104+0000","last_peered":"2024-09-21T09:55:03.412104+0000","last_clean":"2024-09-21T09:55:03.412104+0000","last_became_active":"2024-09-21T09:36:43.115525+0000","last_became_peered":"2024-09-21T09:36:43.115525+0000","last_unstale":"2024-09-21T09:55:03.412104+0000","last_undegraded":"2024-09-21T09:55:03.412104+0000","last_fullsized":"2024-09-21T09:55:03.412104+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:28.396722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1495,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.911833+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T09:55:08.911833+0000","last_peered":"2024-09-21T09:55:08.911833+0000","last_clean":"2024-09-21T09:55:08.911833+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T09:55:08.911833+0000","last_undegraded":"2024-09-21T09:55:08.911833+0000","last_fullsized":"2024-09-21T09:55:08.911833+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1481,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.913152+0000","last_change":"2024-09-21T09:46:47.115788+0000","last_active":"2024-09-21T09:55:08.913152+0000","last_peered":"2024-09-21T09:55:08.913152+0000","last_clean":"2024-09-21T09:55:08.913152+0000","last_became_active":"2024-09-21T09:46:47.114864+0000","last_became_peered":"2024-09-21T09:46:47.114864+0000","last_unstale":"2024-09-21T09:55:08.913152+0000","last_undegraded":"2024-09-21T09:55:08.913152+0000","last_fullsized":"2024-09-21T09:55:08.913152+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:00.034944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1017,"reported_epoch":1075,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.299925+0000","last_change":"2024-09-21T09:45:39.143064+0000","last_active":"2024-09-21T09:55:03.299925+0000","last_peered":"2024-09-21T09:55:03.299925+0000","last_clean":"2024-09-21T09:55:03.299925+0000","last_became_active":"2024-09-21T09:45:39.142699+0000","last_became_peered":"2024-09-21T09:45:39.142699+0000","last_unstale":"2024-09-21T09:55:03.299925+0000","last_undegraded":"2024-09-21T09:55:03.299925+0000","last_fullsized":"2024-09-21T09:55:03.299925+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:14.684721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1577,"reported_epoch":1076,"state":"active+clean","last_fresh":"2024-09-21T09:55:08.914281+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T09:55:08.914281+0000","last_peered":"2024-09-21T09:55:08.914281+0000","last_clean":"2024-09-21T09:55:08.914281+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T09:55:08.914281+0000","last_undegraded":"2024-09-21T09:55:08.914281+0000","last_fullsized":"2024-09-21T09:55:08.914281+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397763,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317368,"kb_used_data":7532,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93427784,"statfs":{"total":95995035648,"available":95670050816,"internally_reserved":0,"allocated":7712768,"data_stored":5600360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593992,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313200,"kb_used_data":7524,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93431952,"statfs":{"total":95995035648,"available":95674318848,"internally_reserved":0,"allocated":7704576,"data_stored":5600225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790221,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316768,"kb_used_data":6936,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93428384,"statfs":{"total":95995035648,"available":95670665216,"internally_reserved":0,"allocated":7102464,"data_stored":5005683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986448,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308460,"kb_used_data":6948,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93436692,"statfs":{"total":95995035648,"available":95679172608,"internally_reserved":0,"allocated":7114752,"data_stored":5009914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999090,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":48744,"kb_used_data":8608,"kb_used_omap":25,"kb_used_meta":40102,"kb_avail":93696408,"statfs":{"total":95995035648,"available":95945121792,"internally_reserved":0,"allocated":8814592,"data_stored":6219336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":41065415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111032,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40984,"kb_used_data":8792,"kb_used_omap":31,"kb_used_meta":31968,"kb_avail":93704168,"statfs":{"total":95995035648,"available":95953068032,"internally_reserved":0,"allocated":9003008,"data_stored":6349770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":32735464},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318222975,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40672,"kb_used_data":8608,"kb_used_omap":26,"kb_used_meta":31845,"kb_avail":93704480,"statfs":{"total":95995035648,"available":95953387520,"internally_reserved":0,"allocated":8814592,"data_stored":6219273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":32609292},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289334918,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67868,"kb_used_data":9372,"kb_used_omap":31,"kb_used_meta":58272,"kb_avail":93677284,"statfs":{"total":95995035648,"available":95925538816,"internally_reserved":0,"allocated":9596928,"data_stored":6940134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":59670624},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:56:08.265 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:56:08.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump 2024-09-21T09:56:08.524 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:08.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:08 smithi017 ceph-mon[102913]: from='client.35247 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:08.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:08 smithi017 ceph-mon[105151]: from='client.35247 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:08 smithi134 ceph-mon[90085]: from='client.35247 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:09.771 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:09 smithi017 ceph-mon[102913]: pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:09.772 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:09 smithi017 ceph-mon[105151]: pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:09 smithi134 ceph-mon[90085]: pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:10.188 INFO:teuthology.orchestra.run.smithi017.stdout:epoch 1076 2024-09-21T09:56:10.188 INFO:teuthology.orchestra.run.smithi017.stdout:fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 2024-09-21T09:56:10.188 INFO:teuthology.orchestra.run.smithi017.stdout:created 2024-09-21T09:29:47.760456+0000 2024-09-21T09:56:10.188 INFO:teuthology.orchestra.run.smithi017.stdout:modified 2024-09-21T09:55:08.309073+0000 2024-09-21T09:56:10.188 INFO:teuthology.orchestra.run.smithi017.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-09-21T09:56:10.188 INFO:teuthology.orchestra.run.smithi017.stdout:crush_version 22 2024-09-21T09:56:10.188 INFO:teuthology.orchestra.run.smithi017.stdout:full_ratio 0.95 2024-09-21T09:56:10.188 INFO:teuthology.orchestra.run.smithi017.stdout:backfillfull_ratio 0.9 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:nearfull_ratio 0.85 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:require_min_compat_client luminous 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:min_compat_client luminous 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:require_osd_release quincy 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:stretch_mode_enabled false 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 58 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 63 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:max_osd 8 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 up in weight 1 up_from 479 up_thru 581 down_at 467 last_clean_interval [8,466) [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120] [v2:172.21.15.17:6804/2105074120,v1:172.21.15.17:6805/2105074120] exists,up bb258889-3128-4c2e-b5da-343281340eb4 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 up in weight 1 up_from 513 up_thru 581 down_at 501 last_clean_interval [13,500) [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230] [v2:172.21.15.17:6812/1078446230,v1:172.21.15.17:6813/1078446230] exists,up efe0af53-c763-4d08-8051-7f781782ab0b 2024-09-21T09:56:10.189 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 up in weight 1 up_from 547 up_thru 581 down_at 535 last_clean_interval [18,534) [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954] [v2:172.21.15.17:6820/323142954,v1:172.21.15.17:6821/323142954] exists,up 59c1433f-c409-4616-a7b9-7a11d8fb28d8 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 up in weight 1 up_from 581 up_thru 581 down_at 569 last_clean_interval [26,568) [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816] [v2:172.21.15.17:6828/3146888816,v1:172.21.15.17:6829/3146888816] exists,up 9fb2cdbe-3662-4ee4-b09d-96dffa74d226 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 up in weight 1 up_from 31 up_thru 581 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249] [v2:172.21.15.134:6802/2226434249,v1:172.21.15.134:6803/2226434249] exists,up 1d6fde21-8847-4673-8b7f-523f387faf50 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 up in weight 1 up_from 37 up_thru 581 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742] [v2:172.21.15.134:6810/852015742,v1:172.21.15.134:6811/852015742] exists,up 2bae1d15-c9ee-4f67-82ed-31fb25c11749 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 up in weight 1 up_from 43 up_thru 569 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667] [v2:172.21.15.134:6818/2227687667,v1:172.21.15.134:6819/2227687667] exists,up dc8f4f0e-887e-424a-bb0c-cb9cd88e6107 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 up in weight 1 up_from 49 up_thru 581 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708] [v2:172.21.15.134:6826/2365723708,v1:172.21.15.134:6827/2365723708] exists,up edff61f5-4a21-484b-9924-e15db0d909a4 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:pg_upmap_items 4.2 [1,2] 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:pg_upmap_items 4.a [1,2] 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:pg_upmap_items 4.1f [1,2] 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3863855841 expires 2024-09-22T09:30:18.002697+0000 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2291861207 expires 2024-09-22T09:35:23.638701+0000 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.134:0/3530815435 expires 2024-09-22T09:40:59.077787+0000 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3512479442 expires 2024-09-22T09:39:24.978166+0000 2024-09-21T09:56:10.190 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3084797810 expires 2024-09-22T09:41:39.758476+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/269146626 expires 2024-09-22T09:30:41.864498+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2599173714 expires 2024-09-22T09:41:39.758476+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/510140310 expires 2024-09-22T09:30:41.864498+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/2894617542 expires 2024-09-22T09:35:23.638701+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/381036987 expires 2024-09-22T09:41:39.758476+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.134:6834/72443452 expires 2024-09-22T09:40:59.077787+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3671582122 expires 2024-09-22T09:39:24.978166+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/2576370885 expires 2024-09-22T09:41:39.758476+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.134:6835/72443452 expires 2024-09-22T09:40:59.077787+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/758458220 expires 2024-09-22T09:41:39.758476+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3206271349 expires 2024-09-22T09:35:23.638701+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3226197858 expires 2024-09-22T09:41:39.758476+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/1944945429 expires 2024-09-22T09:30:41.864498+0000 2024-09-21T09:56:10.191 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/4015678849 expires 2024-09-22T09:30:18.002697+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/3625002403 expires 2024-09-22T09:39:24.978166+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1996088326 expires 2024-09-22T09:30:41.864498+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/1944945429 expires 2024-09-22T09:30:41.864498+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3490773646 expires 2024-09-22T09:30:18.002697+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2266250644 expires 2024-09-22T09:35:23.638701+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.134:0/2350962326 expires 2024-09-22T09:40:59.077787+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.134:0/3275227882 expires 2024-09-22T09:40:59.077787+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/3745862344 expires 2024-09-22T09:30:18.002697+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/3625002403 expires 2024-09-22T09:39:24.978166+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1454017192 expires 2024-09-22T09:35:23.638701+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/877591523 expires 2024-09-22T09:39:24.978166+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/2894617542 expires 2024-09-22T09:35:23.638701+0000 2024-09-21T09:56:10.192 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.134:0/1515484230 expires 2024-09-22T09:40:59.077787+0000 2024-09-21T09:56:10.193 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/2576370885 expires 2024-09-22T09:41:39.758476+0000 2024-09-21T09:56:10.193 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3086297034 expires 2024-09-22T09:39:24.978166+0000 2024-09-21T09:56:10.193 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2023103508 expires 2024-09-22T09:41:39.758476+0000 2024-09-21T09:56:10.193 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/3745862344 expires 2024-09-22T09:30:18.002697+0000 2024-09-21T09:56:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:56:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:56:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:10.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:10 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2705977822' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-21T09:56:10.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:10.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:10 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2705977822' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-21T09:56:10.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:10.799 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 479 up_thru 581 down_at 467 last_clean_interval [8,466) [v2:172.21.15.17:6802/2105074120,v1:172.21.15.17:6803/2105074120] [v2:172.21.15.17:6804/2105074120,v1:172.21.15.17:6805/2105074120] exists,up bb258889-3128-4c2e-b5da-343281340eb4', 'osd.1 up in weight 1 up_from 513 up_thru 581 down_at 501 last_clean_interval [13,500) [v2:172.21.15.17:6810/1078446230,v1:172.21.15.17:6811/1078446230] [v2:172.21.15.17:6812/1078446230,v1:172.21.15.17:6813/1078446230] exists,up efe0af53-c763-4d08-8051-7f781782ab0b', 'osd.2 up in weight 1 up_from 547 up_thru 581 down_at 535 last_clean_interval [18,534) [v2:172.21.15.17:6818/323142954,v1:172.21.15.17:6819/323142954] [v2:172.21.15.17:6820/323142954,v1:172.21.15.17:6821/323142954] exists,up 59c1433f-c409-4616-a7b9-7a11d8fb28d8', 'osd.3 up in weight 1 up_from 581 up_thru 581 down_at 569 last_clean_interval [26,568) [v2:172.21.15.17:6826/3146888816,v1:172.21.15.17:6827/3146888816] [v2:172.21.15.17:6828/3146888816,v1:172.21.15.17:6829/3146888816] exists,up 9fb2cdbe-3662-4ee4-b09d-96dffa74d226', 'osd.4 up in weight 1 up_from 31 up_thru 581 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6800/2226434249,v1:172.21.15.134:6801/2226434249] [v2:172.21.15.134:6802/2226434249,v1:172.21.15.134:6803/2226434249] exists,up 1d6fde21-8847-4673-8b7f-523f387faf50', 'osd.5 up in weight 1 up_from 37 up_thru 581 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6808/852015742,v1:172.21.15.134:6809/852015742] [v2:172.21.15.134:6810/852015742,v1:172.21.15.134:6811/852015742] exists,up 2bae1d15-c9ee-4f67-82ed-31fb25c11749', 'osd.6 up in weight 1 up_from 43 up_thru 569 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6816/2227687667,v1:172.21.15.134:6817/2227687667] [v2:172.21.15.134:6818/2227687667,v1:172.21.15.134:6819/2227687667] exists,up dc8f4f0e-887e-424a-bb0c-cb9cd88e6107', 'osd.7 up in weight 1 up_from 49 up_thru 581 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6824/2365723708,v1:172.21.15.134:6825/2365723708] [v2:172.21.15.134:6826/2365723708,v1:172.21.15.134:6827/2365723708] exists,up edff61f5-4a21-484b-9924-e15db0d909a4'] 2024-09-21T09:56:10.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon mon.a config show 2024-09-21T09:56:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:10 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2705977822' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-21T09:56:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:11.055 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:11 smithi017 ceph-mon[102913]: pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:11 smithi017 ceph-mon[105151]: pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:11 smithi134 ceph-mon[90085]: pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:12.794 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:56:12.795 INFO:teuthology.orchestra.run.smithi017.stdout: "name": "mon.a", 2024-09-21T09:56:12.795 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster": "ceph", 2024-09-21T09:56:12.795 INFO:teuthology.orchestra.run.smithi017.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-09-21T09:56:12.795 INFO:teuthology.orchestra.run.smithi017.stdout: "admin_socket_mode": "", 2024-09-21T09:56:12.795 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-09-21T09:56:12.795 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_client_required": "cephx, none", 2024-09-21T09:56:12.795 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_cluster_required": "cephx", 2024-09-21T09:56:12.795 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_debug": "true", 2024-09-21T09:56:12.795 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_service_required": "cephx", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_service_ticket_ttl": "240.000000", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_supported": "", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio": "true", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio_max_queue_depth": "1024", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio_poll_ms": "250", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio_reap_max": "16", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_async_discard_threads": "0", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_block_size": "4096", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_aio": "false", 2024-09-21T09:56:12.796 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_inflight_ios": "false", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_enable_discard": "false", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_flock_retry": "3", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_flock_retry_interval": "0.100000", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_inject_crash": "0", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_inject_crash_flush_delay": "2", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_ioring": "false", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_ioring_hipri": "false", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_ioring_sqthread_poll": "false", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_read_buffer_alignment": "4096", 2024-09-21T09:56:12.797 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_type": "", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_alloc_size": "1048576", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_allocator": "hybrid", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_buffered_io": "true", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_check_for_zeros": "false", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_check_volume_selector_often": "false", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_compact_log_sync": "false", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-09-21T09:56:12.798 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_log_compact_min_size": "16777216", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_log_replay_check_allocations": "true", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_max_log_runway": "4194304", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_max_prefetch": "1048576", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_min_flush_size": "524288", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_min_log_runway": "1048576", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_replay_recovery": "false", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_shared_alloc_size": "65536", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_sync_write": "false", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-09-21T09:56:12.799 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_allocation_from_file": "true", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_allocator": "hybrid", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_async_db_compaction": "true", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bdev_label_multi": "true", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bdev_label_require_all": "true", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-09-21T09:56:12.800 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_blobid_prealloc": "10240", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_create": "true", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_db_create": "false", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_db_path": "", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_db_size": "0", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_path": "", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_preallocate_file": "false", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_size": "107374182400", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_wal_create": "false", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_wal_path": "", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_wal_size": "100663296", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs": "true", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-09-21T09:56:12.801 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs_env_mirror": "false", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_autotune": "true", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-09-21T09:56:12.802 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_size": "0", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_type": "2q", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_clone_cow": "true", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_algorithm": "snappy", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_max_blob_size": "0", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_min_blob_size": "0", 2024-09-21T09:56:12.803 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_mode": "none", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_csum_type": "crc32c", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_enforce_settings": "default", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_freelist": "false", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_fsck_abort": "false", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_read_err": "false", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_legacy_omap": "false", 2024-09-21T09:56:12.804 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_small_allocations": "0", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_default_buffered_read": "true", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_default_buffered_write": "false", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_deferred_batch_ops": "0", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-09-21T09:56:12.805 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_elastic_shared_blobs": "true", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fail_eio": "false", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-09-21T09:56:12.806 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mkfs": "true", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mount": "false", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_umount": "false", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-09-21T09:56:12.807 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_ignore_data_csum": "false", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_kvbackend": "rocksdb", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_log_op_age": "5.000000", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_alloc_size": "0", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_blob_size": "0", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_defer_interval": "3.000000", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_deferred_txc": "32", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_min_alloc_size": "0", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-09-21T09:56:12.808 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_nid_prealloc": "1024", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_prefer_deferred_size": "0", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_qfsck_on_mount": "true", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_retry_disk_reads": "3", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_cf": "true", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_options_annex": "", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-09-21T09:56:12.809 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_coremask": "0x1", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_io_sleep": "5", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_max_io_completion": "0", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_mem": "512", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_sync_submit_transaction": "false", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_bytes": "67108864", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_cost_per_io": "0", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_tracing": "false", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-09-21T09:56:12.810 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_volume_selection_reserved": "0", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_write_v2": "false", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_write_v2_random": "false", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_zero_block_detection": "false", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_action_update_interval": "2", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-09-21T09:56:12.811 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_mount_timeout": "10", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_cluster_require_signatures": "false", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_cluster_require_version": "2", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_require_signatures": "false", 2024-09-21T09:56:12.812 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_require_version": "2", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_service_require_signatures": "false", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_service_require_version": "2", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_sign_messages": "true", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "chdir": "", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_acl_type": "", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_asio_thread_count": "2", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_cache_mid": "0.750000", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_cache_size": "16384", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_caps_release_delay": "5", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_check_pool_perm": "true", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_collect_and_send_global_metrics": "false", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_force_sync_read": "false", 2024-09-21T09:56:12.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_getattr_caps": "false", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_inject_features": "", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_inject_tick_delay": "0", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_die_on_failed_remount": "false", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_dirsize_rbytes": "true", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_force_lazyio": "false", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_fs": "", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_inject_fixed_oldest_tid": "false", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_inject_release_failure": "false", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_max_inline_size": "4096", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_max_retries_on_remount_failure": "5", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mds_namespace": "", 2024-09-21T09:56:12.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_metadata": "", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mount_gid": "-1", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mount_timeout": "300", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mount_uid": "-1", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mountpoint": "/", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_notify_timeout": "10", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc": "true", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_max_dirty": "104857600", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_max_dirty_age": "5.000000", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_max_objects": "1000", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_size": "209715200", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_target_dirty": "8388608", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_permissions": "true", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_quota": "true", 2024-09-21T09:56:12.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_quota_df": "true", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_readahead_max_bytes": "0", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_readahead_max_periods": "4", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_readahead_min": "131072", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_reconnect_stale": "false", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_shutdown_timeout": "30", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_snapdir": ".snap", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_tick_interval": "1", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_trace": "", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_try_dentry_invalidate": "false", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_use_faked_inos": "false", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_use_random_mds": "false", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_graylog": "false", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-09-21T09:56:12.816 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_graylog_port": "12201", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_monitors": "default=true", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_syslog": "false", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_syslog_level": "info", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster_addr": "-", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster_network": "", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster_network_interface": "", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zlib_isal": "false", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zlib_level": "5", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zlib_winsize": "-15", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zstd_level": "1", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-09-21T09:56:12.817 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_alien_op_num_threads": "6", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_alien_thread_cpu_cores": "", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_osd_obc_lru_size": "512", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_osd_stat_interval": "0", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_seastar_cpu_cores": "", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_seastar_num_threads": "0", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "crush_location": "", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "crush_location_hook": "", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "crush_location_hook_timeout": "10", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "daemonize": "false", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "daos_pool": "tank", 2024-09-21T09:56:12.818 INFO:teuthology.orchestra.run.smithi017.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "dbstore_db_name_prefix": "dbstore", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_alienstore": "0/5", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_allow_any_pool_priority": "false", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_asok": "1/5", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_asok_assert_abort": "false", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_asserts_on_shutdown": "true", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_auth": "1/5", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_bdev": "1/3", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_bluefs": "1/5", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_bluestore": "1/5", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_buffer": "0/1", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_ceph_dedup": "0/5", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_ceph_exporter": "1/5", 2024-09-21T09:56:12.819 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_cephfs_mirror": "0/5", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_cephsqlite": "0/5", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_client": "0/5", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_compressor": "1/5", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_context": "0/1", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_crimson_interrupt": "0/5", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_crush": "1/1", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_crypto": "1/5", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_cyanstore": "0/5", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_deliberately_leak_memory": "false", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_disable_randomized_ping": "false", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_dpdk": "1/5", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_eventtrace": "1/5", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_filer": "0/1", 2024-09-21T09:56:12.820 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_filestore": "1/3", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_finisher": "1/1", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_fuse": "1/5", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_heartbeat_testing_span": "0", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_heartbeatmap": "1/5", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_immutable_obj_cache": "0/5", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_javaclient": "1/5", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_journal": "1/3", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_journaler": "0/5", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_kstore": "1/5", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_lockdep": "0/1", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mclock": "1/5", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds": "1/5", 2024-09-21T09:56:12.821 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_balancer": "1/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_locker": "1/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_log": "1/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_log_expire": "1/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_migrator": "1/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_quiesce": "3/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_memstore": "1/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mgr": "2/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mgrc": "1/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mon": "20/20", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_monc": "0/10", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_ms": "1/1", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_none": "0/5", 2024-09-21T09:56:12.822 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_objclass": "0/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_objectcacher": "0/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_objecter": "0/1", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_optracker": "0/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_osd": "1/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_paxos": "20/20", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_perfcounter": "1/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_prioritycache": "1/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rados": "0/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd": "0/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd_mirror": "0/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd_pwl": "0/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd_replay": "0/5", 2024-09-21T09:56:12.823 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_refs": "0/0", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_reserver": "1/1", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw": "1/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_access": "1/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_datacache": "1/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_dbstore": "1/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_flight": "1/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_lifecycle": "1/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_notification": "1/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_sync": "1/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rocksdb": "4/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore": "0/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_backref": "0/5", 2024-09-21T09:56:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_cache": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_cleaner": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_device": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_epm": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_journal": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_lba": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_odata": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_omap": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_onode": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_t": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_tm": "0/5", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_striper": "0/1", 2024-09-21T09:56:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_test": "0/5", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_throttle": "1/1", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_timer": "0/1", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_tp": "0/5", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_trace": "1/5", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "device_failure_prediction_mode": "none", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_graylog": "false", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_journald": "false", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_stderr": "true", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_syslog": "false", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event_tracing": "false", 2024-09-21T09:56:12.826 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_addr": "0.0.0.0", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_cert_file": "", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_http_port": "9926", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_key_file": "", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_prio_limit": "5", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_sort_metrics": "true", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_stats_period": "5", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "fake_statfs_for_testing": "0", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "fatal_signal_handlers": "true", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "filer_max_purge_ops": "10", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "filer_max_truncate_ops": "128", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_apply_finisher_threads": "1", 2024-09-21T09:56:12.827 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_blackhole": "false", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_btrfs_clone_range": "true", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_btrfs_snap": "true", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_caller_concurrency": "10", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_collect_device_partition_information": "true", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_commit_timeout": "600.000000", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_inject_read_err": "false", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_omap_check": "false", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_verify_split": "false", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_dump_file": "", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fadvise": "true", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fail_eio": "true", 2024-09-21T09:56:12.828 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fd_cache_shards": "16", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fd_cache_size": "128", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fiemap": "false", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fiemap_threshold": "4096", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_index_retry_probability": "0.000000", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_inject_stall": "0", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_journal_parallel": "false", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_journal_trailing": "false", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_journal_writeahead": "false", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_kill_at": "0", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size": "0", 2024-09-21T09:56:12.829 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs": "0", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs_other": "2", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_sync_interval": "5.000000", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size": "0", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_merge_threshold": "-10", 2024-09-21T09:56:12.830 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_min_sync_interval": "0.010000", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_odsync_write": "false", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_omap_backend": "rocksdb", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_omap_backend_path": "", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_omap_header_cache_size": "1024", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_ondisk_finisher_threads": "1", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_op_thread_timeout": "60", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_op_threads": "2", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_punch_hole": "false", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-09-21T09:56:12.831 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_bytes": "104857600", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_ops": "50", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_seek_data_hole": "false", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_sloppy_crc": "false", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_splice": "false", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_split_multiple": "2", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_split_rand_factor": "20", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_update_to": "1000", 2024-09-21T09:56:12.832 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_enable": "true", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-09-21T09:56:12.833 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_xfs_extsize": "false", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_zfs_snap": "false", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fio_dir": "/tmp/fio", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fsid": "ee55f8cc-77fb-11ef-baf6-efdc52797490", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_allow_other": "true", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_atomic_o_trunc": "true", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_big_writes": "true", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_debug": "false", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_default_permissions": "false", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_disable_pagecache": "false", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_max_write": "0", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_multithreaded": "true", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_require_active_mds": "true", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_set_user_groups": "true", 2024-09-21T09:56:12.834 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_splice_move": "true", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_splice_read": "true", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_splice_write": "true", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_syncfs_on_mksnap": "true", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_use_invalidate_cb": "true", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "gss_target_name": "ceph", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "heartbeat_file": "", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "heartbeat_inject_failure": "0", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "heartbeat_interval": "5", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "host": "smithi017", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-09-21T09:56:12.835 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_max_size": "1073741824", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_watermark": "0.900000", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "inject_early_sigterm": "false", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "jaeger_agent_port": "6799", 2024-09-21T09:56:12.836 INFO:teuthology.orchestra.run.smithi017.stdout: "jaeger_tracing_enable": "false", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_aio": "true", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_align_min_size": "65536", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_block_align": "true", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_block_size": "4096", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_dio": "true", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_discard": "false", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_force_aio": "false", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_ignore_corruption": "false", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_max_write_bytes": "10485760", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_max_write_entries": "100", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_replay_from": "0", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_high_multiple": "0.000000", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-09-21T09:56:12.837 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_max_multiple": "0.000000", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_write_header_frequency": "0", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_zero_on_create": "false", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "journaler_prefetch_periods": "10", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "journaler_prezero_periods": "5", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "journaler_write_head_interval": "15", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "key": "", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "keyfile": "", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_backend": "rocksdb", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_default_stripe_size": "65536", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_fsck_on_mount": "false", 2024-09-21T09:56:12.838 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_fsck_on_mount_deep": "true", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_max_bytes": "67108864", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_max_ops": "512", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_nid_prealloc": "1024", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_onode_map_size": "1024", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_sync_submit_transaction": "false", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_sync_transaction": "false", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "librados_thread_count": "2", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "lockdep": "false", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "lockdep_force_backtrace": "false", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "log_coarse_timestamps": "true", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-09-21T09:56:12.839 INFO:teuthology.orchestra.run.smithi017.stdout: "log_flush_on_exit": "false", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_graylog_host": "127.0.0.1", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_graylog_port": "12201", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_max_new": "1000", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_max_recent": "10000", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_stderr_prefix": "", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_stop_at_utilization": "0.970000", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_file": "true", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_graylog": "false", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_journald": "false", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_stderr": "false", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_syslog": "false", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "max_rotating_auth_attempts": "10", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-09-21T09:56:12.840 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_action_on_write_error": "1", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_alternate_name_max": "8192", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_asio_thread_count": "2", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_export_pin": "true", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_dirs": "true", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_interval": "5", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_size_max": "100000", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_idle_threshold": "0.000000", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_interval": "10", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_max": "-1", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_max_until": "-1", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_merge_size": "50", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_midchunk": "0.300000", 2024-09-21T09:56:12.841 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_min_rebalance": "0.100000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_min_start": "0.200000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_minchunk": "0.001000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_mode": "0", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_need_max": "1.200000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_need_min": "0.800000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_overload_epochs": "2", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_sample_interval": "3.000000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_bits": "3", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_rd": "25000.000000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_size": "10000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_wr": "10000.000000", 2024-09-21T09:56:12.842 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_target_decay": "10.000000", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_beacon_grace": "15.000000", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_beacon_interval": "4.000000", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_beacon_mon_down_grace": "60", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_memory_limit": "4294967296", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_mid": "0.700000", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_delay": "0", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_sleep": "200", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_splitauth": "true", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_quiesce_threshold": "524288", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_release_free_interval": "10", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_reservation": "0.050000", 2024-09-21T09:56:12.843 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_trim_interval": "1", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_trim_threshold": "262144", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_client_delegate_inos_pct": "50", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_client_prealloc_inos": "1000", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_connect_bootstrapping": "false", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_damage_table_max_entries": "10000", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_auth_pins": "false", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_frag": "false", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_scatterstat": "false", 2024-09-21T09:56:12.844 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_subtrees": "false", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_decay_halflife": "5.000000", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_default_dir_hash": "2", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_defer_session_stale": "true", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_deny_all_reconnect": "false", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_keys_per_op": "16384", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_max_commit_size": "10", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_max_entries": "0", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_prefetch": "true", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dirstat_min_interval": "1.000000", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_after_rejoin": "false", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_on_map": "false", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_threshold_file": "0", 2024-09-21T09:56:12.845 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_early_reply": "true", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_enable_op_tracker": "true", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_enforce_unique_name": "true", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_distributed": "true", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_random": "true", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_forward_all_requests_to_auth": "false", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-09-21T09:56:12.846 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_health_cache_threshold": "1.500000", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_health_summarize_threshold": "10", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_heartbeat_grace": "15.000000", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_heartbeat_reset_grace": "1000", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_health_dummy": "false", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_migrator_session_race": "false", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_join_fs": "", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_journal_format": "1", 2024-09-21T09:56:12.847 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_create_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_dirfrag_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_export_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_import_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_journal_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_journal_expire_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_journal_replay_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_link_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_mdstable_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_openc_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_rename_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_shutdown_at": "0", 2024-09-21T09:56:12.848 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_event_large_threshold": "524288", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_events_per_segment": "1024", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_major_segment_event_ratio": "12", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_max_events": "-1", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_max_segments": "128", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_pause": "false", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_segment_size": "0", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_skip_corrupt_events": "false", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_skip_unbounded_events": "false", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_trim_threshold": "128", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-09-21T09:56:12.849 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_warn_factor": "2.000000", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_caps_per_client": "1048576", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_completed_flushes": "100000", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_completed_requests": "100000", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_export_size": "20971520", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_file_recover": "32", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_purge_files": "64", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_purge_ops": "8192", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_snaps_per_dir": "100", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_metrics_update_interval": "2", 2024-09-21T09:56:12.850 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_min_caps_per_client": "100", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_min_caps_working_set": "10000", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_numa_node": "-1", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_complaint_time": "30.000000", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_duration": "600", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_size": "20", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_slow_op_size": "20", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_slow_op_threshold": "10", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_log_threshold": "5", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_ping_grace": "15", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_ping_interval": "5", 2024-09-21T09:56:12.851 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_max_caps": "30000", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_max_decay_threshold": "131072", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_warning_threshold": "262144", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_reconnect_timeout": "45.000000", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_replay_interval": "1.000000", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_root_ino_gid": "0", 2024-09-21T09:56:12.852 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_root_ino_uid": "0", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_scrub_stats_review_period": "1", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_blocklist_on_evict": "true", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_blocklist_on_timeout": "true", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_metadata_threshold": "16777216", 2024-09-21T09:56:12.853 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-09-21T09:56:12.854 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_shutdown_check": "0", 2024-09-21T09:56:12.854 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_skip_ino": "0", 2024-09-21T09:56:12.854 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_sleep_rank_change": "0.000000", 2024-09-21T09:56:12.854 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_snap_max_uid": "4294967294", 2024-09-21T09:56:12.854 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_snap_min_uid": "0", 2024-09-21T09:56:12.854 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_snap_rstat": "false", 2024-09-21T09:56:12.855 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_standby_replay_damaged": "false", 2024-09-21T09:56:12.855 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_symlink_recovery": "true", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_task_status_update_interval": "2.000000", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_thrash_exports": "0", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_thrash_fragments": "0", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_tick_interval": "5.000000", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_valgrind_exit": "false", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_verify_backtrace": "1", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_verify_scatter": "false", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_wipe_ino_prealloc": "false", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_wipe_sessions": "false", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "mempool_debug": "false", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_debug_omit_block_device_write": "false", 2024-09-21T09:56:12.856 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_device_bytes": "1073741824", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_page_set": "false", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_page_size": "65536", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_client_bytes": "134217728", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_client_messages": "512", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_connect_retry_interval": "1.000000", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_disabled_modules": "", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_enable_op_tracker": "true", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_max_pg_creating": "1024", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_max_pg_num_change": "128", 2024-09-21T09:56:12.857 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mds_bytes": "134217728", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mds_messages": "128", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mon_bytes": "134217728", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mon_messages": "128", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_num_op_tracker_shard": "32", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_complaint_time": "30.000000", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_history_duration": "600", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_history_size": "20", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_history_slow_op_size": "20", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_op_log_threshold": "5", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_osd_bytes": "536870912", 2024-09-21T09:56:12.858 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_osd_messages": "8192", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_pool": "true", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_service_beacon_grace": "60.000000", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_standby_modules": "true", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_stats_period": "5", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_stats_threshold": "5", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_tick_period": "2", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_accept_timeout_factor": "2.000000", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_allow_pool_delete": "true", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_allow_pool_size_one": "false", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_auth_validate_all_caps": "true", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-09-21T09:56:12.859 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_bytes": "104857600", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_directed_command_retry": "2", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval": "3.000000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_parallel": "3", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_log_interval": "1.000000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_ping_interval": "10.000000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_ping_timeout": "30.000000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_target_rank": "-1", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_clock_drift_allowed": "1.000000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-09-21T09:56:12.860 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_level": "debug", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_file": "true", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_graylog": "false", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_journald": "true", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_stderr": "false", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_compact_on_bootstrap": "false", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_compact_on_start": "false", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_compact_on_trim": "true", 2024-09-21T09:56:12.861 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_con_tracker_persist_interval": "10", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_con_tracker_score_halflife": "43200", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_config_key_max_entry_size": "65536", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cpu_threads": "4", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_crush_min_required_version": "hammer", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_daemon_bytes": "419430400", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data_avail_crit": "5", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data_avail_warn": "5", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data_size_warn": "16106127360", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_block_osdmap_trim": "false", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_dump_json": "false", 2024-09-21T09:56:12.862 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_dump_transactions": "false", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_extra_checks": "false", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_require_reef": "false", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_require_squid": "false", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_delta_reset_interval": "10.000000", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_dns_srv_name": "ceph-mon", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_down_mkfs_grace": "60", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_down_uptime_grace": "60", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_election_default_strategy": "1", 2024-09-21T09:56:12.863 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_election_timeout": "5.000000", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_elector_ping_divisor": "2", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_elector_ping_timeout": "2.000000", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_enable_op_tracker": "true", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_fake_pool_delete": "false", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_force_quorum_join": "false", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_fsmap_prune_threshold": "300", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_globalid_prealloc": "10000", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_detail_to_clog": "true", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_log_update_period": "5", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_max_detail": "50", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_to_clog": "true", 2024-09-21T09:56:12.864 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_to_clog_interval": "600", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_host": "[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_host_override": "", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_initial_members": "", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_keyvaluedb": "rocksdb", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_lease": "5.000000", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_log_full_interval": "50", 2024-09-21T09:56:12.865 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_log_max": "10000", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_log_max_summary": "50", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_log_entries_per_event": "4096", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_log_epochs": "500", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_mdsmap_epochs": "500", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_mgrmap_epochs": "500", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_nvmeof_epochs": "500", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_osd": "10000", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_pg_per_osd": "10000", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_pool_pg_num": "65536", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mds_force_trim_to": "0", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mds_skip_sanity": "true", 2024-09-21T09:56:12.866 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_memory_autotune": "true", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_memory_target": "2147483648", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_beacon_grace": "30", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_digest_period": "5", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_inactive_grace": "60", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_mkfs_grace": "240", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_min_osdmap_epochs": "500", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_complaint_time": "30", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_duration": "600", 2024-09-21T09:56:12.867 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_size": "20", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_slow_op_size": "20", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_slow_op_threshold": "10", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_log_threshold": "5", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_auto_mark_in": "false", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_auto_mark_new_in": "true", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_cache_size": "500", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_cache_size_min": "134217728", 2024-09-21T09:56:12.868 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_crush_smoke_test": "true", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_destroyed_out_interval": "600", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_down_out_interval": "600", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_force_trim_to": "0", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_full_ratio": "0.950000", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_laggy_halflife": "3600", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_laggy_max_interval": "300", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_laggy_weight": "0.300000", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_max_initial_pgs": "1024", 2024-09-21T09:56:12.869 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_min_down_reporters": "2", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_prime_pg_temp": "true", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_report_timeout": "900", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_warn_num_repaired": "10", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_warn_op_age": "32.000000", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-09-21T09:56:12.870 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_interval": "10", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_min": "10000", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_stuck_threshold": "60", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_min_objects": "10000", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_min_per_osd": "0", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pool_quota_crit_threshold": "0", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pool_quota_warn_threshold": "0", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_probe_timeout": "2.000000", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_max_change": "0.050000", 2024-09-21T09:56:12.871 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_max_osds": "4", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_interval": "86400", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_max_keys": "100", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_timeout": "300", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_session_timeout": "300", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_smart_report_timeout": "5", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stat_smooth_intervals": "6", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-09-21T09:56:12.872 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_pool_min_size": "2", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_pool_size": "4", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_subscribe_interval": "86400.000000", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_debug": "false", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_max_payload_keys": "2000", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_max_payload_size": "1048576", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_provider_kill_at": "0", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_requester_kill_at": "0", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_timeout": "60.000000", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_target_pg_per_osd": "100", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_tick_interval": "5", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_timecheck_interval": "300.000000", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-09-21T09:56:12.873 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_use_min_delay_socket": "false", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_older_version_delay": "604800", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_filestore_osds": "true", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_misplaced": "false", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_older_version": "true", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-09-21T09:56:12.874 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_pool_no_app": "true", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_too_few_osds": "false", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "monmap": "", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-09-21T09:56:12.875 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_tracing_enabled": "false", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_op_threads": "3", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_buffer_size": "131072", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_cm": "false", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_device_name": "", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_dscp": "96", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_gid_idx": "0", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_local_gid": "", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_polling_us": "1000", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_port_num": "1", 2024-09-21T09:56:12.876 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_roce_ver": "1", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_send_buffers": "1024", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_sl": "3", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_support_srq": "true", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_type": "ib", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_reap_threshold": "5", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_before_connect": "false", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_ipv4": "true", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_ipv6": "false", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_msgr1": "true", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_msgr2": "true", 2024-09-21T09:56:12.877 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_port_max": "7568", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_port_min": "6800", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_prefer_ipv4": "false", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_retry_count": "3", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_retry_delay": "5", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_client": "false", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_mds": "false", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_mgr": "false", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_mon": "false", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_osd": "false", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_client_mode": "crc secure", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_cluster_mode": "crc secure", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_cluster_type": "", 2024-09-21T09:56:12.878 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_compress_secure": "false", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_connection_idle_timeout": "900", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_connection_ready_timeout": "10", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_crc_data": "true", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_crc_header": "true", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_bad_msg": "false", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_bug": "true", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_old_message": "true", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_skipped_message": "false", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_unhandled_msg": "false", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_coremask": "0xF", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-09-21T09:56:12.879 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_devs_allowlist": "", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_enable_tso": "true", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_hugepages": "", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_hw_flow_control": "true", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_lro": "true", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_memory_channel": "4", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_pmd": "", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_port_id": "0", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-09-21T09:56:12.880 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dump_corrupt_message_level": "1", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dump_on_send": "false", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_initial_backoff": "0.200000", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_delay_max": "1.000000", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_delay_probability": "0.000000", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_delay_type": "", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_internal_delays": "0.000000", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_network_congestion": "0", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_socket_failures": "0", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_learn_addr_from_peer": "true", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_max_accept_failures": "4", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_max_backoff": "15.000000", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_mon_client_mode": "secure crc", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_mon_cluster_mode": "secure crc", 2024-09-21T09:56:12.881 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_mon_service_mode": "secure crc", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_osd_compress_min_size": "1024", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_osd_compress_mode": "none", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_osd_compression_algorithm": "snappy", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_pq_min_cost": "65536", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_public_type": "", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_service_mode": "crc secure", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_listen_backlog": "512", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_nodelay": "true", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_rcvbuf": "0", 2024-09-21T09:56:12.882 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_time_events_min_wait_interval": "1000", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_type": "async+posix", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "no_config_file": "false", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "nvmeof_mon_client_tick_period": "2", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_completion_locks_per_session": "32", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_debug_inject_relock_delay": "false", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_inflight_op_bytes": "104857600", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_inflight_ops": "1024", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_inject_no_watch_ping": "false", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_tick_interval": "5.000000", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_timeout": "10.000000", 2024-09-21T09:56:12.883 INFO:teuthology.orchestra.run.smithi017.stdout: "objectstore_blackhole": "false", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "openssl_engine_opts": "", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_delay_time": "5.000000", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_hist_halflife": "1000", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_max_low_ops": "2", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_max_ops": "4", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_quantize_effort": "0.100000", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_slop": "0.020000", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_allow_recovery_below_min_size": "true", 2024-09-21T09:56:12.884 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_asio_thread_count": "2", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_async_recovery_min_cost": "100", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_auto_mark_unfound_lost": "false", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backfill_retry_interval": "30.000000", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backfill_scan_max": "512", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backfill_scan_min": "64", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backoff_on_degraded": "false", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backoff_on_peering": "false", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backoff_on_unfound": "true", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_beacon_report_interval": "300", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_duration": "30", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_max_block_size": "67108864", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_small_size_max_iops": "100", 2024-09-21T09:56:12.885 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_blkin_trace_all": "false", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_blocked_scrub_grace_period": "120", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_check_for_log_corruption": "false", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_update_on_start": "true", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_message_cap": "256", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_message_size_cap": "524288000", 2024-09-21T09:56:12.886 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_op_priority": "63", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_watch_timeout": "30", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_command_max_records": "256", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_compact_on_start": "false", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_chooseleaf_type": "1", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_initial_weight": "-1.000000", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_update_on_start": "true", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_update_weight_set": "true", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-09-21T09:56:12.887 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_drop_ping_duration": "0", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_feed_pullee": "-1", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_misdirected_ops": "false", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_no_acting_change": "false", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_no_purge_strays": "false", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_op_order": "false", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_pg_log_writeout": "false", 2024-09-21T09:56:12.888 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_pretend_recovery_active": "false", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_shutdown": "false", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_trim_objects": "false", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_cached_snaps": "false", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_missing_on_start": "false", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_snaps": "false", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-09-21T09:56:12.889 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_keys": "1024", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_stride": "524288", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_default_data_pool_replay_window": "45", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_default_notify_timeout": "30", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep": "0.000000", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-09-21T09:56:12.890 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_discard_disconnected_ops": "true", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_ec_partial_reads": "true", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_enable_op_tracker": "true", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_extblkdev_plugins": "vdo", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_info": "true", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_shutdown": "true", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_shutdown_timeout": "15", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-09-21T09:56:12.891 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_function_tracing": "false", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_grace": "20", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_interval": "6", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_min_peers": "10", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_min_size": "2000", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_stale": "600", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_hit_set_max_size": "100000", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_hit_set_min_size": "1000", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-09-21T09:56:12.892 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_journal_flush_on_shutdown": "true", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_journal_size": "5120", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_kill_backfill_at": "0", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_loop_before_reset_tphandle": "64", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_cache_size": "50", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_dedup": "true", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_message_max": "40", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_message_max_bytes": "10485760", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_share_max_epochs": "40", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_attr_name_len": "100", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_attr_size": "0", 2024-09-21T09:56:12.893 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_backfills": "1", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_markdown_count": "5", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_markdown_period": "600", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_object_name_len": "2048", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_object_namespace_len": "256", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_object_size": "134217728", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_omap_entries_per_request": "1024", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pg_blocked_by": "16", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pg_log_entries": "10000", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pgls": "1024", 2024-09-21T09:56:12.894 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_push_cost": "8388608", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_push_objects": "10", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_scrubs": "3", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_trimming_pgs": "2", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_write_op_reply_len": "64", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_write_size": "90", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-09-21T09:56:12.895 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_override_recovery_settings": "false", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_profile": "balanced", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_skip_benchmark": "false", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_base": "805306368", 2024-09-21T09:56:12.896 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_cache_min": "134217728", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_target": "4294967296", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_target_autotune": "false", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_min_pg_log_entries": "250", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_min_recovery_priority": "0", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_heartbeat_interval": "30", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_report_interval": "5", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_report_max_in_flight": "2", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_num_cache_shards": "32", 2024-09-21T09:56:12.897 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_num_op_tracker_shard": "32", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_numa_auto_affinity": "true", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_numa_node": "-1", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_numa_prefer_iface": "true", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objecter_finishers": "1", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore": "bluestore", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore_fuse": "false", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore_ideal_list_max": "64", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore_tracing": "false", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_complaint_time": "30.000000", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_duration": "600", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_size": "20", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_slow_op_size": "20", 2024-09-21T09:56:12.898 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_log_threshold": "5", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_shards": "0", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_shards_hdd": "5", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_shards_ssd": "8", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_threads_per_shard": "0", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_pq_min_cost": "65536", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_queue": "mclock_scheduler", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_queue_cut_off": "high", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_thread_suicide_timeout": "150", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_thread_timeout": "15", 2024-09-21T09:56:12.899 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_open_classes_on_start": "true", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_os_flags": "0", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_peering_op_priority": "255", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_delete_cost": "1048576", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_delete_priority": "5", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_log_dups_tracked": "3000", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_log_trim_max": "10000", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_log_trim_min": "100", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_object_context_cache_count": "64", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-09-21T09:56:12.900 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_crimson": "false", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_crush_rule": "-1", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_ec_fast_read": "false", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_bulk": "false", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-09-21T09:56:12.901 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_nodelete": "false", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flags": "0", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_min_size": "0", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_pg_num": "32", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_pgp_num": "0", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_read_ratio": "70", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_size": "3", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_type": "replicated", 2024-09-21T09:56:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_use_gmt_hitset": "true", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_push_per_object_cost": "1000", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_read_ec_check_for_errors": "false", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recover_clone_overlap": "true", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recover_clone_overlap_limit": "10", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_cost": "20971520", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_delay_start": "0.000000", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_active": "0", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_active_hdd": "3", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_active_ssd": "10", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_chunk": "8388608", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_single_start": "1", 2024-09-21T09:56:12.903 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_op_priority": "3", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_op_warn_multiple": "16", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_priority": "5", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_retry_interval": "30.000000", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep": "0.000000", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_repair_during_recovery": "false", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_requested_scrub_priority": "5", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_rollback_to_cluster_snap": "", 2024-09-21T09:56:12.904 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_auto_repair": "false", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_begin_hour": "0", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_begin_week_day": "0", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_chunk_max": "15", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_chunk_min": "5", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_cost": "52428800", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_during_recovery": "false", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_end_hour": "0", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_end_week_day": "0", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_event_cost": "4096", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-09-21T09:56:12.905 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_invalid_stats": "true", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_load_threshold": "0.500000", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_max_interval": "604800.000000", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_max_preemptions": "5", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_min_interval": "86400.000000", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_priority": "5", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_retry_after_noscrub": "60", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_retry_delay": "30", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_retry_new_interval": "10", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_retry_pg_state": "60", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_retry_trimming": "10", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_sleep": "0.000000", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-09-21T09:56:12.906 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_shutdown_pgref_assert": "false", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_skip_data_digest": "false", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_smart_report_timeout": "5", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_cost": "1048576", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_priority": "5", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep": "0.000000", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_stats_update_period_scrubbing": "15", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-09-21T09:56:12.907 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_target_transaction_size": "30", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_mode": "writeback", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tracing": "false", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_use_stale_snap": "false", 2024-09-21T09:56:12.908 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "osdc_blkin_trace_all": "false", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_kill_at": "0", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_max_join_drift": "10", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_min": "500", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_min_wait": "0.050000", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_propose_interval": "1.000000", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_service_trim_max": "500", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_service_trim_max_multiplier": "20", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_service_trim_min": "250", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_stash_full_interval": "25", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_trim_max": "500", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_trim_min": "250", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "perf": "true", 2024-09-21T09:56:12.909 INFO:teuthology.orchestra.run.smithi017.stdout: "pid_file": "", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "public_addr": "-", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "public_addrv": "", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "public_bind_addr": "-", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "public_network": "172.21.0.0/20", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "public_network_interface": "", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "qat_compressor_busy_polling": "false", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "qat_compressor_enabled": "false", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "qat_compressor_session_max_number": "256", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "rados_mon_op_timeout": "0", 2024-09-21T09:56:12.910 INFO:teuthology.orchestra.run.smithi017.stdout: "rados_osd_op_timeout": "0", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rados_tracing": "false", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_atime_update_interval": "60", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_balance_parent_reads": "false", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_balance_snap_reads": "false", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_blkin_trace_all": "false", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_blocklist_expire_seconds": "0", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_blocklist_on_break_lock": "true", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache": "true", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_block_writes_upfront": "false", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_max_dirty": "25165824", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_max_dirty_object": "0", 2024-09-21T09:56:12.911 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_policy": "writearound", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_size": "33554432", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_target_dirty": "16777216", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_clone_copy_on_read": "false", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_compression_hint": "none", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_concurrent_management_ops": "10", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_clone_format": "auto", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_data_pool": "", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_features": "61", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_format": "2", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_map_options": "", 2024-09-21T09:56:12.912 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_order": "22", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_pool": "rbd", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_stripe_count": "0", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_stripe_unit": "0", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_disable_zero_copy_writes": "true", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_discard_granularity_bytes": "65536", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_enable_alloc_hint": "true", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_io_scheduler": "simple", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_commit_age": "5.000000", 2024-09-21T09:56:12.913 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_flush_interval": "0", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_order": "24", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_pool": "", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_splay_width": "4", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_localize_parent_reads": "false", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_localize_snap_reads": "false", 2024-09-21T09:56:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_type": "simple", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-09-21T09:56:12.915 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_autotune": "true", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_base": "805306368", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_target": "4294967296", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_delete_delay": "0", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-09-21T09:56:12.916 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_replay_delay": "0", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_move_to_trash_on_remove": "false", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mtime_update_interval": "60", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_non_blocking_aio": "true", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_op_thread_timeout": "60", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_op_threads": "1", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_parent_cache_enabled": "false", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_persistent_cache_mode": "disabled", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_persistent_cache_path": "/tmp", 2024-09-21T09:56:12.917 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_persistent_cache_size": "1073741824", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_plugins": "", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_bps_burst": "0", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_bps_limit": "0", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_exclude_ops": "0", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_iops_burst": "0", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_iops_limit": "0", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_bps_burst": "0", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_bps_limit": "0", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_iops_burst": "0", 2024-09-21T09:56:12.918 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_iops_limit": "0", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_schedule_tick_min": "50", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_bps_burst": "0", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_bps_limit": "0", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_iops_burst": "0", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_iops_limit": "0", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_quiesce_notification_attempts": "10", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_read_from_replica_policy": "default", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-09-21T09:56:12.919 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_readahead_max_bytes": "524288", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_readahead_trigger_requests": "10", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_request_timed_out_seconds": "30", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_skip_partial_discard": "true", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_tracing": "false", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_validate_names": "true", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_validate_pool": "true", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "restapi_base_url": "", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "restapi_log_level": "", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_account_default_quota_max_size": "-1", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_acl_grants_max_num": "100", 2024-09-21T09:56:12.920 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_admin_entry": "admin", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_asio_assert_yielding": "false", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_backend_store": "rados", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_barbican_url": "", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_beast_enable_async": "true", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_counters_cache": "false", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_index_max_aio": "128", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-09-21T09:56:12.921 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_quota_ttl": "600", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cache_enabled": "true", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cache_expiry_interval": "900", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cache_lru_size": "25000", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_config_store": "rados", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_content_length_compat": "false", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_copy_obj_progress": "true", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cors_rules_max_num": "100", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cross_domain_policy": "", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_default_encryption_key": "", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_addr": "", 2024-09-21T09:56:12.922 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_ca_path": "", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_client_cert": "", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_client_key": "", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_password": "", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_username": "", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_require_ssl": "true", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-09-21T09:56:12.923 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_suppress_logs": "true", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_addr": "", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_auth": "token", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_namespace": "", 2024-09-21T09:56:12.924 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_prefix": "", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_token_file": "", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_buffersize": "524288", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_low_speed_limit": "1024", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_low_speed_time": "300", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_tcp_keepalive": "0", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-09-21T09:56:12.925 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_fadvise": "4", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-09-21T09:56:12.926 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_l1_fadvise": "4", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_changes_size": "1000", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_num_shards": "128", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_window": "30", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_notify_interval_msec": "0", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_sync_poll_interval": "20", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_sync_spawn_window": "20", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-09-21T09:56:12.927 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_data_log_backing": "fifo", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_region_info_oid": "default.region", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_defer_to_bucket_acls": "", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_disable_s3select": "false", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-09-21T09:56:12.928 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_data_res": "500.000000", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dns_name": "", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dns_s3website_name": "", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dynamic_resharding": "true", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-09-21T09:56:12.929 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_gc_threads": "true", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_lc_threads": "true", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_mdsearch": "true", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_ops_log": "false", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_quota_threads": "true", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_static_website": "false", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_usage_log": "false", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enforce_swift_acls": "true", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_exit_timeout_secs": "120", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_expose_bucket": "false", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_extended_http_attrs": "", 2024-09-21T09:56:12.930 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_filter": "none", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_frontends": "beast port=7480", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_concurrent_io": "10", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_deferred": "50", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_objs": "32", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_queue_size": "134213632", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_trim_chunk": "16", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_obj_min_wait": "7200", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_processor_max_time": "3600", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_processor_period": "3600", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-09-21T09:56:12.931 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_get_obj_window_size": "16777216", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_healthcheck_disabling_path": "", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ignore_get_invalid_range": "false", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_init_timeout": "300", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_kafka_connection_idle": "300", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_kafka_message_timeout": "5000", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_kafka_sleep_timeout": "10", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_domain": "", 2024-09-21T09:56:12.932 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_password": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_password_path": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_project": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_tenant": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_token": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_token_path": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_user": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_domain": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_password": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_project": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_tenant": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_user": "", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-09-21T09:56:12.933 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_implicit_tenants": "false", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_service_token_enabled": "false", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_token_cache_size": "10000", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_url": "", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_verify_ssl": "true", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_debug_interval": "-1", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_lock_max_time": "90", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_objs": "32", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_rules": "1000", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_worker": "3", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_wp_worker": "3", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_thread_delay": "0", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-09-21T09:56:12.934 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_dnattr": "uid", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_searchfilter": "", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_uri": "", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lfuda_sync_frequency": "60", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_http_headers": "", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_nonexistent_bucket": "false", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_object_name_utc": "false", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-09-21T09:56:12.935 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_attr_name_len": "0", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_attr_size": "0", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_attrs_num_in_req": "0", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_chunk_size": "4194304", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_concurrent_requests": "1024", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_dynamic_shards": "1999", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_listing_results": "5000", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_notify_retries": "10", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_objs_per_shard": "100000", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_put_param_size": "1048576", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_put_size": "5368709120", 2024-09-21T09:56:12.936 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_slo_entries": "1000", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_md_log_max_shards": "64", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_md_notify_interval_msec": "200", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_meta_sync_poll_interval": "20", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_meta_sync_spawn_window": "20", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_mp_lock_max_time": "600", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_multipart_min_part_size": "5242880", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_fhcache_size": "2017", 2024-09-21T09:56:12.937 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_lru_lanes": "5", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_max_gc": "300", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_gc_threads": "false", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_lc_threads": "false", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_quota_threads": "false", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_sync_thread": "false", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_num_async_rados_threads": "32", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_num_control_oids": "8", 2024-09-21T09:56:12.938 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_numa_node": "-1", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_obj_stripe_size": "4194304", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_objexp_chunk_size": "100", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_objexp_gc_interval": "600", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_objexp_hints_num_shards": "127", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_op_thread_timeout": "600", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_op_tracing": "false", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_opa_token": "", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_opa_url": "", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_opa_verify_ssl": "true", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-09-21T09:56:12.939 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_rados": "false", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_socket_path": "", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_push_interval": "2.000000", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_push_interval_max": "30.000000", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_root_pool": ".rgw.root", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_cache_lanes": "3", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_cache_max_buckets": "100", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_cache_partitions": "3", 2024-09-21T09:56:12.940 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_print_continue": "true", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_print_prohibited_content_length": "false", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rados_tracing": "false", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_realm": "", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_realm_id": "", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_region": "", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_region_root_pool": ".rgw.root", 2024-09-21T09:56:12.941 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_relaxed_region_enforcement": "false", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_request_uri": "", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_batch_size": "64", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_debug_interval": "-1", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_max_aio": "128", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_num_logs": "16", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_progress_judge_interval": "120", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_progress_judge_ratio": "0.500000", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_thread_interval": "600", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshardlog_threshold": "30000", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_resolve_cname": "false", 2024-09-21T09:56:12.942 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rest_getusage_op_compat": "false", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_run_sync_thread": "true", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_keystone": "false", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_ldap": "false", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_rados": "true", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_sts": "false", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_success_create_obj_status": "0", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_scheduler_type": "throttler", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_script_uri": "", 2024-09-21T09:56:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_service_provider_name": "", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_client_id": "", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_client_secret": "", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_entry": "sts", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_key": "sts", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_max_session_duration": "43200", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_min_session_duration": "900", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_token_introspection_url": "", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_account_in_url": "false", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_auth_entry": "auth", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_auth_url": "", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_custom_header": "", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_enforce_content_length": "false", 2024-09-21T09:56:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_need_stats": "true", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_tenant_name": "", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_token_expiration": "86400", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_url": "", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_url_prefix": "swift", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_versioning_enabled": "false", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_lease_period": "120", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_interval": "1200", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-09-21T09:56:12.945 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_obj_etag_verify": "false", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_trace_history_size": "4096", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_thread_pool_size": "512", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_topic_persistency_max_retries": "0", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_topic_require_publish_policy": "false", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_comment": "", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_createby": "", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_encoding": "", 2024-09-21T09:56:12.946 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_flag": "false", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_max_size": "5368709120", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_origin": "", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_sha_unit": "524288", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_tracker": "", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_trust_forwarded_https": "false", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_log_tick_interval": "30", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_max_shards": "32", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_max_user_shards": "1", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_use_opa_authz": "false", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_counters_cache": "false", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_counters_cache_size": "10000", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-09-21T09:56:12.947 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_default_quota_max_size": "-1", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_max_buckets": "1000", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_policies_max_num": "100", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_sync_interval": "86400", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_unique_email": "true", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_verify_ssl": "true", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_website_routing_rules_max_num": "50", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zone": "", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zone_id": "", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zonegroup": "", 2024-09-21T09:56:12.948 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zonegroup_id": "", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rgwlc_auto_session_clear": "true", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rgwlc_skip_bucket_step": "false", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_block_size": "4096", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_shard_bits": "4", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_size": "536870912", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_type": "binned_lru", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-09-21T09:56:12.949 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_collect_compaction_stats": "false", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_collect_extended_stats": "false", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_collect_memory_stats": "false", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_index_type": "binary_search", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_log_to_ceph_log": "true", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_metadata_block_size": "4096", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_partition_filters": "false", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_perf": "false", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-09-21T09:56:12.950 INFO:teuthology.orchestra.run.smithi017.stdout: "rotating_keys_renewal_timeout": "10", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "run_dir": "/var/run/ceph", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_block_create": "true", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_cache_lru_size": "67108864", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_cbjournal_size": "5368709120", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_data_delta_based_overwrite": "0", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_default_max_object_size": "16777216", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_device_size": "53687091200", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_full_integrity_check": "false", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_batch_capacity": "16", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-09-21T09:56:12.951 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_iodepth_limit": "5", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_main_device_type": "SSD", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_max_concurrent_transactions": "8", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_max_data_allocation_size": "32768", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_segment_size": "67108864", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "set_keepcaps": "false", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "setgroup": "ceph", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "setuser": "ceph", 2024-09-21T09:56:12.952 INFO:teuthology.orchestra.run.smithi017.stdout: "setuser_match_path": "", 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout: "target_max_misplaced_ratio": "0.050000", 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout: "thp": "false", 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout: "threadpool_default_timeout": "60", 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout: "threadpool_empty_queue_max_wait": "2", 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout: "throttler_perf_counter": "true", 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout: "tmp_dir": "/tmp", 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout: "uadk_compressor_enabled": "false", 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout: "uadk_wd_sync_ctx_num": "2" 2024-09-21T09:56:12.953 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:56:13.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-09-21T09:56:13.821 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:13.933 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:13 smithi017 ceph-mon[102913]: pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:13.934 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:13 smithi017 ceph-mon[105151]: pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:13 smithi134 ceph-mon[90085]: pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:56:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:56:14] "GET /metrics HTTP/1.1" 200 34062 "" "Prometheus/2.51.0" 2024-09-21T09:56:15.518 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a: {} 2024-09-21T09:56:15.518 INFO:teuthology.orchestra.run.smithi017.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T09:56:15.519 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b: {} 2024-09-21T09:56:15.519 INFO:teuthology.orchestra.run.smithi017.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T09:56:15.528 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c: {} 2024-09-21T09:56:15.528 INFO:teuthology.orchestra.run.smithi017.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T09:56:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:15 smithi017 ceph-mon[102913]: pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:15 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/343859358' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T09:56:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:15 smithi017 ceph-mon[105151]: pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:15 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/343859358' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T09:56:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:15 smithi134 ceph-mon[90085]: pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:15 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/343859358' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T09:56:16.181 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-09-21T09:56:16.182 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-21T09:56:16.182 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T09:56:16.184 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T09:56:16.184 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-09-21T09:56:16.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_1 16 2024-09-21T09:56:16.185 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-09-21T09:56:16.185 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:56:16.185 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:56:16.185 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-09-21T09:56:16.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:16.186 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-09-21T09:56:16.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:56:16.186 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-09-21T09:56:16.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T09:56:16.187 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-09-21T09:56:16.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T09:56:16.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:16.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:16.318 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:16.350 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:16.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:16.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:16.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:16.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:16.511 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:16.511 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:16.513 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:16.514 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:16.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:16.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:16.711 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:16.737 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:16.838 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:16.863 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:16.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:16.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:16.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:17.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:17.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:17.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:17.213 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:17.237 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:17.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:17.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:17.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:17.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:17.462 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:17.488 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:17.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:17.590 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:17.616 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:17.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:17.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:17.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:17.842 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:17.867 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:17.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:17 smithi017 ceph-mon[102913]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:17 smithi017 ceph-mon[105151]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:17.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:17 smithi134 ceph-mon[90085]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:17.994 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:17.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:18.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:18.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:18.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:18.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:18.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:18.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:18.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:18.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:18.470 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:18.495 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:18.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:18.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:18.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:18.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:18.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:18.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:18.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:18.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:18.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:18.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:18.972 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:18.997 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:19.099 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:19.123 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:19.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:19.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:19.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:19.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:19.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:19.478 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:19.504 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:19.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:19.605 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:19.631 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:19.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:19.733 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:19.757 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:19.859 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:19.885 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:19.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:19 smithi017 ceph-mon[102913]: pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:19 smithi017 ceph-mon[105151]: pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:19.986 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:19.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:19 smithi134 ceph-mon[90085]: pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:20.012 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:20.114 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:20.138 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:20.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:20.240 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:20.265 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:20.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:20.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:20.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:20.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:56:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:56:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:20.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:20.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:20.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:20.647 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:20.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:20.749 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:20.773 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:20.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:20.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:21.000 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:21.025 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:21.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:21.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:21.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:21.256 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:21.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:21.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:21.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:21.511 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:21.537 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:21.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:21.639 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:21.643 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T09:56:21.643 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:56:21.664 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:21.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:21.726 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:56:21.726 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:56:21.726 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:56:21.726 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:56:21.726 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:56:21.726 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:56:21.726 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:56:21.727 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:56:21.727 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:56:21.727 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:56:21.727 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:56:21.727 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:56:21.727 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:56:21.727 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:56:21.727 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:56:21.727 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:56:21.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:21 smithi017 ceph-mon[102913]: pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:21.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:21 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1085599346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:21.728 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:21 smithi017 ceph-mon[105151]: pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:21.728 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:21 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1085599346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:21.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:21.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:21.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:21.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:21.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:21 smithi134 ceph-mon[90085]: pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:21 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1085599346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:22.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:22.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:22.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:22.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:22.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:22.263 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:22.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:22.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:22.391 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:22.416 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:22.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:22.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:22.544 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:22.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:22.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:22.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:22.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:22.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:22.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:22.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:22.924 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:22.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:23.026 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:23.050 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:23.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:23.152 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:23.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:23.279 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:23.302 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:23.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:23.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:23.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:23.529 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:23.554 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:23.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:23.631 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T09:56:23.656 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:23.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T09:56:23.681 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:23.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:23.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:23.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:23.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:23.910 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:23.935 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:23.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:23 smithi017 ceph-mon[102913]: pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:23 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2289555477' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:23 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:23 smithi017 ceph-mon[105151]: pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:23 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2289555477' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:23 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:23 smithi134 ceph-mon[90085]: pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:23 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2289555477' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:23 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:24.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:24.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:24.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:24.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:24.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:24.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:24.312 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:24.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:24.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:56:24.414 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:24.439 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:24.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:24.541 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:24.557 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:24.565 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:24.667 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:24.692 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:24.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:24.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:24.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:24.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:24 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:24.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:24 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:56:24.842 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:24 smithi017 ceph-mon[102913]: osdmap e1077: 8 total, 8 up, 8 in 2024-09-21T09:56:24.842 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:24 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:24.842 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:24 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:56:24.842 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:24 smithi017 ceph-mon[105151]: osdmap e1077: 8 total, 8 up, 8 in 2024-09-21T09:56:24.842 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:56:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:56:24] "GET /metrics HTTP/1.1" 200 34062 "" "Prometheus/2.51.0" 2024-09-21T09:56:24.918 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:24.944 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:24.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:24 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:24 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:56:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:24 smithi134 ceph-mon[90085]: osdmap e1077: 8 total, 8 up, 8 in 2024-09-21T09:56:25.047 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:25.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.073 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:25.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:25.174 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:25.199 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:25.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:25.304 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:25.326 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:25.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:25.428 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:25.452 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:25.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:25.579 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:25.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:25.635 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_1' created 2024-09-21T09:56:25.681 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:25.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:25 smithi017 ceph-mon[102913]: pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:25.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:25 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1874632319' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T09:56:25.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:56:25.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:25 smithi017 ceph-mon[105151]: pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:25 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1874632319' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T09:56:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:56:25.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:25.705 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:25.807 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:25.832 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:25.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:25.933 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:25.959 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:25 smithi134 ceph-mon[90085]: pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:25 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1874632319' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T09:56:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:56:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:26.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:26.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:26.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:26.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:26.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:26.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:26.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T09:56:26.311 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:26.336 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:26.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:26.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:26.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:26.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:26.564 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:26.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-09-21T09:56:26.589 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:26.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:26.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:26.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:26.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:26.793 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:26 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1874632319' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-21T09:56:26.793 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:26 smithi017 ceph-mon[102913]: osdmap e1078: 8 total, 8 up, 8 in 2024-09-21T09:56:26.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:26 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1874632319' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-21T09:56:26.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:26 smithi017 ceph-mon[105151]: osdmap e1078: 8 total, 8 up, 8 in 2024-09-21T09:56:26.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:26.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:26.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:26.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:26.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:26.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:26 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1874632319' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-21T09:56:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:26 smithi134 ceph-mon[90085]: osdmap e1078: 8 total, 8 up, 8 in 2024-09-21T09:56:27.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:27.087 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:27.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:27.189 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:27.214 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:27.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:27.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:27.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:27.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:27.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:27.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:27.567 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:27.593 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:27.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:27.623 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T09:56:27.624 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:56:27.631 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:27.632 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:27.695 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:27.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:27.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:27.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:27.843 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:27.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:27 smithi017 ceph-mon[102913]: pgmap v1238: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:27.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:27 smithi017 ceph-mon[102913]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:56:27.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:27 smithi017 ceph-mon[102913]: osdmap e1079: 8 total, 8 up, 8 in 2024-09-21T09:56:27.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:27 smithi017 ceph-mon[105151]: pgmap v1238: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:27.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:27 smithi017 ceph-mon[105151]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:56:27.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:27 smithi017 ceph-mon[105151]: osdmap e1079: 8 total, 8 up, 8 in 2024-09-21T09:56:27.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:27.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:27 smithi134 ceph-mon[90085]: pgmap v1238: 89 pgs: 8 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:27 smithi134 ceph-mon[90085]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:56:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:27 smithi134 ceph-mon[90085]: osdmap e1079: 8 total, 8 up, 8 in 2024-09-21T09:56:28.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:28.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:28.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:28.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:28.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:28.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T09:56:28.320 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:28.345 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:28.447 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:28.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.473 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:28.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:28.574 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:28.600 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:28.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:28.726 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:28.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:28.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:28.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:28.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:28.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:28 smithi017 ceph-mon[102913]: osdmap e1080: 8 total, 8 up, 8 in 2024-09-21T09:56:28.927 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:28 smithi017 ceph-mon[105151]: osdmap e1080: 8 total, 8 up, 8 in 2024-09-21T09:56:28.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:28.976 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:28.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:28 smithi134 ceph-mon[90085]: osdmap e1080: 8 total, 8 up, 8 in 2024-09-21T09:56:29.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:29.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:29.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:29.204 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:29.230 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:29.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:29.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:29.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:29.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:29.481 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:29.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:29.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:29.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:29.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:29.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:29.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:29.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:29.789 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:56:29.789 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:56:29.789 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:56:29.790 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:56:29.790 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:56:29.790 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:56:29.790 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:56:29.790 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:56:29.790 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:56:29.790 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:56:29.790 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:56:29.791 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:56:29.791 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:56:29.791 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:56:29.791 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:56:29.791 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:56:29.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:29 smithi017 ceph-mon[102913]: pgmap v1241: 89 pgs: 1 creating+activating, 9 creating+peering, 5 unknown, 74 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:29.792 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:29 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3486743191' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:29.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:29 smithi017 ceph-mon[105151]: pgmap v1241: 89 pgs: 1 creating+activating, 9 creating+peering, 5 unknown, 74 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:29.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:29 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3486743191' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:29.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:29.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:29.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:29.978 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:29 smithi134 ceph-mon[90085]: pgmap v1241: 89 pgs: 1 creating+activating, 9 creating+peering, 5 unknown, 74 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:29 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3486743191' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:30.080 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:30.106 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:30.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:30.231 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:30.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:30.333 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:30.358 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:30.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:30.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:30.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:30.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:56:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:56:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:30.584 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:30.610 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:30.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:30.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:30.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:30.835 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:30.861 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:30.963 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:30.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.990 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:30.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:31.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:31.116 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:31.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:31.217 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:31.243 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:31.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:31.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:31.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:31.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:31.470 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:31.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:31.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:31.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:31.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:31 smithi017 ceph-mon[102913]: pgmap v1242: 89 pgs: 1 creating+activating, 9 creating+peering, 3 unknown, 76 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:31.927 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:31 smithi017 ceph-mon[105151]: pgmap v1242: 89 pgs: 1 creating+activating, 9 creating+peering, 3 unknown, 76 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:31.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:31.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:31 smithi134 ceph-mon[90085]: pgmap v1242: 89 pgs: 1 creating+activating, 9 creating+peering, 3 unknown, 76 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:32.028 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:32.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:32.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:32.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:32.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:32.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:32.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:32.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:32.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:32.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:56:32.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:32.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:32.561 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:32.587 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:32.657 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T09:56:32.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:32 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3129658722' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:56:32.670 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:32 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3129658722' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:56:32.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:32.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:32.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:32.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:32.834 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:32.936 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:32.961 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:32.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:32 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3129658722' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:56:33.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:33.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:33.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:33.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:33.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:33.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:33.311 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.338 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:33.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:33.439 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:33.464 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:33.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:33.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:33.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:33.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:33.691 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:33.716 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:33.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:33.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:33.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:33.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:33 smithi017 ceph-mon[102913]: pgmap v1243: 89 pgs: 1 creating+activating, 88 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:33.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:33 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:33.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:33 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3129658722' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:56:33.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:33 smithi017 ceph-mon[102913]: osdmap e1081: 8 total, 8 up, 8 in 2024-09-21T09:56:33.910 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:33 smithi017 ceph-mon[105151]: pgmap v1243: 89 pgs: 1 creating+activating, 88 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:33.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:33 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:33.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:33 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3129658722' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:56:33.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:33 smithi017 ceph-mon[105151]: osdmap e1081: 8 total, 8 up, 8 in 2024-09-21T09:56:33.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:33.971 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:33.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:33 smithi134 ceph-mon[90085]: pgmap v1243: 89 pgs: 1 creating+activating, 88 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:33 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:33 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3129658722' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:56:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:33 smithi134 ceph-mon[90085]: osdmap e1081: 8 total, 8 up, 8 in 2024-09-21T09:56:34.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:34.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:34.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:34.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:34.218 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:34.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:34.320 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:34.346 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:34.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:34.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:34.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:34.573 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:34.598 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:34.663 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-09-21T09:56:34.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:34.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:34.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:34.825 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:34.851 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:34.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:34 smithi017 ceph-mon[102913]: pgmap v1245: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/377802815' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:56:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:34 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:56:34.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:56:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:56:34] "GET /metrics HTTP/1.1" 200 34130 "" "Prometheus/2.51.0" 2024-09-21T09:56:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:34 smithi017 ceph-mon[105151]: pgmap v1245: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/377802815' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:56:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:34 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:56:34.952 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:34.978 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:34 smithi134 ceph-mon[90085]: pgmap v1245: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/377802815' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:56:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:34 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:56:35.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:35.104 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:35.206 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:35.232 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:35.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:35.334 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:35.335 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:56:35.335 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(osd.0.0:322 5.3 e677/479 ondisk, result = 0)", 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:48:22.226471+0000", 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 493.10670388900002, 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012118956, 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226471+0000", 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226471+0000", 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226472+0000", 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.147e-06 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226478+0000", 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3500000000000002e-06 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226492+0000", 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3502e-05 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226500+0000", 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4710000000000007e-06 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.238546+0000", 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012045915000000001 2024-09-21T09:56:35.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.238561+0000", 2024-09-21T09:56:35.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5024e-05 2024-09-21T09:56:35.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.238590+0000", 2024-09-21T09:56:35.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8547e-05 2024-09-21T09:56:35.341 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.341 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.341 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.342 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(osd.4.0:1163 5.e e677/479 5:7c2f69e2:::smithi01785587-33:17f v 677'1194, mlcod=675'1191)", 2024-09-21T09:56:35.342 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:48:22.226521+0000", 2024-09-21T09:56:35.342 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 493.10665359900003, 2024-09-21T09:56:35.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011998101000000001, 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226521+0000", 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226521+0000", 2024-09-21T09:56:35.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226522+0000", 2024-09-21T09:56:35.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8799999999999998e-07 2024-09-21T09:56:35.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226525+0000", 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6119999999999999e-06 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226534+0000", 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1880000000000008e-06 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226542+0000", 2024-09-21T09:56:35.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3509999999999996e-06 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226593+0000", 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1159000000000003e-05 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.226622+0000", 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8683e-05 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.238466+0000", 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011844225 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:48:22.238519+0000", 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3495000000000002e-05 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.347 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.14814.0:7968 5.5 5:aeee7ea3:::smithi01785587-37:head [append 0~0] snapc 1ee=[1ee,1ed,1ec,1eb,1ea,1e9,1e8,1e6,1e5,1e1,1e0,1df,1dc,1da,1d9,1d4,1d2,1ce,1ca,1c6] ondisk+write+known_if_redirected+supports_pool_eio e841)", 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:51:06.233665+0000", 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 329.09951025300001, 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012287298, 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.14814", 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/674371772", 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 7968 2024-09-21T09:56:35.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233665+0000", 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233665+0000", 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233667+0000", 2024-09-21T09:56:35.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.007e-06 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233674+0000", 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9909999999999996e-06 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233688+0000", 2024-09-21T09:56:35.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.452e-05 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233696+0000", 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8299999999999996e-06 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.234006+0000", 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000310186 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2024-09-21T09:56:35.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.234016+0000", 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.01e-05 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.235428+0000", 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014115250000000001 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.235536+0000", 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010844 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-21T09:56:35.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.236364+0000", 2024-09-21T09:56:35.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000827479 2024-09-21T09:56:35.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.241598+0000", 2024-09-21T09:56:35.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052345309999999997 2024-09-21T09:56:35.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.241648+0000", 2024-09-21T09:56:35.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9530000000000002e-05 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.245827+0000", 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041796450000000001 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.245892+0000", 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4987999999999995e-05 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.245952+0000", 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9525999999999999e-05 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.14814.0:7969 5.5 5:aeee7ea3:::smithi01785587-37:head [rollback 1ca] snapc 1ee=[1ee,1ed,1ec,1eb,1ea,1e9,1e8,1e6,1e5,1e1,1e0,1df,1dc,1da,1d9,1d4,1d2,1ce,1ca,1c6] ondisk+write+known_if_redirected+supports_pool_eio e841)", 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:51:06.233726+0000", 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 329.09944899800001, 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013731949, 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.356 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.14814", 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/674371772", 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 7969 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233726+0000", 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233726+0000", 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233727+0000", 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0500000000000002e-07 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233730+0000", 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5530000000000001e-06 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233740+0000", 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9320000000000008e-06 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233747+0000", 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1250000000000004e-06 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.234915+0000", 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011675539999999999 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.234921+0000", 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7289999999999997e-06 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.238707+0000", 2024-09-21T09:56:35.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037861729999999999 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.238815+0000", 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000108311 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.239042+0000", 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022660700000000001 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.246004+0000", 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069626059999999997 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.246019+0000", 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.483e-05 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.247309+0000", 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001290373 2024-09-21T09:56:35.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:56:35.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.247387+0000", 2024-09-21T09:56:35.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7183999999999998e-05 2024-09-21T09:56:35.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.247458+0000", 2024-09-21T09:56:35.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1167000000000005e-05 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.14814.0:7970 5.5 5:aeee7ea3:::smithi01785587-37:head [append 0~0] snapc 1ee=[1ee,1ed,1ec,1eb,1ea,1e9,1e8,1e6,1e5,1e1,1e0,1df,1dc,1da,1d9,1d4,1d2,1ce,1ca,1c6] ondisk+write+known_if_redirected+supports_pool_eio e841)", 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:51:06.233760+0000", 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 329.09941453699997, 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014263704, 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.364 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.14814", 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/674371772", 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 7970 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233760+0000", 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233760+0000", 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233761+0000", 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8299999999999996e-07 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233764+0000", 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1e-06 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233772+0000", 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0399999999999993e-06 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.233778+0000", 2024-09-21T09:56:35.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3080000000000004e-06 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.234948+0000", 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011698050000000001 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.234955+0000", 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.635e-06 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.240458+0000", 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055028960000000002 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.240569+0000", 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111488 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-21T09:56:35.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.240765+0000", 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019577200000000001 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.246025+0000", 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052602660000000004 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.246096+0000", 2024-09-21T09:56:35.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0510000000000001e-05 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.247877+0000", 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017813149999999999 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.247947+0000", 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0239000000000003e-05 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:06.248024+0000", 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6647e-05 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.14814.0:8351 5.3 5:cc9fd6ba:::smithi01785587-27:head [sparse-read 0~2107110] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e876)", 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:51:41.273348+0000", 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 294.05982684600002, 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012448941, 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T09:56:35.372 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.14814", 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/674371772", 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8351 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273348+0000", 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273348+0000", 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273349+0000", 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0800000000000004e-07 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273351+0000", 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.774e-06 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273371+0000", 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.999e-05 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273378+0000", 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.776e-06 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.285353+0000", 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011975065 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.285493+0000", 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014004299999999999 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.285797+0000", 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030358499999999999 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.376 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.14814.0:8354 5.5 5:a1937208:::smithi01785587-39:head [append 0~0] snapc 1fc=[1fc,1fb,1fa,1f9,1f5,1f4,1f0,1ef,1e8,1dc,1da,1d9,1c6] ondisk+write+known_if_redirected+supports_pool_eio e876)", 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:51:41.273518+0000", 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 294.05965724700002, 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013425734, 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.14814", 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/674371772", 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8354 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.377 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273518+0000", 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273518+0000", 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273518+0000", 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3799999999999997e-07 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273520+0000", 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2620000000000001e-06 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273527+0000", 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1179999999999997e-06 2024-09-21T09:56:35.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.273532+0000", 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5740000000000003e-06 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.281148+0000", 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007615564 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.281263+0000", 2024-09-21T09:56:35.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115627 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.281449+0000", 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018522999999999999 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.283214+0000", 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001765165 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.284224+0000", 2024-09-21T09:56:35.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001010548 2024-09-21T09:56:35.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.286769+0000", 2024-09-21T09:56:35.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025443050000000002 2024-09-21T09:56:35.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:56:35.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.286859+0000", 2024-09-21T09:56:35.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0508999999999997e-05 2024-09-21T09:56:35.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:41.286943+0000", 2024-09-21T09:56:35.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4294000000000003e-05 2024-09-21T09:56:35.384 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.384 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.384 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.384 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.14814.0:8425 5.5 5:a1937208:::smithi01785587-39:head [append 0~0] snapc 200=[1ff,1fd,1fc,1fb,1fa,1f9,1f4,1f0,1dc,1da,1c6] ondisk+write+known_if_redirected+supports_pool_eio e886)", 2024-09-21T09:56:35.384 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:51:51.218908+0000", 2024-09-21T09:56:35.384 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 284.11426674400002, 2024-09-21T09:56:35.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012024979, 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.14814", 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/674371772", 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8425 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.218908+0000", 2024-09-21T09:56:35.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.218908+0000", 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.218911+0000", 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.655e-06 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.218920+0000", 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1570000000000006e-06 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.218948+0000", 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8180000000000001e-05 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.218961+0000", 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2471e-05 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219171+0000", 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021018600000000001 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219180+0000", 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4800000000000007e-06 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.222295+0000", 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003114856 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.222403+0000", 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010828300000000001 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.222644+0000", 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024032200000000001 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:56:35.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.228483+0000", 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058390610000000004 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.229959+0000", 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001476111 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.230785+0000", 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082567800000000005 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.230864+0000", 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9108000000000002e-05 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.230933+0000", 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9431000000000007e-05 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.14814.0:8426 5.5 5:a1937208:::smithi01785587-39:head [rollback 1ff] snapc 200=[1ff,1fd,1fc,1fb,1fa,1f9,1f4,1f0,1dc,1da,1c6] ondisk+write+known_if_redirected+supports_pool_eio e886)", 2024-09-21T09:56:35.391 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:51:51.218997+0000", 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 284.11417782400002, 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012153597, 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.14814", 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/674371772", 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8426 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.218997+0000", 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.218997+0000", 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.218998+0000", 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9299999999999996e-07 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219002+0000", 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2899999999999996e-06 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219018+0000", 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6171000000000002e-05 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219026+0000", 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7689999999999996e-06 2024-09-21T09:56:35.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219280+0000", 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025343300000000001 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219287+0000", 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8099999999999998e-06 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.223994+0000", 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047067999999999997 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.224100+0000", 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010597199999999999 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.224337+0000", 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023644200000000001 2024-09-21T09:56:35.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.229001+0000", 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046642610000000003 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.230402+0000", 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014015150000000001 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.231075+0000", 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00067250000000000003 2024-09-21T09:56:35.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.231116+0000", 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1566000000000001e-05 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.231151+0000", 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4174999999999997e-05 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.14814.0:8427 5.5 5:a1937208:::smithi01785587-39:head [append 0~0] snapc 200=[1ff,1fd,1fc,1fb,1fa,1f9,1f4,1f0,1dc,1da,1c6] ondisk+write+known_if_redirected+supports_pool_eio e886)", 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:51:51.219036+0000", 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 284.11413865200001, 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01242798, 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.14814", 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/674371772", 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8427 2024-09-21T09:56:35.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219036+0000", 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219036+0000", 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:56:35.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219037+0000", 2024-09-21T09:56:35.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4899999999999995e-07 2024-09-21T09:56:35.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:56:35.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219039+0000", 2024-09-21T09:56:35.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4719999999999998e-06 2024-09-21T09:56:35.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219052+0000", 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3040000000000001e-05 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219059+0000", 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.866e-06 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219337+0000", 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027745200000000003 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.219346+0000", 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3309999999999993e-06 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:56:35.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.225490+0000", 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061436729999999997 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.225585+0000", 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5858000000000003e-05 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.225774+0000", 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018838500000000001 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.229008+0000", 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032339579999999999 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.230454+0000", 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014459290000000001 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.231333+0000", 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00087921400000000002 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.231408+0000", 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5408999999999996e-05 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:51:51.231464+0000", 2024-09-21T09:56:35.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5844000000000003e-05 2024-09-21T09:56:35.407 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.407 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.407 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.407 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:56:35.407 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:56:35.407 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:56:35.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.408 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:35.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:35.509 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:35.535 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:35.638 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:35.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T09:56:35.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:35.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:35.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:35.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:35.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:35.908 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:35.943 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:35.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:35.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:35 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:56:35.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:35 smithi017 ceph-mon[102913]: osdmap e1082: 8 total, 8 up, 8 in 2024-09-21T09:56:35.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:35 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:56:35.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:35 smithi017 ceph-mon[105151]: osdmap e1082: 8 total, 8 up, 8 in 2024-09-21T09:56:35.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:35 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:56:35.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:35 smithi134 ceph-mon[90085]: osdmap e1082: 8 total, 8 up, 8 in 2024-09-21T09:56:36.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:36.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:36.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:36.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:36.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:36.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:36.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:36.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:36.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:36.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T09:56:36.509 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.510 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:36.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:36.611 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:36.635 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:36.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:36 smithi017 ceph-mon[102913]: pgmap v1247: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:36.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:36 smithi017 ceph-mon[105151]: pgmap v1247: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:36.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:36.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:36.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:36.862 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-09-21T09:56:36.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:36.865 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2024-09-21T09:56:36.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:56:36.868 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T09:56:36.869 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T09:56:36.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:36.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:36.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-21T09:56:36.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T09:56:36.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_149547 2024-09-21T09:56:36.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:56:36.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-21T09:56:36.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:36 smithi134 ceph-mon[90085]: pgmap v1247: 89 pgs: 89 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-21T09:56:37.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:37.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:37.142 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:37.174 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:37.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:37.261 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:37.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:37.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:37.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:37.441 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:37.473 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:37.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:37.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:37.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:37.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:37.717 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:37.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.749 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:37.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:37.760 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:37.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:37.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:37.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3578 3562 222.598 222.625 0.00262618 0.00447919 2024-09-21T09:56:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:37 smithi017 ceph-mon[102913]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:56:37.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:37 smithi017 ceph-mon[105151]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:56:37.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:37 smithi134 ceph-mon[90085]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:56:38.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:38.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:38.154 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:38.188 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:38.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:38.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:38.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:38.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:38.440 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:38.472 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:38.573 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:38.605 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:38.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:38.707 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:38.740 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:38.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:38.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:38.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:38.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:38.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 7038 7022 219.413 216.25 0.00278482 0.00454898 2024-09-21T09:56:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:38 smithi017 ceph-mon[102913]: pgmap v1248: 89 pgs: 89 active+clean; 8.1 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 968 KiB/s wr, 15 op/s 2024-09-21T09:56:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:38 smithi017 ceph-mon[105151]: pgmap v1248: 89 pgs: 89 active+clean; 8.1 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 968 KiB/s wr, 15 op/s 2024-09-21T09:56:38.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:38.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:38 smithi134 ceph-mon[90085]: pgmap v1248: 89 pgs: 89 active+clean; 8.1 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 968 KiB/s wr, 15 op/s 2024-09-21T09:56:39.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:39.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:39.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:39.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:39.265 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:39.300 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:39.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:39.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:39.412 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:39.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:39.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:39.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:39.599 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:39.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:39.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:39.722 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:56:39.723 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:56:39.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:39.765 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:56:39.766 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:56:39.806 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:56:39.806 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:56:39.806 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:39 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/980552889' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:39.807 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:39 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/980552889' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:39.842 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:39.849 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:56:39.849 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:56:39.872 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:39.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:39.882 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:56:39.882 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:56:39.914 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:56:39.915 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:56:39.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 10276 10260 213.728 202.375 0.00612108 0.00466956 2024-09-21T09:56:39.947 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:56:39.947 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:56:39.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:39.982 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:56:39.982 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:56:39.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:39 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/980552889' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:40.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:40.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:40.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:40.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:40.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:40.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:40.410 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:40.439 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:40.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:56:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:56:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:40.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:40.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:40.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:40.684 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:40.715 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:40.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:40.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:40.856 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:40.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:40.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 13353 13337 208.37 192.312 0.00587543 0.00479319 2024-09-21T09:56:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:40 smithi017 ceph-mon[102913]: pgmap v1249: 89 pgs: 89 active+clean; 46 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s wr, 90 op/s 2024-09-21T09:56:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:56:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:40 smithi017 ceph-mon[105151]: pgmap v1249: 89 pgs: 89 active+clean; 46 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s wr, 90 op/s 2024-09-21T09:56:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:56:40.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:40.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:40 smithi134 ceph-mon[90085]: pgmap v1249: 89 pgs: 89 active+clean; 46 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s wr, 90 op/s 2024-09-21T09:56:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:56:41.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:41.111 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:41.143 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:41.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:41.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:41.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:41.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:41.385 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:41.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.418 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:41.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:41.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:41.550 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:41.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:41.652 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:41.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:41.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:41.814 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:41.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:41.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 16474 16458 205.705 195.062 0.00436451 0.00485702 2024-09-21T09:56:41.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:41.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:42.059 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:42.087 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:42.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:42.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:42.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:42.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:42.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:42.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:42.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:42.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:42.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:42.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:42.616 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:42.647 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:42.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:42.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:42.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:42.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:42.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 18974 18958 197.439 156.25 0.0109037 0.00505757 2024-09-21T09:56:42.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:43.038 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:43.065 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:43.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:43.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:43.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:43 smithi017 ceph-mon[102913]: pgmap v1250: 89 pgs: 89 active+clean; 641 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T09:56:43.172 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:43 smithi017 ceph-mon[105151]: pgmap v1250: 89 pgs: 89 active+clean; 641 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T09:56:43.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:43.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:43.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:43.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:43.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:43 smithi134 ceph-mon[90085]: pgmap v1250: 89 pgs: 89 active+clean; 641 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T09:56:43.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:43.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:43.601 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:43.627 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:43.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:43.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:43.757 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:43.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:43.859 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:43.874 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T09:56:43.874 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:56:43.884 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:43.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:43.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 21057 21041 187.829 130.188 0.00738474 0.00531636 2024-09-21T09:56:43.985 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:44.015 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:44.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:44.117 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:44.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.149 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:44.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:44.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:44.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:44.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:44.305 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T09:56:44.388 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:44.417 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:44.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:44.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:44.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:44.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2329939377' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:56:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:44 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:56:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2329939377' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:56:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:44 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:56:44.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:44.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2329939377' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:56:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:44 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:56:44.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:44.833 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:44.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 23519 23503 183.584 153.875 0.00993435 0.00544126 2024-09-21T09:56:44.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:44.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:44.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:44.998 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:56:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:56:44] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.51.0" 2024-09-21T09:56:45.081 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:45.111 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:45.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:45.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:45.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:45.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:45.293 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:56:45.298 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:56:45.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T09:56:45.350 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:45.383 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:45.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:45.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:45.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:45.629 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:45.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:45 smithi017 ceph-mon[102913]: pgmap v1251: 89 pgs: 89 active+clean; 887 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T09:56:45.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:45 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:45.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:45 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:56:45.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:45 smithi017 ceph-mon[102913]: osdmap e1083: 8 total, 8 up, 8 in 2024-09-21T09:56:45.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:45 smithi017 ceph-mon[105151]: pgmap v1251: 89 pgs: 89 active+clean; 887 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T09:56:45.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:45 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:45.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:45 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:56:45.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:45 smithi017 ceph-mon[105151]: osdmap e1083: 8 total, 8 up, 8 in 2024-09-21T09:56:45.659 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:45.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:45 smithi134 ceph-mon[90085]: pgmap v1251: 89 pgs: 89 active+clean; 887 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T09:56:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:45 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:45 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:56:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:45 smithi134 ceph-mon[90085]: osdmap e1083: 8 total, 8 up, 8 in 2024-09-21T09:56:45.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:45.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:45.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:45.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:45.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 25849 25833 179.365 145.625 0.00515998 0.0055697 2024-09-21T09:56:45.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:56:45.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:45.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:46.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:46.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:46.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:46.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:46.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:46.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:46.341 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:46.378 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:46.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:46.479 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:46.510 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:46.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:46.612 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:46.645 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:46.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:46 smithi017 ceph-mon[102913]: from='client.45182 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:46 smithi017 ceph-mon[105151]: from='client.45182 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:46 smithi134 ceph-mon[90085]: from='client.45182 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:46.747 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:46.767 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:46.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:46.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:46.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:46.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:46.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 27948 27932 174.546 131.188 0.0056608 0.0057234 2024-09-21T09:56:47.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:47.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:47.132 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:47.163 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:47.265 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:47.298 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:47.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:47.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:47.538 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:47.571 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:47.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:47.673 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:47.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:47 smithi017 ceph-mon[102913]: pgmap v1253: 89 pgs: 89 active+clean; 1.4 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-21T09:56:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:47 smithi017 ceph-mon[105151]: pgmap v1253: 89 pgs: 89 active+clean; 1.4 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-21T09:56:47.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:47 smithi134 ceph-mon[90085]: pgmap v1253: 89 pgs: 89 active+clean; 1.4 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-21T09:56:47.796 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:47.828 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:47.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:47.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 29831 29815 169.376 117.688 0.00507574 0.00589066 2024-09-21T09:56:47.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:47.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:48.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:48.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:48.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:48.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:48.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:48.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:48.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:48.450 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:48.475 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:48.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:48.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:48.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:48.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:48.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:48.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:48.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:48.835 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:48.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.868 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:48.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:48.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 31356 31340 163.204 95.3125 0.0581392 0.00611999 2024-09-21T09:56:48.970 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:49.001 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:49.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:49.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:49.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:49.246 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:49.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.268 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:49.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:49.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:49.395 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:49.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:49.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:49.517 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:49.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:49.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:49.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:49.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:49 smithi017 ceph-mon[102913]: pgmap v1254: 89 pgs: 89 active+clean; 1.5 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-09-21T09:56:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:49 smithi017 ceph-mon[105151]: pgmap v1254: 89 pgs: 89 active+clean; 1.5 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-09-21T09:56:49.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:49 smithi134 ceph-mon[90085]: pgmap v1254: 89 pgs: 89 active+clean; 1.5 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-09-21T09:56:49.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:49.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:49.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:49.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 32885 32869 158 95.5625 0.00787666 0.00632273 2024-09-21T09:56:49.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:49.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:50.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:50.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:50.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:50.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:50.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:50.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:50.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:50.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:50.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:50.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:50.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:56:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:56:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:50.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:50.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:50.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:50.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:50.732 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:50.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:50.833 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:50.866 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:50.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 34655 34639 154.616 110.625 0.00423842 0.0064626 2024-09-21T09:56:50.968 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:50.998 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:50.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:51.100 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:51.129 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:51.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:51.231 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:51.263 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:51.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:51.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:51.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:51.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:51.505 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:51.534 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:51.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:51.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:51.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:51 smithi017 ceph-mon[102913]: pgmap v1255: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-09-21T09:56:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:51 smithi017 ceph-mon[105151]: pgmap v1255: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-09-21T09:56:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:51 smithi134 ceph-mon[90085]: pgmap v1255: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-09-21T09:56:51.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:51.800 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:51.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:51.902 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:51.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 36589 36573 152.366 120.875 0.0120958 0.00655798 2024-09-21T09:56:51.932 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:51.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:52.034 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:52.067 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:52.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:52.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:52.190 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:52.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:52.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:52.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:52.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:52.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:52.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:52.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:52.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:52.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:52.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:52.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:52.849 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:52.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.881 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:52.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:52.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 38261 38245 149.374 104.5 0.00440576 0.00668962 2024-09-21T09:56:52.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:53.014 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:53.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:53.116 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:53.148 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:53.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:53.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:53.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:53.392 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:53.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:53.485 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1252,"stamp":"2024-09-21T09:56:44.299854+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":929896397,"num_objects":14206,"num_object_clones":0,"num_object_copies":42618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14206,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":14392,"num_write_kb":909179,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":14338,"ondisk_log_size":14338,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3827680,"kb_used_data":2431456,"kb_used_omap":122,"kb_used_meta":1395973,"kb_avail":746133536,"statfs":{"total":767960285184,"available":764040740864,"internally_reserved":0,"allocated":2489810944,"data_stored":2472006551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125317,"internal_metadata":1429476987},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,5,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":929300480,"num_objects":14180,"num_object_clones":0,"num_object_copies":42540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14180,"num_write_kb":907520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.641612"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1498,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.166215+0000","last_change":"2024-09-21T09:46:47.114206+0000","last_active":"2024-09-21T09:56:35.166215+0000","last_peered":"2024-09-21T09:56:35.166215+0000","last_clean":"2024-09-21T09:56:35.166215+0000","last_became_active":"2024-09-21T09:46:47.113819+0000","last_became_peered":"2024-09-21T09:46:47.113819+0000","last_unstale":"2024-09-21T09:56:35.166215+0000","last_undegraded":"2024-09-21T09:56:35.166215+0000","last_fullsized":"2024-09-21T09:56:35.166215+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:07.629544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1007,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658030+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T09:56:32.658030+0000","last_peered":"2024-09-21T09:56:32.658030+0000","last_clean":"2024-09-21T09:56:32.658030+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T09:56:32.658030+0000","last_undegraded":"2024-09-21T09:56:32.658030+0000","last_fullsized":"2024-09-21T09:56:32.658030+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1026,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658300+0000","last_change":"2024-09-21T09:46:47.115484+0000","last_active":"2024-09-21T09:56:32.658300+0000","last_peered":"2024-09-21T09:56:32.658300+0000","last_clean":"2024-09-21T09:56:32.658300+0000","last_became_active":"2024-09-21T09:46:47.115080+0000","last_became_peered":"2024-09-21T09:46:47.115080+0000","last_unstale":"2024-09-21T09:56:32.658300+0000","last_undegraded":"2024-09-21T09:56:32.658300+0000","last_fullsized":"2024-09-21T09:56:32.658300+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:56.851805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1582,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.659991+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T09:56:34.659991+0000","last_peered":"2024-09-21T09:56:34.659991+0000","last_clean":"2024-09-21T09:56:34.659991+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T09:56:34.659991+0000","last_undegraded":"2024-09-21T09:56:34.659991+0000","last_fullsized":"2024-09-21T09:56:34.659991+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1583,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.660695+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T09:56:34.660695+0000","last_peered":"2024-09-21T09:56:34.660695+0000","last_clean":"2024-09-21T09:56:34.660695+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T09:56:34.660695+0000","last_undegraded":"2024-09-21T09:56:34.660695+0000","last_fullsized":"2024-09-21T09:56:34.660695+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":988,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658093+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:56:32.658093+0000","last_peered":"2024-09-21T09:56:32.658093+0000","last_clean":"2024-09-21T09:56:32.658093+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:56:32.658093+0000","last_undegraded":"2024-09-21T09:56:32.658093+0000","last_fullsized":"2024-09-21T09:56:32.658093+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1498,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.166422+0000","last_change":"2024-09-21T09:46:47.117580+0000","last_active":"2024-09-21T09:56:35.166422+0000","last_peered":"2024-09-21T09:56:35.166422+0000","last_clean":"2024-09-21T09:56:35.166422+0000","last_became_active":"2024-09-21T09:46:47.113030+0000","last_became_peered":"2024-09-21T09:46:47.113030+0000","last_unstale":"2024-09-21T09:56:35.166422+0000","last_undegraded":"2024-09-21T09:56:35.166422+0000","last_fullsized":"2024-09-21T09:56:35.166422+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:28.220180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":1014,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.188934+0000","last_change":"2024-09-21T09:45:05.080014+0000","last_active":"2024-09-21T09:56:33.188934+0000","last_peered":"2024-09-21T09:56:33.188934+0000","last_clean":"2024-09-21T09:56:33.188934+0000","last_became_active":"2024-09-21T09:45:05.079618+0000","last_became_peered":"2024-09-21T09:45:05.079618+0000","last_unstale":"2024-09-21T09:56:33.188934+0000","last_undegraded":"2024-09-21T09:56:33.188934+0000","last_fullsized":"2024-09-21T09:56:33.188934+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:42.160292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1082'812","reported_seq":824,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.877227+0000","last_change":"2024-09-21T09:56:27.225991+0000","last_active":"2024-09-21T09:56:40.877227+0000","last_peered":"2024-09-21T09:56:40.877227+0000","last_clean":"2024-09-21T09:56:40.877227+0000","last_became_active":"2024-09-21T09:56:27.225305+0000","last_became_peered":"2024-09-21T09:56:27.225305+0000","last_unstale":"2024-09-21T09:56:40.877227+0000","last_undegraded":"2024-09-21T09:56:40.877227+0000","last_fullsized":"2024-09-21T09:56:40.877227+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":812,"log_dups_size":0,"ondisk_log_size":812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:33:11.007162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53215232,"num_objects":812,"num_object_clones":0,"num_object_copies":2436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":812,"num_write_kb":51968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1006,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658377+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T09:56:32.658377+0000","last_peered":"2024-09-21T09:56:32.658377+0000","last_clean":"2024-09-21T09:56:32.658377+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T09:56:32.658377+0000","last_undegraded":"2024-09-21T09:56:32.658377+0000","last_fullsized":"2024-09-21T09:56:32.658377+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1010,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.657392+0000","last_change":"2024-09-21T09:36:43.389124+0000","last_active":"2024-09-21T09:56:32.657392+0000","last_peered":"2024-09-21T09:56:32.657392+0000","last_clean":"2024-09-21T09:56:32.657392+0000","last_became_active":"2024-09-21T09:36:43.388588+0000","last_became_peered":"2024-09-21T09:36:43.388588+0000","last_unstale":"2024-09-21T09:56:32.657392+0000","last_undegraded":"2024-09-21T09:56:32.657392+0000","last_fullsized":"2024-09-21T09:56:32.657392+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:57.562156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1082'720","reported_seq":739,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.166998+0000","last_change":"2024-09-21T09:56:26.652941+0000","last_active":"2024-09-21T09:56:40.166998+0000","last_peered":"2024-09-21T09:56:40.166998+0000","last_clean":"2024-09-21T09:56:40.166998+0000","last_became_active":"2024-09-21T09:56:26.652635+0000","last_became_peered":"2024-09-21T09:56:26.652635+0000","last_unstale":"2024-09-21T09:56:40.166998+0000","last_undegraded":"2024-09-21T09:56:40.166998+0000","last_fullsized":"2024-09-21T09:56:40.166998+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":720,"log_dups_size":0,"ondisk_log_size":720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:25:26.606125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47185920,"num_objects":720,"num_object_clones":0,"num_object_copies":2160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":720,"num_write_kb":46080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1007,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658065+0000","last_change":"2024-09-21T09:45:39.143190+0000","last_active":"2024-09-21T09:56:32.658065+0000","last_peered":"2024-09-21T09:56:32.658065+0000","last_clean":"2024-09-21T09:56:32.658065+0000","last_became_active":"2024-09-21T09:45:39.142826+0000","last_became_peered":"2024-09-21T09:45:39.142826+0000","last_unstale":"2024-09-21T09:56:32.658065+0000","last_undegraded":"2024-09-21T09:56:32.658065+0000","last_fullsized":"2024-09-21T09:56:32.658065+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:45.070945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1040,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.533687+0000","last_change":"2024-09-21T09:46:47.119247+0000","last_active":"2024-09-21T09:56:33.533687+0000","last_peered":"2024-09-21T09:56:33.533687+0000","last_clean":"2024-09-21T09:56:33.533687+0000","last_became_active":"2024-09-21T09:46:47.118980+0000","last_became_peered":"2024-09-21T09:46:47.118980+0000","last_unstale":"2024-09-21T09:56:33.533687+0000","last_undegraded":"2024-09-21T09:56:33.533687+0000","last_fullsized":"2024-09-21T09:56:33.533687+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.140253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1082'826","reported_seq":837,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.880322+0000","last_change":"2024-09-21T09:56:26.650240+0000","last_active":"2024-09-21T09:56:40.880322+0000","last_peered":"2024-09-21T09:56:40.880322+0000","last_clean":"2024-09-21T09:56:40.880322+0000","last_became_active":"2024-09-21T09:56:26.649902+0000","last_became_peered":"2024-09-21T09:56:26.649902+0000","last_unstale":"2024-09-21T09:56:40.880322+0000","last_undegraded":"2024-09-21T09:56:40.880322+0000","last_fullsized":"2024-09-21T09:56:40.880322+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":826,"log_dups_size":0,"ondisk_log_size":826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:53:25.701315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54132736,"num_objects":826,"num_object_clones":0,"num_object_copies":2478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":826,"num_write_kb":52864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":989,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658476+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:56:32.658476+0000","last_peered":"2024-09-21T09:56:32.658476+0000","last_clean":"2024-09-21T09:56:32.658476+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:56:32.658476+0000","last_undegraded":"2024-09-21T09:56:32.658476+0000","last_fullsized":"2024-09-21T09:56:32.658476+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1015,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.189057+0000","last_change":"2024-09-21T09:46:13.260744+0000","last_active":"2024-09-21T09:56:33.189057+0000","last_peered":"2024-09-21T09:56:33.189057+0000","last_clean":"2024-09-21T09:56:33.189057+0000","last_became_active":"2024-09-21T09:46:13.260443+0000","last_became_peered":"2024-09-21T09:46:13.260443+0000","last_unstale":"2024-09-21T09:56:33.189057+0000","last_undegraded":"2024-09-21T09:56:33.189057+0000","last_fullsized":"2024-09-21T09:56:33.189057+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:22.699110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1082'1257","reported_seq":1276,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:43.204144+0000","last_change":"2024-09-21T09:56:26.651524+0000","last_active":"2024-09-21T09:56:43.204144+0000","last_peered":"2024-09-21T09:56:43.204144+0000","last_clean":"2024-09-21T09:56:43.204144+0000","last_became_active":"2024-09-21T09:56:26.651233+0000","last_became_peered":"2024-09-21T09:56:26.651233+0000","last_unstale":"2024-09-21T09:56:43.204144+0000","last_undegraded":"2024-09-21T09:56:43.204144+0000","last_fullsized":"2024-09-21T09:56:43.204144+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":1257,"log_dups_size":0,"ondisk_log_size":1257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:09.126343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378752,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1257,"num_write_kb":80448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1028,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658064+0000","last_change":"2024-09-21T09:46:47.115343+0000","last_active":"2024-09-21T09:56:32.658064+0000","last_peered":"2024-09-21T09:56:32.658064+0000","last_clean":"2024-09-21T09:56:32.658064+0000","last_became_active":"2024-09-21T09:46:47.115017+0000","last_became_peered":"2024-09-21T09:46:47.115017+0000","last_unstale":"2024-09-21T09:56:32.658064+0000","last_undegraded":"2024-09-21T09:56:32.658064+0000","last_fullsized":"2024-09-21T09:56:32.658064+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.820662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1584,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.660054+0000","last_change":"2024-09-21T09:45:39.138289+0000","last_active":"2024-09-21T09:56:34.660054+0000","last_peered":"2024-09-21T09:56:34.660054+0000","last_clean":"2024-09-21T09:56:34.660054+0000","last_became_active":"2024-09-21T09:45:39.137902+0000","last_became_peered":"2024-09-21T09:45:39.137902+0000","last_unstale":"2024-09-21T09:56:34.660054+0000","last_undegraded":"2024-09-21T09:56:34.660054+0000","last_fullsized":"2024-09-21T09:56:34.660054+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:40:16.383906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1082'836","reported_seq":848,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.864354+0000","last_change":"2024-09-21T09:56:26.650365+0000","last_active":"2024-09-21T09:56:40.864354+0000","last_peered":"2024-09-21T09:56:40.864354+0000","last_clean":"2024-09-21T09:56:40.864354+0000","last_became_active":"2024-09-21T09:56:26.649684+0000","last_became_peered":"2024-09-21T09:56:26.649684+0000","last_unstale":"2024-09-21T09:56:40.864354+0000","last_undegraded":"2024-09-21T09:56:40.864354+0000","last_fullsized":"2024-09-21T09:56:40.864354+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":836,"log_dups_size":0,"ondisk_log_size":836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:04.800486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54788096,"num_objects":836,"num_object_clones":0,"num_object_copies":2508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":836,"num_write_kb":53504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1508,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.166072+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T09:56:35.166072+0000","last_peered":"2024-09-21T09:56:35.166072+0000","last_clean":"2024-09-21T09:56:35.166072+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T09:56:35.166072+0000","last_undegraded":"2024-09-21T09:56:35.166072+0000","last_fullsized":"2024-09-21T09:56:35.166072+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1023,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.533965+0000","last_change":"2024-09-21T09:46:47.114293+0000","last_active":"2024-09-21T09:56:33.533965+0000","last_peered":"2024-09-21T09:56:33.533965+0000","last_clean":"2024-09-21T09:56:33.533965+0000","last_became_active":"2024-09-21T09:46:47.113993+0000","last_became_peered":"2024-09-21T09:46:47.113993+0000","last_unstale":"2024-09-21T09:56:33.533965+0000","last_undegraded":"2024-09-21T09:56:33.533965+0000","last_fullsized":"2024-09-21T09:56:33.533965+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:30.941931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1082'866","reported_seq":884,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:41.304850+0000","last_change":"2024-09-21T09:56:26.651596+0000","last_active":"2024-09-21T09:56:41.304850+0000","last_peered":"2024-09-21T09:56:41.304850+0000","last_clean":"2024-09-21T09:56:41.304850+0000","last_became_active":"2024-09-21T09:56:26.651284+0000","last_became_peered":"2024-09-21T09:56:26.651284+0000","last_unstale":"2024-09-21T09:56:41.304850+0000","last_undegraded":"2024-09-21T09:56:41.304850+0000","last_fullsized":"2024-09-21T09:56:41.304850+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":866,"log_dups_size":0,"ondisk_log_size":866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:07.072468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56754176,"num_objects":866,"num_object_clones":0,"num_object_copies":2598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":866,"num_write_kb":55424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1006,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.189297+0000","last_change":"2024-09-21T09:45:39.127186+0000","last_active":"2024-09-21T09:56:33.189297+0000","last_peered":"2024-09-21T09:56:33.189297+0000","last_clean":"2024-09-21T09:56:33.189297+0000","last_became_active":"2024-09-21T09:45:39.126940+0000","last_became_peered":"2024-09-21T09:45:39.126940+0000","last_unstale":"2024-09-21T09:56:33.189297+0000","last_undegraded":"2024-09-21T09:56:33.189297+0000","last_fullsized":"2024-09-21T09:56:33.189297+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:52.205703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1040,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.534223+0000","last_change":"2024-09-21T09:46:13.259814+0000","last_active":"2024-09-21T09:56:33.534223+0000","last_peered":"2024-09-21T09:56:33.534223+0000","last_clean":"2024-09-21T09:56:33.534223+0000","last_became_active":"2024-09-21T09:46:13.259527+0000","last_became_peered":"2024-09-21T09:46:13.259527+0000","last_unstale":"2024-09-21T09:56:33.534223+0000","last_undegraded":"2024-09-21T09:56:33.534223+0000","last_fullsized":"2024-09-21T09:56:33.534223+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:00.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1082'720","reported_seq":739,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.165773+0000","last_change":"2024-09-21T09:56:27.225575+0000","last_active":"2024-09-21T09:56:40.165773+0000","last_peered":"2024-09-21T09:56:40.165773+0000","last_clean":"2024-09-21T09:56:40.165773+0000","last_became_active":"2024-09-21T09:56:27.225236+0000","last_became_peered":"2024-09-21T09:56:27.225236+0000","last_unstale":"2024-09-21T09:56:40.165773+0000","last_undegraded":"2024-09-21T09:56:40.165773+0000","last_fullsized":"2024-09-21T09:56:40.165773+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":720,"log_dups_size":0,"ondisk_log_size":720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:18.594048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47185920,"num_objects":720,"num_object_clones":0,"num_object_copies":2160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":720,"num_write_kb":46080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1024,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.533778+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T09:56:33.533778+0000","last_peered":"2024-09-21T09:56:33.533778+0000","last_clean":"2024-09-21T09:56:33.533778+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T09:56:33.533778+0000","last_undegraded":"2024-09-21T09:56:33.533778+0000","last_fullsized":"2024-09-21T09:56:33.533778+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1023,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.534319+0000","last_change":"2024-09-21T09:45:39.143302+0000","last_active":"2024-09-21T09:56:33.534319+0000","last_peered":"2024-09-21T09:56:33.534319+0000","last_clean":"2024-09-21T09:56:33.534319+0000","last_became_active":"2024-09-21T09:45:39.142964+0000","last_became_peered":"2024-09-21T09:45:39.142964+0000","last_unstale":"2024-09-21T09:56:33.534319+0000","last_undegraded":"2024-09-21T09:56:33.534319+0000","last_fullsized":"2024-09-21T09:56:33.534319+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:54.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1082'757","reported_seq":768,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.872417+0000","last_change":"2024-09-21T09:56:26.650435+0000","last_active":"2024-09-21T09:56:40.872417+0000","last_peered":"2024-09-21T09:56:40.872417+0000","last_clean":"2024-09-21T09:56:40.872417+0000","last_became_active":"2024-09-21T09:56:26.650085+0000","last_became_peered":"2024-09-21T09:56:26.650085+0000","last_unstale":"2024-09-21T09:56:40.872417+0000","last_undegraded":"2024-09-21T09:56:40.872417+0000","last_fullsized":"2024-09-21T09:56:40.872417+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":757,"log_dups_size":0,"ondisk_log_size":757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:56.071328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49610752,"num_objects":757,"num_object_clones":0,"num_object_copies":2271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":757,"num_write_kb":48448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1499,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.165940+0000","last_change":"2024-09-21T09:46:47.121643+0000","last_active":"2024-09-21T09:56:35.165940+0000","last_peered":"2024-09-21T09:56:35.165940+0000","last_clean":"2024-09-21T09:56:35.165940+0000","last_became_active":"2024-09-21T09:46:47.120528+0000","last_became_peered":"2024-09-21T09:46:47.120528+0000","last_unstale":"2024-09-21T09:56:35.165940+0000","last_undegraded":"2024-09-21T09:56:35.165940+0000","last_fullsized":"2024-09-21T09:56:35.165940+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:07.989377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1545,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.195712+0000","last_change":"2024-09-21T09:46:47.121159+0000","last_active":"2024-09-21T09:56:35.195712+0000","last_peered":"2024-09-21T09:56:35.195712+0000","last_clean":"2024-09-21T09:56:35.195712+0000","last_became_active":"2024-09-21T09:46:47.120829+0000","last_became_peered":"2024-09-21T09:46:47.120829+0000","last_unstale":"2024-09-21T09:56:35.195712+0000","last_undegraded":"2024-09-21T09:56:35.195712+0000","last_fullsized":"2024-09-21T09:56:35.195712+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:43.393840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1082'825","reported_seq":837,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.876946+0000","last_change":"2024-09-21T09:56:26.648574+0000","last_active":"2024-09-21T09:56:40.876946+0000","last_peered":"2024-09-21T09:56:40.876946+0000","last_clean":"2024-09-21T09:56:40.876946+0000","last_became_active":"2024-09-21T09:56:26.648254+0000","last_became_peered":"2024-09-21T09:56:26.648254+0000","last_unstale":"2024-09-21T09:56:40.876946+0000","last_undegraded":"2024-09-21T09:56:40.876946+0000","last_fullsized":"2024-09-21T09:56:40.876946+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":825,"log_dups_size":0,"ondisk_log_size":825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:28.331074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54067200,"num_objects":825,"num_object_clones":0,"num_object_copies":2475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":825,"num_write_kb":52800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1122,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.657673+0000","last_change":"2024-09-21T09:45:05.087928+0000","last_active":"2024-09-21T09:56:32.657673+0000","last_peered":"2024-09-21T09:56:32.657673+0000","last_clean":"2024-09-21T09:56:32.657673+0000","last_became_active":"2024-09-21T09:45:05.087478+0000","last_became_peered":"2024-09-21T09:45:05.087478+0000","last_unstale":"2024-09-21T09:56:32.657673+0000","last_undegraded":"2024-09-21T09:56:32.657673+0000","last_fullsized":"2024-09-21T09:56:32.657673+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:48.651314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":1041,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.657810+0000","last_change":"2024-09-21T09:46:13.262337+0000","last_active":"2024-09-21T09:56:32.657810+0000","last_peered":"2024-09-21T09:56:32.657810+0000","last_clean":"2024-09-21T09:56:32.657810+0000","last_became_active":"2024-09-21T09:46:13.262005+0000","last_became_peered":"2024-09-21T09:46:13.262005+0000","last_unstale":"2024-09-21T09:56:32.657810+0000","last_undegraded":"2024-09-21T09:56:32.657810+0000","last_fullsized":"2024-09-21T09:56:32.657810+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:32.413210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028803299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1481,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.166135+0000","last_change":"2024-09-21T09:46:47.116045+0000","last_active":"2024-09-21T09:56:35.166135+0000","last_peered":"2024-09-21T09:56:35.166135+0000","last_clean":"2024-09-21T09:56:35.166135+0000","last_became_active":"2024-09-21T09:46:47.115526+0000","last_became_peered":"2024-09-21T09:46:47.115526+0000","last_unstale":"2024-09-21T09:56:35.166135+0000","last_undegraded":"2024-09-21T09:56:35.166135+0000","last_fullsized":"2024-09-21T09:56:35.166135+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:43:35.293486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":988,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.189604+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:56:33.189604+0000","last_peered":"2024-09-21T09:56:33.189604+0000","last_clean":"2024-09-21T09:56:33.189604+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:56:33.189604+0000","last_undegraded":"2024-09-21T09:56:33.189604+0000","last_fullsized":"2024-09-21T09:56:33.189604+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1082'1081","reported_seq":1100,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:42.176433+0000","last_change":"2024-09-21T09:56:26.649185+0000","last_active":"2024-09-21T09:56:42.176433+0000","last_peered":"2024-09-21T09:56:42.176433+0000","last_clean":"2024-09-21T09:56:42.176433+0000","last_became_active":"2024-09-21T09:56:26.648886+0000","last_became_peered":"2024-09-21T09:56:26.648886+0000","last_unstale":"2024-09-21T09:56:42.176433+0000","last_undegraded":"2024-09-21T09:56:42.176433+0000","last_fullsized":"2024-09-21T09:56:42.176433+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":1081,"log_dups_size":0,"ondisk_log_size":1081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:48.286996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70844416,"num_objects":1081,"num_object_clones":0,"num_object_copies":3243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1081,"num_write_kb":69184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1034,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.657666+0000","last_change":"2024-09-21T09:45:39.129555+0000","last_active":"2024-09-21T09:56:32.657666+0000","last_peered":"2024-09-21T09:56:32.657666+0000","last_clean":"2024-09-21T09:56:32.657666+0000","last_became_active":"2024-09-21T09:45:39.129205+0000","last_became_peered":"2024-09-21T09:45:39.129205+0000","last_unstale":"2024-09-21T09:56:32.657666+0000","last_undegraded":"2024-09-21T09:56:32.657666+0000","last_fullsized":"2024-09-21T09:56:32.657666+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:07.229126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1582,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.661311+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T09:56:34.661311+0000","last_peered":"2024-09-21T09:56:34.661311+0000","last_clean":"2024-09-21T09:56:34.661311+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T09:56:34.661311+0000","last_undegraded":"2024-09-21T09:56:34.661311+0000","last_fullsized":"2024-09-21T09:56:34.661311+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1595,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.660522+0000","last_change":"2024-09-21T09:46:13.273573+0000","last_active":"2024-09-21T09:56:34.660522+0000","last_peered":"2024-09-21T09:56:34.660522+0000","last_clean":"2024-09-21T09:56:34.660522+0000","last_became_active":"2024-09-21T09:46:13.273297+0000","last_became_peered":"2024-09-21T09:46:13.273297+0000","last_unstale":"2024-09-21T09:56:34.660522+0000","last_undegraded":"2024-09-21T09:56:34.660522+0000","last_fullsized":"2024-09-21T09:56:34.660522+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:38.957745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1082'847","reported_seq":859,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.870741+0000","last_change":"2024-09-21T09:56:27.226675+0000","last_active":"2024-09-21T09:56:40.870741+0000","last_peered":"2024-09-21T09:56:40.870741+0000","last_clean":"2024-09-21T09:56:40.870741+0000","last_became_active":"2024-09-21T09:56:27.226362+0000","last_became_peered":"2024-09-21T09:56:27.226362+0000","last_unstale":"2024-09-21T09:56:40.870741+0000","last_undegraded":"2024-09-21T09:56:40.870741+0000","last_fullsized":"2024-09-21T09:56:40.870741+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":847,"log_dups_size":0,"ondisk_log_size":847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:25.080551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55508992,"num_objects":847,"num_object_clones":0,"num_object_copies":2541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":847,"num_write_kb":54208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1557,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.659936+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T09:56:34.659936+0000","last_peered":"2024-09-21T09:56:34.659936+0000","last_clean":"2024-09-21T09:56:34.659936+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T09:56:34.659936+0000","last_undegraded":"2024-09-21T09:56:34.659936+0000","last_fullsized":"2024-09-21T09:56:34.659936+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1547,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.195248+0000","last_change":"2024-09-21T09:46:13.255342+0000","last_active":"2024-09-21T09:56:35.195248+0000","last_peered":"2024-09-21T09:56:35.195248+0000","last_clean":"2024-09-21T09:56:35.195248+0000","last_became_active":"2024-09-21T09:46:13.255014+0000","last_became_peered":"2024-09-21T09:46:13.255014+0000","last_unstale":"2024-09-21T09:56:35.195248+0000","last_undegraded":"2024-09-21T09:56:35.195248+0000","last_fullsized":"2024-09-21T09:56:35.195248+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:53.012266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1558,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.194974+0000","last_change":"2024-09-21T09:45:39.134465+0000","last_active":"2024-09-21T09:56:35.194974+0000","last_peered":"2024-09-21T09:56:35.194974+0000","last_clean":"2024-09-21T09:56:35.194974+0000","last_became_active":"2024-09-21T09:45:39.129692+0000","last_became_peered":"2024-09-21T09:45:39.129692+0000","last_unstale":"2024-09-21T09:56:35.194974+0000","last_undegraded":"2024-09-21T09:56:35.194974+0000","last_fullsized":"2024-09-21T09:56:35.194974+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:14.288180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1082'664","reported_seq":682,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.164847+0000","last_change":"2024-09-21T09:56:26.650186+0000","last_active":"2024-09-21T09:56:40.164847+0000","last_peered":"2024-09-21T09:56:40.164847+0000","last_clean":"2024-09-21T09:56:40.164847+0000","last_became_active":"2024-09-21T09:56:26.649826+0000","last_became_peered":"2024-09-21T09:56:26.649826+0000","last_unstale":"2024-09-21T09:56:40.164847+0000","last_undegraded":"2024-09-21T09:56:40.164847+0000","last_fullsized":"2024-09-21T09:56:40.164847+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":664,"log_dups_size":0,"ondisk_log_size":664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:19:59.016344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43515904,"num_objects":664,"num_object_clones":0,"num_object_copies":1992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":664,"num_write_kb":42496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1042,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.657766+0000","last_change":"2024-09-21T09:45:39.131285+0000","last_active":"2024-09-21T09:56:32.657766+0000","last_peered":"2024-09-21T09:56:32.657766+0000","last_clean":"2024-09-21T09:56:32.657766+0000","last_became_active":"2024-09-21T09:45:39.130505+0000","last_became_peered":"2024-09-21T09:45:39.130505+0000","last_unstale":"2024-09-21T09:56:32.657766+0000","last_undegraded":"2024-09-21T09:56:32.657766+0000","last_fullsized":"2024-09-21T09:56:32.657766+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:37.523277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1024,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.534126+0000","last_change":"2024-09-21T09:45:05.097518+0000","last_active":"2024-09-21T09:56:33.534126+0000","last_peered":"2024-09-21T09:56:33.534126+0000","last_clean":"2024-09-21T09:56:33.534126+0000","last_became_active":"2024-09-21T09:45:05.096918+0000","last_became_peered":"2024-09-21T09:45:05.096918+0000","last_unstale":"2024-09-21T09:56:33.534126+0000","last_undegraded":"2024-09-21T09:56:33.534126+0000","last_fullsized":"2024-09-21T09:56:33.534126+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:13.781847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1557,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.195430+0000","last_change":"2024-09-21T09:46:47.113687+0000","last_active":"2024-09-21T09:56:35.195430+0000","last_peered":"2024-09-21T09:56:35.195430+0000","last_clean":"2024-09-21T09:56:35.195430+0000","last_became_active":"2024-09-21T09:46:47.113235+0000","last_became_peered":"2024-09-21T09:46:47.113235+0000","last_unstale":"2024-09-21T09:56:35.195430+0000","last_undegraded":"2024-09-21T09:56:35.195430+0000","last_fullsized":"2024-09-21T09:56:35.195430+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:50.851616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1082'1212","reported_seq":1229,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:43.213828+0000","last_change":"2024-09-21T09:56:26.651484+0000","last_active":"2024-09-21T09:56:43.213828+0000","last_peered":"2024-09-21T09:56:43.213828+0000","last_clean":"2024-09-21T09:56:43.213828+0000","last_became_active":"2024-09-21T09:56:26.651170+0000","last_became_peered":"2024-09-21T09:56:26.651170+0000","last_unstale":"2024-09-21T09:56:43.213828+0000","last_undegraded":"2024-09-21T09:56:43.213828+0000","last_fullsized":"2024-09-21T09:56:43.213828+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":1212,"log_dups_size":0,"ondisk_log_size":1212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:21.955853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79429632,"num_objects":1212,"num_object_clones":0,"num_object_copies":3636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1212,"num_write_kb":77568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1044,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658408+0000","last_change":"2024-09-21T09:46:47.115673+0000","last_active":"2024-09-21T09:56:32.658408+0000","last_peered":"2024-09-21T09:56:32.658408+0000","last_clean":"2024-09-21T09:56:32.658408+0000","last_became_active":"2024-09-21T09:46:47.115166+0000","last_became_peered":"2024-09-21T09:46:47.115166+0000","last_unstale":"2024-09-21T09:56:32.658408+0000","last_undegraded":"2024-09-21T09:56:32.658408+0000","last_fullsized":"2024-09-21T09:56:32.658408+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:16.660834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1571,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.194976+0000","last_change":"2024-09-21T09:45:39.132441+0000","last_active":"2024-09-21T09:56:35.194976+0000","last_peered":"2024-09-21T09:56:35.194976+0000","last_clean":"2024-09-21T09:56:35.194976+0000","last_became_active":"2024-09-21T09:45:39.129666+0000","last_became_peered":"2024-09-21T09:45:39.129666+0000","last_unstale":"2024-09-21T09:56:35.194976+0000","last_undegraded":"2024-09-21T09:56:35.194976+0000","last_fullsized":"2024-09-21T09:56:35.194976+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:34.450160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1510,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.659948+0000","last_change":"2024-09-21T09:46:13.258244+0000","last_active":"2024-09-21T09:56:34.659948+0000","last_peered":"2024-09-21T09:56:34.659948+0000","last_clean":"2024-09-21T09:56:34.659948+0000","last_became_active":"2024-09-21T09:46:13.257820+0000","last_became_peered":"2024-09-21T09:46:13.257820+0000","last_unstale":"2024-09-21T09:56:34.659948+0000","last_undegraded":"2024-09-21T09:56:34.659948+0000","last_fullsized":"2024-09-21T09:56:34.659948+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:04.618304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.1","version":"1082'1105","reported_seq":1124,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:42.179981+0000","last_change":"2024-09-21T09:56:27.225552+0000","last_active":"2024-09-21T09:56:42.179981+0000","last_peered":"2024-09-21T09:56:42.179981+0000","last_clean":"2024-09-21T09:56:42.179981+0000","last_became_active":"2024-09-21T09:56:27.225232+0000","last_became_peered":"2024-09-21T09:56:27.225232+0000","last_unstale":"2024-09-21T09:56:42.179981+0000","last_undegraded":"2024-09-21T09:56:42.179981+0000","last_fullsized":"2024-09-21T09:56:42.179981+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":1105,"log_dups_size":0,"ondisk_log_size":1105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:04:57.034718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72417280,"num_objects":1105,"num_object_clones":0,"num_object_copies":3315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1105,"num_write_kb":70720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1547,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.195634+0000","last_change":"2024-09-21T09:46:13.260666+0000","last_active":"2024-09-21T09:56:35.195634+0000","last_peered":"2024-09-21T09:56:35.195634+0000","last_clean":"2024-09-21T09:56:35.195634+0000","last_became_active":"2024-09-21T09:46:13.260324+0000","last_became_peered":"2024-09-21T09:46:13.260324+0000","last_unstale":"2024-09-21T09:56:35.195634+0000","last_undegraded":"2024-09-21T09:56:35.195634+0000","last_fullsized":"2024-09-21T09:56:35.195634+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:25.140797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1017,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658494+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T09:56:32.658494+0000","last_peered":"2024-09-21T09:56:32.658494+0000","last_clean":"2024-09-21T09:56:32.658494+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T09:56:32.658494+0000","last_undegraded":"2024-09-21T09:56:32.658494+0000","last_fullsized":"2024-09-21T09:56:32.658494+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1573,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.660141+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T09:56:34.660141+0000","last_peered":"2024-09-21T09:56:34.660141+0000","last_clean":"2024-09-21T09:56:34.660141+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T09:56:34.660141+0000","last_undegraded":"2024-09-21T09:56:34.660141+0000","last_fullsized":"2024-09-21T09:56:34.660141+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1082'834","reported_seq":846,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.873847+0000","last_change":"2024-09-21T09:56:26.650069+0000","last_active":"2024-09-21T09:56:40.873847+0000","last_peered":"2024-09-21T09:56:40.873847+0000","last_clean":"2024-09-21T09:56:40.873847+0000","last_became_active":"2024-09-21T09:56:26.649705+0000","last_became_peered":"2024-09-21T09:56:26.649705+0000","last_unstale":"2024-09-21T09:56:40.873847+0000","last_undegraded":"2024-09-21T09:56:40.873847+0000","last_fullsized":"2024-09-21T09:56:40.873847+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":834,"log_dups_size":0,"ondisk_log_size":834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:01:29.211110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54657024,"num_objects":834,"num_object_clones":0,"num_object_copies":2502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":834,"num_write_kb":53376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1508,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.166516+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T09:56:35.166516+0000","last_peered":"2024-09-21T09:56:35.166516+0000","last_clean":"2024-09-21T09:56:35.166516+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T09:56:35.166516+0000","last_undegraded":"2024-09-21T09:56:35.166516+0000","last_fullsized":"2024-09-21T09:56:35.166516+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1545,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.195675+0000","last_change":"2024-09-21T09:45:39.130168+0000","last_active":"2024-09-21T09:56:35.195675+0000","last_peered":"2024-09-21T09:56:35.195675+0000","last_clean":"2024-09-21T09:56:35.195675+0000","last_became_active":"2024-09-21T09:45:39.129498+0000","last_became_peered":"2024-09-21T09:45:39.129498+0000","last_unstale":"2024-09-21T09:56:35.195675+0000","last_undegraded":"2024-09-21T09:56:35.195675+0000","last_fullsized":"2024-09-21T09:56:35.195675+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:58.310876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1505,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.165797+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T09:56:35.165797+0000","last_peered":"2024-09-21T09:56:35.165797+0000","last_clean":"2024-09-21T09:56:35.165797+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T09:56:35.165797+0000","last_undegraded":"2024-09-21T09:56:35.165797+0000","last_fullsized":"2024-09-21T09:56:35.165797+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1082'818","reported_seq":830,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:40.874466+0000","last_change":"2024-09-21T09:56:27.225608+0000","last_active":"2024-09-21T09:56:40.874466+0000","last_peered":"2024-09-21T09:56:40.874466+0000","last_clean":"2024-09-21T09:56:40.874466+0000","last_became_active":"2024-09-21T09:56:27.225275+0000","last_became_peered":"2024-09-21T09:56:27.225275+0000","last_unstale":"2024-09-21T09:56:40.874466+0000","last_undegraded":"2024-09-21T09:56:40.874466+0000","last_fullsized":"2024-09-21T09:56:40.874466+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":818,"log_dups_size":0,"ondisk_log_size":818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:01:58.643767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53608448,"num_objects":818,"num_object_clones":0,"num_object_copies":2454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":818,"num_write_kb":52352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1586,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.660788+0000","last_change":"2024-09-21T09:45:39.140762+0000","last_active":"2024-09-21T09:56:34.660788+0000","last_peered":"2024-09-21T09:56:34.660788+0000","last_clean":"2024-09-21T09:56:34.660788+0000","last_became_active":"2024-09-21T09:45:39.140402+0000","last_became_peered":"2024-09-21T09:45:39.140402+0000","last_unstale":"2024-09-21T09:56:34.660788+0000","last_undegraded":"2024-09-21T09:56:34.660788+0000","last_fullsized":"2024-09-21T09:56:34.660788+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:06.545428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1018,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.189614+0000","last_change":"2024-09-21T09:46:13.260656+0000","last_active":"2024-09-21T09:56:33.189614+0000","last_peered":"2024-09-21T09:56:33.189614+0000","last_clean":"2024-09-21T09:56:33.189614+0000","last_became_active":"2024-09-21T09:46:13.260368+0000","last_became_peered":"2024-09-21T09:46:13.260368+0000","last_unstale":"2024-09-21T09:56:33.189614+0000","last_undegraded":"2024-09-21T09:56:33.189614+0000","last_fullsized":"2024-09-21T09:56:33.189614+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:22.059036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1006,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.533585+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:56:33.533585+0000","last_peered":"2024-09-21T09:56:33.533585+0000","last_clean":"2024-09-21T09:56:33.533585+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:56:33.533585+0000","last_undegraded":"2024-09-21T09:56:33.533585+0000","last_fullsized":"2024-09-21T09:56:33.533585+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1023,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.534036+0000","last_change":"2024-09-21T09:45:39.129538+0000","last_active":"2024-09-21T09:56:33.534036+0000","last_peered":"2024-09-21T09:56:33.534036+0000","last_clean":"2024-09-21T09:56:33.534036+0000","last_became_active":"2024-09-21T09:45:39.129198+0000","last_became_peered":"2024-09-21T09:45:39.129198+0000","last_unstale":"2024-09-21T09:56:33.534036+0000","last_undegraded":"2024-09-21T09:56:33.534036+0000","last_fullsized":"2024-09-21T09:56:33.534036+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.798277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1029,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.534035+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:56:33.534035+0000","last_peered":"2024-09-21T09:56:33.534035+0000","last_clean":"2024-09-21T09:56:33.534035+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:56:33.534035+0000","last_undegraded":"2024-09-21T09:56:33.534035+0000","last_fullsized":"2024-09-21T09:56:33.534035+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1545,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.195286+0000","last_change":"2024-09-21T09:46:13.254014+0000","last_active":"2024-09-21T09:56:35.195286+0000","last_peered":"2024-09-21T09:56:35.195286+0000","last_clean":"2024-09-21T09:56:35.195286+0000","last_became_active":"2024-09-21T09:46:13.253673+0000","last_became_peered":"2024-09-21T09:46:13.253673+0000","last_unstale":"2024-09-21T09:56:35.195286+0000","last_undegraded":"2024-09-21T09:56:35.195286+0000","last_fullsized":"2024-09-21T09:56:35.195286+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:02.784613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1008,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658412+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T09:56:32.658412+0000","last_peered":"2024-09-21T09:56:32.658412+0000","last_clean":"2024-09-21T09:56:32.658412+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T09:56:32.658412+0000","last_undegraded":"2024-09-21T09:56:32.658412+0000","last_fullsized":"2024-09-21T09:56:32.658412+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1028,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.657573+0000","last_change":"2024-09-21T09:45:39.130886+0000","last_active":"2024-09-21T09:56:32.657573+0000","last_peered":"2024-09-21T09:56:32.657573+0000","last_clean":"2024-09-21T09:56:32.657573+0000","last_became_active":"2024-09-21T09:45:39.130559+0000","last_became_peered":"2024-09-21T09:45:39.130559+0000","last_unstale":"2024-09-21T09:56:32.657573+0000","last_undegraded":"2024-09-21T09:56:32.657573+0000","last_fullsized":"2024-09-21T09:56:32.657573+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:39.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1481,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.165815+0000","last_change":"2024-09-21T09:46:47.122643+0000","last_active":"2024-09-21T09:56:35.165815+0000","last_peered":"2024-09-21T09:56:35.165815+0000","last_clean":"2024-09-21T09:56:35.165815+0000","last_became_active":"2024-09-21T09:46:47.122364+0000","last_became_peered":"2024-09-21T09:46:47.122364+0000","last_unstale":"2024-09-21T09:56:35.165815+0000","last_undegraded":"2024-09-21T09:56:35.165815+0000","last_fullsized":"2024-09-21T09:56:35.165815+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:29.222333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1581,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.661070+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T09:56:34.661070+0000","last_peered":"2024-09-21T09:56:34.661070+0000","last_clean":"2024-09-21T09:56:34.661070+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T09:56:34.661070+0000","last_undegraded":"2024-09-21T09:56:34.661070+0000","last_fullsized":"2024-09-21T09:56:34.661070+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1498,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.165761+0000","last_change":"2024-09-21T09:46:47.121360+0000","last_active":"2024-09-21T09:56:35.165761+0000","last_peered":"2024-09-21T09:56:35.165761+0000","last_clean":"2024-09-21T09:56:35.165761+0000","last_became_active":"2024-09-21T09:46:47.120494+0000","last_became_peered":"2024-09-21T09:46:47.120494+0000","last_unstale":"2024-09-21T09:56:35.165761+0000","last_undegraded":"2024-09-21T09:56:35.165761+0000","last_fullsized":"2024-09-21T09:56:35.165761+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:45.862984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1015,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.189586+0000","last_change":"2024-09-21T09:46:13.260037+0000","last_active":"2024-09-21T09:56:33.189586+0000","last_peered":"2024-09-21T09:56:33.189586+0000","last_clean":"2024-09-21T09:56:33.189586+0000","last_became_active":"2024-09-21T09:46:13.259690+0000","last_became_peered":"2024-09-21T09:46:13.259690+0000","last_unstale":"2024-09-21T09:56:33.189586+0000","last_undegraded":"2024-09-21T09:56:33.189586+0000","last_fullsized":"2024-09-21T09:56:33.189586+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:57.466762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1516,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.165879+0000","last_change":"2024-09-21T09:46:47.120830+0000","last_active":"2024-09-21T09:56:35.165879+0000","last_peered":"2024-09-21T09:56:35.165879+0000","last_clean":"2024-09-21T09:56:35.165879+0000","last_became_active":"2024-09-21T09:46:47.118419+0000","last_became_peered":"2024-09-21T09:46:47.118419+0000","last_unstale":"2024-09-21T09:56:35.165879+0000","last_undegraded":"2024-09-21T09:56:35.165879+0000","last_fullsized":"2024-09-21T09:56:35.165879+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:13.789806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1584,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.660611+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T09:56:34.660611+0000","last_peered":"2024-09-21T09:56:34.660611+0000","last_clean":"2024-09-21T09:56:34.660611+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T09:56:34.660611+0000","last_undegraded":"2024-09-21T09:56:34.660611+0000","last_fullsized":"2024-09-21T09:56:34.660611+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1546,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.195319+0000","last_change":"2024-09-21T09:46:47.121724+0000","last_active":"2024-09-21T09:56:35.195319+0000","last_peered":"2024-09-21T09:56:35.195319+0000","last_clean":"2024-09-21T09:56:35.195319+0000","last_became_active":"2024-09-21T09:46:47.121406+0000","last_became_peered":"2024-09-21T09:46:47.121406+0000","last_unstale":"2024-09-21T09:56:35.195319+0000","last_undegraded":"2024-09-21T09:56:35.195319+0000","last_fullsized":"2024-09-21T09:56:35.195319+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:55.493704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1497,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.166188+0000","last_change":"2024-09-21T09:46:47.113184+0000","last_active":"2024-09-21T09:56:35.166188+0000","last_peered":"2024-09-21T09:56:35.166188+0000","last_clean":"2024-09-21T09:56:35.166188+0000","last_became_active":"2024-09-21T09:46:47.112857+0000","last_became_peered":"2024-09-21T09:46:47.112857+0000","last_unstale":"2024-09-21T09:56:35.166188+0000","last_undegraded":"2024-09-21T09:56:35.166188+0000","last_fullsized":"2024-09-21T09:56:35.166188+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:46.783986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1042,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.533949+0000","last_change":"2024-09-21T09:46:13.262555+0000","last_active":"2024-09-21T09:56:33.533949+0000","last_peered":"2024-09-21T09:56:33.533949+0000","last_clean":"2024-09-21T09:56:33.533949+0000","last_became_active":"2024-09-21T09:46:13.261474+0000","last_became_peered":"2024-09-21T09:46:13.261474+0000","last_unstale":"2024-09-21T09:56:33.533949+0000","last_undegraded":"2024-09-21T09:56:33.533949+0000","last_fullsized":"2024-09-21T09:56:33.533949+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:44.865988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1536,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.660047+0000","last_change":"2024-09-21T09:46:47.114406+0000","last_active":"2024-09-21T09:56:34.660047+0000","last_peered":"2024-09-21T09:56:34.660047+0000","last_clean":"2024-09-21T09:56:34.660047+0000","last_became_active":"2024-09-21T09:46:47.113986+0000","last_became_peered":"2024-09-21T09:46:47.113986+0000","last_unstale":"2024-09-21T09:56:34.660047+0000","last_undegraded":"2024-09-21T09:56:34.660047+0000","last_fullsized":"2024-09-21T09:56:34.660047+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:32.292201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1574,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.660228+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T09:56:34.660228+0000","last_peered":"2024-09-21T09:56:34.660228+0000","last_clean":"2024-09-21T09:56:34.660228+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T09:56:34.660228+0000","last_undegraded":"2024-09-21T09:56:34.660228+0000","last_fullsized":"2024-09-21T09:56:34.660228+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1040,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.533948+0000","last_change":"2024-09-21T09:46:47.114188+0000","last_active":"2024-09-21T09:56:33.533948+0000","last_peered":"2024-09-21T09:56:33.533948+0000","last_clean":"2024-09-21T09:56:33.533948+0000","last_became_active":"2024-09-21T09:46:47.113859+0000","last_became_peered":"2024-09-21T09:46:47.113859+0000","last_unstale":"2024-09-21T09:56:33.533948+0000","last_undegraded":"2024-09-21T09:56:33.533948+0000","last_fullsized":"2024-09-21T09:56:33.533948+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:22.670368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":1030,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.657918+0000","last_change":"2024-09-21T09:45:39.131124+0000","last_active":"2024-09-21T09:56:32.657918+0000","last_peered":"2024-09-21T09:56:32.657918+0000","last_clean":"2024-09-21T09:56:32.657918+0000","last_became_active":"2024-09-21T09:45:39.130756+0000","last_became_peered":"2024-09-21T09:45:39.130756+0000","last_unstale":"2024-09-21T09:56:32.657918+0000","last_undegraded":"2024-09-21T09:56:32.657918+0000","last_fullsized":"2024-09-21T09:56:32.657918+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:49.130698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1040,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.533791+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T09:56:33.533791+0000","last_peered":"2024-09-21T09:56:33.533791+0000","last_clean":"2024-09-21T09:56:33.533791+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T09:56:33.533791+0000","last_undegraded":"2024-09-21T09:56:33.533791+0000","last_fullsized":"2024-09-21T09:56:33.533791+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":1012,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:32.658402+0000","last_change":"2024-09-21T09:36:43.115791+0000","last_active":"2024-09-21T09:56:32.658402+0000","last_peered":"2024-09-21T09:56:32.658402+0000","last_clean":"2024-09-21T09:56:32.658402+0000","last_became_active":"2024-09-21T09:36:43.115525+0000","last_became_peered":"2024-09-21T09:36:43.115525+0000","last_unstale":"2024-09-21T09:56:32.658402+0000","last_undegraded":"2024-09-21T09:56:32.658402+0000","last_fullsized":"2024-09-21T09:56:32.658402+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:28.396722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1508,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.166004+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T09:56:35.166004+0000","last_peered":"2024-09-21T09:56:35.166004+0000","last_clean":"2024-09-21T09:56:35.166004+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T09:56:35.166004+0000","last_undegraded":"2024-09-21T09:56:35.166004+0000","last_fullsized":"2024-09-21T09:56:35.166004+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1493,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:35.166342+0000","last_change":"2024-09-21T09:46:47.115788+0000","last_active":"2024-09-21T09:56:35.166342+0000","last_peered":"2024-09-21T09:56:35.166342+0000","last_clean":"2024-09-21T09:56:35.166342+0000","last_became_active":"2024-09-21T09:46:47.114864+0000","last_became_peered":"2024-09-21T09:46:47.114864+0000","last_unstale":"2024-09-21T09:56:35.166342+0000","last_undegraded":"2024-09-21T09:56:35.166342+0000","last_fullsized":"2024-09-21T09:56:35.166342+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:00.034944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1023,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:56:33.533536+0000","last_change":"2024-09-21T09:45:39.143064+0000","last_active":"2024-09-21T09:56:33.533536+0000","last_peered":"2024-09-21T09:56:33.533536+0000","last_clean":"2024-09-21T09:56:33.533536+0000","last_became_active":"2024-09-21T09:45:39.142699+0000","last_became_peered":"2024-09-21T09:45:39.142699+0000","last_unstale":"2024-09-21T09:56:33.533536+0000","last_undegraded":"2024-09-21T09:56:33.533536+0000","last_fullsized":"2024-09-21T09:56:33.533536+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:14.684721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1589,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:56:34.661166+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T09:56:34.661166+0000","last_peered":"2024-09-21T09:56:34.661166+0000","last_clean":"2024-09-21T09:56:34.661166+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T09:56:34.661166+0000","last_undegraded":"2024-09-21T09:56:34.661166+0000","last_fullsized":"2024-09-21T09:56:34.661166+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":929300480,"num_objects":14180,"num_object_clones":0,"num_object_copies":42540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14180,"num_write_kb":907520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2734227456,"data_stored":2734227456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":14180,"ondisk_log_size":14180,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397770,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":611580,"kb_used_data":314228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93133572,"statfs":{"total":95995035648,"available":95368777728,"internally_reserved":0,"allocated":321769472,"data_stored":319633310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683593999,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":531060,"kb_used_data":229548,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93214092,"statfs":{"total":95995035648,"available":95451230208,"internally_reserved":0,"allocated":235057152,"data_stored":232929047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790228,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":528744,"kb_used_data":223072,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93216408,"statfs":{"total":95995035648,"available":95453601792,"internally_reserved":0,"allocated":228425728,"data_stored":226305193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986455,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":547124,"kb_used_data":249708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93198028,"statfs":{"total":95995035648,"available":95434780672,"internally_reserved":0,"allocated":255700992,"data_stored":253572400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999097,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":395192,"kb_used_data":359216,"kb_used_omap":25,"kb_used_meta":35942,"kb_avail":93349960,"statfs":{"total":95995035648,"available":95590359040,"internally_reserved":0,"allocated":367837184,"data_stored":365544090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":36805575},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111040,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":471536,"kb_used_data":421480,"kb_used_omap":31,"kb_used_meta":50016,"kb_avail":93273616,"statfs":{"total":95995035648,"available":95512182784,"internally_reserved":0,"allocated":431595520,"data_stored":429244924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":51216616},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318222983,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":379576,"kb_used_data":329648,"kb_used_omap":26,"kb_used_meta":49893,"kb_avail":93365576,"statfs":{"total":95995035648,"available":95606349824,"internally_reserved":0,"allocated":337559552,"data_stored":335266395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":51090444},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289334926,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362868,"kb_used_data":304556,"kb_used_omap":31,"kb_used_meta":58272,"kb_avail":93382284,"statfs":{"total":95995035648,"available":95623458816,"internally_reserved":0,"allocated":311865344,"data_stored":309511192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":59670624},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":314179584,"data_stored":314179584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":350224384,"data_stored":350224384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":481034240,"data_stored":481034240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":452329472,"data_stored":452329472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266665984,"data_stored":266665984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":267845632,"data_stored":267845632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":274792448,"data_stored":274792448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327155712,"data_stored":327155712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:56:53.489 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T09:56:53.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_1 pg_num 26 2024-09-21T09:56:53.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:53.550 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:53.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:53.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:53.618 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:53.691 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:53.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:53 smithi017 ceph-mon[102913]: pgmap v1256: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2024-09-21T09:56:53.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:53 smithi017 ceph-mon[105151]: pgmap v1256: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2024-09-21T09:56:53.720 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:53.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:53.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:53 smithi134 ceph-mon[90085]: pgmap v1256: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2024-09-21T09:56:53.822 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:53.851 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:53.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:53.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 40127 40111 147.439 116.625 0.00827518 0.00677679 2024-09-21T09:56:53.953 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:53.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:54.087 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:54.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.123 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:54.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:54.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T09:56:54.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:54.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:54.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:54.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:54.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:54.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:54.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:54.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:54.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:54.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:54.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:54.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:54.921 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 42006 41990 145.772 117.438 0.00470635 0.0068553 2024-09-21T09:56:54.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:55.002 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:55.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:55.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:55.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:55.189 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:56:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:56:54] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.51.0" 2024-09-21T09:56:55.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:55.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:55.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:55.386 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:55.418 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:55 smithi017 ceph-mon[102913]: pgmap v1257: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-21T09:56:55.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:55.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:55 smithi017 ceph-mon[105151]: pgmap v1257: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-21T09:56:55.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:55.520 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:55.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:55.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:55.687 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:55 smithi134 ceph-mon[90085]: pgmap v1257: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-21T09:56:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:55.788 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:55.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:55.921 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:55.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 43839 43823 144.129 114.562 0.0109636 0.00693305 2024-09-21T09:56:55.952 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:55.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:56.054 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:56.087 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:56.189 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:56.220 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:56.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:56.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:56.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:56.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:56.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:56.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:56.615 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:56.643 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:56.745 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:56.775 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:56.877 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:56.905 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:56.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:56.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T09:56:56.920916+0000 min lat: 0.00126159 max lat: 0.172889 avg lat: 0.00699053 2024-09-21T09:56:56.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:56:56.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 45765 45749 142.94 120.375 0.00602688 0.00699053 2024-09-21T09:56:57.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:57.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:57.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:57.147 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:57.175 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:57.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:57.264 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:56:57.264 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:56:57.277 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:57.306 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:57.312 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:56:57.313 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:56:57.350 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:56:57.350 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:56:57.397 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:56:57.397 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:56:57.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:57.434 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:57.439 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:56:57.439 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:56:57.472 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:56:57.472 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:56:57.492 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:56:57.492 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T09:56:57.492 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T09:56:57.492 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T09:56:57.492 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:56:57.507 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:57.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:57 smithi017 ceph-mon[102913]: pgmap v1258: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-21T09:56:57.523 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3019413265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:57.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:57 smithi017 ceph-mon[105151]: pgmap v1258: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-21T09:56:57.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3019413265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:57.524 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:56:57.524 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:56:57.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:57.563 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:56:57.563 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:56:57.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:57.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:57.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:57.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:57.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:57.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:57 smithi134 ceph-mon[90085]: pgmap v1258: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-21T09:56:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3019413265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:57.812 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:57.842 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:57.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:57.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 47396 47380 140.987 101.938 0.00953128 0.00708681 2024-09-21T09:56:57.943 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:57.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:57.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:58.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:58.116 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:58.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:58.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:58.254 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:58.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:58.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:58.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:58 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:58.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:58 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:58.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:58.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T09:56:58.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:58.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:58.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:58.628 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:58 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:58.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:58.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:58.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:58.819 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:58.852 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:58.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:58.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 49101 49085 139.422 106.562 0.00472213 0.00716712 2024-09-21T09:56:58.953 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:59.004 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:59.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:59.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:59.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:59.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:59.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:59.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:59.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:59.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:59.433 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:59.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:59.535 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:59.552 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:56:59 smithi017 ceph-mon[102913]: pgmap v1259: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-09-21T09:56:59.552 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:56:59 smithi017 ceph-mon[105151]: pgmap v1259: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-09-21T09:56:59.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:56:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:59.654 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:59.682 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:59.688 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:56:59 smithi134 ceph-mon[90085]: pgmap v1259: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-09-21T09:56:59.711 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:59.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:59.844 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:59.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:59.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 50850 50834 138.112 109.312 0.0187438 0.00723502 2024-09-21T09:56:59.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:59.989 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:56:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:00.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:00.125 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:00.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:00.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:00.237 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:00.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:00.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:00.387 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:00.421 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:00.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:57:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:57:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:00.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:00.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:00.665 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:00.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.697 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:00.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:00.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:00.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:00.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 52457 52441 136.542 100.438 0.00750639 0.00731872 2024-09-21T09:57:00.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:00.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:01.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:01.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:01.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:01.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:01.270 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:01.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:01.399 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:01.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:01.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:01.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:01.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:01.648 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:01 smithi134 ceph-mon[90085]: pgmap v1260: 89 pgs: 89 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-21T09:57:01.685 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:01 smithi017 ceph-mon[102913]: pgmap v1260: 89 pgs: 89 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-21T09:57:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:01 smithi017 ceph-mon[105151]: pgmap v1260: 89 pgs: 89 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-21T09:57:01.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:01.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:01.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:01.922 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 54045 54029 135.05 99.25 0.00850971 0.00739985 2024-09-21T09:57:01.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:02.064 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:02.101 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:02.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:57:02.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:02.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:02.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:02.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:02.372 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T09:57:02.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:02.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:02.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:02.550 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:02.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:02.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3096551532' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:02.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:02 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:02.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3096551532' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:02.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:02 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:02.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:02.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3096551532' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:02 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:02.796 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:02.829 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:02.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:02.923 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 55626 55610 133.654 98.8125 0.00763825 0.00747708 2024-09-21T09:57:02.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:02.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:02.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:03.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:03.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:03.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:03.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:03.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:03.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:03.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:03.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:03.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:03.507 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:03.535 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:03.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[102913]: pgmap v1261: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[102913]: osdmap e1084: 8 total, 8 up, 8 in 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[105151]: pgmap v1261: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T09:57:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[105151]: osdmap e1084: 8 total, 8 up, 8 in 2024-09-21T09:57:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:03.740 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:03 smithi134 ceph-mon[90085]: pgmap v1261: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-21T09:57:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:03 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:57:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:03 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T09:57:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:03 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:03 smithi134 ceph-mon[90085]: osdmap e1084: 8 total, 8 up, 8 in 2024-09-21T09:57:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.779 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:03.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:03.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:03.919 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:03.923 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 57220 57204 132.394 99.625 0.0105161 0.0075472 2024-09-21T09:57:04.021 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:04.053 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:04.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:04.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:04.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:04.299 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:04.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:04.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:04.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:04.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:04.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:04.605 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:04.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:04.706 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:04.730 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:04.832 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:04.863 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:04.869 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:57:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:57:04] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.51.0" 2024-09-21T09:57:04.923 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 58798 58782 131.188 98.625 0.0049594 0.00761785 2024-09-21T09:57:04.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:05.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:05.104 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:05.134 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:05.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:05.236 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:05.262 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:05.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:05.378 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 6 pg_num to 26 2024-09-21T09:57:05.420 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:05.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:05 smithi017 ceph-mon[102913]: pgmap v1263: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-21T09:57:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:05 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3911860462' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T09:57:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:05 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T09:57:05.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:05 smithi017 ceph-mon[105151]: pgmap v1263: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-21T09:57:05.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:05 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3911860462' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T09:57:05.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:05 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T09:57:05.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:05.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:05.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:05.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:05.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:05 smithi134 ceph-mon[90085]: pgmap v1263: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-21T09:57:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:05 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3911860462' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T09:57:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:05 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T09:57:05.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:05.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:05.923 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 60305 60289 129.911 94.1875 0.0147368 0.00769298 2024-09-21T09:57:05.948 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:05.979 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:05.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:06.081 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:06.115 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:06.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:06.216 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:06.237 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:06.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:06.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:06.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:06.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:06.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:06.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:06.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:06.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:06 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-09-21T09:57:06.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:06 smithi017 ceph-mon[102913]: osdmap e1085: 8 total, 8 up, 8 in 2024-09-21T09:57:06.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:06 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T09:57:06.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:06 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-09-21T09:57:06.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:06 smithi017 ceph-mon[105151]: osdmap e1085: 8 total, 8 up, 8 in 2024-09-21T09:57:06.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:06 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T09:57:06.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:06 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-09-21T09:57:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:06 smithi134 ceph-mon[90085]: osdmap e1085: 8 total, 8 up, 8 in 2024-09-21T09:57:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:06 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T09:57:06.752 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:06.773 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:06.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:06.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:06.881 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:06.885 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:57:06.885 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T09:57:06.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T09:57:06.885 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T09:57:06.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:46548 6.4 6:2510008e:::benchmark_data_smithi017_149547_object46547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:56:57.364380+0000", 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 9.5177259270000008, 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026042709000000001, 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 46548 2024-09-21T09:57:06.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.364380+0000", 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.364380+0000", 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.364381+0000", 2024-09-21T09:57:06.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3900000000000004e-07 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.364407+0000", 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5559e-05 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.364414+0000", 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7179999999999996e-06 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.364420+0000", 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8939999999999998e-06 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.369197+0000", 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047770859999999998 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.369440+0000", 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024291699999999999 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.369578+0000", 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137393 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.373630+0000", 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040517419999999997 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.373651+0000", 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1120000000000001e-05 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.390305+0000", 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016654287 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.390375+0000", 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0124999999999997e-05 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.390423+0000", 2024-09-21T09:57:06.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8028999999999997e-05 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:46603 6.4 6:28331fcc:::benchmark_data_smithi017_149547_object46602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:56:57.400935+0000", 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 9.4811712210000003, 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029943131000000001, 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.892 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 46603 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.400935+0000", 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.400935+0000", 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.400937+0000", 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4959999999999999e-06 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.400959+0000", 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2881e-05 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.400970+0000", 2024-09-21T09:57:06.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0849000000000001e-05 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.400977+0000", 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.126e-06 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.403647+0000", 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026694879999999998 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.403769+0000", 2024-09-21T09:57:06.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012220899999999999 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.403902+0000", 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000132819 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.416089+0000", 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012187567 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.416292+0000", 2024-09-21T09:57:06.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000202826 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.425950+0000", 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0096580539999999993 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.430714+0000", 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047631349999999999 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.430878+0000", 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016468099999999999 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:46616 6.4 6:26fca3d9:::benchmark_data_smithi017_149547_object46615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:56:57.406542+0000", 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 9.4755644960000005, 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.024594135, 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.898 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.899 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.899 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.899 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.899 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 46616 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.406542+0000", 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.406542+0000", 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.406544+0000", 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.379e-06 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.406567+0000", 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.321e-05 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.406581+0000", 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3844e-05 2024-09-21T09:57:06.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.406589+0000", 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1170000000000004e-06 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.409386+0000", 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027967999999999999 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.409523+0000", 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013653999999999999 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.409676+0000", 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015369099999999999 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.416483+0000", 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068066259999999996 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.430914+0000", 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014430689 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.430963+0000", 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9104999999999998e-05 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.431094+0000", 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131691 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.431136+0000", 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1443000000000003e-05 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:46684 6.4 6:288e0881:::benchmark_data_smithi017_149547_object46683:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:56:57.464657+0000", 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 9.4174496370000007, 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025177639000000002, 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.905 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 46684 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.464657+0000", 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.464657+0000", 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.464657+0000", 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3900000000000005e-07 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.464680+0000", 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2937000000000001e-05 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.464688+0000", 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0940000000000008e-06 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.464694+0000", 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.592e-06 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.470064+0000", 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053705719999999997 2024-09-21T09:57:06.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.470219+0000", 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015482899999999999 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.470367+0000", 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014757799999999999 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.472946+0000", 2024-09-21T09:57:06.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025788510000000001 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.474179+0000", 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001233095 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.488999+0000", 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014820359 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.489777+0000", 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00077819200000000003 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:57.489834+0000", 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7000999999999999e-05 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:48523 6.4 6:20d32780:::benchmark_data_smithi017_149547_object48522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:56:58.577443+0000", 2024-09-21T09:57:06.911 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.3046633209999996, 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025964511999999999, 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 48523 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577443+0000", 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577443+0000", 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577445+0000", 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8530000000000001e-06 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577471+0000", 2024-09-21T09:57:06.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6027e-05 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577490+0000", 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8913e-05 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577499+0000", 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9199999999999993e-06 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.584369+0000", 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068701109999999999 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.584957+0000", 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00058786799999999996 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.586485+0000", 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015285629999999999 2024-09-21T09:57:06.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.591362+0000", 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048763690000000002 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.591541+0000", 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017910300000000001 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.603232+0000", 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011691113 2024-09-21T09:57:06.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.603366+0000", 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134518 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.603407+0000", 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1153999999999999e-05 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:48525 6.4 6:2d87d71c:::benchmark_data_smithi017_149547_object48524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:56:58.577732+0000", 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.3043747579999998, 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028235731999999999, 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 48525 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577732+0000", 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577732+0000", 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577733+0000", 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.850000000000001e-07 2024-09-21T09:57:06.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577756+0000", 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3163000000000002e-05 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577768+0000", 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2139e-05 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.577776+0000", 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7510000000000008e-06 2024-09-21T09:57:06.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.588119+0000", 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010343089 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.588252+0000", 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013373599999999999 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.588387+0000", 2024-09-21T09:57:06.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013484200000000001 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.591366+0000", 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029787070000000001 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.593636+0000", 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022699119999999998 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.603450+0000", 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0098142339999999998 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.605776+0000", 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023255810000000002 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.605967+0000", 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019159299999999999 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.923 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:48682 6.4 6:2ce86e38:::benchmark_data_smithi017_149547_object48681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:56:58.676811+0000", 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.2052948299999997, 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.024749429, 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 48682 2024-09-21T09:57:06.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.676811+0000", 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.676811+0000", 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.676813+0000", 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.779e-06 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.676837+0000", 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3841000000000001e-05 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.676853+0000", 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6362e-05 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.676862+0000", 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1810000000000001e-06 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.677100+0000", 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000237965 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.677232+0000", 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013262899999999999 2024-09-21T09:57:06.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.677366+0000", 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013423600000000001 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.680124+0000", 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027573979999999999 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.683576+0000", 2024-09-21T09:57:06.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034518800000000001 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.701459+0000", 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017883408 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.701519+0000", 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9861999999999997e-05 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:58.701561+0000", 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1888000000000003e-05 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:50370 6.4 6:2cfd45fa:::benchmark_data_smithi017_149547_object50369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:56:59.640793+0000", 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 7.2413137289999998, 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023922253000000001, 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.930 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50370 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.640793+0000", 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.640793+0000", 2024-09-21T09:57:06.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.640795+0000", 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8049999999999998e-06 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.640834+0000", 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8151000000000001e-05 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.640855+0000", 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1217999999999999e-05 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.640864+0000", 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2110000000000004e-06 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.641206+0000", 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034240099999999999 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.641348+0000", 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014182299999999999 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.641491+0000", 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014250499999999999 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.646271+0000", 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.004779841 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.658303+0000", 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012032645999999999 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.664612+0000", 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063085049999999998 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.664669+0000", 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7309000000000003e-05 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:56:59.664715+0000", 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5837999999999997e-05 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:51852 6.4 6:2a3cd43f:::benchmark_data_smithi017_149547_object51851:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:00.535483+0000", 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 6.346623482, 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028497880999999999, 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.936 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51852 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.535483+0000", 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.535483+0000", 2024-09-21T09:57:06.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.535483+0000", 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0900000000000002e-07 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.535503+0000", 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9258000000000001e-05 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.535510+0000", 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0999999999999998e-06 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.535515+0000", 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2480000000000004e-06 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.538201+0000", 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026864689999999999 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.538365+0000", 2024-09-21T09:57:06.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:06.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000163649 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.538511+0000", 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014563799999999999 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.541269+0000", 2024-09-21T09:57:06.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027580619999999999 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.542931+0000", 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001662516 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.563702+0000", 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020770423999999999 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.563928+0000", 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022619099999999999 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.563981+0000", 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2816999999999999e-05 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.944 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:52461 6.4 6:2b2315bd:::benchmark_data_smithi017_149547_object52460:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:00.922715+0000", 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 5.959391589, 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026209626999999999, 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 52461 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.922715+0000", 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.922715+0000", 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.922716+0000", 2024-09-21T09:57:06.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7230000000000001e-06 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.922740+0000", 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3907e-05 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.922753+0000", 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2371000000000001e-05 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.922760+0000", 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9070000000000001e-06 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.923871+0000", 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001111391 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.924002+0000", 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013091599999999999 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.924154+0000", 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000151905 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.929010+0000", 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048564059999999997 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.930179+0000", 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001168676 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.948818+0000", 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018638688 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.948876+0000", 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8350999999999997e-05 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:00.948924+0000", 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8386000000000001e-05 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.950 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:52996 6.1 6:83b68a6d:::benchmark_data_smithi017_149547_object52995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1083)", 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:01.274104+0000", 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 5.6080025180000002, 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026674953000000001, 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 52996 2024-09-21T09:57:06.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.274104+0000", 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.274104+0000", 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.274105+0000", 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6780000000000001e-06 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.274130+0000", 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4522999999999999e-05 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.274140+0000", 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0278999999999999e-05 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.274146+0000", 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2129999999999996e-06 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.274264+0000", 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011729499999999999 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.274385+0000", 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012115799999999999 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.274532+0000", 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000147381 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.279545+0000", 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050123240000000003 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.281475+0000", 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019300980000000001 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.300677+0000", 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019202752 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.300732+0000", 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4565999999999999e-05 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:01.300779+0000", 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6686e-05 2024-09-21T09:57:06.956 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:57184 6.1 6:83f7e7d7:::benchmark_data_smithi017_149547_object57183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1084)", 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:03.899205+0000", 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.982900823, 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023175406999999999, 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.957 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57184 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.899205+0000", 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.899205+0000", 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.899207+0000", 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5290000000000001e-06 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.899231+0000", 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3890999999999999e-05 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.899243+0000", 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1807999999999999e-05 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.899250+0000", 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6070000000000002e-06 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.899367+0000", 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116666 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.899557+0000", 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019008100000000001 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.899695+0000", 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137761 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.908743+0000", 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0090482159999999996 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.908787+0000", 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3739999999999998e-05 2024-09-21T09:57:06.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.922288+0000", 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013501167 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.922339+0000", 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0921000000000003e-05 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:03.922381+0000", 2024-09-21T09:57:06.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2020000000000001e-05 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:57699 6.4 6:2bf88733:::benchmark_data_smithi017_149547_object57698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1084)", 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:04.219700+0000", 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.6624059440000001, 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025696292999999999, 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.963 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57699 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.219700+0000", 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.219700+0000", 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.219702+0000", 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6810000000000001e-06 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.219728+0000", 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5490000000000002e-05 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.219744+0000", 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6835000000000001e-05 2024-09-21T09:57:06.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.219752+0000", 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8679999999999999e-06 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.227570+0000", 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0078173319999999998 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.227714+0000", 2024-09-21T09:57:06.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014427099999999999 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.227858+0000", 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144487 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.234247+0000", 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063884780000000004 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.239777+0000", 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055306569999999996 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.245297+0000", 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00551908 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.245351+0000", 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4187000000000001e-05 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.245397+0000", 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5927000000000003e-05 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:57702 6.4 6:2f8b3e10:::benchmark_data_smithi017_149547_object57701:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1084)", 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:04.222161+0000", 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.659945515, 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023330712, 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.969 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57702 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.222161+0000", 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.222161+0000", 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.222162+0000", 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.203e-06 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.222186+0000", 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3907e-05 2024-09-21T09:57:06.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.222198+0000", 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2283000000000001e-05 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.222206+0000", 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5390000000000002e-06 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.230450+0000", 2024-09-21T09:57:06.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082447909999999996 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.230836+0000", 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038568400000000002 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.231382+0000", 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000545357 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.239734+0000", 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0083520160000000003 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.239817+0000", 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3727000000000007e-05 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.245411+0000", 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055936789999999998 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.245455+0000", 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3835999999999998e-05 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.245491+0000", 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6690000000000003e-05 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:58496 6.4 6:2bb8afc1:::benchmark_data_smithi017_149547_object58495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1084)", 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:04.700464+0000", 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1816418, 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027591026000000001, 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 58496 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700464+0000", 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700464+0000", 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700466+0000", 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.488e-06 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700488+0000", 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2523000000000001e-05 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700500+0000", 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1909000000000001e-05 2024-09-21T09:57:06.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700508+0000", 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8869999999999992e-06 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700564+0000", 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5560000000000003e-05 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700963+0000", 2024-09-21T09:57:06.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00039964500000000001 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.701084+0000", 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120093 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.711297+0000", 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010213177 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.711357+0000", 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0297999999999998e-05 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.727941+0000", 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016584196999999998 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.728004+0000", 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2953999999999994e-05 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.728056+0000", 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1295000000000003e-05 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:58497 6.4 6:291fc78d:::benchmark_data_smithi017_149547_object58496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1084)", 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:04.700749+0000", 2024-09-21T09:57:06.982 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1813570219999998, 2024-09-21T09:57:06.983 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 60928 60912 126.879 38.9375 0.199325 0.00774749 2024-09-21T09:57:06.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027418133000000001, 2024-09-21T09:57:06.983 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.983 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.983 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.983 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.983 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 58497 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700749+0000", 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700749+0000", 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700750+0000", 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.105e-06 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700804+0000", 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3940000000000002e-05 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700815+0000", 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0812000000000001e-05 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.700822+0000", 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8260000000000004e-06 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.704344+0000", 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003521999 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.704571+0000", 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000226708 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.706673+0000", 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021019110000000001 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.711342+0000", 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046696910000000001 2024-09-21T09:57:06.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.711707+0000", 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036431000000000001 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.728093+0000", 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016386123999999998 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.728136+0000", 2024-09-21T09:57:06.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3741e-05 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:04.728167+0000", 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0966000000000001e-05 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:59449 6.4 6:2d3b3fd4:::benchmark_data_smithi017_149547_object59448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1084)", 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:05.368315+0000", 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.513791605, 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022682539000000002, 2024-09-21T09:57:06.989 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 59449 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.368315+0000", 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.368315+0000", 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.368315+0000", 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7899999999999999e-07 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.368334+0000", 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8893999999999999e-05 2024-09-21T09:57:06.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.368342+0000", 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8250000000000005e-06 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.368347+0000", 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.242e-06 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.373867+0000", 2024-09-21T09:57:06.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055197850000000001 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.374013+0000", 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000146057 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.374167+0000", 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015411299999999999 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.384525+0000", 2024-09-21T09:57:06.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010358255 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.384554+0000", 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8806000000000002e-05 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.390913+0000", 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063587740000000002 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:06.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.390960+0000", 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7584999999999999e-05 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.390997+0000", 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6724000000000003e-05 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:59467 6.1 6:84243471:::benchmark_data_smithi017_149547_object59466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:05.391776+0000", 2024-09-21T09:57:06.995 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.4903303430000001, 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027352950000000001, 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 59467 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.391776+0000", 2024-09-21T09:57:06.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.391776+0000", 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.391777+0000", 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5510000000000001e-06 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:06.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.391804+0000", 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6678000000000001e-05 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.391818+0000", 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3740000000000001e-05 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.391826+0000", 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1620000000000008e-06 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.392100+0000", 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000273808 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.392274+0000", 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017438800000000001 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.392420+0000", 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014622499999999999 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:06.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.394970+0000", 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025497409999999999 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.400427+0000", 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054570039999999997 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.418985+0000", 2024-09-21T09:57:07.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018557509999999999 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.419081+0000", 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6634999999999999e-05 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:05.419129+0000", 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7508e-05 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:60908 6.4 6:2d50697e:::benchmark_data_smithi017_149547_object60907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.391733+0000", 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.49037313700000001, 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.094915599000000003, 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60908 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.002 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.391733+0000", 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.391733+0000", 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.391735+0000", 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6309999999999999e-06 2024-09-21T09:57:07.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.391759+0000", 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4012999999999999e-05 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.391775+0000", 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6382e-05 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.391782+0000", 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9879999999999998e-06 2024-09-21T09:57:07.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.391830+0000", 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7716999999999997e-05 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.391954+0000", 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012381999999999999 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.392081+0000", 2024-09-21T09:57:07.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012688 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.396472+0000", 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043917549999999998 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.396679+0000", 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020679300000000001 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:07.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.486633+0000", 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.089953626999999994 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.486649+0000", 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5993000000000001e-05 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:60916 6.4 6.87279344 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.400570+0000", 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.48153590499999999, 2024-09-21T09:57:07.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.086187106999999999, 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60916 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400570+0000", 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400570+0000", 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400572+0000", 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7859999999999999e-06 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400602+0000", 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0088999999999999e-05 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400624+0000", 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1421999999999999e-05 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400632+0000", 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1589999999999993e-06 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.486750+0000", 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.086118181000000002 2024-09-21T09:57:07.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.486757+0000", 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4699999999999996e-06 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:57:07.011 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:57:07.042 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:07.063 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:07.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:07.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:07.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:07.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:07.311 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:07.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:07.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:07.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:07.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:07.691 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:07 smithi017 ceph-mon[102913]: pgmap v1265: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T09:57:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T09:57:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:07 smithi017 ceph-mon[102913]: osdmap e1086: 8 total, 8 up, 8 in 2024-09-21T09:57:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:07 smithi017 ceph-mon[105151]: pgmap v1265: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T09:57:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T09:57:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:07 smithi017 ceph-mon[105151]: osdmap e1086: 8 total, 8 up, 8 in 2024-09-21T09:57:07.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T09:57:07.722 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:07.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:07.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:07 smithi134 ceph-mon[90085]: pgmap v1265: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-21T09:57:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T09:57:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:07 smithi134 ceph-mon[90085]: osdmap e1086: 8 total, 8 up, 8 in 2024-09-21T09:57:07.824 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:07.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:07.923 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 61806 61790 124.556 54.875 0.00340448 0.00802346 2024-09-21T09:57:07.957 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:07.986 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:07.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:08.088 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:08.120 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:08.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:08.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:08.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:08.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:08.367 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:08.400 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:08.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:08.502 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:08.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.537 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:08.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:08.638 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:08.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.674 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:08.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:08.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:08 smithi017 ceph-mon[102913]: osdmap e1087: 8 total, 8 up, 8 in 2024-09-21T09:57:08.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:08 smithi017 ceph-mon[105151]: osdmap e1087: 8 total, 8 up, 8 in 2024-09-21T09:57:08.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T09:57:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:08 smithi134 ceph-mon[90085]: osdmap e1087: 8 total, 8 up, 8 in 2024-09-21T09:57:08.775 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:08.802 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:08.807 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:08.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:08.909 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:08.923 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 63738 63722 124.437 120.75 0.00620346 0.00803115 2024-09-21T09:57:08.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.942 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:08.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:09.043 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:09.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.078 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:09.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:09.179 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:09.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.210 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:09.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:09.311 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:09.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:09.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:09.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:09.608 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:09.639 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:09.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:09 smithi017 ceph-mon[102913]: pgmap v1268: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T09:57:09.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:09 smithi017 ceph-mon[105151]: pgmap v1268: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T09:57:09.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:09.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:09 smithi134 ceph-mon[90085]: pgmap v1268: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T09:57:09.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:09.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:09.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:09.927 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 65141 65125 123.323 87.6875 0.00696667 0.00810387 2024-09-21T09:57:09.948 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:10.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:10.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:10.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:10.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:10.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.258 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:10.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:10.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:10.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:10.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:10.488 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:57:10.488 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:57:10.496 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:57:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:57:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:10.497 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:10 smithi134 ceph-mon[90085]: 6.3 deep-scrub starts 2024-09-21T09:57:10.497 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:10 smithi134 ceph-mon[90085]: 6.1 scrub starts 2024-09-21T09:57:10.497 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:10 smithi134 ceph-mon[90085]: 6.9 scrub starts 2024-09-21T09:57:10.497 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:10 smithi134 ceph-mon[90085]: 6.7 scrub starts 2024-09-21T09:57:10.497 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:10 smithi134 ceph-mon[90085]: 6.1 scrub ok 2024-09-21T09:57:10.497 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:10 smithi134 ceph-mon[90085]: 6.9 scrub ok 2024-09-21T09:57:10.497 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:10.497 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:10.515 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:10.525 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:57:10.525 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:57:10.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[102913]: 6.3 deep-scrub starts 2024-09-21T09:57:10.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[102913]: 6.1 scrub starts 2024-09-21T09:57:10.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[102913]: 6.9 scrub starts 2024-09-21T09:57:10.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[102913]: 6.7 scrub starts 2024-09-21T09:57:10.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[102913]: 6.1 scrub ok 2024-09-21T09:57:10.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[102913]: 6.9 scrub ok 2024-09-21T09:57:10.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:10.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:10.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[105151]: 6.3 deep-scrub starts 2024-09-21T09:57:10.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[105151]: 6.1 scrub starts 2024-09-21T09:57:10.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[105151]: 6.9 scrub starts 2024-09-21T09:57:10.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[105151]: 6.7 scrub starts 2024-09-21T09:57:10.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[105151]: 6.1 scrub ok 2024-09-21T09:57:10.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[105151]: 6.9 scrub ok 2024-09-21T09:57:10.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:10.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:10.546 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:10.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:10.561 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:57:10.561 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:57:10.604 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:57:10.604 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:57:10.632 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:57:10.633 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:57:10.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:10.670 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:57:10.670 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:57:10.690 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:10.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:10.698 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:57:10.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:57:10.731 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:57:10.731 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:57:10.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:10.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:10.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:10.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 66719 66703 122.596 98.625 0.00586313 0.00815097 2024-09-21T09:57:10.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:11.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:11.109 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:11.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.148 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:11.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:11.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:11.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:11.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:11.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:11.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:11.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:11.554 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:11.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.560 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:11.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:11.642 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:57:11.642 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:57:11.642 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-21T09:57:11.642 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2024-09-21T09:57:11.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 5 0 2024-09-21T09:57:11.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:11.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[102913]: 6.6 scrub starts 2024-09-21T09:57:11.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[102913]: 6.7 scrub ok 2024-09-21T09:57:11.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[102913]: pgmap v1269: 99 pgs: 2 peering, 8 unknown, 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 11 KiB/s, 0 objects/s recovering 2024-09-21T09:57:11.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[102913]: 6.6 scrub ok 2024-09-21T09:57:11.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[102913]: 6.17 scrub starts 2024-09-21T09:57:11.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/34480095' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:11.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[102913]: 6.19 scrub starts 2024-09-21T09:57:11.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[105151]: 6.6 scrub starts 2024-09-21T09:57:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[105151]: 6.7 scrub ok 2024-09-21T09:57:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[105151]: pgmap v1269: 99 pgs: 2 peering, 8 unknown, 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 11 KiB/s, 0 objects/s recovering 2024-09-21T09:57:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[105151]: 6.6 scrub ok 2024-09-21T09:57:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[105151]: 6.17 scrub starts 2024-09-21T09:57:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/34480095' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:11 smithi017 ceph-mon[105151]: 6.19 scrub starts 2024-09-21T09:57:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:11 smithi134 ceph-mon[90085]: 6.6 scrub starts 2024-09-21T09:57:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:11 smithi134 ceph-mon[90085]: 6.7 scrub ok 2024-09-21T09:57:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:11 smithi134 ceph-mon[90085]: pgmap v1269: 99 pgs: 2 peering, 8 unknown, 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 11 KiB/s, 0 objects/s recovering 2024-09-21T09:57:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:11 smithi134 ceph-mon[90085]: 6.6 scrub ok 2024-09-21T09:57:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:11 smithi134 ceph-mon[90085]: 6.17 scrub starts 2024-09-21T09:57:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:11 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/34480095' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:11 smithi134 ceph-mon[90085]: 6.19 scrub starts 2024-09-21T09:57:11.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:11.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:11.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:11.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:11.923 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 68138 68122 121.627 88.6875 0.00454509 0.00821683 2024-09-21T09:57:12.008 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:12.036 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:12.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:12.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:12.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:12.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:12.344 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:12.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:12.446 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:12.475 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:12.577 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:12.606 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[102913]: 6.16 scrub starts 2024-09-21T09:57:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[102913]: 6.17 scrub ok 2024-09-21T09:57:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[102913]: 6.11 scrub starts 2024-09-21T09:57:12.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[102913]: 6.16 scrub ok 2024-09-21T09:57:12.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[102913]: 6.11 scrub ok 2024-09-21T09:57:12.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[102913]: 6.19 scrub ok 2024-09-21T09:57:12.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[102913]: 6.0 scrub starts 2024-09-21T09:57:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[105151]: 6.16 scrub starts 2024-09-21T09:57:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[105151]: 6.17 scrub ok 2024-09-21T09:57:12.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[105151]: 6.11 scrub starts 2024-09-21T09:57:12.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[105151]: 6.16 scrub ok 2024-09-21T09:57:12.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[105151]: 6.11 scrub ok 2024-09-21T09:57:12.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[105151]: 6.19 scrub ok 2024-09-21T09:57:12.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:12 smithi017 ceph-mon[105151]: 6.0 scrub starts 2024-09-21T09:57:12.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:12.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:12.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:12 smithi134 ceph-mon[90085]: 6.16 scrub starts 2024-09-21T09:57:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:12 smithi134 ceph-mon[90085]: 6.17 scrub ok 2024-09-21T09:57:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:12 smithi134 ceph-mon[90085]: 6.11 scrub starts 2024-09-21T09:57:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:12 smithi134 ceph-mon[90085]: 6.16 scrub ok 2024-09-21T09:57:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:12 smithi134 ceph-mon[90085]: 6.11 scrub ok 2024-09-21T09:57:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:12 smithi134 ceph-mon[90085]: 6.19 scrub ok 2024-09-21T09:57:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:12 smithi134 ceph-mon[90085]: 6.0 scrub starts 2024-09-21T09:57:12.839 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:12.860 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:12.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 69161 69145 120.025 63.9375 0.00246589 0.00828166 2024-09-21T09:57:12.962 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:12.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.995 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:12.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:13.097 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:13.127 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:13.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:13.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:13.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:13.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:13.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:13.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:13.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:13.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:13.558 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:13.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:13.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[102913]: 6.2 scrub starts 2024-09-21T09:57:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[102913]: 6.3 deep-scrub ok 2024-09-21T09:57:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[102913]: 6.4 scrub starts 2024-09-21T09:57:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[102913]: pgmap v1270: 99 pgs: 2 peering, 1 active+clean+scrubbing+deep, 2 unknown, 94 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:57:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[102913]: 6.2 scrub ok 2024-09-21T09:57:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[102913]: 6.4 scrub ok 2024-09-21T09:57:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[102913]: 6.0 scrub ok 2024-09-21T09:57:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[105151]: 6.2 scrub starts 2024-09-21T09:57:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[105151]: 6.3 deep-scrub ok 2024-09-21T09:57:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[105151]: 6.4 scrub starts 2024-09-21T09:57:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[105151]: pgmap v1270: 99 pgs: 2 peering, 1 active+clean+scrubbing+deep, 2 unknown, 94 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:57:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[105151]: 6.2 scrub ok 2024-09-21T09:57:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[105151]: 6.4 scrub ok 2024-09-21T09:57:13.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:13 smithi017 ceph-mon[105151]: 6.0 scrub ok 2024-09-21T09:57:13.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.703 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:13.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:13 smithi134 ceph-mon[90085]: 6.2 scrub starts 2024-09-21T09:57:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:13 smithi134 ceph-mon[90085]: 6.3 deep-scrub ok 2024-09-21T09:57:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:13 smithi134 ceph-mon[90085]: 6.4 scrub starts 2024-09-21T09:57:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:13 smithi134 ceph-mon[90085]: pgmap v1270: 99 pgs: 2 peering, 1 active+clean+scrubbing+deep, 2 unknown, 94 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:57:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:13 smithi134 ceph-mon[90085]: 6.2 scrub ok 2024-09-21T09:57:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:13 smithi134 ceph-mon[90085]: 6.4 scrub ok 2024-09-21T09:57:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:13 smithi134 ceph-mon[90085]: 6.0 scrub ok 2024-09-21T09:57:13.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:13.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:13.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 70636 70620 119.272 92.1875 0.00375732 0.00837921 2024-09-21T09:57:13.956 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:13.990 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:14.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:14.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:14.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:14.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:57:14.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:14.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:14.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:14.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:14.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:14.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:14.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:14.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:14.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:14.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:14.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[102913]: 6.12 scrub starts 2024-09-21T09:57:14.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[102913]: 6.12 scrub ok 2024-09-21T09:57:14.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[102913]: 6.5 scrub starts 2024-09-21T09:57:14.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T09:57:14.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[102913]: 6.10 scrub starts 2024-09-21T09:57:14.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[105151]: 6.12 scrub starts 2024-09-21T09:57:14.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[105151]: 6.12 scrub ok 2024-09-21T09:57:14.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[105151]: 6.5 scrub starts 2024-09-21T09:57:14.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T09:57:14.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-mon[105151]: 6.10 scrub starts 2024-09-21T09:57:14.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:14 smithi134 ceph-mon[90085]: 6.12 scrub starts 2024-09-21T09:57:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:14 smithi134 ceph-mon[90085]: 6.12 scrub ok 2024-09-21T09:57:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:14 smithi134 ceph-mon[90085]: 6.5 scrub starts 2024-09-21T09:57:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:14 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T09:57:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:14 smithi134 ceph-mon[90085]: 6.10 scrub starts 2024-09-21T09:57:14.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:14.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:14.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:14.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 71707 71691 117.895 66.9375 0.00302777 0.00841713 2024-09-21T09:57:14.944 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:14.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:15.046 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:15.068 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:15.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:15.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:15.191 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:15.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:15.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:57:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:57:14] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-21T09:57:15.293 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:15.314 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:15.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:15.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:15.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:15.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:15.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:15.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:15.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:15.664 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:15.686 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:15.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:15 smithi134 ceph-mon[90085]: 6.5 scrub ok 2024-09-21T09:57:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:15 smithi134 ceph-mon[90085]: 6.8 scrub starts 2024-09-21T09:57:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:15 smithi134 ceph-mon[90085]: pgmap v1271: 99 pgs: 1 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 97 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:57:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:15 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T09:57:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:15 smithi134 ceph-mon[90085]: osdmap e1088: 8 total, 8 up, 8 in 2024-09-21T09:57:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:15 smithi134 ceph-mon[90085]: 6.8 scrub ok 2024-09-21T09:57:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:15 smithi134 ceph-mon[90085]: 6.15 scrub starts 2024-09-21T09:57:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:15 smithi134 ceph-mon[90085]: 6.15 scrub ok 2024-09-21T09:57:15.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:15.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:15.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:15.912 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:15.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 71707 71691 114.872 0 - 0.00841713 2024-09-21T09:57:15.934 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:15.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[102913]: 6.5 scrub ok 2024-09-21T09:57:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[102913]: 6.8 scrub starts 2024-09-21T09:57:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[102913]: pgmap v1271: 99 pgs: 1 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 97 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:57:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T09:57:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[102913]: osdmap e1088: 8 total, 8 up, 8 in 2024-09-21T09:57:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[102913]: 6.8 scrub ok 2024-09-21T09:57:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[102913]: 6.15 scrub starts 2024-09-21T09:57:15.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[102913]: 6.15 scrub ok 2024-09-21T09:57:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[105151]: 6.5 scrub ok 2024-09-21T09:57:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[105151]: 6.8 scrub starts 2024-09-21T09:57:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[105151]: pgmap v1271: 99 pgs: 1 active+clean+scrubbing, 1 active+clean+scrubbing+deep, 97 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:57:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T09:57:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[105151]: osdmap e1088: 8 total, 8 up, 8 in 2024-09-21T09:57:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[105151]: 6.8 scrub ok 2024-09-21T09:57:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[105151]: 6.15 scrub starts 2024-09-21T09:57:15.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:15 smithi017 ceph-mon[105151]: 6.15 scrub ok 2024-09-21T09:57:16.036 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:16.056 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:16.158 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:16.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:16.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:16.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:16.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:16.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:16.407 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:16.428 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:16.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:16.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:16.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:16.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:16.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:16.681 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:16.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:16 smithi134 ceph-mon[90085]: osdmap e1089: 8 total, 8 up, 8 in 2024-09-21T09:57:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:16 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T09:57:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:16 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T09:57:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:16 smithi134 ceph-mon[90085]: osdmap e1090: 8 total, 8 up, 8 in 2024-09-21T09:57:16.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:16.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:16.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:16.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:16.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T09:57:16.923469+0000 min lat: 0.00126159 max lat: 2.09184 avg lat: 0.00883182 2024-09-21T09:57:16.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:57:16.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 72155 72139 112.7 14 0.00344957 0.00883182 2024-09-21T09:57:16.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:16.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:16 smithi017 ceph-mon[105151]: osdmap e1089: 8 total, 8 up, 8 in 2024-09-21T09:57:16.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:16 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T09:57:16.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:16 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T09:57:16.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:16 smithi017 ceph-mon[105151]: osdmap e1090: 8 total, 8 up, 8 in 2024-09-21T09:57:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:16 smithi017 ceph-mon[102913]: osdmap e1089: 8 total, 8 up, 8 in 2024-09-21T09:57:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:16 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T09:57:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:16 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T09:57:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:16 smithi017 ceph-mon[102913]: osdmap e1090: 8 total, 8 up, 8 in 2024-09-21T09:57:17.043 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:17.064 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:17.165 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:17.187 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:17.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:17.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:17.310 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:17.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:17.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:17.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:17.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:17.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:17.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:17.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:17 smithi134 ceph-mon[90085]: pgmap v1274: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 24 KiB/s, 0 objects/s recovering 2024-09-21T09:57:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:17 smithi134 ceph-mon[90085]: 6.14 scrub starts 2024-09-21T09:57:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:17 smithi134 ceph-mon[90085]: 6.14 scrub ok 2024-09-21T09:57:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:17 smithi134 ceph-mon[90085]: osdmap e1091: 8 total, 8 up, 8 in 2024-09-21T09:57:17.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:17.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:17.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:17.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:17.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 72164 72148 109.965 0.5625 0.00284175 0.00883107 2024-09-21T09:57:17.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:17.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:17 smithi017 ceph-mon[102913]: pgmap v1274: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 24 KiB/s, 0 objects/s recovering 2024-09-21T09:57:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:17 smithi017 ceph-mon[102913]: 6.14 scrub starts 2024-09-21T09:57:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:17 smithi017 ceph-mon[102913]: 6.14 scrub ok 2024-09-21T09:57:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:17 smithi017 ceph-mon[102913]: osdmap e1091: 8 total, 8 up, 8 in 2024-09-21T09:57:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:17 smithi017 ceph-mon[105151]: pgmap v1274: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 24 KiB/s, 0 objects/s recovering 2024-09-21T09:57:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:17 smithi017 ceph-mon[105151]: 6.14 scrub starts 2024-09-21T09:57:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:17 smithi017 ceph-mon[105151]: 6.14 scrub ok 2024-09-21T09:57:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:17 smithi017 ceph-mon[105151]: osdmap e1091: 8 total, 8 up, 8 in 2024-09-21T09:57:18.035 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:18.056 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:18.158 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:18.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.180 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:18.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:18.281 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:18.302 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:18.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:18.425 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:18.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:18.466 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T09:57:18.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:18.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:18.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:18.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:18.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:18 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2538230287' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:57:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:18 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T09:57:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:18 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:18 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2538230287' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:57:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:18 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T09:57:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:18 smithi134 ceph-mon[90085]: osdmap e1092: 8 total, 8 up, 8 in 2024-09-21T09:57:18.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:18.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:18.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:18.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:18.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 72164 72148 107.347 0 - 0.00883107 2024-09-21T09:57:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2538230287' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:57:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T09:57:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2538230287' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:57:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T09:57:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[102913]: osdmap e1092: 8 total, 8 up, 8 in 2024-09-21T09:57:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2538230287' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:57:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T09:57:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2538230287' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:57:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T09:57:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:18 smithi017 ceph-mon[105151]: osdmap e1092: 8 total, 8 up, 8 in 2024-09-21T09:57:19.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:19.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:19.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:19.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:19.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:19.274 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:19.301 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:19.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:19.403 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:19.434 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:19.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:19.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:19.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:19.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:19.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:19.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:19.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:19.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:19 smithi134 ceph-mon[90085]: pgmap v1277: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail 2024-09-21T09:57:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:19 smithi134 ceph-mon[90085]: osdmap e1093: 8 total, 8 up, 8 in 2024-09-21T09:57:19.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:19.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:19.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 72674 72658 105.592 15.9375 0.00742593 0.00936704 2024-09-21T09:57:19.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:19.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:19 smithi017 ceph-mon[102913]: pgmap v1277: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail 2024-09-21T09:57:19.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:19 smithi017 ceph-mon[102913]: osdmap e1093: 8 total, 8 up, 8 in 2024-09-21T09:57:19.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:19 smithi017 ceph-mon[105151]: pgmap v1277: 99 pgs: 99 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail 2024-09-21T09:57:19.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:19 smithi017 ceph-mon[105151]: osdmap e1093: 8 total, 8 up, 8 in 2024-09-21T09:57:19.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:20.052 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:20.074 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:20.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:20.175 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:20.196 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:20.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:20.298 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:20.319 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:20.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:20.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:20.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:20.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:20.480 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:57:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:57:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:20.545 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:20.574 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:20.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:20.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:20.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:20.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:20.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:20 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T09:57:20.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:20 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:20.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:20 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T09:57:20.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:20 smithi134 ceph-mon[90085]: osdmap e1094: 8 total, 8 up, 8 in 2024-09-21T09:57:20.800 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:20.828 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:20.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 73154 73138 103.874 30 0.00763026 0.00959388 2024-09-21T09:57:20.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T09:57:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T09:57:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:20 smithi017 ceph-mon[102913]: osdmap e1094: 8 total, 8 up, 8 in 2024-09-21T09:57:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T09:57:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T09:57:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:20 smithi017 ceph-mon[105151]: osdmap e1094: 8 total, 8 up, 8 in 2024-09-21T09:57:20.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:20.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:21.063 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:21.087 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:21.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:21.188 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:21.212 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:21.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:21.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:21.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:21.439 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:21.459 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:21.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:21.561 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:21.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:21.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:21.685 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:21.706 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:21.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:21 smithi134 ceph-mon[90085]: pgmap v1280: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 4230/214923 objects misplaced (1.968%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T09:57:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:21 smithi134 ceph-mon[90085]: osdmap e1095: 8 total, 8 up, 8 in 2024-09-21T09:57:21.808 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:21.830 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:21.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:21.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 73360 73344 101.852 12.875 0.00320201 0.00957898 2024-09-21T09:57:21.931 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:21 smithi017 ceph-mon[102913]: pgmap v1280: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 4230/214923 objects misplaced (1.968%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T09:57:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:21 smithi017 ceph-mon[102913]: osdmap e1095: 8 total, 8 up, 8 in 2024-09-21T09:57:21.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:21 smithi017 ceph-mon[105151]: pgmap v1280: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 4230/214923 objects misplaced (1.968%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T09:57:21.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:21 smithi017 ceph-mon[105151]: osdmap e1095: 8 total, 8 up, 8 in 2024-09-21T09:57:21.952 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:21.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:21.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T09:57:22.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:22.079 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:22.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:22.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:22.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:22.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:22.309 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:22.330 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:22.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:22.432 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:22.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.454 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:22.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:22.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:22.576 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:22.678 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:22.706 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:22.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T09:57:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T09:57:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:22 smithi134 ceph-mon[90085]: osdmap e1096: 8 total, 8 up, 8 in 2024-09-21T09:57:22.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:22.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:22.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:22.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 15 73700 73685 100.101 21.3125 0.00345866 0.00996763 2024-09-21T09:57:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T09:57:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T09:57:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:22 smithi017 ceph-mon[102913]: osdmap e1096: 8 total, 8 up, 8 in 2024-09-21T09:57:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T09:57:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T09:57:22.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:22 smithi017 ceph-mon[105151]: osdmap e1096: 8 total, 8 up, 8 in 2024-09-21T09:57:22.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:22.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:22.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:23.080 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:23.104 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:23.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:23.205 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:23.227 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:23.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:23.329 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:23.351 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:23.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:23.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:23.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:23.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:23.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:23.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:23 smithi017 ceph-mon[102913]: pgmap v1283: 99 pgs: 2 active+recovering+undersized+remapped, 97 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 20 MiB/s wr, 316 op/s; 6366/217851 objects misplaced (2.922%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T09:57:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:23 smithi017 ceph-mon[102913]: osdmap e1097: 8 total, 8 up, 8 in 2024-09-21T09:57:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:23 smithi017 ceph-mon[105151]: pgmap v1283: 99 pgs: 2 active+recovering+undersized+remapped, 97 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 20 MiB/s wr, 316 op/s; 6366/217851 objects misplaced (2.922%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T09:57:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:23 smithi017 ceph-mon[105151]: osdmap e1097: 8 total, 8 up, 8 in 2024-09-21T09:57:23.705 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:23.726 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:23.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:23 smithi134 ceph-mon[90085]: pgmap v1283: 99 pgs: 2 active+recovering+undersized+remapped, 97 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 20 MiB/s wr, 316 op/s; 6366/217851 objects misplaced (2.922%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T09:57:23.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:23 smithi134 ceph-mon[90085]: osdmap e1097: 8 total, 8 up, 8 in 2024-09-21T09:57:23.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:23.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:23.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:23.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 73863 73847 98.1867 10.125 0.00300312 0.00995529 2024-09-21T09:57:23.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:23.976 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:23.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:24.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:24.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:24.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:24.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:24.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:24.331 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:24.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.354 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:24.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:24.455 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:24.477 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:24.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:24.578 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:24.602 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:24.703 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:24.731 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:24.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:24.832 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:24.860 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:24.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:24.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 74163 74147 96.5319 18.75 0.0106331 0.0103538 2024-09-21T09:57:24.961 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:24.987 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:24.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:25.044 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:25.056 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:25.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:25.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:25.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:25.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:57:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:57:24] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-21T09:57:25.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:25.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:25.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:25.405 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:25.430 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:25.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:25.532 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:25.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.559 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:25.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:25.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[102913]: pgmap v1286: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 MiB/s wr, 361 op/s; 4492/219264 objects degraded (2.049%); 7502/219264 objects misplaced (3.421%); 10 MiB/s, 163 objects/s recovering 2024-09-21T09:57:25.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[102913]: osdmap e1098: 8 total, 8 up, 8 in 2024-09-21T09:57:25.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 4492/219264 objects degraded (2.049%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:57:25.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:25.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:25.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[105151]: pgmap v1286: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 MiB/s wr, 361 op/s; 4492/219264 objects degraded (2.049%); 7502/219264 objects misplaced (3.421%); 10 MiB/s, 163 objects/s recovering 2024-09-21T09:57:25.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[105151]: osdmap e1098: 8 total, 8 up, 8 in 2024-09-21T09:57:25.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 4492/219264 objects degraded (2.049%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:57:25.580 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:25.580 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:25.660 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:25.690 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:25.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:25 smithi134 ceph-mon[90085]: pgmap v1286: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 MiB/s wr, 361 op/s; 4492/219264 objects degraded (2.049%); 7502/219264 objects misplaced (3.421%); 10 MiB/s, 163 objects/s recovering 2024-09-21T09:57:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:25 smithi134 ceph-mon[90085]: osdmap e1098: 8 total, 8 up, 8 in 2024-09-21T09:57:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:25 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 4492/219264 objects degraded (2.049%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:57:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:25.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:25.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:25.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:25.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 75405 75389 96.1459 77.625 0.00575482 0.0103959 2024-09-21T09:57:25.948 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:25.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:25.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:26.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:26.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:26.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:26.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:26.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:26.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:26.365 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:26.394 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:26.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:26.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:26.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:26.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:26.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:26.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:26.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:26.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:26.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 76721 76705 95.8678 82.25 0.0122355 0.0104254 2024-09-21T09:57:26.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:26.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:26.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:27.074 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:27.109 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:27.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:27.211 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:27.243 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:27.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:27.344 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:27.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:27.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:27.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:27.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:27.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:27.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:27.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:27 smithi017 ceph-mon[102913]: pgmap v1288: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 25 MiB/s wr, 401 op/s; 4492/223332 objects degraded (2.011%); 7502/223332 objects misplaced (3.359%); 7.6 MiB/s, 122 objects/s recovering 2024-09-21T09:57:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:27 smithi017 ceph-mon[105151]: pgmap v1288: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 25 MiB/s wr, 401 op/s; 4492/223332 objects degraded (2.011%); 7502/223332 objects misplaced (3.359%); 7.6 MiB/s, 122 objects/s recovering 2024-09-21T09:57:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:27 smithi134 ceph-mon[90085]: pgmap v1288: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 25 MiB/s wr, 401 op/s; 4492/223332 objects degraded (2.011%); 7502/223332 objects misplaced (3.359%); 7.6 MiB/s, 122 objects/s recovering 2024-09-21T09:57:27.761 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:27.791 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:27.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:27.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.915 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:27.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:27.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 77949 77933 95.4927 76.75 0.00466767 0.010458 2024-09-21T09:57:28.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:28.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:28.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:28.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:28.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:28.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:28.328 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:28.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:28.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:28.485 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:28.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:28.587 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:28.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:28.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:28.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:28.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:28.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:28.926 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 79170 79154 95.123 76.3125 0.00442977 0.0105076 2024-09-21T09:57:28.939 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:28.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:29.041 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:29.067 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:29.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:29.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:29.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:29.315 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T09:57:29.315 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:57:29.329 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:29.355 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:29.388 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[102913]: pgmap v1289: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 27 MiB/s wr, 435 op/s; 11400/225696 objects degraded (5.051%); 11311/225696 objects misplaced (5.012%); 14 MiB/s, 226 objects/s recovering 2024-09-21T09:57:29.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:29.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:29.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:29.598 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:29.630 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:29.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:29.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:29.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:29.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:29.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:29.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[105151]: pgmap v1289: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 27 MiB/s wr, 435 op/s; 11400/225696 objects degraded (5.051%); 11311/225696 objects misplaced (5.012%); 14 MiB/s, 226 objects/s recovering 2024-09-21T09:57:29.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:29.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:29.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:29.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:29.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:29.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:29 smithi134 ceph-mon[90085]: pgmap v1289: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 27 MiB/s wr, 435 op/s; 11400/225696 objects degraded (5.051%); 11311/225696 objects misplaced (5.012%); 14 MiB/s, 226 objects/s recovering 2024-09-21T09:57:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:57:29.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:29.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:29.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:29.919 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:29.926 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 80378 80362 94.7526 75.5 0.0123679 0.0105478 2024-09-21T09:57:29.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T09:57:30.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:30.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:30.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:30.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:30.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:30.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:30.375 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:30.402 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:30 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 11400/227445 objects degraded (5.012%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:57:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:57:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:30.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:30.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:30.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:30.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:30 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 11400/227445 objects degraded (5.012%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:30.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:30 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 11400/227445 objects degraded (5.012%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:30.680 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:30.709 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:30.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:30.810 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:30.839 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:30.926 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 81655 81639 94.4758 79.8125 0.0139875 0.0105797 2024-09-21T09:57:30.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:30.975 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:30.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:31.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:31.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:31.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:31.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:31.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:31.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:31.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:31.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:31.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:31.545 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:31.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:31.678 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:31 smithi017 ceph-mon[105151]: pgmap v1290: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 24 MiB/s wr, 388 op/s; 11400/227445 objects degraded (5.012%); 10497/227445 objects misplaced (4.615%); 15 MiB/s, 233 objects/s recovering 2024-09-21T09:57:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:31 smithi017 ceph-mon[102913]: pgmap v1290: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 24 MiB/s wr, 388 op/s; 11400/227445 objects degraded (5.012%); 10497/227445 objects misplaced (4.615%); 15 MiB/s, 233 objects/s recovering 2024-09-21T09:57:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.710 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:31.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:31.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:31 smithi134 ceph-mon[90085]: pgmap v1290: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 24 MiB/s wr, 388 op/s; 11400/227445 objects degraded (5.012%); 10497/227445 objects misplaced (4.615%); 15 MiB/s, 233 objects/s recovering 2024-09-21T09:57:31.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:31.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:31.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:31.926 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 15 82904 82889 94.1782 78.125 0.00907497 0.0106129 2024-09-21T09:57:31.965 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:31.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.995 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:31.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:32.096 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:32.125 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:32.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:32.227 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:32.259 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:32.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:32.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:32.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:32.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:32.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:32.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:32.598 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:57:32.598 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:57:32.642 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:57:32.643 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:57:32.664 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:32.689 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:32.690 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:57:32.690 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:57:32.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:32.716 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:32.739 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:57:32.739 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:57:32.779 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:57:32.779 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:57:32.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:32.832 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:57:32.832 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:57:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.849 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:32.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:32.871 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:57:32.871 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:57:32.903 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:57:32.903 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:57:32.926 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 84041 84025 93.7642 71 0.00754953 0.01066 2024-09-21T09:57:32.949 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:32.978 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:32.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:33.080 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:33.107 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:33.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:33.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:33.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:33.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:33.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:33.464 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T09:57:33.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:33.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:33.652 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:33.678 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:33.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:33 smithi017 ceph-mon[102913]: pgmap v1291: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 11400/241686 objects degraded (4.717%); 10030/241686 objects misplaced (4.150%); 12 MiB/s, 192 objects/s recovering 2024-09-21T09:57:33.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:33 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1126527587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:33.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:33 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/529397279' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:33.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:33 smithi017 ceph-mon[105151]: pgmap v1291: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 11400/241686 objects degraded (4.717%); 10030/241686 objects misplaced (4.150%); 12 MiB/s, 192 objects/s recovering 2024-09-21T09:57:33.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:33 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1126527587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:33.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:33 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/529397279' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:33.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:33 smithi134 ceph-mon[90085]: pgmap v1291: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 11400/241686 objects degraded (4.717%); 10030/241686 objects misplaced (4.150%); 12 MiB/s, 192 objects/s recovering 2024-09-21T09:57:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:33 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1126527587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:33 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/529397279' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:33.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:33.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:33.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:33.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:33.927 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 85290 85274 93.4887 78.0625 0.00452504 0.010691 2024-09-21T09:57:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:33.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:34.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:34.116 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:34.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:34.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:34.239 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:34.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:34.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:34.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:34.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:34.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:34.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:34.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.665 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:34.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:34 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/529397279' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:34 smithi017 ceph-mon[102913]: osdmap e1099: 8 total, 8 up, 8 in 2024-09-21T09:57:34.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:34 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:34.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/529397279' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:34.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:34 smithi017 ceph-mon[105151]: osdmap e1099: 8 total, 8 up, 8 in 2024-09-21T09:57:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:34 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/529397279' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:34 smithi134 ceph-mon[90085]: osdmap e1099: 8 total, 8 up, 8 in 2024-09-21T09:57:34.766 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:34.795 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:34.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:34.926 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 86457 86441 93.1343 72.9375 0.00884633 0.0107315 2024-09-21T09:57:34.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:34.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:35.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:35.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:35.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:35.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:57:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:57:34] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-21T09:57:35.212 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:35.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:35.314 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:35.342 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:35.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:35.471 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:35.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:35.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:35.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:35.719 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:35.733 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:35 smithi134 ceph-mon[90085]: pgmap v1293: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 11400/246150 objects degraded (4.631%); 9444/246150 objects misplaced (3.837%); 13 MiB/s, 202 objects/s recovering 2024-09-21T09:57:35.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:57:35.752 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:35.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:35 smithi017 ceph-mon[102913]: pgmap v1293: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 11400/246150 objects degraded (4.631%); 9444/246150 objects misplaced (3.837%); 13 MiB/s, 202 objects/s recovering 2024-09-21T09:57:35.778 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:35 smithi017 ceph-mon[105151]: pgmap v1293: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 11400/246150 objects degraded (4.631%); 9444/246150 objects misplaced (3.837%); 13 MiB/s, 202 objects/s recovering 2024-09-21T09:57:35.854 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:35.882 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:35.927 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 87644 87628 92.813 74.1875 0.0104067 0.0107684 2024-09-21T09:57:35.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:36.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:36.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:36.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:36.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:36.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:36.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:36.432 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:36.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.467 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:36.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:36.568 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:36.601 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:36.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:36.703 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:36.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:36.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:36.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:36 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 11400/257373 objects degraded (4.429%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:36.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:36.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:36.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:36.893 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:36 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 11400/257373 objects degraded (4.429%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:36.894 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:36 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 11400/257373 objects degraded (4.429%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:36.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T09:57:36.927575+0000 min lat: 0.00126159 max lat: 2.57071 avg lat: 0.0107954 2024-09-21T09:57:36.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:57:36.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 88906 88890 92.578 78.875 0.0118347 0.0107954 2024-09-21T09:57:36.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:37.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:37.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:37.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:37.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.192 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:37.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:37.293 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:37.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:37.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:37.428 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:37.458 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:37.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:37.560 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:37.596 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:37.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:37.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:37.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:37.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:37.857 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:37.858 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:37 smithi017 ceph-mon[102913]: pgmap v1294: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 11400/257373 objects degraded (4.429%); 9444/257373 objects misplaced (3.669%); 12 MiB/s, 196 objects/s recovering 2024-09-21T09:57:37.858 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:37 smithi017 ceph-mon[105151]: pgmap v1294: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 11400/257373 objects degraded (4.429%); 9444/257373 objects misplaced (3.669%); 12 MiB/s, 196 objects/s recovering 2024-09-21T09:57:37.886 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:37.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:37.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 90156 90140 92.341 78.125 0.013707 0.0108241 2024-09-21T09:57:37.975 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:37.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:37.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:37 smithi134 ceph-mon[90085]: pgmap v1294: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 11400/257373 objects degraded (4.429%); 9444/257373 objects misplaced (3.669%); 12 MiB/s, 196 objects/s recovering 2024-09-21T09:57:38.013 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:38.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:38.115 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:38.143 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:38.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:38.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:38.275 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:38.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:38.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:38.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:38.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T09:57:38.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:38.538 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.5 primary-affinity to 0 (802) 2024-09-21T09:57:38.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:38.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:38 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4255721324' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-21T09:57:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:38 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-21T09:57:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:38 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4255721324' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-21T09:57:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:38 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-21T09:57:38.709 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:38.811 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:38.832 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:38.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:38.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 90889 90873 91.5905 45.8125 0.00382556 0.0108514 2024-09-21T09:57:38.933 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:38.954 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:38.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:38.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:38 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4255721324' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-21T09:57:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:38 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-21T09:57:39.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:39.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:39.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:39.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:39.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:39.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:39.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:39.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:39.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:39.438 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:39.460 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:39.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:39.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:39.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:39.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:39.697 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:39 smithi017 ceph-mon[102913]: pgmap v1295: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 11400/261969 objects degraded (4.352%); 8899/261969 objects misplaced (3.397%); 11 MiB/s, 171 objects/s recovering 2024-09-21T09:57:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:39 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-09-21T09:57:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:39 smithi017 ceph-mon[102913]: osdmap e1100: 8 total, 8 up, 8 in 2024-09-21T09:57:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:39 smithi017 ceph-mon[105151]: pgmap v1295: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 11400/261969 objects degraded (4.352%); 8899/261969 objects misplaced (3.397%); 11 MiB/s, 171 objects/s recovering 2024-09-21T09:57:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:39 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-09-21T09:57:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:39 smithi017 ceph-mon[105151]: osdmap e1100: 8 total, 8 up, 8 in 2024-09-21T09:57:39.725 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:39.828 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:39.860 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:39.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:39.939 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 91224 91208 90.4691 20.9375 0.00707254 0.0110341 2024-09-21T09:57:39.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:39.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:39.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:39 smithi134 ceph-mon[90085]: pgmap v1295: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 11400/261969 objects degraded (4.352%); 8899/261969 objects misplaced (3.397%); 11 MiB/s, 171 objects/s recovering 2024-09-21T09:57:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:39 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-09-21T09:57:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:39 smithi134 ceph-mon[90085]: osdmap e1100: 8 total, 8 up, 8 in 2024-09-21T09:57:40.088 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:40.113 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:40.215 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:40.238 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:40.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:40.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:40.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:40.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:40.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:40.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:57:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:57:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:40.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:40.634 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:40.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:40.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:40.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:40.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:40.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:40.899 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:40.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:40.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 91664 91648 89.4852 27.5 0.00225505 0.0111403 2024-09-21T09:57:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:40 smithi017 ceph-mon[102913]: osdmap e1101: 8 total, 8 up, 8 in 2024-09-21T09:57:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:40 smithi017 ceph-mon[105151]: osdmap e1101: 8 total, 8 up, 8 in 2024-09-21T09:57:40.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:40 smithi134 ceph-mon[90085]: osdmap e1101: 8 total, 8 up, 8 in 2024-09-21T09:57:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:41.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:41.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:41.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:41.136 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:41.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:41.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:41.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:41.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:41.393 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:41.417 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:41.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:41.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:41.540 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:41.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:41.642 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:41.663 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:41.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:41.765 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:41.786 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:41.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:41.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:41 smithi017 ceph-mon[102913]: pgmap v1298: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s; 11400/264312 objects degraded (4.313%); 8205/264312 objects misplaced (3.104%); 8.5 MiB/s, 136 objects/s recovering 2024-09-21T09:57:41.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:41 smithi017 ceph-mon[102913]: osdmap e1102: 8 total, 8 up, 8 in 2024-09-21T09:57:41.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:41 smithi017 ceph-mon[105151]: pgmap v1298: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s; 11400/264312 objects degraded (4.313%); 8205/264312 objects misplaced (3.104%); 8.5 MiB/s, 136 objects/s recovering 2024-09-21T09:57:41.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:41 smithi017 ceph-mon[105151]: osdmap e1102: 8 total, 8 up, 8 in 2024-09-21T09:57:41.888 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:41.918 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:41.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:41.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 92138 92122 88.5643 29.625 0.00454483 0.0112856 2024-09-21T09:57:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:41 smithi134 ceph-mon[90085]: pgmap v1298: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s; 11400/264312 objects degraded (4.313%); 8205/264312 objects misplaced (3.104%); 8.5 MiB/s, 136 objects/s recovering 2024-09-21T09:57:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:41 smithi134 ceph-mon[90085]: osdmap e1102: 8 total, 8 up, 8 in 2024-09-21T09:57:42.020 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:42.046 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:42.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:42.148 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:42.174 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:42.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:42.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:42.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:42.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:42.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:42.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:42.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:42.574 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:42.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:42.676 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:42.703 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:42.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:42.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:42.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:42.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 93141 93125 88.1722 62.6875 0.00365922 0.0113346 2024-09-21T09:57:42.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:42 smithi017 ceph-mon[102913]: osdmap e1103: 8 total, 8 up, 8 in 2024-09-21T09:57:42.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:42 smithi017 ceph-mon[105151]: osdmap e1103: 8 total, 8 up, 8 in 2024-09-21T09:57:42.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:42.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:42 smithi134 ceph-mon[90085]: osdmap e1103: 8 total, 8 up, 8 in 2024-09-21T09:57:43.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:43.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:43.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:43.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:43.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:43.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:43.367 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:43.392 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:43.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:43.494 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:43.519 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:43.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:43.621 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:43.648 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:43.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:43.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:43.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:43.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:43.896 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:43.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:43.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:43.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 94172 94156 87.8179 64.4375 0.00817282 0.011381 2024-09-21T09:57:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:43 smithi017 ceph-mon[102913]: pgmap v1301: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 11400/272622 objects degraded (4.182%); 7883/272622 objects misplaced (2.892%); 10 MiB/s, 167 objects/s recovering 2024-09-21T09:57:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:43 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 11400/272622 objects degraded (4.182%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:43 smithi017 ceph-mon[105151]: pgmap v1301: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 11400/272622 objects degraded (4.182%); 7883/272622 objects misplaced (2.892%); 10 MiB/s, 167 objects/s recovering 2024-09-21T09:57:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:43 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 11400/272622 objects degraded (4.182%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:43 smithi134 ceph-mon[90085]: pgmap v1301: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 94 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 11400/272622 objects degraded (4.182%); 7883/272622 objects misplaced (2.892%); 10 MiB/s, 167 objects/s recovering 2024-09-21T09:57:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:43 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 11400/272622 objects degraded (4.182%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:44.023 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:44.033 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:57:44.034 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T09:57:44.034 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T09:57:44.034 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T09:57:44.034 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:57:44.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:44.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:44.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:44.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:44.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:44.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:44.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:44.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:44.426 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:44.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.455 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:44.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:44.556 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:44.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:44.685 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:44.716 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:44.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:44.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:44.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:44.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T09:57:44.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 95331 95315 87.5917 72.4375 0.0199782 0.0114112 2024-09-21T09:57:44.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:45.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:45.125 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:45.151 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:45.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:57:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:57:44] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-21T09:57:45.253 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:45.279 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:45.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:45.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:45.403 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:45.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:45.570 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:45.596 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:45.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:45.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:45.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:45.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:45 smithi017 ceph-mon[102913]: pgmap v1302: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 6908/275835 objects degraded (2.504%); 11021/275835 objects misplaced (3.996%); 11 MiB/s, 172 objects/s recovering 2024-09-21T09:57:45.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:45 smithi017 ceph-mon[105151]: pgmap v1302: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 6908/275835 objects degraded (2.504%); 11021/275835 objects misplaced (3.996%); 11 MiB/s, 172 objects/s recovering 2024-09-21T09:57:45.856 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:45.883 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:45.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:45.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 96456 96440 87.3412 70.3125 0.0117915 0.0114441 2024-09-21T09:57:45.984 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:45.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:45 smithi134 ceph-mon[90085]: pgmap v1302: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 6908/275835 objects degraded (2.504%); 11021/275835 objects misplaced (3.996%); 11 MiB/s, 172 objects/s recovering 2024-09-21T09:57:46.010 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:46.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:46.112 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:46.139 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:46.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:46.205 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:46.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:46.284 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:57:46.284 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:57:46.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:46.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:46.323 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:57:46.323 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:57:46.356 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:57:46.356 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:57:46.392 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:57:46.392 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:57:46.394 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:46.422 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:46.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:46.424 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:57:46.424 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:57:46.455 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:57:46.456 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:57:46.490 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:57:46.490 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:57:46.522 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:57:46.523 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:57:46.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:46.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:46.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:46.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:46.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:46.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:46.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:46.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:46.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:46.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 97576 97560 87.0934 70 0.00410694 0.0114748 2024-09-21T09:57:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1509981606' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1509981606' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:46.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:46.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:46.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1509981606' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:47.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:47.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:47.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:47.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:47.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:47.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:47.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:47.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:47.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:47.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:47.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:47.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:47.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:47.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:47.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:47.815 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:47.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:47.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 98555 98539 86.7284 61.1875 0.00543258 0.0115236 2024-09-21T09:57:47.940 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:47.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:47 smithi017 ceph-mon[102913]: pgmap v1303: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 6908/284082 objects degraded (2.432%); 11021/284082 objects misplaced (3.880%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T09:57:47.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:47 smithi017 ceph-mon[105151]: pgmap v1303: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 6908/284082 objects degraded (2.432%); 11021/284082 objects misplaced (3.880%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T09:57:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:47 smithi134 ceph-mon[90085]: pgmap v1303: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 6908/284082 objects degraded (2.432%); 11021/284082 objects misplaced (3.880%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T09:57:48.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:48.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:48.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:48.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:48.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:48.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:48.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:48.351 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:48.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:48.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:48.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:48.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:48.601 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:48.631 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:48.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:48.733 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:48.762 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:48.864 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:48.893 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:48.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:48.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 99630 99614 86.4569 67.1875 0.0117333 0.011561 2024-09-21T09:57:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:48 smithi017 ceph-mon[102913]: pgmap v1304: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 6908/288549 objects degraded (2.394%); 10632/288549 objects misplaced (3.685%); 8.4 MiB/s, 134 objects/s recovering 2024-09-21T09:57:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:48 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6908/284082 objects degraded (2.432%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:48.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:48 smithi017 ceph-mon[105151]: pgmap v1304: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 6908/288549 objects degraded (2.394%); 10632/288549 objects misplaced (3.685%); 8.4 MiB/s, 134 objects/s recovering 2024-09-21T09:57:48.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:48 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6908/284082 objects degraded (2.432%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:48 smithi134 ceph-mon[90085]: pgmap v1304: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 6908/288549 objects degraded (2.394%); 10632/288549 objects misplaced (3.685%); 8.4 MiB/s, 134 objects/s recovering 2024-09-21T09:57:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:48 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6908/284082 objects degraded (2.432%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:48.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:49.020 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:49.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:49.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:49.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:49.170 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:57:49.170 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:57:49.170 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-21T09:57:49.170 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T09:57:49.170 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T09:57:49.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-21T09:57:49.267 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:49.294 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:49.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:49.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:49.423 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:49.525 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:49.556 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:49.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:49.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:49.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:49.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:49.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:49.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:49.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:49.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 100726 100710 86.2109 68.5 0.0140621 0.0115944 2024-09-21T09:57:49.961 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:49.989 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:50.091 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:50.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.118 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:50.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:50.219 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:50.246 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:50.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:50.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:50.373 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:50.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:50.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:57:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:57:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:50.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:50.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:50.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:50.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:50.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:50.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:50.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:50.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:50.931 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 101898 101882 86.0353 73.25 0.0133235 0.0116184 2024-09-21T09:57:50.944 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:50.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:51.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:51.090 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:51.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:51.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:51.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:51.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:51.340 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:51.372 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:51.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:51.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:51 smithi017 ceph-mon[102913]: pgmap v1305: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 6908/290463 objects degraded (2.378%); 10632/290463 objects misplaced (3.660%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T09:57:51.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:51 smithi017 ceph-mon[105151]: pgmap v1305: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 6908/290463 objects degraded (2.378%); 10632/290463 objects misplaced (3.660%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T09:57:51.474 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:51.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:51 smithi134 ceph-mon[90085]: pgmap v1305: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 6908/290463 objects degraded (2.378%); 10632/290463 objects misplaced (3.660%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T09:57:51.500 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:51.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:51.602 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:51.633 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:51.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:51.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:51.764 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:51.866 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:51.891 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:51.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:51.930 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 103113 103097 85.9004 75.9375 0.00677316 0.0116358 2024-09-21T09:57:51.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:52.015 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:52.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:52.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:52.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:52.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:52.311 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:52.338 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:52.440 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:52.469 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:52.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:52.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:52.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:52.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:52.713 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:52.743 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:52.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:52.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:52.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:52.930 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 104237 104221 85.6944 70.25 0.0129874 0.0116646 2024-09-21T09:57:52.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:53.026 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:53.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:53.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:53.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:53.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:53.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:57:53.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:53.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:53.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:53 smithi017 ceph-mon[102913]: pgmap v1306: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 6908/301599 objects degraded (2.290%); 10632/301599 objects misplaced (3.525%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T09:57:53.348 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:53 smithi017 ceph-mon[105151]: pgmap v1306: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 6908/301599 objects degraded (2.290%); 10632/301599 objects misplaced (3.525%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T09:57:53.419 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:53.447 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:53 smithi134 ceph-mon[90085]: pgmap v1306: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 6908/301599 objects degraded (2.290%); 10632/301599 objects misplaced (3.525%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T09:57:53.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:53.570 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:53.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:53.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:53.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:53.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:53.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:53.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:53.930 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 105218 105202 85.3777 61.3125 0.016787 0.0117076 2024-09-21T09:57:53.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:53.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.999 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:53.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:54.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:54.139 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:54.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:54.241 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:54.268 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:54.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:54.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:54.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:54.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:54 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6908/301599 objects degraded (2.290%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:54.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:54 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6908/301599 objects degraded (2.290%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:54 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6908/301599 objects degraded (2.290%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:54.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:54.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:54.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:54.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:54.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:54.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:54.786 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:54.812 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:54.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:54.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:54.930 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 106327 106311 85.1716 69.3125 0.0123264 0.0117348 2024-09-21T09:57:54.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:57:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:57:54] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-21T09:57:54.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:55.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:55.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:55.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:55.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:55.350 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:55.379 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:55.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:55.466 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T09:57:55.466 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:57:55.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:55.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:55.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:55.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:55 smithi017 ceph-mon[102913]: pgmap v1307: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 6908/307029 objects degraded (2.250%); 10176/307029 objects misplaced (3.314%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T09:57:55.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:55.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:55 smithi017 ceph-mon[105151]: pgmap v1307: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 6908/307029 objects degraded (2.250%); 10176/307029 objects misplaced (3.314%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T09:57:55.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:55.635 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:55.675 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:55.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:55.747 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:55 smithi134 ceph-mon[90085]: pgmap v1307: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 6908/307029 objects degraded (2.250%); 10176/307029 objects misplaced (3.314%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T09:57:55.747 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:55.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:55.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:55.931 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 107366 107350 84.9154 64.9375 0.016525 0.0117713 2024-09-21T09:57:55.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.972 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:55.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:56.074 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:56.105 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:56.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:56.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:56.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:56.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:56.332 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T09:57:56.364 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:56.391 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:56 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/366139218' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:56.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:56 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:56 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/366139218' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:56.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:56 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:56.493 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:56.523 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:56.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:56.625 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:56.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:56.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:56.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T09:57:56.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:56 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/366139218' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:56 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:56.753 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:56.781 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:56.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:56.932 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T09:57:56.931428+0000 min lat: 0.00126159 max lat: 2.57071 avg lat: 0.0117973 2024-09-21T09:57:56.932 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:57:56.932 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 108467 108451 84.7125 68.8125 0.0252128 0.0117973 2024-09-21T09:57:56.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:56.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:57.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:57.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:57.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:57.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:57.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:57.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:57.330 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:57.359 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:57.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:57.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:57.489 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:57.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:57.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:57.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:57 smithi017 ceph-mon[102913]: pgmap v1308: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 6908/316896 objects degraded (2.180%); 10176/316896 objects misplaced (3.211%); 1.5 MiB/s, 23 objects/s recovering 2024-09-21T09:57:57.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:57 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:57.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:57 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:57:57.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:57 smithi017 ceph-mon[102913]: osdmap e1104: 8 total, 8 up, 8 in 2024-09-21T09:57:57.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:57 smithi017 ceph-mon[105151]: pgmap v1308: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 6908/316896 objects degraded (2.180%); 10176/316896 objects misplaced (3.211%); 1.5 MiB/s, 23 objects/s recovering 2024-09-21T09:57:57.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:57 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:57.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:57 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:57:57.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:57 smithi017 ceph-mon[105151]: osdmap e1104: 8 total, 8 up, 8 in 2024-09-21T09:57:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.634 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:57.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:57.696 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:57.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:57.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:57 smithi134 ceph-mon[90085]: pgmap v1308: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 6908/316896 objects degraded (2.180%); 10176/316896 objects misplaced (3.211%); 1.5 MiB/s, 23 objects/s recovering 2024-09-21T09:57:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:57 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:57 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:57:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:57 smithi134 ceph-mon[90085]: osdmap e1104: 8 total, 8 up, 8 in 2024-09-21T09:57:57.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:57.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:57.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:57.933 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 109500 109484 84.4636 64.5625 0.0184567 0.0118343 2024-09-21T09:57:57.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:57.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:57.983 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:57:57.983 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T09:57:57.984 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:57:58.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:58.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:58.186 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:58.211 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:58.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:58.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:58.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:58.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:58 smithi017 ceph-mon[102913]: 6.18 scrub starts 2024-09-21T09:57:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:58 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6908/316896 objects degraded (2.180%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:58 smithi017 ceph-mon[105151]: 6.18 scrub starts 2024-09-21T09:57:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:58 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6908/316896 objects degraded (2.180%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:58.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:58.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.491 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:58.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:58.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:58.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:58.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:58.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:58.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:58 smithi134 ceph-mon[90085]: 6.18 scrub starts 2024-09-21T09:57:58.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:58 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6908/316896 objects degraded (2.180%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:58.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.763 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:58.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:58.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:58.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:58.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:58.932 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 110612 110596 84.281 69.5 0.00775421 0.0118598 2024-09-21T09:57:59.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:59.044 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:59.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:59.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:59.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:59.297 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:59.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:59.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:59.428 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:59.457 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:59.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:59.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T09:57:59.559 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:59.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:59 smithi017 ceph-mon[102913]: pgmap v1310: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 6908/321438 objects degraded (2.149%); 9714/321438 objects misplaced (3.022%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:57:59.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:57:59 smithi017 ceph-mon[102913]: 6.18 scrub ok 2024-09-21T09:57:59.585 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:59 smithi017 ceph-mon[105151]: pgmap v1310: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 6908/321438 objects degraded (2.149%); 9714/321438 objects misplaced (3.022%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:57:59.585 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:57:59 smithi017 ceph-mon[105151]: 6.18 scrub ok 2024-09-21T09:57:59.587 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:57:59.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:59.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:59.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:59.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:59 smithi134 ceph-mon[90085]: pgmap v1310: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 6908/321438 objects degraded (2.149%); 9714/321438 objects misplaced (3.022%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:57:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:57:59 smithi134 ceph-mon[90085]: 6.18 scrub ok 2024-09-21T09:57:59.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:59.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:57:59.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:59.933 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 111727 111711 84.105 69.6875 0.00948408 0.011885 2024-09-21T09:57:59.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:00.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:00.121 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:00.154 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:00.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:00.256 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:00.284 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:00.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:00.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:58:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:58:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:00.530 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:00.558 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:00.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:00.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:00.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:00.812 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:00.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:00.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:00.933 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 112855 112839 83.943 70.5 0.0078266 0.011908 2024-09-21T09:58:00.958 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:00.991 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:01.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:01.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:01.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:01.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:01.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:01.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:01.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:01.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:01.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:01.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:01.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:01.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:01 smithi017 ceph-mon[102913]: pgmap v1311: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 6908/323265 objects degraded (2.137%); 9714/323265 objects misplaced (3.005%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:58:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:01 smithi017 ceph-mon[105151]: pgmap v1311: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 6908/323265 objects degraded (2.137%); 9714/323265 objects misplaced (3.005%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:58:01.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:01.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:01.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:01 smithi134 ceph-mon[90085]: pgmap v1311: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 6908/323265 objects degraded (2.137%); 9714/323265 objects misplaced (3.005%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:58:01.753 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:01.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:01.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:01.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:01.910 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:01.933 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 113938 113922 83.7508 67.6875 0.0102223 0.0119346 2024-09-21T09:58:01.965 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:01.993 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:01.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:02.095 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:02.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:02.223 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:02.252 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:02.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:02.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:02.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:02.499 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:02.525 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:02.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:02.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:02.655 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:02.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:02.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:02.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:02.902 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:02.933 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:02.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:02.934 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 115043 115027 83.58 69.0625 0.00860878 0.0119597 2024-09-21T09:58:03.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:03.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:03.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:03.177 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:03.205 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:03.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:03.307 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:03.332 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:03.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:03.434 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:03.463 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:03.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:03.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:03.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:03 smithi017 ceph-mon[102913]: pgmap v1312: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s; 6908/334647 objects degraded (2.064%); 9714/334647 objects misplaced (2.903%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:58:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:03 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6908/334647 objects degraded (2.064%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:03 smithi017 ceph-mon[105151]: pgmap v1312: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s; 6908/334647 objects degraded (2.064%); 9714/334647 objects misplaced (2.903%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:58:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:03 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6908/334647 objects degraded (2.064%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:03.709 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:03.737 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:03.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:03 smithi134 ceph-mon[90085]: pgmap v1312: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s; 6908/334647 objects degraded (2.064%); 9714/334647 objects misplaced (2.903%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:58:03.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:03 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6908/334647 objects degraded (2.064%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:03.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:03.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:03.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:03.934 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 115951 115935 83.2715 56.75 0.00222797 0.0120002 2024-09-21T09:58:03.965 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.995 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:03.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:04.096 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:04.120 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:04.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:04.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:04.253 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:04.355 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:04.373 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:58:04.373 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:58:04.374 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-21T09:58:04.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:58:04.381 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:04.484 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:04.511 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:04.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:04.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:04.639 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:04.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:04.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:04.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:04.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:04.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:04.920 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:04.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:04.934 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 116952 116936 83.0361 62.5625 0.0087749 0.012033 2024-09-21T09:58:04.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:58:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:58:04] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-21T09:58:05.022 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:05.051 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:05.153 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:05.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.184 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:05.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:05.285 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:05.315 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:05.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:05.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:05 smithi017 ceph-mon[102913]: pgmap v1313: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 6908/339726 objects degraded (2.033%); 9222/339726 objects misplaced (2.715%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T09:58:05.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:05 smithi017 ceph-mon[105151]: pgmap v1313: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 6908/339726 objects degraded (2.033%); 9222/339726 objects misplaced (2.715%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T09:58:05.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:05.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:05.562 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:58:05.562 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:05.563 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:05.591 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:05.617 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:58:05.617 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:05.672 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:58:05.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:05.693 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:05.716 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:58:05.716 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:05.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:05.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:05 smithi134 ceph-mon[90085]: pgmap v1313: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 6908/339726 objects degraded (2.033%); 9222/339726 objects misplaced (2.715%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T09:58:05.747 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:58:05.747 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:05.780 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:58:05.780 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:05.812 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:58:05.813 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:05.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:05.845 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:58:05.845 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:05.849 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:05.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:05.934 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 117982 117966 82.8263 64.375 0.0104741 0.0120685 2024-09-21T09:58:05.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:05.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:05.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:06.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:06.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:06.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:06.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:06.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:06.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:06.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:06.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:06.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:06.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:06.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:06.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:06.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:06 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1165519222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:06.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:06 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1165519222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:06.680 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:06.704 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:06.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:06 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1165519222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:06.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:06.838 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:06.934 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0075 2024-09-21T09:58:06.934 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 119076 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 82.6848 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 45.3376 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 222.625 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1322 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 725.402 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3562 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0120897 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0531885 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 2.57071 2024-09-21T09:58:06.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00126159 2024-09-21T09:58:06.938 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T09:58:06.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:06.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:07.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:07.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:07.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:07.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:07.257 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:07.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:07.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:07.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:07.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:07.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:07.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:07.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:07.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:07 smithi017 ceph-mon[102913]: pgmap v1314: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 6908/348984 objects degraded (1.979%); 9222/348984 objects misplaced (2.643%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T09:58:07.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:07 smithi017 ceph-mon[105151]: pgmap v1314: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 6908/348984 objects degraded (1.979%); 9222/348984 objects misplaced (2.643%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T09:58:07.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:07.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:07 smithi134 ceph-mon[90085]: pgmap v1314: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 6908/348984 objects degraded (1.979%); 9222/348984 objects misplaced (2.643%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T09:58:07.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:07.833 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:07.907 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:07.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:07.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:07.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:08.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:08.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:08.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:08.223 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:08.250 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:08.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:08.352 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:08.378 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:08.480 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:08.504 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:08.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:08.559 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:08 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6908/348984 objects degraded (1.979%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:08.559 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:08 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6908/348984 objects degraded (1.979%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:08.606 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:08.633 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:08.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:58:08.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:08.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:08 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6908/348984 objects degraded (1.979%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:08.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:08.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:08.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:08.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:09.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:09.040 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:09.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:09.151 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:58:09.151 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T09:58:09.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.151 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:121002 6.b 6:db81d5ef:::benchmark_data_smithi017_149547_object1923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:09.151 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:08.700630+0000", 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.015671396000000001, 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015708295000000001, 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 121002 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.152 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.700630+0000", 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.700630+0000", 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:09.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.700631+0000", 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3000000000000001e-07 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.700634+0000", 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.959e-06 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.700642+0000", 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.96e-06 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.700648+0000", 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0950000000000001e-06 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.701518+0000", 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00087044200000000003 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.701640+0000", 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121625 2024-09-21T09:58:09.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,7", 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.701747+0000", 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010740599999999999 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.707625+0000", 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058778950000000002 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.712021+0000", 2024-09-21T09:58:09.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00439566 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPush(6.11 1104/1091 [PushOp(6:8f95fdf9:::benchmark_data_smithi017_149547_object27954:head, version: 1083'1791, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:8f95fdf9:::benchmark_data_smithi017_149547_object27954:head@1083'1791, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:08.711775+0000", 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0045267149999999997, 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046752039999999996, 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:09.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711775+0000", 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711775+0000", 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711776+0000", 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8199999999999998e-07 2024-09-21T09:58:09.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711818+0000", 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1684000000000003e-05 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711842+0000", 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4524000000000001e-05 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711859+0000", 2024-09-21T09:58:09.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7067000000000001e-05 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:121011 6.11 e1104/1091 6:8ba201dc:::benchmark_data_smithi017_149547_object1932:head v 1104'5607, mlcod=1083'1789)", 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:08.711887+0000", 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0044149849999999997, 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046060169999999996, 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:09.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711887+0000", 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711887+0000", 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711887+0000", 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5999999999999999e-07 2024-09-21T09:58:09.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711891+0000", 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.647e-06 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711897+0000", 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6880000000000002e-06 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711902+0000", 2024-09-21T09:58:09.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3900000000000003e-06 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.712103+0000", 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020087100000000001 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.712125+0000", 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2714e-05 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:121010 6.6 6:640bde8e:::benchmark_data_smithi017_149547_object1931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:08.711988+0000", 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0043140679999999999, 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045615680000000002, 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 121010 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.164 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711988+0000", 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711988+0000", 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711989+0000", 2024-09-21T09:58:09.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.032e-06 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.711993+0000", 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.811e-06 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.712001+0000", 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2670000000000006e-06 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:09.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.712006+0000", 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6489999999999996e-06 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.712124+0000", 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117698 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.712237+0000", 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011260999999999999 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.712344+0000", 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000107593 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.714672+0000", 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023274049999999998 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715824+0000", 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001151815 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.168 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:121013 6.7 6:e1ed5f96:::benchmark_data_smithi017_149547_object1934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:08.715581+0000", 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00072115999999999999, 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001047039, 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:09.169 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 121013 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715581+0000", 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715581+0000", 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715582+0000", 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.736e-06 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715589+0000", 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3890000000000002e-06 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715613+0000", 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4658999999999999e-05 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715622+0000", 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6459999999999999e-06 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715866+0000", 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000244229 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715987+0000", 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120647 2024-09-21T09:58:09.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.716103+0000", 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116054 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:121014 6.5 e1104/1086)", 2024-09-21T09:58:09.173 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:08.715664+0000", 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00063791300000000004, 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001025689, 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715664+0000", 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:09.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715664+0000", 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715665+0000", 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1900000000000002e-07 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715669+0000", 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2250000000000002e-06 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715678+0000", 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8049999999999996e-06 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.715683+0000", 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1789999999999999e-06 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45173.0:121010 6.6 e1104/1086 ondisk, result = 0)", 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:08.716264+0000", 2024-09-21T09:58:09.176 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 3.8226000000000003e-05, 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00046621199999999997, 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.716264+0000", 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.716264+0000", 2024-09-21T09:58:09.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.716264+0000", 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3099999999999997e-07 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.716271+0000", 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4849999999999997e-06 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:09.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.716280+0000", 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8090000000000004e-06 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.716287+0000", 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7939999999999997e-06 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.716520+0000", 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000233129 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:09.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:08.716528+0000", 2024-09-21T09:58:09.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0080000000000002e-06 2024-09-21T09:58:09.180 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.180 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:09.180 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.180 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:09.180 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-21T09:58:09.180 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 7 2024-09-21T09:58:09.180 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:58:09.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:09.292 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:09.322 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:09.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:09.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:09.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:09.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:09.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:09 smithi017 ceph-mon[102913]: pgmap v1315: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 87 B/s rd, 46 MiB/s wr, 741 op/s; 6888/352794 objects degraded (1.952%); 8722/352794 objects misplaced (2.472%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T09:58:09.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:09 smithi017 ceph-mon[105151]: pgmap v1315: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 87 B/s rd, 46 MiB/s wr, 741 op/s; 6888/352794 objects degraded (1.952%); 8722/352794 objects misplaced (2.472%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T09:58:09.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:09.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:09.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:09.694 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:09.721 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:09.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:09 smithi134 ceph-mon[90085]: pgmap v1315: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 87 B/s rd, 46 MiB/s wr, 741 op/s; 6888/352794 objects degraded (1.952%); 8722/352794 objects misplaced (2.472%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T09:58:09.823 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:09.848 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:09.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:09.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:09.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:09.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:10.095 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:10.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:10.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:10.223 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:10.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:10.351 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:10.378 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:10.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:10.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:10.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:58:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:58:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:10.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:10.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:10.626 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:10.647 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:10.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:10.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:10.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:10.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:10.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:10.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:10.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:10.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:11.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:11.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:11.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:11.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:11.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:11.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:11.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:11.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:11.422 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:11.444 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:11.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:11.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:11.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:11.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:11 smithi017 ceph-mon[102913]: pgmap v1316: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 652 op/s; 6888/353586 objects degraded (1.948%); 8722/353586 objects misplaced (2.467%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T09:58:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:11 smithi017 ceph-mon[105151]: pgmap v1316: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 652 op/s; 6888/353586 objects degraded (1.948%); 8722/353586 objects misplaced (2.467%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T09:58:11.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:11.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:11 smithi134 ceph-mon[90085]: pgmap v1316: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 652 op/s; 6888/353586 objects degraded (1.948%); 8722/353586 objects misplaced (2.467%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T09:58:11.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:11.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:11.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:11.959 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:11.981 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:11.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:12.083 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:12.108 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:12.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:12.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:12.232 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:12.334 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:12.362 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:12.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:12.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:12.601 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:12.628 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:12.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:12.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:12.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:12.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T09:58:12.883 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:12.907 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:13.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:13.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:13.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:13.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:13.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:13.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:13.372 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:13.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:13.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:13.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:13.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:13.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:13 smithi017 ceph-mon[102913]: pgmap v1317: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 897 op/s; 6888/348195 objects degraded (1.978%); 8722/348195 objects misplaced (2.505%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T09:58:13.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:13 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6888/348195 objects degraded (1.978%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:13.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:13 smithi017 ceph-mon[105151]: pgmap v1317: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 897 op/s; 6888/348195 objects degraded (1.978%); 8722/348195 objects misplaced (2.505%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T09:58:13.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:13 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6888/348195 objects degraded (1.978%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:13.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.661 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:13.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:13 smithi134 ceph-mon[90085]: pgmap v1317: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 897 op/s; 6888/348195 objects degraded (1.978%); 8722/348195 objects misplaced (2.505%); 1.7 MiB/s, 27 objects/s recovering 2024-09-21T09:58:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:13 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6888/348195 objects degraded (1.978%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:13.761 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:13.783 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:13.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:13.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:13.950 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:13.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:13.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:14.068 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:14.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.097 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:14.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:14.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:14.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:14.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:14.357 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:14.386 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:14.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:14.411 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T09:58:14.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:14.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:14.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:14.640 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:14.670 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:14.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1135145467' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:14.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:14 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:14.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1135145467' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:14.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:14 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1135145467' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:14 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:14.772 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:14.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.799 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:14.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:14.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:14.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:14.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:15.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:15.081 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:15.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:15.094 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:58:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:58:14] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-21T09:58:15.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:15.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:15.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:15.346 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:15.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:15.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:15.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:15.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.518 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:15.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:15.578 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:15.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:15.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:15 smithi017 ceph-mon[102913]: pgmap v1318: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 987 op/s; 6594/343899 objects degraded (1.917%); 8123/343899 objects misplaced (2.362%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T09:58:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:15 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:58:15.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:15 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:15.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:15 smithi017 ceph-mon[102913]: osdmap e1105: 8 total, 8 up, 8 in 2024-09-21T09:58:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:15 smithi017 ceph-mon[105151]: pgmap v1318: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 987 op/s; 6594/343899 objects degraded (1.917%); 8123/343899 objects misplaced (2.362%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T09:58:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:15 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:58:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:15 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:15 smithi017 ceph-mon[105151]: osdmap e1105: 8 total, 8 up, 8 in 2024-09-21T09:58:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:15 smithi134 ceph-mon[90085]: pgmap v1318: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 987 op/s; 6594/343899 objects degraded (1.917%); 8123/343899 objects misplaced (2.362%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T09:58:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:15 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:58:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:15 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:15 smithi134 ceph-mon[90085]: osdmap e1105: 8 total, 8 up, 8 in 2024-09-21T09:58:15.765 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:15.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:15.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:15.939 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:15.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:15.992 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:58:15.995 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:58:16.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:16.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:16.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:16.192 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:16.220 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:16.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:16.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:16.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:16.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:16.482 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:16.512 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:16.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:16.659 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:16.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:16.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:16.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:16.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:16.863 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1319,"stamp":"2024-09-21T09:58:14.404123+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7511414757,"num_objects":114633,"num_object_clones":0,"num_object_copies":343899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6594,"num_objects_misplaced":8123,"num_objects_unfound":0,"num_objects_dirty":114633,"num_whiteouts":0,"num_read":307,"num_read_kb":760,"num_write":84178,"num_write_kb":5094716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5720,"num_bytes_recovered":375325216,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158434,"ondisk_log_size":158434,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":308,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24784804,"kb_used_data":23073252,"kb_used_omap":122,"kb_used_meta":1711109,"kb_avail":725176412,"statfs":{"total":767960285184,"available":742580645888,"internally_reserved":0,"allocated":23627010048,"data_stored":23609341231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125317,"internal_metadata":1752176251},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,5,8,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":91095064,"num_objects":1391,"num_object_clones":0,"num_object_copies":4173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-314,"num_objects_misplaced":-1099,"num_objects_unfound":0,"num_objects_dirty":1391,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10163,"num_write_kb":369665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":471,"num_bytes_recovered":30867456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.383618"},"pg_stats":[{"pgid":"6.16","version":"1104'5670","reported_seq":5777,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.283584+0000","last_change":"2024-09-21T09:57:11.311033+0000","last_active":"2024-09-21T09:58:10.283584+0000","last_peered":"2024-09-21T09:58:10.283584+0000","last_clean":"2024-09-21T09:58:10.283584+0000","last_became_active":"2024-09-21T09:57:07.444641+0000","last_became_peered":"2024-09-21T09:57:07.444641+0000","last_unstale":"2024-09-21T09:58:10.283584+0000","last_undegraded":"2024-09-21T09:58:10.283584+0000","last_fullsized":"2024-09-21T09:58:10.283584+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'4023","last_scrub_stamp":"2024-09-21T09:57:11.310699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:11.310699+0000","objects_scrubbed":2106,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:52.271305+0000","scrub_duration":650,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232521728,"num_objects":3548,"num_object_clones":0,"num_object_copies":10644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1662,"num_write_kb":99328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1542,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.345373+0000","last_change":"2024-09-21T09:46:47.114206+0000","last_active":"2024-09-21T09:57:56.345373+0000","last_peered":"2024-09-21T09:57:56.345373+0000","last_clean":"2024-09-21T09:57:56.345373+0000","last_became_active":"2024-09-21T09:46:47.113819+0000","last_became_peered":"2024-09-21T09:46:47.113819+0000","last_unstale":"2024-09-21T09:57:56.345373+0000","last_undegraded":"2024-09-21T09:57:56.345373+0000","last_fullsized":"2024-09-21T09:57:56.345373+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:07.629544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1029,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.641997+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T09:57:41.641997+0000","last_peered":"2024-09-21T09:57:41.641997+0000","last_clean":"2024-09-21T09:57:41.641997+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T09:57:41.641997+0000","last_undegraded":"2024-09-21T09:57:41.641997+0000","last_fullsized":"2024-09-21T09:57:41.641997+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.17","version":"1104'5812","reported_seq":5959,"reported_epoch":1104,"state":"active+recovery_wait","last_fresh":"2024-09-21T09:58:10.390052+0000","last_change":"2024-09-21T09:57:39.551603+0000","last_active":"2024-09-21T09:58:10.390052+0000","last_peered":"2024-09-21T09:58:10.390052+0000","last_clean":"2024-09-21T09:57:38.481321+0000","last_became_active":"2024-09-21T09:57:39.544853+0000","last_became_peered":"2024-09-21T09:57:39.544853+0000","last_unstale":"2024-09-21T09:58:10.390052+0000","last_undegraded":"2024-09-21T09:58:10.390052+0000","last_fullsized":"2024-09-21T09:58:10.390052+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'4025","last_scrub_stamp":"2024-09-21T09:57:10.848092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:10.848092+0000","objects_scrubbed":2113,"log_size":5812,"log_dups_size":0,"ondisk_log_size":5812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":0.575755185,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241762304,"num_objects":3689,"num_object_clones":0,"num_object_copies":11067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1796,"num_write_kb":107904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":3689}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":537,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.642073+0000","last_change":"2024-09-21T09:57:39.541887+0000","last_active":"2024-09-21T09:57:41.642073+0000","last_peered":"2024-09-21T09:57:41.642073+0000","last_clean":"2024-09-21T09:57:41.642073+0000","last_became_active":"2024-09-21T09:57:39.541598+0000","last_became_peered":"2024-09-21T09:57:39.541598+0000","last_unstale":"2024-09-21T09:57:41.642073+0000","last_undegraded":"2024-09-21T09:57:41.642073+0000","last_fullsized":"2024-09-21T09:57:41.642073+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:41.615525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1626,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.352717+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T09:57:56.352717+0000","last_peered":"2024-09-21T09:57:56.352717+0000","last_clean":"2024-09-21T09:57:56.352717+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T09:57:56.352717+0000","last_undegraded":"2024-09-21T09:57:56.352717+0000","last_fullsized":"2024-09-21T09:57:56.352717+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"1104'5743","reported_seq":6483,"reported_epoch":1104,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:11.949015+0000","last_change":"2024-09-21T09:57:24.631233+0000","last_active":"2024-09-21T09:58:11.949015+0000","last_peered":"2024-09-21T09:58:11.949015+0000","last_clean":"2024-09-21T09:57:22.236005+0000","last_became_active":"2024-09-21T09:57:24.625791+0000","last_became_peered":"2024-09-21T09:57:24.625791+0000","last_unstale":"2024-09-21T09:58:11.949015+0000","last_undegraded":"2024-09-21T09:57:24.385357+0000","last_fullsized":"2024-09-21T09:57:24.301747+0000","mapping_epoch":1097,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1088'4137","last_scrub_stamp":"2024-09-21T09:57:16.338288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:16.338288+0000","objects_scrubbed":2242,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":308,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231604224,"num_objects":3534,"num_object_clones":0,"num_object_copies":10602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4280,"num_objects_misplaced":2140,"num_objects_unfound":0,"num_objects_dirty":3534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1606,"num_write_kb":92736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":10289152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[1,2],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,2","objects":1394},{"shards":"1,3,5","objects":2140}],"blocked_by":[],"up_primary":2,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1628,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.350871+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T09:57:56.350871+0000","last_peered":"2024-09-21T09:57:56.350871+0000","last_clean":"2024-09-21T09:57:56.350871+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T09:57:56.350871+0000","last_undegraded":"2024-09-21T09:57:56.350871+0000","last_fullsized":"2024-09-21T09:57:56.350871+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1010,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.641678+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:57:41.641678+0000","last_peered":"2024-09-21T09:57:41.641678+0000","last_clean":"2024-09-21T09:57:41.641678+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:57:41.641678+0000","last_undegraded":"2024-09-21T09:57:41.641678+0000","last_fullsized":"2024-09-21T09:57:41.641678+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.15","version":"1104'5788","reported_seq":5932,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.829427+0000","last_change":"2024-09-21T09:57:15.230133+0000","last_active":"2024-09-21T09:58:10.829427+0000","last_peered":"2024-09-21T09:58:10.829427+0000","last_clean":"2024-09-21T09:58:10.829427+0000","last_became_active":"2024-09-21T09:57:07.439001+0000","last_became_peered":"2024-09-21T09:57:07.439001+0000","last_unstale":"2024-09-21T09:58:10.829427+0000","last_undegraded":"2024-09-21T09:58:10.829427+0000","last_fullsized":"2024-09-21T09:58:10.829427+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1088'4200","last_scrub_stamp":"2024-09-21T09:57:15.230067+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:15.230067+0000","objects_scrubbed":2263,"log_size":5788,"log_dups_size":0,"ondisk_log_size":5788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:22.500492+0000","scrub_duration":0.27387100399999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236650496,"num_objects":3611,"num_object_clones":0,"num_object_copies":10833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1588,"num_write_kb":93952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1542,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.348365+0000","last_change":"2024-09-21T09:46:47.117580+0000","last_active":"2024-09-21T09:57:56.348365+0000","last_peered":"2024-09-21T09:57:56.348365+0000","last_clean":"2024-09-21T09:57:56.348365+0000","last_became_active":"2024-09-21T09:46:47.113030+0000","last_became_peered":"2024-09-21T09:46:47.113030+0000","last_unstale":"2024-09-21T09:57:56.348365+0000","last_undegraded":"2024-09-21T09:57:56.348365+0000","last_fullsized":"2024-09-21T09:57:56.348365+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:28.220180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":1036,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.674137+0000","last_change":"2024-09-21T09:45:05.080014+0000","last_active":"2024-09-21T09:57:41.674137+0000","last_peered":"2024-09-21T09:57:41.674137+0000","last_clean":"2024-09-21T09:57:41.674137+0000","last_became_active":"2024-09-21T09:45:05.079618+0000","last_became_peered":"2024-09-21T09:45:05.079618+0000","last_unstale":"2024-09-21T09:57:41.674137+0000","last_undegraded":"2024-09-21T09:57:41.674137+0000","last_fullsized":"2024-09-21T09:57:41.674137+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:42.160292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1104'7632","reported_seq":7673,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.866254+0000","last_change":"2024-09-21T09:57:39.645463+0000","last_active":"2024-09-21T09:58:10.866254+0000","last_peered":"2024-09-21T09:58:10.866254+0000","last_clean":"2024-09-21T09:58:10.866254+0000","last_became_active":"2024-09-21T09:57:39.634848+0000","last_became_peered":"2024-09-21T09:57:39.634848+0000","last_unstale":"2024-09-21T09:58:10.866254+0000","last_undegraded":"2024-09-21T09:58:10.866254+0000","last_fullsized":"2024-09-21T09:58:10.866254+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":7632,"log_dups_size":0,"ondisk_log_size":7632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:51:28.212787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":468320256,"num_objects":7146,"num_object_clones":0,"num_object_copies":21438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7632,"num_write_kb":472896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1028,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.642298+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T09:57:41.642298+0000","last_peered":"2024-09-21T09:57:41.642298+0000","last_clean":"2024-09-21T09:57:41.642298+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T09:57:41.642298+0000","last_undegraded":"2024-09-21T09:57:41.642298+0000","last_fullsized":"2024-09-21T09:57:41.642298+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":16,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.641387+0000","last_change":"2024-09-21T09:57:39.636116+0000","last_active":"2024-09-21T09:57:41.641387+0000","last_peered":"2024-09-21T09:57:41.641387+0000","last_clean":"2024-09-21T09:57:41.641387+0000","last_became_active":"2024-09-21T09:57:39.635802+0000","last_became_peered":"2024-09-21T09:57:39.635802+0000","last_unstale":"2024-09-21T09:57:41.641387+0000","last_undegraded":"2024-09-21T09:57:41.641387+0000","last_fullsized":"2024-09-21T09:57:41.641387+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:43:40.045268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"1104'7693","reported_seq":7762,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.396216+0000","last_change":"2024-09-21T09:56:26.652941+0000","last_active":"2024-09-21T09:58:10.396216+0000","last_peered":"2024-09-21T09:58:10.396216+0000","last_clean":"2024-09-21T09:58:10.396216+0000","last_became_active":"2024-09-21T09:56:26.652635+0000","last_became_peered":"2024-09-21T09:56:26.652635+0000","last_unstale":"2024-09-21T09:58:10.396216+0000","last_undegraded":"2024-09-21T09:58:10.396216+0000","last_fullsized":"2024-09-21T09:58:10.396216+0000","mapping_epoch":1078,"log_start":"1083'1600","ondisk_log_start":"1083'1600","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":6093,"log_dups_size":0,"ondisk_log_size":6093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:00.304863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":472449024,"num_objects":7209,"num_object_clones":0,"num_object_copies":21627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7693,"num_write_kb":476864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1029,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.642274+0000","last_change":"2024-09-21T09:45:39.143190+0000","last_active":"2024-09-21T09:57:41.642274+0000","last_peered":"2024-09-21T09:57:41.642274+0000","last_clean":"2024-09-21T09:57:41.642274+0000","last_became_active":"2024-09-21T09:45:39.142826+0000","last_became_peered":"2024-09-21T09:45:39.142826+0000","last_unstale":"2024-09-21T09:57:41.642274+0000","last_undegraded":"2024-09-21T09:57:41.642274+0000","last_fullsized":"2024-09-21T09:57:41.642274+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:45.070945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1062,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667311+0000","last_change":"2024-09-21T09:46:47.119247+0000","last_active":"2024-09-21T09:57:41.667311+0000","last_peered":"2024-09-21T09:57:41.667311+0000","last_clean":"2024-09-21T09:57:41.667311+0000","last_became_active":"2024-09-21T09:46:47.118980+0000","last_became_peered":"2024-09-21T09:46:47.118980+0000","last_unstale":"2024-09-21T09:57:41.667311+0000","last_undegraded":"2024-09-21T09:57:41.667311+0000","last_fullsized":"2024-09-21T09:57:41.667311+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.140253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1104'5844","reported_seq":5992,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.883424+0000","last_change":"2024-09-21T09:57:14.741073+0000","last_active":"2024-09-21T09:58:10.883424+0000","last_peered":"2024-09-21T09:58:10.883424+0000","last_clean":"2024-09-21T09:58:10.883424+0000","last_became_active":"2024-09-21T09:57:07.438175+0000","last_became_peered":"2024-09-21T09:57:07.438175+0000","last_unstale":"2024-09-21T09:58:10.883424+0000","last_undegraded":"2024-09-21T09:58:10.883424+0000","last_fullsized":"2024-09-21T09:58:10.883424+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1088'4206","last_scrub_stamp":"2024-09-21T09:57:14.741002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:14.741002+0000","objects_scrubbed":2270,"log_size":5844,"log_dups_size":0,"ondisk_log_size":5844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:30.124559+0000","scrub_duration":0.80346662300000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239992832,"num_objects":3662,"num_object_clones":0,"num_object_copies":10986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1654,"num_write_kb":97472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":1011,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.642196+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:57:41.642196+0000","last_peered":"2024-09-21T09:57:41.642196+0000","last_clean":"2024-09-21T09:57:41.642196+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:57:41.642196+0000","last_undegraded":"2024-09-21T09:57:41.642196+0000","last_fullsized":"2024-09-21T09:57:41.642196+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1037,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.674462+0000","last_change":"2024-09-21T09:46:13.260744+0000","last_active":"2024-09-21T09:57:41.674462+0000","last_peered":"2024-09-21T09:57:41.674462+0000","last_clean":"2024-09-21T09:57:41.674462+0000","last_became_active":"2024-09-21T09:46:13.260443+0000","last_became_peered":"2024-09-21T09:46:13.260443+0000","last_unstale":"2024-09-21T09:57:41.674462+0000","last_undegraded":"2024-09-21T09:57:41.674462+0000","last_fullsized":"2024-09-21T09:57:41.674462+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:22.699110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1104'5789","reported_seq":5894,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:13.157596+0000","last_change":"2024-09-21T09:57:09.604301+0000","last_active":"2024-09-21T09:58:13.157596+0000","last_peered":"2024-09-21T09:58:13.157596+0000","last_clean":"2024-09-21T09:58:13.157596+0000","last_became_active":"2024-09-21T09:57:07.402981+0000","last_became_peered":"2024-09-21T09:57:07.402981+0000","last_unstale":"2024-09-21T09:58:13.157596+0000","last_undegraded":"2024-09-21T09:58:13.157596+0000","last_fullsized":"2024-09-21T09:58:13.157596+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'3957","last_scrub_stamp":"2024-09-21T09:57:09.604140+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:09.604140+0000","objects_scrubbed":2037,"log_size":5789,"log_dups_size":0,"ondisk_log_size":5789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:54.970082+0000","scrub_duration":518,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229834752,"num_objects":3507,"num_object_clones":0,"num_object_copies":10521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1846,"num_write_kb":106112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":542,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.348445+0000","last_change":"2024-09-21T09:57:39.641367+0000","last_active":"2024-09-21T09:57:56.348445+0000","last_peered":"2024-09-21T09:57:56.348445+0000","last_clean":"2024-09-21T09:57:56.348445+0000","last_became_active":"2024-09-21T09:57:39.641057+0000","last_became_peered":"2024-09-21T09:57:39.641057+0000","last_unstale":"2024-09-21T09:57:56.348445+0000","last_undegraded":"2024-09-21T09:57:56.348445+0000","last_fullsized":"2024-09-21T09:57:56.348445+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:22.196149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1628,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.349539+0000","last_change":"2024-09-21T09:45:39.138289+0000","last_active":"2024-09-21T09:57:56.349539+0000","last_peered":"2024-09-21T09:57:56.349539+0000","last_clean":"2024-09-21T09:57:56.349539+0000","last_became_active":"2024-09-21T09:45:39.137902+0000","last_became_peered":"2024-09-21T09:45:39.137902+0000","last_unstale":"2024-09-21T09:57:56.349539+0000","last_undegraded":"2024-09-21T09:57:56.349539+0000","last_fullsized":"2024-09-21T09:57:56.349539+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:40:16.383906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1104'7606","reported_seq":7640,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.887604+0000","last_change":"2024-09-21T09:56:26.650365+0000","last_active":"2024-09-21T09:58:10.887604+0000","last_peered":"2024-09-21T09:58:10.887604+0000","last_clean":"2024-09-21T09:58:10.887604+0000","last_became_active":"2024-09-21T09:56:26.649684+0000","last_became_peered":"2024-09-21T09:56:26.649684+0000","last_unstale":"2024-09-21T09:58:10.887604+0000","last_undegraded":"2024-09-21T09:58:10.887604+0000","last_fullsized":"2024-09-21T09:58:10.887604+0000","mapping_epoch":1078,"log_start":"1082'400","ondisk_log_start":"1082'400","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":7206,"log_dups_size":0,"ondisk_log_size":7206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:04.800486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":465567744,"num_objects":7104,"num_object_clones":0,"num_object_copies":21312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7606,"num_write_kb":470720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1553,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.350664+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T09:57:56.350664+0000","last_peered":"2024-09-21T09:57:56.350664+0000","last_clean":"2024-09-21T09:57:56.350664+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T09:57:56.350664+0000","last_undegraded":"2024-09-21T09:57:56.350664+0000","last_fullsized":"2024-09-21T09:57:56.350664+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1045,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.669542+0000","last_change":"2024-09-21T09:46:47.114293+0000","last_active":"2024-09-21T09:57:41.669542+0000","last_peered":"2024-09-21T09:57:41.669542+0000","last_clean":"2024-09-21T09:57:41.669542+0000","last_became_active":"2024-09-21T09:46:47.113993+0000","last_became_peered":"2024-09-21T09:46:47.113993+0000","last_unstale":"2024-09-21T09:57:41.669542+0000","last_undegraded":"2024-09-21T09:57:41.669542+0000","last_fullsized":"2024-09-21T09:57:41.669542+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:30.941931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1104'7733","reported_seq":7803,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:11.266384+0000","last_change":"2024-09-21T09:56:26.651596+0000","last_active":"2024-09-21T09:58:11.266384+0000","last_peered":"2024-09-21T09:58:11.266384+0000","last_clean":"2024-09-21T09:58:11.266384+0000","last_became_active":"2024-09-21T09:56:26.651284+0000","last_became_peered":"2024-09-21T09:56:26.651284+0000","last_unstale":"2024-09-21T09:58:11.266384+0000","last_undegraded":"2024-09-21T09:58:11.266384+0000","last_fullsized":"2024-09-21T09:58:11.266384+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":7733,"log_dups_size":0,"ondisk_log_size":7733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:19:05.103239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":471924736,"num_objects":7201,"num_object_clones":0,"num_object_copies":21603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7733,"num_write_kb":477888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1028,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.674591+0000","last_change":"2024-09-21T09:45:39.127186+0000","last_active":"2024-09-21T09:57:41.674591+0000","last_peered":"2024-09-21T09:57:41.674591+0000","last_clean":"2024-09-21T09:57:41.674591+0000","last_became_active":"2024-09-21T09:45:39.126940+0000","last_became_peered":"2024-09-21T09:45:39.126940+0000","last_unstale":"2024-09-21T09:57:41.674591+0000","last_undegraded":"2024-09-21T09:57:41.674591+0000","last_fullsized":"2024-09-21T09:57:41.674591+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:52.205703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1062,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667660+0000","last_change":"2024-09-21T09:46:13.259814+0000","last_active":"2024-09-21T09:57:41.667660+0000","last_peered":"2024-09-21T09:57:41.667660+0000","last_clean":"2024-09-21T09:57:41.667660+0000","last_became_active":"2024-09-21T09:46:13.259527+0000","last_became_peered":"2024-09-21T09:46:13.259527+0000","last_unstale":"2024-09-21T09:57:41.667660+0000","last_undegraded":"2024-09-21T09:57:41.667660+0000","last_fullsized":"2024-09-21T09:57:41.667660+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:00.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1104'7754","reported_seq":7824,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.387085+0000","last_change":"2024-09-21T09:56:27.225575+0000","last_active":"2024-09-21T09:58:10.387085+0000","last_peered":"2024-09-21T09:58:10.387085+0000","last_clean":"2024-09-21T09:58:10.387085+0000","last_became_active":"2024-09-21T09:56:27.225236+0000","last_became_peered":"2024-09-21T09:56:27.225236+0000","last_unstale":"2024-09-21T09:58:10.387085+0000","last_undegraded":"2024-09-21T09:58:10.387085+0000","last_fullsized":"2024-09-21T09:58:10.387085+0000","mapping_epoch":1078,"log_start":"1083'1700","ondisk_log_start":"1083'1700","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":6054,"log_dups_size":0,"ondisk_log_size":6054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:03.692657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477888512,"num_objects":7292,"num_object_clones":0,"num_object_copies":21876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7754,"num_write_kb":481472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1046,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667346+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T09:57:41.667346+0000","last_peered":"2024-09-21T09:57:41.667346+0000","last_clean":"2024-09-21T09:57:41.667346+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T09:57:41.667346+0000","last_undegraded":"2024-09-21T09:57:41.667346+0000","last_fullsized":"2024-09-21T09:57:41.667346+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1045,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667918+0000","last_change":"2024-09-21T09:45:39.143302+0000","last_active":"2024-09-21T09:57:41.667918+0000","last_peered":"2024-09-21T09:57:41.667918+0000","last_clean":"2024-09-21T09:57:41.667918+0000","last_became_active":"2024-09-21T09:45:39.142964+0000","last_became_peered":"2024-09-21T09:45:39.142964+0000","last_unstale":"2024-09-21T09:57:41.667918+0000","last_undegraded":"2024-09-21T09:57:41.667918+0000","last_fullsized":"2024-09-21T09:57:41.667918+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:54.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1104'7608","reported_seq":7654,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:12.191108+0000","last_change":"2024-09-21T09:57:39.553065+0000","last_active":"2024-09-21T09:58:12.191108+0000","last_peered":"2024-09-21T09:58:12.191108+0000","last_clean":"2024-09-21T09:58:12.191108+0000","last_became_active":"2024-09-21T09:57:39.545966+0000","last_became_peered":"2024-09-21T09:57:39.545966+0000","last_unstale":"2024-09-21T09:58:12.191108+0000","last_undegraded":"2024-09-21T09:58:12.191108+0000","last_fullsized":"2024-09-21T09:58:12.191108+0000","mapping_epoch":1100,"log_start":"1082'100","ondisk_log_start":"1082'100","created":1078,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":7508,"log_dups_size":0,"ondisk_log_size":7508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:54.066959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":460259352,"num_objects":7024,"num_object_clones":0,"num_object_copies":21072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7024,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7608,"num_write_kb":468161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1544,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.349391+0000","last_change":"2024-09-21T09:46:47.121643+0000","last_active":"2024-09-21T09:57:56.349391+0000","last_peered":"2024-09-21T09:57:56.349391+0000","last_clean":"2024-09-21T09:57:56.349391+0000","last_became_active":"2024-09-21T09:46:47.120528+0000","last_became_peered":"2024-09-21T09:46:47.120528+0000","last_unstale":"2024-09-21T09:57:56.349391+0000","last_undegraded":"2024-09-21T09:57:56.349391+0000","last_fullsized":"2024-09-21T09:57:56.349391+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:07.989377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1590,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.334928+0000","last_change":"2024-09-21T09:46:47.121159+0000","last_active":"2024-09-21T09:57:56.334928+0000","last_peered":"2024-09-21T09:57:56.334928+0000","last_clean":"2024-09-21T09:57:56.334928+0000","last_became_active":"2024-09-21T09:46:47.120829+0000","last_became_peered":"2024-09-21T09:46:47.120829+0000","last_unstale":"2024-09-21T09:57:56.334928+0000","last_undegraded":"2024-09-21T09:57:56.334928+0000","last_fullsized":"2024-09-21T09:57:56.334928+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:43.393840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1104'5794","reported_seq":5941,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.359856+0000","last_change":"2024-09-21T09:57:39.547586+0000","last_active":"2024-09-21T09:58:10.359856+0000","last_peered":"2024-09-21T09:58:10.359856+0000","last_clean":"2024-09-21T09:58:10.359856+0000","last_became_active":"2024-09-21T09:57:39.544701+0000","last_became_peered":"2024-09-21T09:57:39.544701+0000","last_unstale":"2024-09-21T09:58:10.359856+0000","last_undegraded":"2024-09-21T09:58:10.359856+0000","last_fullsized":"2024-09-21T09:58:10.359856+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'3934","last_scrub_stamp":"2024-09-21T09:57:09.713484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:09.713484+0000","objects_scrubbed":1997,"log_size":5794,"log_dups_size":0,"ondisk_log_size":5794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:10:42.424723+0000","scrub_duration":0.46009026200000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238092288,"num_objects":3633,"num_object_clones":0,"num_object_copies":10899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1866,"num_write_kb":112064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1144,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.641705+0000","last_change":"2024-09-21T09:45:05.087928+0000","last_active":"2024-09-21T09:57:41.641705+0000","last_peered":"2024-09-21T09:57:41.641705+0000","last_clean":"2024-09-21T09:57:41.641705+0000","last_became_active":"2024-09-21T09:45:05.087478+0000","last_became_peered":"2024-09-21T09:45:05.087478+0000","last_unstale":"2024-09-21T09:57:41.641705+0000","last_undegraded":"2024-09-21T09:57:41.641705+0000","last_fullsized":"2024-09-21T09:57:41.641705+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:48.651314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":29,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.344927+0000","last_change":"2024-09-21T09:57:39.541461+0000","last_active":"2024-09-21T09:57:56.344927+0000","last_peered":"2024-09-21T09:57:56.344927+0000","last_clean":"2024-09-21T09:57:56.344927+0000","last_became_active":"2024-09-21T09:57:39.540927+0000","last_became_peered":"2024-09-21T09:57:39.540927+0000","last_unstale":"2024-09-21T09:57:56.344927+0000","last_undegraded":"2024-09-21T09:57:56.344927+0000","last_fullsized":"2024-09-21T09:57:56.344927+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:14.815783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1526,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.345045+0000","last_change":"2024-09-21T09:46:47.116045+0000","last_active":"2024-09-21T09:57:56.345045+0000","last_peered":"2024-09-21T09:57:56.345045+0000","last_clean":"2024-09-21T09:57:56.345045+0000","last_became_active":"2024-09-21T09:46:47.115526+0000","last_became_peered":"2024-09-21T09:46:47.115526+0000","last_unstale":"2024-09-21T09:57:56.345045+0000","last_undegraded":"2024-09-21T09:57:56.345045+0000","last_fullsized":"2024-09-21T09:57:56.345045+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:43:35.293486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1010,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.674778+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:57:41.674778+0000","last_peered":"2024-09-21T09:57:41.674778+0000","last_clean":"2024-09-21T09:57:41.674778+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:57:41.674778+0000","last_undegraded":"2024-09-21T09:57:41.674778+0000","last_fullsized":"2024-09-21T09:57:41.674778+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1104'5749","reported_seq":5855,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:12.154915+0000","last_change":"2024-09-21T09:57:12.585329+0000","last_active":"2024-09-21T09:58:12.154915+0000","last_peered":"2024-09-21T09:58:12.154915+0000","last_clean":"2024-09-21T09:58:12.154915+0000","last_became_active":"2024-09-21T09:57:07.409230+0000","last_became_peered":"2024-09-21T09:57:07.409230+0000","last_unstale":"2024-09-21T09:58:12.154915+0000","last_undegraded":"2024-09-21T09:58:12.154915+0000","last_fullsized":"2024-09-21T09:58:12.154915+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4057","last_scrub_stamp":"2024-09-21T09:57:12.582965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:12.582965+0000","objects_scrubbed":2117,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:10.909855+0000","scrub_duration":679,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231014400,"num_objects":3525,"num_object_clones":0,"num_object_copies":10575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1704,"num_write_kb":99584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1056,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.641540+0000","last_change":"2024-09-21T09:45:39.129555+0000","last_active":"2024-09-21T09:57:41.641540+0000","last_peered":"2024-09-21T09:57:41.641540+0000","last_clean":"2024-09-21T09:57:41.641540+0000","last_became_active":"2024-09-21T09:45:39.129205+0000","last_became_peered":"2024-09-21T09:45:39.129205+0000","last_unstale":"2024-09-21T09:57:41.641540+0000","last_undegraded":"2024-09-21T09:57:41.641540+0000","last_fullsized":"2024-09-21T09:57:41.641540+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:07.229126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1627,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.349729+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T09:57:56.349729+0000","last_peered":"2024-09-21T09:57:56.349729+0000","last_clean":"2024-09-21T09:57:56.349729+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T09:57:56.349729+0000","last_undegraded":"2024-09-21T09:57:56.349729+0000","last_fullsized":"2024-09-21T09:57:56.349729+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1640,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.349810+0000","last_change":"2024-09-21T09:46:13.273573+0000","last_active":"2024-09-21T09:57:56.349810+0000","last_peered":"2024-09-21T09:57:56.349810+0000","last_clean":"2024-09-21T09:57:56.349810+0000","last_became_active":"2024-09-21T09:46:13.273297+0000","last_became_peered":"2024-09-21T09:46:13.273297+0000","last_unstale":"2024-09-21T09:57:56.349810+0000","last_undegraded":"2024-09-21T09:57:56.349810+0000","last_fullsized":"2024-09-21T09:57:56.349810+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:38.957745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1104'5827","reported_seq":5970,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.837967+0000","last_change":"2024-09-21T09:57:13.591883+0000","last_active":"2024-09-21T09:58:10.837967+0000","last_peered":"2024-09-21T09:58:10.837967+0000","last_clean":"2024-09-21T09:58:10.837967+0000","last_became_active":"2024-09-21T09:57:07.409458+0000","last_became_peered":"2024-09-21T09:57:07.409458+0000","last_unstale":"2024-09-21T09:58:10.837967+0000","last_undegraded":"2024-09-21T09:58:10.837967+0000","last_fullsized":"2024-09-21T09:58:10.837967+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4144","last_scrub_stamp":"2024-09-21T09:57:13.591791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:13.591791+0000","objects_scrubbed":2205,"log_size":5827,"log_dups_size":0,"ondisk_log_size":5827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:57:10.786770+0000","scrub_duration":0.62276720100000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240517120,"num_objects":3670,"num_object_clones":0,"num_object_copies":11010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1691,"num_write_kb":100992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1601,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.347642+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T09:57:56.347642+0000","last_peered":"2024-09-21T09:57:56.347642+0000","last_clean":"2024-09-21T09:57:56.347642+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T09:57:56.347642+0000","last_undegraded":"2024-09-21T09:57:56.347642+0000","last_fullsized":"2024-09-21T09:57:56.347642+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1592,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.335053+0000","last_change":"2024-09-21T09:46:13.255342+0000","last_active":"2024-09-21T09:57:56.335053+0000","last_peered":"2024-09-21T09:57:56.335053+0000","last_clean":"2024-09-21T09:57:56.335053+0000","last_became_active":"2024-09-21T09:46:13.255014+0000","last_became_peered":"2024-09-21T09:46:13.255014+0000","last_unstale":"2024-09-21T09:57:56.335053+0000","last_undegraded":"2024-09-21T09:57:56.335053+0000","last_fullsized":"2024-09-21T09:57:56.335053+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:53.012266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1602,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.334823+0000","last_change":"2024-09-21T09:45:39.134465+0000","last_active":"2024-09-21T09:57:56.334823+0000","last_peered":"2024-09-21T09:57:56.334823+0000","last_clean":"2024-09-21T09:57:56.334823+0000","last_became_active":"2024-09-21T09:45:39.129692+0000","last_became_peered":"2024-09-21T09:45:39.129692+0000","last_unstale":"2024-09-21T09:57:56.334823+0000","last_undegraded":"2024-09-21T09:57:56.334823+0000","last_fullsized":"2024-09-21T09:57:56.334823+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:14.288180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1104'5739","reported_seq":5844,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.154531+0000","last_change":"2024-09-21T09:57:10.193508+0000","last_active":"2024-09-21T09:58:10.154531+0000","last_peered":"2024-09-21T09:58:10.154531+0000","last_clean":"2024-09-21T09:58:10.154531+0000","last_became_active":"2024-09-21T09:57:07.410893+0000","last_became_peered":"2024-09-21T09:57:07.410893+0000","last_unstale":"2024-09-21T09:58:10.154531+0000","last_undegraded":"2024-09-21T09:58:10.154531+0000","last_fullsized":"2024-09-21T09:58:10.154531+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'3968","last_scrub_stamp":"2024-09-21T09:57:10.193203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:10.193203+0000","objects_scrubbed":2031,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:17.441883+0000","scrub_duration":533,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234815488,"num_objects":3583,"num_object_clones":0,"num_object_copies":10749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1786,"num_write_kb":106816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":35,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.334826+0000","last_change":"2024-09-21T09:57:39.646219+0000","last_active":"2024-09-21T09:57:56.334826+0000","last_peered":"2024-09-21T09:57:56.334826+0000","last_clean":"2024-09-21T09:57:56.334826+0000","last_became_active":"2024-09-21T09:57:39.645883+0000","last_became_peered":"2024-09-21T09:57:39.645883+0000","last_unstale":"2024-09-21T09:57:56.334826+0000","last_undegraded":"2024-09-21T09:57:56.334826+0000","last_fullsized":"2024-09-21T09:57:56.334826+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:07.561254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1046,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667488+0000","last_change":"2024-09-21T09:45:05.097518+0000","last_active":"2024-09-21T09:57:41.667488+0000","last_peered":"2024-09-21T09:57:41.667488+0000","last_clean":"2024-09-21T09:57:41.667488+0000","last_became_active":"2024-09-21T09:45:05.096918+0000","last_became_peered":"2024-09-21T09:45:05.096918+0000","last_unstale":"2024-09-21T09:57:41.667488+0000","last_undegraded":"2024-09-21T09:57:41.667488+0000","last_fullsized":"2024-09-21T09:57:41.667488+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:13.781847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1602,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.334840+0000","last_change":"2024-09-21T09:46:47.113687+0000","last_active":"2024-09-21T09:57:56.334840+0000","last_peered":"2024-09-21T09:57:56.334840+0000","last_clean":"2024-09-21T09:57:56.334840+0000","last_became_active":"2024-09-21T09:46:47.113235+0000","last_became_peered":"2024-09-21T09:46:47.113235+0000","last_unstale":"2024-09-21T09:57:56.334840+0000","last_undegraded":"2024-09-21T09:57:56.334840+0000","last_fullsized":"2024-09-21T09:57:56.334840+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:50.851616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1104'5829","reported_seq":5933,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:13.239617+0000","last_change":"2024-09-21T09:57:12.654933+0000","last_active":"2024-09-21T09:58:13.239617+0000","last_peered":"2024-09-21T09:58:13.239617+0000","last_clean":"2024-09-21T09:58:13.239617+0000","last_became_active":"2024-09-21T09:57:07.435577+0000","last_became_peered":"2024-09-21T09:57:07.435577+0000","last_unstale":"2024-09-21T09:58:13.239617+0000","last_undegraded":"2024-09-21T09:58:13.239617+0000","last_fullsized":"2024-09-21T09:58:13.239617+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4081","last_scrub_stamp":"2024-09-21T09:57:12.654694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:12.654694+0000","objects_scrubbed":2087,"log_size":5829,"log_dups_size":0,"ondisk_log_size":5829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:02.588703+0000","scrub_duration":664,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229179392,"num_objects":3497,"num_object_clones":0,"num_object_copies":10491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1760,"num_write_kb":101440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":558,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.345358+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T09:57:56.345358+0000","last_peered":"2024-09-21T09:57:56.345358+0000","last_clean":"2024-09-21T09:57:56.345358+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T09:57:56.345358+0000","last_undegraded":"2024-09-21T09:57:56.345358+0000","last_fullsized":"2024-09-21T09:57:56.345358+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1616,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.334943+0000","last_change":"2024-09-21T09:45:39.132441+0000","last_active":"2024-09-21T09:57:56.334943+0000","last_peered":"2024-09-21T09:57:56.334943+0000","last_clean":"2024-09-21T09:57:56.334943+0000","last_became_active":"2024-09-21T09:45:39.129666+0000","last_became_peered":"2024-09-21T09:45:39.129666+0000","last_unstale":"2024-09-21T09:57:56.334943+0000","last_undegraded":"2024-09-21T09:57:56.334943+0000","last_fullsized":"2024-09-21T09:57:56.334943+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:34.450160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1554,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.344735+0000","last_change":"2024-09-21T09:46:13.258244+0000","last_active":"2024-09-21T09:57:56.344735+0000","last_peered":"2024-09-21T09:57:56.344735+0000","last_clean":"2024-09-21T09:57:56.344735+0000","last_became_active":"2024-09-21T09:46:13.257820+0000","last_became_peered":"2024-09-21T09:46:13.257820+0000","last_unstale":"2024-09-21T09:57:56.344735+0000","last_undegraded":"2024-09-21T09:57:56.344735+0000","last_fullsized":"2024-09-21T09:57:56.344735+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:04.618304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.1","version":"1104'5704","reported_seq":5809,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:12.155575+0000","last_change":"2024-09-21T09:57:09.426376+0000","last_active":"2024-09-21T09:58:12.155575+0000","last_peered":"2024-09-21T09:58:12.155575+0000","last_clean":"2024-09-21T09:58:12.155575+0000","last_became_active":"2024-09-21T09:57:07.399441+0000","last_became_peered":"2024-09-21T09:57:07.399441+0000","last_unstale":"2024-09-21T09:58:12.155575+0000","last_undegraded":"2024-09-21T09:58:12.155575+0000","last_fullsized":"2024-09-21T09:58:12.155575+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'3855","last_scrub_stamp":"2024-09-21T09:57:09.426224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:09.426224+0000","objects_scrubbed":1962,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:08:05.610483+0000","scrub_duration":531,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228655104,"num_objects":3489,"num_object_clones":0,"num_object_copies":10467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1863,"num_write_kb":108480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1592,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.335375+0000","last_change":"2024-09-21T09:46:13.260666+0000","last_active":"2024-09-21T09:57:56.335375+0000","last_peered":"2024-09-21T09:57:56.335375+0000","last_clean":"2024-09-21T09:57:56.335375+0000","last_became_active":"2024-09-21T09:46:13.260324+0000","last_became_peered":"2024-09-21T09:46:13.260324+0000","last_unstale":"2024-09-21T09:57:56.335375+0000","last_undegraded":"2024-09-21T09:57:56.335375+0000","last_fullsized":"2024-09-21T09:57:56.335375+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:25.140797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1039,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.642226+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T09:57:41.642226+0000","last_peered":"2024-09-21T09:57:41.642226+0000","last_clean":"2024-09-21T09:57:41.642226+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T09:57:41.642226+0000","last_undegraded":"2024-09-21T09:57:41.642226+0000","last_fullsized":"2024-09-21T09:57:41.642226+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1618,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.349763+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T09:57:56.349763+0000","last_peered":"2024-09-21T09:57:56.349763+0000","last_clean":"2024-09-21T09:57:56.349763+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T09:57:56.349763+0000","last_undegraded":"2024-09-21T09:57:56.349763+0000","last_fullsized":"2024-09-21T09:57:56.349763+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1104'5833","reported_seq":5979,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.889929+0000","last_change":"2024-09-21T09:57:12.078248+0000","last_active":"2024-09-21T09:58:10.889929+0000","last_peered":"2024-09-21T09:58:10.889929+0000","last_clean":"2024-09-21T09:58:10.889929+0000","last_became_active":"2024-09-21T09:57:07.419410+0000","last_became_peered":"2024-09-21T09:57:07.419410+0000","last_unstale":"2024-09-21T09:58:10.889929+0000","last_undegraded":"2024-09-21T09:58:10.889929+0000","last_fullsized":"2024-09-21T09:58:10.889929+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4085","last_scrub_stamp":"2024-09-21T09:57:12.078156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:12.078156+0000","objects_scrubbed":2168,"log_size":5833,"log_dups_size":0,"ondisk_log_size":5833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:31:56.497563+0000","scrub_duration":0.59593628200000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239206400,"num_objects":3650,"num_object_clones":0,"num_object_copies":10950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1758,"num_write_kb":103680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1553,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.348960+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T09:57:56.348960+0000","last_peered":"2024-09-21T09:57:56.348960+0000","last_clean":"2024-09-21T09:57:56.348960+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T09:57:56.348960+0000","last_undegraded":"2024-09-21T09:57:56.348960+0000","last_fullsized":"2024-09-21T09:57:56.348960+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1589,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.338652+0000","last_change":"2024-09-21T09:45:39.130168+0000","last_active":"2024-09-21T09:57:56.338652+0000","last_peered":"2024-09-21T09:57:56.338652+0000","last_clean":"2024-09-21T09:57:56.338652+0000","last_became_active":"2024-09-21T09:45:39.129498+0000","last_became_peered":"2024-09-21T09:45:39.129498+0000","last_unstale":"2024-09-21T09:57:56.338652+0000","last_undegraded":"2024-09-21T09:57:56.338652+0000","last_fullsized":"2024-09-21T09:57:56.338652+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:58.310876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1550,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.349160+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T09:57:56.349160+0000","last_peered":"2024-09-21T09:57:56.349160+0000","last_clean":"2024-09-21T09:57:56.349160+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T09:57:56.349160+0000","last_undegraded":"2024-09-21T09:57:56.349160+0000","last_fullsized":"2024-09-21T09:57:56.349160+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1104'6170","reported_seq":6311,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.843888+0000","last_change":"2024-09-21T09:57:11.657896+0000","last_active":"2024-09-21T09:58:10.843888+0000","last_peered":"2024-09-21T09:58:10.843888+0000","last_clean":"2024-09-21T09:58:10.843888+0000","last_became_active":"2024-09-21T09:57:07.419558+0000","last_became_peered":"2024-09-21T09:57:07.419558+0000","last_unstale":"2024-09-21T09:58:10.843888+0000","last_undegraded":"2024-09-21T09:58:10.843888+0000","last_fullsized":"2024-09-21T09:58:10.843888+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4365","last_scrub_stamp":"2024-09-21T09:57:11.657796+0000","last_deep_scrub":"1087'4365","last_deep_scrub_stamp":"2024-09-21T09:57:11.657796+0000","last_clean_scrub_stamp":"2024-09-21T09:57:11.657796+0000","objects_scrubbed":2040,"log_size":6170,"log_dups_size":0,"ondisk_log_size":6170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":3,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:08.924721+0000","scrub_duration":2.8729851470000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240058368,"num_objects":3663,"num_object_clones":0,"num_object_copies":10989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1875,"num_write_kb":111936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1631,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.351005+0000","last_change":"2024-09-21T09:45:39.140762+0000","last_active":"2024-09-21T09:57:56.351005+0000","last_peered":"2024-09-21T09:57:56.351005+0000","last_clean":"2024-09-21T09:57:56.351005+0000","last_became_active":"2024-09-21T09:45:39.140402+0000","last_became_peered":"2024-09-21T09:45:39.140402+0000","last_unstale":"2024-09-21T09:57:56.351005+0000","last_undegraded":"2024-09-21T09:57:56.351005+0000","last_fullsized":"2024-09-21T09:57:56.351005+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:06.545428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1040,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.674452+0000","last_change":"2024-09-21T09:46:13.260656+0000","last_active":"2024-09-21T09:57:41.674452+0000","last_peered":"2024-09-21T09:57:41.674452+0000","last_clean":"2024-09-21T09:57:41.674452+0000","last_became_active":"2024-09-21T09:46:13.260368+0000","last_became_peered":"2024-09-21T09:46:13.260368+0000","last_unstale":"2024-09-21T09:57:41.674452+0000","last_undegraded":"2024-09-21T09:57:41.674452+0000","last_fullsized":"2024-09-21T09:57:41.674452+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:22.059036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1028,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667262+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:57:41.667262+0000","last_peered":"2024-09-21T09:57:41.667262+0000","last_clean":"2024-09-21T09:57:41.667262+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:57:41.667262+0000","last_undegraded":"2024-09-21T09:57:41.667262+0000","last_fullsized":"2024-09-21T09:57:41.667262+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.11","version":"1104'5722","reported_seq":9920,"reported_epoch":1104,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-21T09:58:12.285332+0000","last_change":"2024-09-21T09:57:40.110836+0000","last_active":"2024-09-21T09:58:12.285332+0000","last_peered":"2024-09-21T09:58:12.285332+0000","last_clean":"2024-09-21T09:57:15.457720+0000","last_became_active":"2024-09-21T09:57:19.166511+0000","last_became_peered":"2024-09-21T09:57:19.166511+0000","last_unstale":"2024-09-21T09:58:12.285332+0000","last_undegraded":"2024-09-21T09:58:12.285332+0000","last_fullsized":"2024-09-21T09:57:18.457573+0000","mapping_epoch":1091,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'3973","last_scrub_stamp":"2024-09-21T09:57:11.468690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:11.468690+0000","objects_scrubbed":2101,"log_size":5722,"log_dups_size":0,"ondisk_log_size":5722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":570,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229048320,"num_objects":3495,"num_object_clones":0,"num_object_copies":10485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3669,"num_objects_unfound":0,"num_objects_dirty":3495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1760,"num_write_kb":100928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1023,"num_bytes_recovered":67043328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[1,0],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"0,1","objects":2272},{"shards":"1,2,6","objects":1223}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1045,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667425+0000","last_change":"2024-09-21T09:45:39.129538+0000","last_active":"2024-09-21T09:57:41.667425+0000","last_peered":"2024-09-21T09:57:41.667425+0000","last_clean":"2024-09-21T09:57:41.667425+0000","last_became_active":"2024-09-21T09:45:39.129198+0000","last_became_peered":"2024-09-21T09:45:39.129198+0000","last_unstale":"2024-09-21T09:57:41.667425+0000","last_undegraded":"2024-09-21T09:57:41.667425+0000","last_fullsized":"2024-09-21T09:57:41.667425+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.798277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1051,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.669601+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:57:41.669601+0000","last_peered":"2024-09-21T09:57:41.669601+0000","last_clean":"2024-09-21T09:57:41.669601+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:57:41.669601+0000","last_undegraded":"2024-09-21T09:57:41.669601+0000","last_fullsized":"2024-09-21T09:57:41.669601+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.10","version":"1104'5864","reported_seq":12869,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:13.236002+0000","last_change":"2024-09-21T09:57:41.648481+0000","last_active":"2024-09-21T09:58:13.236002+0000","last_peered":"2024-09-21T09:58:13.236002+0000","last_clean":"2024-09-21T09:58:13.236002+0000","last_became_active":"2024-09-21T09:57:41.645466+0000","last_became_peered":"2024-09-21T09:57:41.645466+0000","last_unstale":"2024-09-21T09:58:13.236002+0000","last_undegraded":"2024-09-21T09:58:13.236002+0000","last_fullsized":"2024-09-21T09:58:13.236002+0000","mapping_epoch":1102,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":1099,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232390656,"num_objects":3546,"num_object_clones":0,"num_object_copies":10638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3942,"num_write_kb":239616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2302,"num_bytes_recovered":150863872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1590,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.334960+0000","last_change":"2024-09-21T09:46:13.254014+0000","last_active":"2024-09-21T09:57:56.334960+0000","last_peered":"2024-09-21T09:57:56.334960+0000","last_clean":"2024-09-21T09:57:56.334960+0000","last_became_active":"2024-09-21T09:46:13.253673+0000","last_became_peered":"2024-09-21T09:46:13.253673+0000","last_unstale":"2024-09-21T09:57:56.334960+0000","last_undegraded":"2024-09-21T09:57:56.334960+0000","last_fullsized":"2024-09-21T09:57:56.334960+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:02.784613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1030,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.642145+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T09:57:41.642145+0000","last_peered":"2024-09-21T09:57:41.642145+0000","last_clean":"2024-09-21T09:57:41.642145+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T09:57:41.642145+0000","last_undegraded":"2024-09-21T09:57:41.642145+0000","last_fullsized":"2024-09-21T09:57:41.642145+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.13","version":"1104'5771","reported_seq":5828,"reported_epoch":1104,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:11.253502+0000","last_change":"2024-09-21T09:57:22.557056+0000","last_active":"2024-09-21T09:58:11.253502+0000","last_peered":"2024-09-21T09:58:11.253502+0000","last_clean":"2024-09-21T09:57:19.631866+0000","last_became_active":"2024-09-21T09:57:22.552789+0000","last_became_peered":"2024-09-21T09:57:22.552789+0000","last_unstale":"2024-09-21T09:58:11.253502+0000","last_undegraded":"2024-09-21T09:57:22.550233+0000","last_fullsized":"2024-09-21T09:57:22.463332+0000","mapping_epoch":1095,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":5771,"log_dups_size":0,"ondisk_log_size":5771,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236060672,"num_objects":3602,"num_object_clones":0,"num_object_copies":10806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2314,"num_objects_misplaced":2314,"num_objects_unfound":0,"num_objects_dirty":3602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3872,"num_write_kb":239168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[2,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6","objects":1288},{"shards":"2,6,7","objects":2179}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":474,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.642287+0000","last_change":"2024-09-21T09:57:39.544714+0000","last_active":"2024-09-21T09:57:41.642287+0000","last_peered":"2024-09-21T09:57:41.642287+0000","last_clean":"2024-09-21T09:57:41.642287+0000","last_became_active":"2024-09-21T09:57:39.544459+0000","last_became_peered":"2024-09-21T09:57:39.544459+0000","last_unstale":"2024-09-21T09:57:41.642287+0000","last_undegraded":"2024-09-21T09:57:41.642287+0000","last_fullsized":"2024-09-21T09:57:41.642287+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:21.280135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1525,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.345018+0000","last_change":"2024-09-21T09:46:47.122643+0000","last_active":"2024-09-21T09:57:56.345018+0000","last_peered":"2024-09-21T09:57:56.345018+0000","last_clean":"2024-09-21T09:57:56.345018+0000","last_became_active":"2024-09-21T09:46:47.122364+0000","last_became_peered":"2024-09-21T09:46:47.122364+0000","last_unstale":"2024-09-21T09:57:56.345018+0000","last_undegraded":"2024-09-21T09:57:56.345018+0000","last_fullsized":"2024-09-21T09:57:56.345018+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:29.222333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.12","version":"1104'5773","reported_seq":10404,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.774104+0000","last_change":"2024-09-21T09:57:40.561013+0000","last_active":"2024-09-21T09:58:10.774104+0000","last_peered":"2024-09-21T09:58:10.774104+0000","last_clean":"2024-09-21T09:58:10.774104+0000","last_became_active":"2024-09-21T09:57:40.544862+0000","last_became_peered":"2024-09-21T09:57:40.544862+0000","last_unstale":"2024-09-21T09:58:10.774104+0000","last_undegraded":"2024-09-21T09:58:10.774104+0000","last_fullsized":"2024-09-21T09:58:10.774104+0000","mapping_epoch":1101,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1102,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'4093","last_scrub_stamp":"2024-09-21T09:57:12.883129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:12.883129+0000","objects_scrubbed":2133,"log_size":5773,"log_dups_size":0,"ondisk_log_size":5773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:27.825610+0000","scrub_duration":0.369013857,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233963520,"num_objects":3570,"num_object_clones":0,"num_object_copies":10710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1681,"num_write_kb":99776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2235,"num_bytes_recovered":146472960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1626,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.350650+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T09:57:56.350650+0000","last_peered":"2024-09-21T09:57:56.350650+0000","last_clean":"2024-09-21T09:57:56.350650+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T09:57:56.350650+0000","last_undegraded":"2024-09-21T09:57:56.350650+0000","last_fullsized":"2024-09-21T09:57:56.350650+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1542,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.345760+0000","last_change":"2024-09-21T09:46:47.121360+0000","last_active":"2024-09-21T09:57:56.345760+0000","last_peered":"2024-09-21T09:57:56.345760+0000","last_clean":"2024-09-21T09:57:56.345760+0000","last_became_active":"2024-09-21T09:46:47.120494+0000","last_became_peered":"2024-09-21T09:46:47.120494+0000","last_unstale":"2024-09-21T09:57:56.345760+0000","last_undegraded":"2024-09-21T09:57:56.345760+0000","last_fullsized":"2024-09-21T09:57:56.345760+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:45.862984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1037,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.674380+0000","last_change":"2024-09-21T09:46:13.260037+0000","last_active":"2024-09-21T09:57:41.674380+0000","last_peered":"2024-09-21T09:57:41.674380+0000","last_clean":"2024-09-21T09:57:41.674380+0000","last_became_active":"2024-09-21T09:46:13.259690+0000","last_became_peered":"2024-09-21T09:46:13.259690+0000","last_unstale":"2024-09-21T09:57:41.674380+0000","last_undegraded":"2024-09-21T09:57:41.674380+0000","last_fullsized":"2024-09-21T09:57:41.674380+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:57.466762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1560,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.349299+0000","last_change":"2024-09-21T09:46:47.120830+0000","last_active":"2024-09-21T09:57:56.349299+0000","last_peered":"2024-09-21T09:57:56.349299+0000","last_clean":"2024-09-21T09:57:56.349299+0000","last_became_active":"2024-09-21T09:46:47.118419+0000","last_became_peered":"2024-09-21T09:46:47.118419+0000","last_unstale":"2024-09-21T09:57:56.349299+0000","last_undegraded":"2024-09-21T09:57:56.349299+0000","last_fullsized":"2024-09-21T09:57:56.349299+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:13.789806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1629,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.349882+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T09:57:56.349882+0000","last_peered":"2024-09-21T09:57:56.349882+0000","last_clean":"2024-09-21T09:57:56.349882+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T09:57:56.349882+0000","last_undegraded":"2024-09-21T09:57:56.349882+0000","last_fullsized":"2024-09-21T09:57:56.349882+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1591,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.338217+0000","last_change":"2024-09-21T09:46:47.121724+0000","last_active":"2024-09-21T09:57:56.338217+0000","last_peered":"2024-09-21T09:57:56.338217+0000","last_clean":"2024-09-21T09:57:56.338217+0000","last_became_active":"2024-09-21T09:46:47.121406+0000","last_became_peered":"2024-09-21T09:46:47.121406+0000","last_unstale":"2024-09-21T09:57:56.338217+0000","last_undegraded":"2024-09-21T09:57:56.338217+0000","last_fullsized":"2024-09-21T09:57:56.338217+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:55.493704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1542,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.345138+0000","last_change":"2024-09-21T09:46:47.113184+0000","last_active":"2024-09-21T09:57:56.345138+0000","last_peered":"2024-09-21T09:57:56.345138+0000","last_clean":"2024-09-21T09:57:56.345138+0000","last_became_active":"2024-09-21T09:46:47.112857+0000","last_became_peered":"2024-09-21T09:46:47.112857+0000","last_unstale":"2024-09-21T09:57:56.345138+0000","last_undegraded":"2024-09-21T09:57:56.345138+0000","last_fullsized":"2024-09-21T09:57:56.345138+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:46.783986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1064,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667528+0000","last_change":"2024-09-21T09:46:13.262555+0000","last_active":"2024-09-21T09:57:41.667528+0000","last_peered":"2024-09-21T09:57:41.667528+0000","last_clean":"2024-09-21T09:57:41.667528+0000","last_became_active":"2024-09-21T09:46:13.261474+0000","last_became_peered":"2024-09-21T09:46:13.261474+0000","last_unstale":"2024-09-21T09:57:41.667528+0000","last_undegraded":"2024-09-21T09:57:41.667528+0000","last_fullsized":"2024-09-21T09:57:41.667528+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:44.865988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1580,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.344419+0000","last_change":"2024-09-21T09:46:47.114406+0000","last_active":"2024-09-21T09:57:56.344419+0000","last_peered":"2024-09-21T09:57:56.344419+0000","last_clean":"2024-09-21T09:57:56.344419+0000","last_became_active":"2024-09-21T09:46:47.113986+0000","last_became_peered":"2024-09-21T09:46:47.113986+0000","last_unstale":"2024-09-21T09:57:56.344419+0000","last_undegraded":"2024-09-21T09:57:56.344419+0000","last_fullsized":"2024-09-21T09:57:56.344419+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:32.292201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1619,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.349807+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T09:57:56.349807+0000","last_peered":"2024-09-21T09:57:56.349807+0000","last_clean":"2024-09-21T09:57:56.349807+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T09:57:56.349807+0000","last_undegraded":"2024-09-21T09:57:56.349807+0000","last_fullsized":"2024-09-21T09:57:56.349807+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"1104'5817","reported_seq":5922,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:13.239742+0000","last_change":"2024-09-21T09:57:11.595394+0000","last_active":"2024-09-21T09:58:13.239742+0000","last_peered":"2024-09-21T09:58:13.239742+0000","last_clean":"2024-09-21T09:58:13.239742+0000","last_became_active":"2024-09-21T09:57:07.422449+0000","last_became_peered":"2024-09-21T09:57:07.422449+0000","last_unstale":"2024-09-21T09:58:13.239742+0000","last_undegraded":"2024-09-21T09:58:13.239742+0000","last_fullsized":"2024-09-21T09:58:13.239742+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'4025","last_scrub_stamp":"2024-09-21T09:57:11.595114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:11.595114+0000","objects_scrubbed":2093,"log_size":5817,"log_dups_size":0,"ondisk_log_size":5817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:49.720861+0000","scrub_duration":566,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228392960,"num_objects":3485,"num_object_clones":0,"num_object_copies":10455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1804,"num_write_kb":102272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1062,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667366+0000","last_change":"2024-09-21T09:46:47.114188+0000","last_active":"2024-09-21T09:57:41.667366+0000","last_peered":"2024-09-21T09:57:41.667366+0000","last_clean":"2024-09-21T09:57:41.667366+0000","last_became_active":"2024-09-21T09:46:47.113859+0000","last_became_peered":"2024-09-21T09:46:47.113859+0000","last_unstale":"2024-09-21T09:57:41.667366+0000","last_undegraded":"2024-09-21T09:57:41.667366+0000","last_fullsized":"2024-09-21T09:57:41.667366+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:22.670368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":26,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.669518+0000","last_change":"2024-09-21T09:57:39.543638+0000","last_active":"2024-09-21T09:57:41.669518+0000","last_peered":"2024-09-21T09:57:41.669518+0000","last_clean":"2024-09-21T09:57:41.669518+0000","last_became_active":"2024-09-21T09:57:39.543224+0000","last_became_peered":"2024-09-21T09:57:39.543224+0000","last_unstale":"2024-09-21T09:57:41.669518+0000","last_undegraded":"2024-09-21T09:57:41.669518+0000","last_fullsized":"2024-09-21T09:57:41.669518+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:06.898662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"1104'5812","reported_seq":6006,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:58:10.787499+0000","last_change":"2024-09-21T09:57:58.103549+0000","last_active":"2024-09-21T09:58:10.787499+0000","last_peered":"2024-09-21T09:58:10.787499+0000","last_clean":"2024-09-21T09:58:10.787499+0000","last_became_active":"2024-09-21T09:57:07.454647+0000","last_became_peered":"2024-09-21T09:57:07.454647+0000","last_unstale":"2024-09-21T09:58:10.787499+0000","last_undegraded":"2024-09-21T09:58:10.787499+0000","last_fullsized":"2024-09-21T09:58:10.787499+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1104'5399","last_scrub_stamp":"2024-09-21T09:57:58.103403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:58.103403+0000","objects_scrubbed":3448,"log_size":5812,"log_dups_size":0,"ondisk_log_size":5812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:15.110839+0000","scrub_duration":1.1907735719999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240648192,"num_objects":3672,"num_object_clones":0,"num_object_copies":11016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":426,"num_write_kb":20800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1062,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667478+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T09:57:41.667478+0000","last_peered":"2024-09-21T09:57:41.667478+0000","last_clean":"2024-09-21T09:57:41.667478+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T09:57:41.667478+0000","last_undegraded":"2024-09-21T09:57:41.667478+0000","last_fullsized":"2024-09-21T09:57:41.667478+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":26,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667582+0000","last_change":"2024-09-21T09:57:39.541586+0000","last_active":"2024-09-21T09:57:41.667582+0000","last_peered":"2024-09-21T09:57:41.667582+0000","last_clean":"2024-09-21T09:57:41.667582+0000","last_became_active":"2024-09-21T09:57:39.541229+0000","last_became_peered":"2024-09-21T09:57:39.541229+0000","last_unstale":"2024-09-21T09:57:41.667582+0000","last_undegraded":"2024-09-21T09:57:41.667582+0000","last_fullsized":"2024-09-21T09:57:41.667582+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:20.209058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1552,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.345317+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T09:57:56.345317+0000","last_peered":"2024-09-21T09:57:56.345317+0000","last_clean":"2024-09-21T09:57:56.345317+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T09:57:56.345317+0000","last_undegraded":"2024-09-21T09:57:56.345317+0000","last_fullsized":"2024-09-21T09:57:56.345317+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1538,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.345676+0000","last_change":"2024-09-21T09:46:47.115788+0000","last_active":"2024-09-21T09:57:56.345676+0000","last_peered":"2024-09-21T09:57:56.345676+0000","last_clean":"2024-09-21T09:57:56.345676+0000","last_became_active":"2024-09-21T09:46:47.114864+0000","last_became_peered":"2024-09-21T09:46:47.114864+0000","last_unstale":"2024-09-21T09:57:56.345676+0000","last_undegraded":"2024-09-21T09:57:56.345676+0000","last_fullsized":"2024-09-21T09:57:56.345676+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:00.034944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1045,"reported_epoch":1103,"state":"active+clean","last_fresh":"2024-09-21T09:57:41.667174+0000","last_change":"2024-09-21T09:45:39.143064+0000","last_active":"2024-09-21T09:57:41.667174+0000","last_peered":"2024-09-21T09:57:41.667174+0000","last_clean":"2024-09-21T09:57:41.667174+0000","last_became_active":"2024-09-21T09:45:39.142699+0000","last_became_peered":"2024-09-21T09:45:39.142699+0000","last_unstale":"2024-09-21T09:57:41.667174+0000","last_undegraded":"2024-09-21T09:57:41.667174+0000","last_fullsized":"2024-09-21T09:57:41.667174+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:14.684721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1634,"reported_epoch":1104,"state":"active+clean","last_fresh":"2024-09-21T09:57:56.350741+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T09:57:56.350741+0000","last_peered":"2024-09-21T09:57:56.350741+0000","last_clean":"2024-09-21T09:57:56.350741+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T09:57:56.350741+0000","last_undegraded":"2024-09-21T09:57:56.350741+0000","last_fullsized":"2024-09-21T09:57:56.350741+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":7510818840,"num_objects":114607,"num_object_clones":0,"num_object_copies":343821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6594,"num_objects_misplaced":8123,"num_objects_unfound":0,"num_objects_dirty":114607,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":83966,"num_write_kb":5093057,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5718,"num_bytes_recovered":374734848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":23505612800,"data_stored":23505600584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":158276,"ondisk_log_size":158276,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397788,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3338884,"kb_used_data":3032956,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":90406268,"statfs":{"total":95995035648,"available":92576018432,"internally_reserved":0,"allocated":3105746944,"data_stored":3103525678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594017,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2553212,"kb_used_data":2247540,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91191940,"statfs":{"total":95995035648,"available":93380546560,"internally_reserved":0,"allocated":2301480960,"data_stored":2299267751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790246,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2904052,"kb_used_data":2606700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":90841100,"statfs":{"total":95995035648,"available":93021286400,"internally_reserved":0,"allocated":2669260800,"data_stored":2667051089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986473,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2711612,"kb_used_data":2401780,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91033540,"statfs":{"total":95995035648,"available":93218344960,"internally_reserved":0,"allocated":2459422720,"data_stored":2457209024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999115,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4779048,"kb_used_data":4634784,"kb_used_omap":25,"kb_used_meta":144230,"kb_avail":88966104,"statfs":{"total":95995035648,"available":91101290496,"internally_reserved":0,"allocated":4746018816,"data_stored":4743820812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":147692487},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111058,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3743148,"kb_used_data":3607012,"kb_used_omap":31,"kb_used_meta":136096,"kb_avail":90002004,"statfs":{"total":95995035648,"available":92162052096,"internally_reserved":0,"allocated":3693580288,"data_stored":3691378710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":139362536},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318223001,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2598444,"kb_used_data":2494372,"kb_used_omap":26,"kb_used_meta":104037,"kb_avail":91146708,"statfs":{"total":95995035648,"available":93334228992,"internally_reserved":0,"allocated":2554236928,"data_stored":2552034789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":106533900},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289334944,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2156404,"kb_used_data":2048108,"kb_used_omap":31,"kb_used_meta":108256,"kb_avail":91588748,"statfs":{"total":95995035648,"available":93786877952,"internally_reserved":0,"allocated":2097262592,"data_stored":2095053378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":110854240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2085294080,"data_stored":2085290008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2546208768,"data_stored":2546204696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3680763904,"data_stored":3680763904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4731174912,"data_stored":4731174912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2447769600,"data_stored":2447769600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2658537472,"data_stored":2658533400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2278359040,"data_stored":2278359040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3077505024,"data_stored":3077505024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:58:16.869 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T09:58:16.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pg_num 11 2024-09-21T09:58:16.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:16.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:17.070 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:17.096 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:17.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:17.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T09:58:17.198 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:17.225 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:17.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:17.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:17.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:17.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:17.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:17.488 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:17 smithi017 ceph-mon[102913]: from='client.54868 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:17.489 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:17 smithi017 ceph-mon[102913]: pgmap v1320: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 974 op/s; 6594/335877 objects degraded (1.963%); 8123/335877 objects misplaced (2.418%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T09:58:17.489 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:17 smithi017 ceph-mon[105151]: from='client.54868 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:17.489 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:17 smithi017 ceph-mon[105151]: pgmap v1320: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 974 op/s; 6594/335877 objects degraded (1.963%); 8123/335877 objects misplaced (2.418%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T09:58:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:17.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:17.650 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.679 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:17.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:17 smithi134 ceph-mon[90085]: from='client.54868 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:17 smithi134 ceph-mon[90085]: pgmap v1320: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 17 MiB/s wr, 974 op/s; 6594/335877 objects degraded (1.963%); 8123/335877 objects misplaced (2.418%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T09:58:17.780 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:17.804 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:17.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:17.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:17.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:18.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:18.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:18.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:18.182 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:18.209 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:18.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:18.311 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:18.338 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:18.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:18.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:18.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:18.589 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:18.617 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:18.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:18 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6594/335877 objects degraded (1.963%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:18.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:18 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6594/335877 objects degraded (1.963%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:18.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:18.727 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:18 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6594/335877 objects degraded (1.963%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:18.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:18.820 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:58:18.821 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:18.863 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:58:18.863 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:18.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:18.897 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:58:18.897 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:18.919 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:18.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:18.943 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:58:18.944 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:18.977 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:58:18.977 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:19.019 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:58:19.019 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:19.022 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:19.044 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:19.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:58:19.059 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:19.093 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:58:19.093 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:19.146 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:19.172 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:19.274 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:19.303 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:19.405 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.432 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:19.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:19.533 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:19.546 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:19 smithi017 ceph-mon[105151]: pgmap v1321: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 8.4 MiB/s wr, 959 op/s; 6300/332082 objects degraded (1.897%); 7364/332082 objects misplaced (2.218%); 4.1 MiB/s, 64 objects/s recovering 2024-09-21T09:58:19.546 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:19 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2710958174' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:19.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:19 smithi017 ceph-mon[102913]: pgmap v1321: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 8.4 MiB/s wr, 959 op/s; 6300/332082 objects degraded (1.897%); 7364/332082 objects misplaced (2.218%); 4.1 MiB/s, 64 objects/s recovering 2024-09-21T09:58:19.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:19 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2710958174' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:19.559 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:19.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:19.583 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:19.583 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:19.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:19.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:19 smithi134 ceph-mon[90085]: pgmap v1321: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 8.4 MiB/s wr, 959 op/s; 6300/332082 objects degraded (1.897%); 7364/332082 objects misplaced (2.218%); 4.1 MiB/s, 64 objects/s recovering 2024-09-21T09:58:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:19 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2710958174' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:19.805 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:19.835 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:19.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:19.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:19.980 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:19.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:20.082 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:20.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:20.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:20.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:20.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:20.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:20.356 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:20.382 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:20.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:20.483 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:20.493 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:58:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:58:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:20.508 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:20.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:20.609 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:20.633 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:20.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:20.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:20.774 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:20.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:20.872 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:20.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:20.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:21.024 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:21.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:21.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:21.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:21.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:21.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:21.306 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:21.332 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:21.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:21.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:21.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:21.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:21.576 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:21.608 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:21.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:21.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:21 smithi017 ceph-mon[102913]: pgmap v1322: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.5 MiB/s wr, 953 op/s; 6300/330228 objects degraded (1.908%); 7364/330228 objects misplaced (2.230%); 4.1 MiB/s, 64 objects/s recovering 2024-09-21T09:58:21.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:21 smithi017 ceph-mon[105151]: pgmap v1322: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.5 MiB/s wr, 953 op/s; 6300/330228 objects degraded (1.908%); 7364/330228 objects misplaced (2.230%); 4.1 MiB/s, 64 objects/s recovering 2024-09-21T09:58:21.710 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:21.736 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:21.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:21.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:21 smithi134 ceph-mon[90085]: pgmap v1322: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.5 MiB/s wr, 953 op/s; 6300/330228 objects degraded (1.908%); 7364/330228 objects misplaced (2.230%); 4.1 MiB/s, 64 objects/s recovering 2024-09-21T09:58:21.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:21.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:21.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:21.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:22.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:22.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:22.137 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:22.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:22.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:22.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:22.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:22.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:22.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:58:22.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:22.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:22.730 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:22.757 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:22.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:22.859 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:22.882 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:22.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:22.984 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:23.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.012 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:23.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:23.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:23.141 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:23.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:23.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:23.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:23.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:23.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:23.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:23.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:23.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:23.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:23.652 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:23.682 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:23.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:23 smithi017 ceph-mon[102913]: pgmap v1323: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 6300/319713 objects degraded (1.971%); 7364/319713 objects misplaced (2.303%); 4.1 MiB/s, 64 objects/s recovering 2024-09-21T09:58:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:23 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6300/319713 objects degraded (1.971%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:58:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:23 smithi017 ceph-mon[105151]: pgmap v1323: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 6300/319713 objects degraded (1.971%); 7364/319713 objects misplaced (2.303%); 4.1 MiB/s, 64 objects/s recovering 2024-09-21T09:58:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:23 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6300/319713 objects degraded (1.971%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:58:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:23 smithi134 ceph-mon[90085]: pgmap v1323: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 6300/319713 objects degraded (1.971%); 7364/319713 objects misplaced (2.303%); 4.1 MiB/s, 64 objects/s recovering 2024-09-21T09:58:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:23 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6300/319713 objects degraded (1.971%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:58:23.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:23.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:23.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:23.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:23.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:23.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:24.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:24.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:24.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:24.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:24.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T09:58:24.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:24.380 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:24.409 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:24.458 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pg_num to 11 2024-09-21T09:58:24.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:24.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:24.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:24.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2338140364' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T09:58:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:24 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T09:58:24.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2338140364' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T09:58:24.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:24 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T09:58:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:24.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:24.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2338140364' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T09:58:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:24 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T09:58:24.821 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:24.847 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:24.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:24.948 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:24.978 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:24.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:25.079 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:25.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:25.126 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:58:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:58:24] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-21T09:58:25.213 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:25.241 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:25.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:25.342 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:25.369 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:25.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:25.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:25.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:25.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:25.613 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:25.642 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:25.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[102913]: pgmap v1324: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 5986/314901 objects degraded (1.901%); 6598/314901 objects misplaced (2.095%); 4.4 MiB/s, 71 objects/s recovering 2024-09-21T09:58:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-21T09:58:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[102913]: osdmap e1106: 8 total, 8 up, 8 in 2024-09-21T09:58:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[105151]: pgmap v1324: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 5986/314901 objects degraded (1.901%); 6598/314901 objects misplaced (2.095%); 4.4 MiB/s, 71 objects/s recovering 2024-09-21T09:58:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-21T09:58:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[105151]: osdmap e1106: 8 total, 8 up, 8 in 2024-09-21T09:58:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:25 smithi134 ceph-mon[90085]: pgmap v1324: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 5986/314901 objects degraded (1.901%); 6598/314901 objects misplaced (2.095%); 4.4 MiB/s, 71 objects/s recovering 2024-09-21T09:58:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:25 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-21T09:58:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:25 smithi134 ceph-mon[90085]: osdmap e1106: 8 total, 8 up, 8 in 2024-09-21T09:58:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:25.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:25.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:25.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:25.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:25.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:25.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:26.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:26.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:26.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:26.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:26.206 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:26.307 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:26.333 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:26.434 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:26.626 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:26.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-21T09:58:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-21T09:58:26.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:26.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-21T09:58:26.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:26.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:26.882 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:26.908 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:26.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:27.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:27.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:27.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:27.149 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:27.181 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:27.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:27.310 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:27.412 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:27.438 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:27.475 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:27.484 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:27.504 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T09:58:27.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:27.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:27.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:27.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[105151]: pgmap v1326: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 5986/305895 objects degraded (1.957%); 6598/305895 objects misplaced (2.157%); 4.4 MiB/s, 71 objects/s recovering 2024-09-21T09:58:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-21T09:58:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[105151]: osdmap e1107: 8 total, 8 up, 8 in 2024-09-21T09:58:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1612554356' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[102913]: pgmap v1326: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 5986/305895 objects degraded (1.957%); 6598/305895 objects misplaced (2.157%); 4.4 MiB/s, 71 objects/s recovering 2024-09-21T09:58:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-21T09:58:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[102913]: osdmap e1107: 8 total, 8 up, 8 in 2024-09-21T09:58:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1612554356' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:27 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:27.723 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:27.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:27 smithi134 ceph-mon[90085]: pgmap v1326: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 5986/305895 objects degraded (1.957%); 6598/305895 objects misplaced (2.157%); 4.4 MiB/s, 71 objects/s recovering 2024-09-21T09:58:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-21T09:58:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:27 smithi134 ceph-mon[90085]: osdmap e1107: 8 total, 8 up, 8 in 2024-09-21T09:58:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:27 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1612554356' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:27 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:27.825 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:27.846 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:27.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:27.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:27.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:27.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:28.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:28.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:28.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:28.243 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:28.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:28.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:28.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:28.486 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:28.513 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:28.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:28.615 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:28.642 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:28.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:28 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:28.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:28 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:58:28.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:28 smithi134 ceph-mon[90085]: osdmap e1108: 8 total, 8 up, 8 in 2024-09-21T09:58:28.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:28 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5986/305895 objects degraded (1.957%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:28.746 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:28.772 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:28.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:28.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:28.894 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:28.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:28 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:28 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:58:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:28 smithi017 ceph-mon[102913]: osdmap e1108: 8 total, 8 up, 8 in 2024-09-21T09:58:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:28 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5986/305895 objects degraded (1.957%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:28 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:28 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:58:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:28 smithi017 ceph-mon[105151]: osdmap e1108: 8 total, 8 up, 8 in 2024-09-21T09:58:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:28 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5986/305895 objects degraded (1.957%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:28.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:29.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:29.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:29.137 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:29.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:29.260 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:29.286 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:29.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:29.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:29.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:29.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:29.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:29.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:29.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:29.693 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:29.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:29.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:29.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:29.940 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:29.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:29 smithi017 ceph-mon[102913]: pgmap v1329: 109 pgs: 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 5668/301812 objects degraded (1.878%); 5817/301812 objects misplaced (1.927%); 7.6 MiB/s, 120 objects/s recovering 2024-09-21T09:58:29.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:29 smithi017 ceph-mon[105151]: pgmap v1329: 109 pgs: 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 5668/301812 objects degraded (1.878%); 5817/301812 objects misplaced (1.927%); 7.6 MiB/s, 120 objects/s recovering 2024-09-21T09:58:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:29.960 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:29.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:29 smithi134 ceph-mon[90085]: pgmap v1329: 109 pgs: 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 5668/301812 objects degraded (1.878%); 5817/301812 objects misplaced (1.927%); 7.6 MiB/s, 120 objects/s recovering 2024-09-21T09:58:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:30.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:30.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:30.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:30.167 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:58:30.167 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:58:30.167 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-21T09:58:30.167 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T09:58:30.167 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T09:58:30.167 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-21T09:58:30.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:30.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:30.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:30.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:30.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:30.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:30.478 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:58:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:58:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:30.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:30.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:30.643 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:30.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:30.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:30.779 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:30.800 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:30.902 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:30.933 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:30.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:30.978 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:31.035 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:31.067 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:31.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:31.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:31.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:31.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:31.308 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:31.345 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:31.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:31.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:31.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:31.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:31.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:31.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:31 smithi134 ceph-mon[90085]: pgmap v1330: 109 pgs: 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 5668/300102 objects degraded (1.889%); 5817/300102 objects misplaced (1.938%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T09:58:31.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:31.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:31.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:31.916 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:31 smithi017 ceph-mon[102913]: pgmap v1330: 109 pgs: 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 5668/300102 objects degraded (1.889%); 5817/300102 objects misplaced (1.938%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T09:58:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:31 smithi017 ceph-mon[105151]: pgmap v1330: 109 pgs: 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 5668/300102 objects degraded (1.889%); 5817/300102 objects misplaced (1.938%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T09:58:32.019 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:32.060 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:32.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:32.162 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:32.185 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:32.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:32.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:32.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:32.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:32.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:32.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:32.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:32.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:32.602 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:32.704 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:32.730 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:32.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:32.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:32.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:32.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:32.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:33.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:33.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:33.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:33.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:33.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:33.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:33.270 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:33.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:33.402 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:33.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:33.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:33.644 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:33.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:33.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:33 smithi017 ceph-mon[102913]: pgmap v1331: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 105 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 5668/291042 objects degraded (1.947%); 5817/291042 objects misplaced (1.999%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T09:58:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5668/291042 objects degraded (1.947%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:33 smithi017 ceph-mon[105151]: pgmap v1331: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 105 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 5668/291042 objects degraded (1.947%); 5817/291042 objects misplaced (1.999%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T09:58:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5668/291042 objects degraded (1.947%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:33.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:33.813 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:33 smithi134 ceph-mon[90085]: pgmap v1331: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 105 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 5668/291042 objects degraded (1.947%); 5817/291042 objects misplaced (1.999%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T09:58:33.813 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5668/291042 objects degraded (1.947%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:33.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:33.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:33.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:34.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:34.092 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:58:34.093 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T09:58:34.093 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:58:34.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:34.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:34.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:34.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:34.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:34.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:34.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:34.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:34.518 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:34.547 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:34.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:34.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:34.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:34.791 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:34.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.825 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:34.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:34.897 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:58:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:58:34] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.51.0" 2024-09-21T09:58:34.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:34.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:34.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:35.074 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:35.107 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:35.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:35.209 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:35.236 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:35.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:35.338 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:35.370 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:35.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:35.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:35.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:35.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:35.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:35.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:35.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:35.755 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:35.784 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:35.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:35.915 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:35.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:35 smithi017 ceph-mon[102913]: pgmap v1332: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 105 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 5358/286935 objects degraded (1.867%); 5032/286935 objects misplaced (1.754%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T09:58:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:35 smithi017 ceph-mon[105151]: pgmap v1332: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 105 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 5358/286935 objects degraded (1.867%); 5032/286935 objects misplaced (1.754%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T09:58:35.955 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:35 smithi134 ceph-mon[90085]: pgmap v1332: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 105 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 5358/286935 objects degraded (1.867%); 5032/286935 objects misplaced (1.754%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T09:58:36.017 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:36.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T09:58:36.044 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:36.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:36.145 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:36.173 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:36.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:36.275 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:36.301 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:36.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:36.403 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:36.430 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:36.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:36.532 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:36.562 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:36.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:36.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:36.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:36.779 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:58:36.780 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:36.781 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:58:36.781 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:36.810 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:36.817 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:58:36.817 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:36.844 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:36.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:36.850 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:58:36.850 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:36.884 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:58:36.884 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:36.917 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:58:36.918 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:36.946 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:36.954 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:58:36.955 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:36.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:36.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:36.991 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:58:36.991 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:37.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:37.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:37.213 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:37.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:37.299 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:58:37.299 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:60908 6.4 e1085/1078)", 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.397099+0000", 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.645889353000001, 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.177237539, 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397099+0000", 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397099+0000", 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397100+0000", 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9999999999999995e-07 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397157+0000", 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6965000000000001e-05 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397167+0000", 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0339999999999999e-05 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397176+0000", 2024-09-21T09:58:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7460000000000006e-06 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.574318+0000", 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17714152499999999 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.574337+0000", 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8963000000000001e-05 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:60910 6.8 e1085/1078)", 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.398359+0000", 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.644628991999994, 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18159845899999999, 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.398359+0000", 2024-09-21T09:58:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.398359+0000", 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.398360+0000", 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7300000000000005e-07 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.398395+0000", 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4366000000000003e-05 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.398404+0000", 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9609999999999994e-06 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.398410+0000", 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5069999999999997e-06 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.579947+0000", 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18153715000000001 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.579958+0000", 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0601999999999999e-05 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:60914 6.8 e1085/1078)", 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.400933+0000", 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.642055913999997, 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17906060900000001, 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400933+0000", 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400933+0000", 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400933+0000", 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7700000000000003e-07 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400963+0000", 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9306999999999999e-05 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400974+0000", 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0842999999999999e-05 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.400980+0000", 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3330000000000002e-06 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.579981+0000", 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17900143099999999 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.579993+0000", 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1817999999999999e-05 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:60915 6.0 e1085/1078)", 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.402593+0000", 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.640395756000004, 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17736068299999999, 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402593+0000", 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402593+0000", 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402594+0000", 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0839999999999999e-06 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402625+0000", 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1220999999999998e-05 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402639+0000", 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3757999999999999e-05 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402646+0000", 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5820000000000004e-06 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.579937+0000", 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17729115300000001 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.579953+0000", 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5885000000000001e-05 2024-09-21T09:58:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:60913 6.f e1085/1078 6:fefdacc2:::benchmark_data_smithi017_149547_object60912:head v 1085'3830, mlcod=1085'3828)", 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.405062+0000", 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.637926257999993, 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.191724845, 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.405062+0000", 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.405062+0000", 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.405062+0000", 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5100000000000001e-07 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.405088+0000", 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5218000000000001e-05 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.405098+0000", 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0399e-05 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.405106+0000", 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3760000000000008e-06 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.564717+0000", 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.159610326 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.569290+0000", 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045727859999999997 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.596729+0000", 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027439341999999999 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.596787+0000", 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8146999999999997e-05 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:60919 6.6 6:60165c15:::benchmark_data_smithi017_149547_object60918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.425071+0000", 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.617917121000005, 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.015076965, 2024-09-21T09:58:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60919 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425071+0000", 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425071+0000", 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425073+0000", 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0320000000000002e-06 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425111+0000", 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7629000000000002e-05 2024-09-21T09:58:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425134+0000", 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2532e-05 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425144+0000", 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0376e-05 2024-09-21T09:58:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.573307+0000", 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.148163245 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.573317+0000", 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0359e-05 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.424167+0000", 2024-09-21T09:58:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.85084939599999998 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.424309+0000", 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014259799999999999 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.424460+0000", 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000150216 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.432200+0000", 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077398700000000003 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.435328+0000", 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031281989999999999 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440031+0000", 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047034499999999996 2024-09-21T09:58:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440094+0000", 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2502000000000006e-05 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440148+0000", 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4561000000000002e-05 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:60922 6.6 6:63c23454:::benchmark_data_smithi017_149547_object60921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.425160+0000", 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.617828684000003, 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0152000210000001, 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60922 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425160+0000", 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425160+0000", 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425160+0000", 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6400000000000002e-07 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425186+0000", 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6190000000000002e-05 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425194+0000", 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2300000000000002e-06 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425199+0000", 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1379999999999998e-06 2024-09-21T09:58:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.573706+0000", 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14850719300000001 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.573718+0000", 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1591e-05 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.430125+0000", 2024-09-21T09:58:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.85640711700000005 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.430297+0000", 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017236899999999999 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.430476+0000", 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017851100000000001 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.435369+0000", 2024-09-21T09:58:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048929619999999998 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.435391+0000", 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2847e-05 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440270+0000", 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048781950000000001 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440321+0000", 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1585999999999997e-05 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440360+0000", 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8528000000000001e-05 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90824 6.17 6:e9ce8460:::benchmark_data_smithi017_149547_object90823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.553125+0000", 2024-09-21T09:58:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 58.489863178, 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99456902999999997, 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90824 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553125+0000", 2024-09-21T09:58:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553125+0000", 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553127+0000", 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0099999999999998e-06 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553155+0000", 2024-09-21T09:58:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7645e-05 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553167+0000", 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2021999999999999e-05 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553176+0000", 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4769999999999992e-06 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.554991+0000", 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001814782 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.554999+0000", 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.276e-06 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547582+0000", 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99258215699999996 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547694+0000", 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112661 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90827 6.17 6:e8bdeb5b:::benchmark_data_smithi017_149547_object90826:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.553190+0000", 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 58.489798831000002, 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99469125599999997, 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:58:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90827 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553190+0000", 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553190+0000", 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553190+0000", 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0399999999999996e-07 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553211+0000", 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1285999999999999e-05 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553219+0000", 2024-09-21T09:58:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1690000000000003e-06 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553224+0000", 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4940000000000003e-06 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555018+0000", 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017938120000000001 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555023+0000", 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5169999999999999e-06 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547798+0000", 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99277433599999998 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547881+0000", 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3138000000000003e-05 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90829 6.7 6:e6b999dd:::benchmark_data_smithi017_149547_object90828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.553231+0000", 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 58.489756948, 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99863575900000001, 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90829 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553231+0000", 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553231+0000", 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553232+0000", 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5499999999999999e-07 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553253+0000", 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0780000000000001e-05 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553259+0000", 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7429999999999996e-06 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553264+0000", 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9189999999999998e-06 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555044+0000", 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001780139 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555052+0000", 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.593e-06 2024-09-21T09:58:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551791+0000", 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99673869800000003 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551867+0000", 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6532000000000005e-05 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90837 6.17 6:e97fb59f:::benchmark_data_smithi017_149547_object90836:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.553271+0000", 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 58.489717237999997, 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99816365399999996, 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90837 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553271+0000", 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553271+0000", 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553272+0000", 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3300000000000003e-07 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553293+0000", 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0883e-05 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553299+0000", 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6730000000000003e-06 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553304+0000", 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0649999999999998e-06 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555072+0000", 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017676949999999999 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555079+0000", 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7669999999999998e-06 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547928+0000", 2024-09-21T09:58:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99284903599999996 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.548012+0000", 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4330000000000001e-05 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551317+0000", 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033052939999999999 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551435+0000", 2024-09-21T09:58:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117478 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90846 6.17 6:ea2175a0:::benchmark_data_smithi017_149547_object90845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.568722+0000", 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 58.474266479000001, 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98300621899999996, 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90846 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568722+0000", 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568722+0000", 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568724+0000", 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9489999999999999e-06 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568758+0000", 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4072000000000002e-05 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568772+0000", 2024-09-21T09:58:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4515000000000001e-05 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568781+0000", 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.602e-06 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.570949+0000", 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002168279 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.570960+0000", 2024-09-21T09:58:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0526999999999999e-05 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.548054+0000", 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97709458800000004 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.548176+0000", 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001211 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T09:58:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.548342+0000", 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166222 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551477+0000", 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031355599999999999 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551492+0000", 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4851999999999999e-05 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551538+0000", 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6167e-05 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551601+0000", 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2989000000000003e-05 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551728+0000", 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126797 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90859 6.7 6:e6380ed1:::benchmark_data_smithi017_149547_object90858:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.586953+0000", 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 58.456035477, 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96900693999999998, 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90859 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.586953+0000", 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.586953+0000", 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.586953+0000", 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0100000000000005e-07 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.586998+0000", 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4981000000000002e-05 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587006+0000", 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2790000000000001e-06 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587021+0000", 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4873e-05 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587910+0000", 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000889914 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587922+0000", 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1012999999999999e-05 2024-09-21T09:58:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551891+0000", 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96396979800000004 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551995+0000", 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103677 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.552141+0000", 2024-09-21T09:58:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000145966 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.554708+0000", 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002566615 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555663+0000", 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00095549899999999995 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555876+0000", 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000212742 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555926+0000", 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0299000000000001e-05 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555960+0000", 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3782999999999998e-05 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90861 6.7 6:e1281cf9:::benchmark_data_smithi017_149547_object90860:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.587101+0000", 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 58.455887666000002, 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97326060400000003, 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90861 2024-09-21T09:58:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587101+0000", 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587101+0000", 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587101+0000", 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2e-07 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587123+0000", 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.154e-05 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587133+0000", 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0246e-05 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587138+0000", 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8330000000000001e-06 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587947+0000", 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00080937800000000003 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587954+0000", 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1130000000000003e-06 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555986+0000", 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96803142600000003 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.556102+0000", 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011658 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.556236+0000", 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133449 2024-09-21T09:58:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560018+0000", 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037822099999999998 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560063+0000", 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5349000000000002e-05 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560200+0000", 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013658799999999999 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560313+0000", 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112865 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560361+0000", 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8606999999999997e-05 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90865 6.7 6:e3d3d708:::benchmark_data_smithi017_149547_object90864:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.594875+0000", 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 58.448112938999998, 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96661452800000003, 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90865 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594875+0000", 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594875+0000", 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594876+0000", 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9000000000000005e-07 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594899+0000", 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2316000000000001e-05 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594914+0000", 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5498999999999998e-05 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594922+0000", 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7829999999999999e-06 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.595060+0000", 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013757799999999999 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.595071+0000", 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1616e-05 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.558004+0000", 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96293270500000006 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.558199+0000", 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000194811 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.558344+0000", 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014554800000000001 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560148+0000", 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018039060000000001 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560373+0000", 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000225028 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.561105+0000", 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00073150000000000005 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.561187+0000", 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1772000000000002e-05 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.561490+0000", 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030347600000000001 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90877 6.17 6:ede7eb83:::benchmark_data_smithi017_149547_object90876:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:58:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.614443+0000", 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 58.428545253000003, 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94139699700000001, 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90877 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.614443+0000", 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.614443+0000", 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.614445+0000", 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9120000000000001e-06 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.614468+0000", 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3030000000000001e-05 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.614484+0000", 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6237999999999999e-05 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.614494+0000", 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2629999999999992e-06 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.614574+0000", 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0128000000000003e-05 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.614586+0000", 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2013e-05 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551820+0000", 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.93723404200000004 2024-09-21T09:58:37.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551994+0000", 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017430599999999999 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.552151+0000", 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015705499999999999 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.554454+0000", 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023027379999999999 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.554468+0000", 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4226000000000001e-05 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555698+0000", 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012299279999999999 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555753+0000", 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4886000000000003e-05 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555840+0000", 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7231999999999994e-05 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:123412 6.c 6:3fdceaf1:::benchmark_data_smithi017_149547_object4333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:11.279482+0000", 2024-09-21T09:58:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.763506235000001, 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17119648600000001, 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123412 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.279482+0000", 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.279482+0000", 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.279483+0000", 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6699999999999999e-07 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.279485+0000", 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1440000000000001e-06 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.279491+0000", 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4930000000000003e-06 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.279497+0000", 2024-09-21T09:58:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2299999999999999e-06 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.442629+0000", 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.163132902 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.442745+0000", 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115241 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.442859+0000", 2024-09-21T09:58:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114084 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.450506+0000", 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076469049999999998 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.450517+0000", 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1199999999999999e-05 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.450549+0000", 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2623999999999998e-05 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.450636+0000", 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6291000000000004e-05 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.450679+0000", 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2904999999999997e-05 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:123425 6.c 6:305b0e62:::benchmark_data_smithi017_149547_object4346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:11.290083+0000", 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.75290493, 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16802410400000001, 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123425 2024-09-21T09:58:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.290083+0000", 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.290083+0000", 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.290085+0000", 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.455e-06 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.290088+0000", 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.365e-06 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.290096+0000", 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5399999999999998e-06 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.290101+0000", 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7060000000000003e-06 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.447204+0000", 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15710373899999999 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.447306+0000", 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102021 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.447408+0000", 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101655 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.453991+0000", 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065834769999999999 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.457907+0000", 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039156190000000004 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.458002+0000", 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5297000000000006e-05 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.458062+0000", 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9369e-05 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.458108+0000", 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5861e-05 2024-09-21T09:58:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:123429 6.c 6:3af1a56e:::benchmark_data_smithi017_149547_object4350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:11.293488+0000", 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.749500201, 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16482122499999999, 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123429 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.293488+0000", 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.293488+0000", 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.293489+0000", 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1710000000000001e-06 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.293493+0000", 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4999999999999999e-06 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.293502+0000", 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.363e-06 2024-09-21T09:58:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.295216+0000", 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017132510000000001 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.450709+0000", 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.155493619 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.450807+0000", 2024-09-21T09:58:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8202000000000006e-05 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.450947+0000", 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000139235 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.457920+0000", 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006973656 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.457960+0000", 2024-09-21T09:58:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9920999999999999e-05 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.458197+0000", 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023708799999999999 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.458278+0000", 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0578e-05 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.458309+0000", 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1640999999999997e-05 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:123434 6.c 6:386a209c:::benchmark_data_smithi017_149547_object4355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:11.295533+0000", 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.747455016, 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.176299814, 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:58:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123434 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.295533+0000", 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.295533+0000", 2024-09-21T09:58:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.295534+0000", 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4899999999999995e-07 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.295537+0000", 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9069999999999999e-06 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.295544+0000", 2024-09-21T09:58:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5379999999999997e-06 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.295550+0000", 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4920000000000001e-06 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.454096+0000", 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.158546301 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:58:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.454218+0000", 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121902 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.454349+0000", 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130927 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.458152+0000", 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038027989999999999 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.458343+0000", 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019130899999999999 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.471744+0000", 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013400406 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.471791+0000", 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7568000000000003e-05 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:58:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T09:58:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.471833+0000", 2024-09-21T09:58:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2116000000000001e-05 2024-09-21T09:58:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:58:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:58:37.412 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:58:37.412 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:37.436 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:37.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:37.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:37.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:37.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:37.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:37 smithi017 ceph-mon[102913]: pgmap v1333: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 105 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 5358/278226 objects degraded (1.926%); 5032/278226 objects misplaced (1.809%); 4.8 MiB/s, 76 objects/s recovering 2024-09-21T09:58:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:37 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4189137374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:37 smithi017 ceph-mon[105151]: pgmap v1333: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 105 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 5358/278226 objects degraded (1.926%); 5032/278226 objects misplaced (1.809%); 4.8 MiB/s, 76 objects/s recovering 2024-09-21T09:58:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:37 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4189137374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:37.723 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:37.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:37.742 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:37 smithi134 ceph-mon[90085]: pgmap v1333: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 105 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 5358/278226 objects degraded (1.926%); 5032/278226 objects misplaced (1.809%); 4.8 MiB/s, 76 objects/s recovering 2024-09-21T09:58:37.742 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:37 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4189137374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:37.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:37.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:37.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:37.972 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:38.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:38.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:38.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:38.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:38.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:38.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:38.272 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:38.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:38.373 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:38.406 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:38.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:38.508 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:38.542 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:38.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:38.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:38.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:38.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5358/278226 objects degraded (1.926%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:38.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5358/278226 objects degraded (1.926%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:38.783 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:38.804 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:38.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:38.842 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5358/278226 objects degraded (1.926%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:38.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:38.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:38.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:39.044 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:39.073 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:39.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:39.175 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:39.201 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:39.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:39.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:39.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:39.448 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:39.476 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:39.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:39.577 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:39.611 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:39.713 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:39.738 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:39.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:39.798 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:58:39.798 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:58:39.798 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-21T09:58:39.798 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T09:58:39.798 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T09:58:39.799 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-21T09:58:39.820 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:39 smithi017 ceph-mon[102913]: pgmap v1334: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 105 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 5030/274101 objects degraded (1.835%); 4011/274101 objects misplaced (1.463%); 4.8 MiB/s, 77 objects/s recovering 2024-09-21T09:58:39.820 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:39.820 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:39.820 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:39 smithi017 ceph-mon[105151]: pgmap v1334: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 105 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 5030/274101 objects degraded (1.835%); 4011/274101 objects misplaced (1.463%); 4.8 MiB/s, 77 objects/s recovering 2024-09-21T09:58:39.820 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:39.820 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:39.823 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:39 smithi134 ceph-mon[90085]: pgmap v1334: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 105 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 5030/274101 objects degraded (1.835%); 4011/274101 objects misplaced (1.463%); 4.8 MiB/s, 77 objects/s recovering 2024-09-21T09:58:39.824 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:39.824 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:39.840 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:39.863 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T09:58:39.866 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:39.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:40.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:40.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:40.104 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:40.131 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:40.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.132 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:40.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:40.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:40.255 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:40.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:40.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:40.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:40.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:40.484 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:58:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:58:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:40.506 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:40.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:40.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:40.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:40.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:40.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:40.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:40.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:40.811 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:40 smithi017 ceph-mon[102913]: osdmap e1109: 8 total, 8 up, 8 in 2024-09-21T09:58:40.812 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:40.812 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:40.813 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:40 smithi017 ceph-mon[105151]: osdmap e1109: 8 total, 8 up, 8 in 2024-09-21T09:58:40.813 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:40.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:40.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:40.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:40.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:40 smithi134 ceph-mon[90085]: osdmap e1109: 8 total, 8 up, 8 in 2024-09-21T09:58:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:41.024 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:41.050 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:41.116 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:41.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:41.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:41.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:41.264 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:41.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:41.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:41.491 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:41.520 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:41.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:41.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:41.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:41.801 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:41.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:41.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:41 smithi017 ceph-mon[102913]: pgmap v1336: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 105 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 5030/272403 objects degraded (1.847%); 4011/272403 objects misplaced (1.472%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T09:58:41.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:41 smithi017 ceph-mon[102913]: osdmap e1110: 8 total, 8 up, 8 in 2024-09-21T09:58:41.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:41 smithi017 ceph-mon[105151]: pgmap v1336: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 105 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 5030/272403 objects degraded (1.847%); 4011/272403 objects misplaced (1.472%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T09:58:41.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:41 smithi017 ceph-mon[105151]: osdmap e1110: 8 total, 8 up, 8 in 2024-09-21T09:58:41.834 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:41 smithi134 ceph-mon[90085]: pgmap v1336: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 105 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 5030/272403 objects degraded (1.847%); 4011/272403 objects misplaced (1.472%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T09:58:41.834 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:41 smithi134 ceph-mon[90085]: osdmap e1110: 8 total, 8 up, 8 in 2024-09-21T09:58:41.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:41.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:42.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:42.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:42.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:42.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:42.210 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:42.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:42.339 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:42.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:42.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:42.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:42.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:42.633 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:42.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:42.732 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:58:42.732 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T09:58:42.732 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:58:42.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:42.763 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:42.865 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:42.890 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:42.992 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:43.017 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:43.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:43.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:43.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:43.269 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:43.303 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:43.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:43.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:58:43.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:43.429 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:43.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:43.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:43.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:43.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:43.713 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:43 smithi134 ceph-mon[90085]: pgmap v1338: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 969 op/s; 3689/263661 objects degraded (1.399%); 4759/263661 objects misplaced (1.805%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T09:58:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:43 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 3689/263661 objects degraded (1.399%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:43.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:43.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:43 smithi017 ceph-mon[105151]: pgmap v1338: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 969 op/s; 3689/263661 objects degraded (1.399%); 4759/263661 objects misplaced (1.805%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T09:58:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:43 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 3689/263661 objects degraded (1.399%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:43 smithi017 ceph-mon[102913]: pgmap v1338: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 969 op/s; 3689/263661 objects degraded (1.399%); 4759/263661 objects misplaced (1.805%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T09:58:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:43 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 3689/263661 objects degraded (1.399%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:43.975 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:44.001 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:44.103 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:44.132 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:44.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:44.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:44.245 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:44.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:44.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:44.389 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:44.417 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:44.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:44.548 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:44.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:44.594 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T09:58:44.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/989923940' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:44 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:44.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/989923940' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:44.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:44 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:44.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:44.755 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:58:44.755 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T09:58:44.755 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.755 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:154218 6.e 6:7f0c9e08:::benchmark_data_smithi017_149547_object35139:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-21T09:58:44.755 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:44.747670+0000", 2024-09-21T09:58:44.755 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0043931200000000004, 2024-09-21T09:58:44.755 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0044307410000000002, 2024-09-21T09:58:44.755 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:44.755 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:58:44.756 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:44.756 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:44.756 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:44.756 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 154218 2024-09-21T09:58:44.756 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.756 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:44.756 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.756 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:44.756 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.747670+0000", 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.747670+0000", 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.747669+0000", 2024-09-21T09:58:44.757 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:44.758 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.758 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.758 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:44.758 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.747674+0000", 2024-09-21T09:58:44.758 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4949999999999999e-06 2024-09-21T09:58:44.758 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.758 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.747691+0000", 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7158000000000001e-05 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.747698+0000", 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8589999999999999e-06 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:44.759 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.747743+0000", 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4696e-05 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.747863+0000", 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000119988 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.747981+0000", 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000117636 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.760 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.750707+0000", 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0027259620000000002 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:154199 6.e 6:77b9b2aa:::benchmark_data_smithi017_149547_object35120:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:44.729760+0000", 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.022303730000000001, 2024-09-21T09:58:44.761 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.022422788999999999, 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 154199 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.762 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.729760+0000", 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.729760+0000", 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.729759+0000", 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:44.763 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.729764+0000", 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9529999999999996e-06 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.729775+0000", 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0967e-05 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.729782+0000", 2024-09-21T09:58:44.764 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7970000000000003e-06 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.729838+0000", 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5829999999999999e-05 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.729984+0000", 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014581699999999999 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.765 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.730104+0000", 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000120381 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.733811+0000", 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003707017 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.737455+0000", 2024-09-21T09:58:44.766 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0036437629999999999 2024-09-21T09:58:44.767 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:44.767 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:44.767 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:44.767 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.767 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.767 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:154212 6.2 6:45064b5b:::benchmark_data_smithi017_149547_object35133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:44.740921+0000", 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.011142641, 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011337274, 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 154212 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.768 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.740921+0000", 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.740921+0000", 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.740919+0000", 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.769 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.740941+0000", 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1532999999999999e-05 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741159+0000", 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00021808 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741167+0000", 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.174e-06 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.770 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741246+0000", 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8943000000000006e-05 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741345+0000", 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.9193999999999999e-05 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741447+0000", 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010165100000000001 2024-09-21T09:58:44.771 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.746702+0000", 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0052550619999999996 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.749162+0000", 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002460683 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.772 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:154214 6.2 6:467ead88:::benchmark_data_smithi017_149547_object35135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:44.741191+0000", 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.010872704, 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011141606, 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 154214 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.773 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741191+0000", 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741191+0000", 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741190+0000", 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.774 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741195+0000", 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6120000000000001e-06 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741204+0000", 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8890000000000008e-06 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.741209+0000", 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0629999999999996e-06 2024-09-21T09:58:44.775 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.744190+0000", 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029815250000000001 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.744308+0000", 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011770899999999999 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.744422+0000", 2024-09-21T09:58:44.776 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000113927 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.746886+0000", 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0024645460000000002 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:44.749199+0000", 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00231262 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: ], 2024-09-21T09:58:44.778 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 4 2024-09-21T09:58:44.778 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:58:44.807 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:44.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:44.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:44.938 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:44.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.965 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:44.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:44.982 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:58:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:58:44] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-21T09:58:44.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/989923940' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:44.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:44 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:45.067 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:45.096 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:45.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:45.200 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:45.230 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:45.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:45.332 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:45.458 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:45.485 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:45.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:45.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T09:58:45.586 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:45.622 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:45.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:45.724 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:45.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:45.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:45.904 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:45.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:45.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:45.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:45 smithi017 ceph-mon[102913]: pgmap v1339: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 creating+peering, 106 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 973/260238 objects degraded (0.374%); 6809/260238 objects misplaced (2.616%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T09:58:45.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:45 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:45.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:45 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:45.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:45 smithi017 ceph-mon[102913]: osdmap e1111: 8 total, 8 up, 8 in 2024-09-21T09:58:45.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:45 smithi017 ceph-mon[105151]: pgmap v1339: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 creating+peering, 106 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 973/260238 objects degraded (0.374%); 6809/260238 objects misplaced (2.616%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T09:58:45.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:45 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:45.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:45 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:45.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:45 smithi017 ceph-mon[105151]: osdmap e1111: 8 total, 8 up, 8 in 2024-09-21T09:58:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:45 smithi134 ceph-mon[90085]: pgmap v1339: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 creating+peering, 106 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 973/260238 objects degraded (0.374%); 6809/260238 objects misplaced (2.616%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T09:58:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:45 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:45 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:45 smithi134 ceph-mon[90085]: osdmap e1111: 8 total, 8 up, 8 in 2024-09-21T09:58:46.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:46.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:46.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:46.159 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:46.186 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:46.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:46.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:46.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:46.414 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:46.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.445 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:46.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:46.546 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:46.572 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:46.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:46.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:46.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.702 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:46.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:46.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:46.831 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:46.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:46.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:46.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:46.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:47.074 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:47.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.105 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:47.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:47.206 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:47.234 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:47.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T09:58:47.336 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:47.367 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:47.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:47.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:47.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:47.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:47.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.685 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:47.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:47.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:47 smithi017 ceph-mon[102913]: pgmap v1341: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 973/251931 objects degraded (0.386%); 6809/251931 objects misplaced (2.703%); 4.4 MiB/s, 71 objects/s recovering 2024-09-21T09:58:47.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:47 smithi017 ceph-mon[105151]: pgmap v1341: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 973/251931 objects degraded (0.386%); 6809/251931 objects misplaced (2.703%); 4.4 MiB/s, 71 objects/s recovering 2024-09-21T09:58:47.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:47.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:47.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:47.854 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:47 smithi134 ceph-mon[90085]: pgmap v1341: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 973/251931 objects degraded (0.386%); 6809/251931 objects misplaced (2.703%); 4.4 MiB/s, 71 objects/s recovering 2024-09-21T09:58:47.922 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:58:47.922 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T09:58:47.922 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T09:58:47.923 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T09:58:47.923 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:58:47.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:47.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:48.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:48.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:48.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:48.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:48.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:48.357 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:48.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:48.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:48.490 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:48.519 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:48.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:48.621 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:48.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:48.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:48.663 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T09:58:48.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:48.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:48.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:48.858 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:48 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 973/251931 objects degraded (0.386%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:48.858 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:48 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 973/251931 objects degraded (0.386%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:48.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:48.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:48.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:48 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 973/251931 objects degraded (0.386%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:49.059 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:49.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.106 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:49.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:49.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:49.215 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:49.219 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:58:49.220 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:58:49.220 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-21T09:58:49.220 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T09:58:49.220 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T09:58:49.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-21T09:58:49.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:49.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:49.370 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:49.399 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:49.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:49.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:49.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:49.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:49 smithi017 ceph-mon[102913]: pgmap v1342: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 994 op/s; 973/248517 objects degraded (0.392%); 5916/248517 objects misplaced (2.381%); 6.3 MiB/s, 101 objects/s recovering 2024-09-21T09:58:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:49 smithi017 ceph-mon[105151]: pgmap v1342: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 994 op/s; 973/248517 objects degraded (0.392%); 5916/248517 objects misplaced (2.381%); 6.3 MiB/s, 101 objects/s recovering 2024-09-21T09:58:49.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:49.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:49.830 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:49.859 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:49.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:49.961 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:49.989 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:49.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:49 smithi134 ceph-mon[90085]: pgmap v1342: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 994 op/s; 973/248517 objects degraded (0.392%); 5916/248517 objects misplaced (2.381%); 6.3 MiB/s, 101 objects/s recovering 2024-09-21T09:58:50.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:50.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:50.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:50.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:50.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:50.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:50.349 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:58:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:58:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:50.379 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:50.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.408 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:50.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:50.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:50.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:50.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:50.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:50.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:50.879 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:50.908 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:51.009 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:58:51.009 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T09:58:51.009 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T09:58:51.009 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T09:58:51.009 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60703 6.2 6:4b87b783:::benchmark_data_smithi017_149547_object60702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.175116+0000", 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 104.828597327, 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.097218917000000002, 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60703 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.010 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.175116+0000", 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.175116+0000", 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.175115+0000", 2024-09-21T09:58:51.011 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.175260+0000", 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000144955 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.175271+0000", 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0973e-05 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.012 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.175276+0000", 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.366e-06 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.175310+0000", 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3395999999999997e-05 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.175431+0000", 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012181200000000001 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.013 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.175555+0000", 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012314199999999999 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.177838+0000", 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0022828940000000002 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.178522+0000", 2024-09-21T09:58:51.014 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00068423400000000004 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.272224+0000", 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.093702474999999993 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.272300+0000", 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5417e-05 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.015 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.272335+0000", 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5481999999999997e-05 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60714 6.e 6:79b28daa:::benchmark_data_smithi017_149547_object60713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.181657+0000", 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 104.82205675199999, 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.094967274000000004, 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.016 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60714 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.181657+0000", 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.017 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.181657+0000", 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.181656+0000", 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.181715+0000", 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8862e-05 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.018 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.181729+0000", 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4583000000000001e-05 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.181733+0000", 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0940000000000001e-06 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.181785+0000", 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1323000000000003e-05 2024-09-21T09:58:51.019 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.181910+0000", 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000124824 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.182043+0000", 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000133511 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.184094+0000", 2024-09-21T09:58:51.020 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020507680000000001 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.185289+0000", 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001194978 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.276546+0000", 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.091257260000000007 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.021 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.276588+0000", 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1668000000000002e-05 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.276624+0000", 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6347999999999998e-05 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60917 6.2 6:45cf8ae7:::benchmark_data_smithi017_149547_object60916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.398649+0000", 2024-09-21T09:58:51.022 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 104.60506450699999, 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13833842199999999, 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60917 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398649+0000", 2024-09-21T09:58:51.023 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398649+0000", 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398648+0000", 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.024 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398706+0000", 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8044000000000002e-05 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398721+0000", 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5298000000000002e-05 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398727+0000", 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7679999999999997e-06 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.025 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398765+0000", 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7812999999999999e-05 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398864+0000", 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.9423e-05 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398982+0000", 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000118461 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.026 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.401903+0000", 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029207220000000002 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.536966+0000", 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13506279700000001 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.536987+0000", 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1508000000000001e-05 2024-09-21T09:58:51.027 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.028 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.028 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.028 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.028 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.028 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60925 6.2 6.45ad44e2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T09:58:51.028 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.419283+0000", 2024-09-21T09:58:51.028 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 104.584430169, 2024-09-21T09:58:51.029 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.118468055, 2024-09-21T09:58:51.029 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.029 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T09:58:51.029 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.029 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60925 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419283+0000", 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419283+0000", 2024-09-21T09:58:51.030 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419282+0000", 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419334+0000", 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2682999999999997e-05 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.031 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419347+0000", 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2378000000000001e-05 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419353+0000", 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7899999999999996e-06 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537738+0000", 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.118385267 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.032 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537751+0000", 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3498e-05 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60917 6.2 6:45cf8ae7:::benchmark_data_smithi017_149547_object60916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.430482+0000", 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 104.573231161, 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.00240142, 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.033 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60917 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430482+0000", 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430482+0000", 2024-09-21T09:58:51.034 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430480+0000", 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430535+0000", 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5269999999999998e-05 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.035 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430560+0000", 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.5000000000000001e-05 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430569+0000", 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7290000000000006e-06 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537803+0000", 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10723382400000001 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T09:58:51.036 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537811+0000", 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.6440000000000006e-06 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.420962+0000", 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.88315087199999998 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.421705+0000", 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00074260199999999998 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.037 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.038 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432810+0000", 2024-09-21T09:58:51.038 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011104852 2024-09-21T09:58:51.038 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.038 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.038 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.038 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432884+0000", 2024-09-21T09:58:51.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:51.039 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4006999999999998e-05 2024-09-21T09:58:51.039 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.039 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60925 6.2 6:4722b5a2:::benchmark_data_smithi017_149547_object60924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.430577+0000", 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 104.57313616, 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0022010509999999, 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.040 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60925 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430577+0000", 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430577+0000", 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.041 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430577+0000", 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430619+0000", 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2605000000000003e-05 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430627+0000", 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2289999999999997e-06 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.042 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430632+0000", 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9099999999999996e-06 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537828+0000", 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.107196871 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537833+0000", 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6090000000000003e-06 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.043 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.421894+0000", 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.88406131600000004 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.422024+0000", 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000129207 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.422186+0000", 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000161982 2024-09-21T09:58:51.044 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.423391+0000", 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0012054660000000001 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.427355+0000", 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0039639389999999997 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432653+0000", 2024-09-21T09:58:51.045 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0052981959999999998 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432704+0000", 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0963e-05 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432778+0000", 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4288000000000004e-05 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.046 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:91930 6.2 6:42d23dfa:::benchmark_data_smithi017_149547_object91929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:41.674378+0000", 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 69.329335094000001, 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11810723400000001, 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.047 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 91930 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674378+0000", 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674378+0000", 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.048 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674368+0000", 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999895 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674413+0000", 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5049000000000002e-05 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674430+0000", 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7016000000000001e-05 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.049 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674437+0000", 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.669e-06 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674482+0000", 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5361000000000002e-05 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674592+0000", 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010995 2024-09-21T09:58:51.050 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674711+0000", 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000118763 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.676478+0000", 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0017673910000000001 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.679252+0000", 2024-09-21T09:58:51.051 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0027736190000000002 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.792402+0000", 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.113149972 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.792442+0000", 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0438000000000001e-05 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.792486+0000", 2024-09-21T09:58:51.052 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3560000000000003e-05 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:92446 6.e 6:7f22a4f7:::benchmark_data_smithi017_149547_object92445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:42.167363+0000", 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 68.836350139000004, 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.120846588, 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.053 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 92446 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167363+0000", 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167363+0000", 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.054 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167362+0000", 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167417+0000", 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5041999999999999e-05 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167430+0000", 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3580999999999999e-05 2024-09-21T09:58:51.055 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167436+0000", 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1460000000000001e-06 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167486+0000", 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9453000000000002e-05 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167626+0000", 2024-09-21T09:58:51.056 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013969899999999999 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167787+0000", 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016156499999999999 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.170662+0000", 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0028746520000000001 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.057 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.170722+0000", 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0239999999999999e-05 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288106+0000", 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11738346299999999 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288164+0000", 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8634000000000001e-05 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.058 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288210+0000", 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5766999999999997e-05 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:92450 6.e 6:742c5309:::benchmark_data_smithi017_149547_object92449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:42.172337+0000", 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 68.831376246000005, 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11601592300000001, 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.059 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 92450 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172337+0000", 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172337+0000", 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.060 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172336+0000", 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172384+0000", 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8196999999999999e-05 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172396+0000", 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1904e-05 2024-09-21T09:58:51.061 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172400+0000", 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3460000000000004e-06 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172445+0000", 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4499999999999997e-05 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172582+0000", 2024-09-21T09:58:51.062 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000137176 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172735+0000", 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000152967 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.175058+0000", 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0023232909999999999 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.063 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.175223+0000", 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016430199999999999 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288272+0000", 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.113049715 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288319+0000", 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6632999999999999e-05 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.064 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288353+0000", 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4209999999999999e-05 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:122941 6.e 6:72cd0250:::benchmark_data_smithi017_149547_object3862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:10.576941+0000", 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 40.426772583000002, 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.155614635, 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.065 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122941 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576941+0000", 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576941+0000", 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.066 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576940+0000", 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576946+0000", 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.241e-06 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576961+0000", 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4987999999999999e-05 2024-09-21T09:58:51.067 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576967+0000", 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4130000000000003e-06 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577049+0000", 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1457000000000003e-05 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577161+0000", 2024-09-21T09:58:51.068 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000112606 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577280+0000", 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000119129 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.583018+0000", 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0057380390000000003 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.069 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.584726+0000", 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0017071790000000001 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732474+0000", 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.14774837599999999 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732518+0000", 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3837000000000001e-05 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.070 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732556+0000", 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7710000000000003e-05 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:122943 6.e 6:7dbda6e0:::benchmark_data_smithi017_149547_object3864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:10.577994+0000", 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 40.425719362000002, 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.154652334, 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.071 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122943 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577994+0000", 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577994+0000", 2024-09-21T09:58:51.072 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577993+0000", 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577998+0000", 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9420000000000003e-06 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.073 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.578010+0000", 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1796e-05 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.578015+0000", 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4029999999999998e-06 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580136+0000", 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021202069999999998 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.074 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580239+0000", 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000103319 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580310+0000", 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0637999999999997e-05 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.584691+0000", 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0043812809999999999 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.075 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.586116+0000", 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0014252010000000001 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732585+0000", 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.14646848300000001 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732618+0000", 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3581999999999998e-05 2024-09-21T09:58:51.076 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732646+0000", 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8347000000000002e-05 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:122947 6.e 6:7d385fde:::benchmark_data_smithi017_149547_object3868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:10.580733+0000", 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 40.422980621000001, 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.15199675800000001, 2024-09-21T09:58:51.077 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122947 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580733+0000", 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.078 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580733+0000", 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580732+0000", 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580739+0000", 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3119999999999996e-06 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.079 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580750+0000", 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0719e-05 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580755+0000", 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4369999999999998e-06 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.583033+0000", 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0022775339999999999 2024-09-21T09:58:51.080 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.583109+0000", 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5663999999999999e-05 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.583185+0000", 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6848999999999996e-05 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.584792+0000", 2024-09-21T09:58:51.081 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001606892 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.586161+0000", 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0013685310000000001 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732671+0000", 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.14651046500000001 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.082 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732704+0000", 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2187999999999997e-05 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732730+0000", 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.6088000000000002e-05 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:123433 6.e 6:7f54b96a:::benchmark_data_smithi017_149547_object4354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:11.295471+0000", 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 39.708242034999998, 2024-09-21T09:58:51.083 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.150257261, 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 123433 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295471+0000", 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.084 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295471+0000", 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295470+0000", 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295477+0000", 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5880000000000003e-06 2024-09-21T09:58:51.085 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295491+0000", 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4024e-05 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295498+0000", 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9600000000000003e-06 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295546+0000", 2024-09-21T09:58:51.086 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8374000000000001e-05 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295626+0000", 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0322999999999997e-05 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295695+0000", 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8498999999999996e-05 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.087 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.298939+0000", 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032442619999999999 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.299749+0000", 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00081013199999999997 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.445644+0000", 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.145894317 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.088 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.445690+0000", 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5998999999999998e-05 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.445729+0000", 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9221999999999998e-05 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:124124 6.e 6:70f8299d:::benchmark_data_smithi017_149547_object5045:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:12.198188+0000", 2024-09-21T09:58:51.089 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 38.805525205999999, 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10971036300000001, 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 124124 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198188+0000", 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.090 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198188+0000", 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198187+0000", 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198194+0000", 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7669999999999998e-06 2024-09-21T09:58:51.091 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198208+0000", 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4262e-05 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198214+0000", 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5579999999999997e-06 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198262+0000", 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7599000000000003e-05 2024-09-21T09:58:51.092 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.093 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.093 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.093 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198366+0000", 2024-09-21T09:58:51.093 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000104261 2024-09-21T09:58:51.093 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.093 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.094 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.094 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198479+0000", 2024-09-21T09:58:51.094 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000112327 2024-09-21T09:58:51.094 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.094 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.094 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.094 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.202028+0000", 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0035496740000000001 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.202182+0000", 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000154203 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.307794+0000", 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.105611815 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.095 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.307841+0000", 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6826000000000002e-05 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.307899+0000", 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7590999999999998e-05 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:149580 6.e 6:751bd8da:::benchmark_data_smithi017_149547_object30501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:51.096 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:38.761914+0000", 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 12.241799124, 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.101136616, 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 149580 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.761914+0000", 2024-09-21T09:58:51.097 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.761914+0000", 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.761913+0000", 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.098 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.761921+0000", 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.96e-06 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.761936+0000", 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5483999999999999e-05 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.761943+0000", 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0990000000000001e-06 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.099 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.762010+0000", 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6264000000000001e-05 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.762145+0000", 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013568200000000001 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.762257+0000", 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000111903 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.100 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767385+0000", 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0051277570000000001 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.768094+0000", 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00070886400000000002 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.862981+0000", 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.094887349999999995 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.101 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.863015+0000", 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3689000000000002e-05 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.863051+0000", 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6235999999999997e-05 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.102 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:149588 6.2 6:44f4b2f7:::benchmark_data_smithi017_149547_object30509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:38.767018+0000", 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 12.236695499, 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.097252179999999994, 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 149588 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767018+0000", 2024-09-21T09:58:51.103 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767018+0000", 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767016+0000", 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767023+0000", 2024-09-21T09:58:51.104 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9829999999999999e-06 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767035+0000", 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1294999999999999e-05 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767040+0000", 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6799999999999998e-06 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767108+0000", 2024-09-21T09:58:51.105 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7600000000000003e-05 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767219+0000", 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011123 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.767324+0000", 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010476000000000001 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.106 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.769963+0000", 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0026390200000000002 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.837764+0000", 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.067800703000000004 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.864186+0000", 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.026422085000000001 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.107 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.864235+0000", 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9302999999999999e-05 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:38.864270+0000", 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5055000000000002e-05 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:150148 6.e 6:7101fe91:::benchmark_data_smithi017_149547_object31069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:51.108 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:39.399817+0000", 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 11.603896388000001, 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13629029300000001, 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 150148 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399817+0000", 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399817+0000", 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399816+0000", 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.110 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399822+0000", 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3009999999999997e-06 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399832+0000", 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0672999999999999e-05 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399837+0000", 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2420000000000002e-06 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.111 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399873+0000", 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6791000000000001e-05 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399995+0000", 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000122125 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.400082+0000", 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.6749999999999994e-05 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.112 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.403983+0000", 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003900761 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.417968+0000", 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013985205000000001 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536010+0000", 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.118041418 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.113 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536069+0000", 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9500000000000003e-05 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536107+0000", 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8294999999999998e-05 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.114 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:150149 6.e 6:7280da22:::benchmark_data_smithi017_149547_object31070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:39.404830+0000", 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 11.598883892, 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13136800500000001, 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 150149 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404830+0000", 2024-09-21T09:58:51.115 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404830+0000", 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404828+0000", 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404837+0000", 2024-09-21T09:58:51.116 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.9789999999999999e-06 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404854+0000", 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6361000000000001e-05 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404861+0000", 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0010000000000004e-06 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.117 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404953+0000", 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.2493999999999999e-05 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.405123+0000", 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016946999999999999 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.405236+0000", 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000113758 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.118 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.410836+0000", 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0055995100000000003 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.418175+0000", 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0073390479999999999 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536137+0000", 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11796179599999999 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.119 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536169+0000", 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2088999999999997e-05 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536198+0000", 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8858000000000001e-05 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:150156 6.e 6:79b93165:::benchmark_data_smithi017_149547_object31077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:51.120 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:39.404992+0000", 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 11.598721833999999, 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13130247, 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 150156 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404992+0000", 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.121 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404992+0000", 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404991+0000", 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404996+0000", 2024-09-21T09:58:51.122 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7090000000000001e-06 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.405004+0000", 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.67e-06 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.405009+0000", 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0069999999999997e-06 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.123 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.408315+0000", 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0033059320000000001 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.408396+0000", 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0297999999999996e-05 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.408472+0000", 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6533999999999996e-05 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.124 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.410988+0000", 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002515485 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.418351+0000", 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0073635569999999997 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536236+0000", 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11788446599999999 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.125 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536267+0000", 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1146999999999998e-05 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536294+0000", 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7419999999999998e-05 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:150170 6.e 6:7571e895:::benchmark_data_smithi017_149547_object31091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:51.126 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:39.431315+0000", 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 11.572398807999999, 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.105067644, 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 150170 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431315+0000", 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.127 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431315+0000", 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431313+0000", 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431321+0000", 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4370000000000001e-06 2024-09-21T09:58:51.128 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431333+0000", 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2401999999999999e-05 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431339+0000", 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7529999999999998e-06 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431377+0000", 2024-09-21T09:58:51.129 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8614e-05 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431469+0000", 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.2052000000000006e-05 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431551+0000", 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1301e-05 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.434274+0000", 2024-09-21T09:58:51.130 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002722801 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.434473+0000", 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019968899999999999 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536319+0000", 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.101845463 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536354+0000", 2024-09-21T09:58:51.131 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5369e-05 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536382+0000", 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8260999999999999e-05 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:58:51.132 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:58:51.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:51.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:51.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:51.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:51.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:51.418 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:51.428 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:58:51.429 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:51.444 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:51.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:51.475 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:58:51.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:51.512 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:58:51.512 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:51.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:51.548 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:58:51.549 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:51.582 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:58:51.583 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:51.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.601 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:51.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:51.621 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:58:51.622 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:51.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:58:51.656 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:51.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:51 smithi017 ceph-mon[102913]: pgmap v1343: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 973/246804 objects degraded (0.394%); 5916/246804 objects misplaced (2.397%); 4.6 MiB/s, 73 objects/s recovering 2024-09-21T09:58:51.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:51.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:51.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:51 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3359117051' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:51.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:51 smithi017 ceph-mon[105151]: pgmap v1343: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 973/246804 objects degraded (0.394%); 5916/246804 objects misplaced (2.397%); 4.6 MiB/s, 73 objects/s recovering 2024-09-21T09:58:51.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:51.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:51.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:51 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3359117051' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:51.692 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:58:51.693 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:51.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:51.727 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:51.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:51 smithi134 ceph-mon[90085]: pgmap v1343: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 973/246804 objects degraded (0.394%); 5916/246804 objects misplaced (2.397%); 4.6 MiB/s, 73 objects/s recovering 2024-09-21T09:58:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:51 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3359117051' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:51.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:51.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:51.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:51.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:52.016 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T09:58:52.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:52.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:52.126 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:52.167 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:52.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:52.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:52.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:52.407 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:52.415 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:52.452 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:52.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:52.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:52.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:52.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:52.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:52.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:52.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:52.847 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:52.875 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:52.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:52.876 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:52 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:52.877 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:52 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:52.877 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:52 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:52 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:52.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:52 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:52 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:52 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:52 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:52 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:58:52.977 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:53.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:53.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:53.105 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:53.133 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:53.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:53.234 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:53.260 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:53.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:53.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:53.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:53.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:53.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:53.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:53.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:53.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:53.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:53 smithi134 ceph-mon[90085]: pgmap v1344: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 973/236490 objects degraded (0.411%); 5625/236490 objects misplaced (2.379%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T09:58:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:53 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 973/236490 objects degraded (0.411%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:53.802 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:53.829 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:53.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:53.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:53 smithi017 ceph-mon[102913]: pgmap v1344: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 973/236490 objects degraded (0.411%); 5625/236490 objects misplaced (2.379%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T09:58:53.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:53 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 973/236490 objects degraded (0.411%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:53.845 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:53 smithi017 ceph-mon[105151]: pgmap v1344: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 973/236490 objects degraded (0.411%); 5625/236490 objects misplaced (2.379%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T09:58:53.845 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:53 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 973/236490 objects degraded (0.411%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:53.926 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:53.930 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:53.957 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:53.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:54.059 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:54.087 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:54.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:54.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:54.239 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:54.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:54.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:54.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:54.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:54.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:54.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:54.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:54.548 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:54.548 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:54.628 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T09:58:54.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:54.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:54.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:54.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:54 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3602936178' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:58:54.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:54 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3602936178' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:58:54.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:54.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:54.893 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:54 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3602936178' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:58:54.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:54.940 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:54.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:55.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:55.075 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:58:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:58:54] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-09-21T09:58:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:55.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:55.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:58:55.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:55.242 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T09:58:55.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:55.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:55.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:55.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:55.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:55.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:55.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:55.617 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:55.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:55.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:55.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:55.835 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:55.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:55.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:55.927 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:55 smithi134 ceph-mon[90085]: pgmap v1345: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 926 op/s; 973/232449 objects degraded (0.419%); 4986/232449 objects misplaced (2.145%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:58:55.927 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:55 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:55.927 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:55 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3602936178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:58:55.927 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:55 smithi134 ceph-mon[90085]: osdmap e1112: 8 total, 8 up, 8 in 2024-09-21T09:58:55.927 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[102913]: pgmap v1345: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 926 op/s; 973/232449 objects degraded (0.419%); 4986/232449 objects misplaced (2.145%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:58:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3602936178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:58:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[102913]: osdmap e1112: 8 total, 8 up, 8 in 2024-09-21T09:58:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:55.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[105151]: pgmap v1345: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 926 op/s; 973/232449 objects degraded (0.419%); 4986/232449 objects misplaced (2.145%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:58:55.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:55.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3602936178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:58:55.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[105151]: osdmap e1112: 8 total, 8 up, 8 in 2024-09-21T09:58:55.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:55.962 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:55.995 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:55.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:56.097 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:56.131 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:56.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:56.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.265 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:56.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:56.367 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:56.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.398 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:56.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:56.499 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:56.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.528 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:56.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:56.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:56.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:56.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:56.755 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:56.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.786 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:56.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:56.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:56.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:56.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:57.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:57.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:57.177 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:57.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:57.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:57.326 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:57.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:57.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T09:58:57.429 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:57.456 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:57.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:57.558 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:57.589 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:57.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:57.690 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:57 smithi017 ceph-mon[102913]: pgmap v1347: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 973/223869 objects degraded (0.435%); 4986/223869 objects misplaced (2.227%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:58:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:57 smithi017 ceph-mon[105151]: pgmap v1347: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 973/223869 objects degraded (0.435%); 4986/223869 objects misplaced (2.227%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:58:57.723 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:57.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:57.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:57.826 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:57 smithi134 ceph-mon[90085]: pgmap v1347: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 973/223869 objects degraded (0.435%); 4986/223869 objects misplaced (2.227%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:58:57.846 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:57.846 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:57.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:57.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:57.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:58.001 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:58.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:58.102 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:58.129 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:58.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:58.231 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:58.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.257 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:58.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:58.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:58.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:58.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:58.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:58.641 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:58.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:58.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:58.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:58.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:58.943 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:58 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 973/223869 objects degraded (0.435%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:58.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:58 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 973/223869 objects degraded (0.435%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:58 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 973/223869 objects degraded (0.435%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:59.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:59.078 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:59.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:59.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T09:58:59.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.218 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:59.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:59.319 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:59.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.361 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:59.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:59.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:59.501 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:59.501 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:59.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:59.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:59.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:59.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:59.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:59.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:58:59.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:59.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:59.933 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:58:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:59.938 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:58:59 smithi017 ceph-mon[102913]: pgmap v1348: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 933 op/s; 973/220512 objects degraded (0.441%); 3800/220512 objects misplaced (1.723%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:58:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:58:59 smithi017 ceph-mon[105151]: pgmap v1348: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 933 op/s; 973/220512 objects degraded (0.441%); 3800/220512 objects misplaced (1.723%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:58:59.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:58:59 smithi134 ceph-mon[90085]: pgmap v1348: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 933 op/s; 973/220512 objects degraded (0.441%); 3800/220512 objects misplaced (1.723%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:58:59.964 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:00.035 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:00.064 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:00.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:00.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:00.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:00.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:00.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:00.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:00.454 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:00.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:00.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:59:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:59:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:00.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:00.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:00.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:00.727 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:00.753 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:00.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:00.822 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:59:00.822 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T09:59:00.822 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:59:00.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:00.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:00.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:01.004 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:01.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:01.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:01.147 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:01.175 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:01.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:01.277 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:01.304 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:01.406 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:01.444 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:01.472 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T09:59:01.472 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:59:01.546 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:01.577 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:01.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:01.723 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:01.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:01.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:01.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:01.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:01.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:01 smithi017 ceph-mon[102913]: pgmap v1349: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 973/218811 objects degraded (0.445%); 3800/218811 objects misplaced (1.737%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:59:01.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:01 smithi017 ceph-mon[105151]: pgmap v1349: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 973/218811 objects degraded (0.445%); 3800/218811 objects misplaced (1.737%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:59:01.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:01 smithi134 ceph-mon[90085]: pgmap v1349: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 973/218811 objects degraded (0.445%); 3800/218811 objects misplaced (1.737%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:59:02.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:02.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:02.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:02.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:02.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:02.253 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:02.284 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T09:59:02.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:02.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:02.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:02.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:02.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:02.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:02.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:02.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:02.646 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:59:02.675 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:02.701 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:02.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:02.829 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:02.930 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:02.954 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:02.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:03.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:03.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:03.201 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:03.232 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:03.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:03.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:03.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:03.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:03.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:03.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:03.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:03.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:03.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:03.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.812 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:03.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:03.913 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:03.937 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:03.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:03 smithi017 ceph-mon[102913]: pgmap v1350: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 973/209316 objects degraded (0.465%); 3508/209316 objects misplaced (1.676%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:59:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:03 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 973/209316 objects degraded (0.465%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:03 smithi017 ceph-mon[105151]: pgmap v1350: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 973/209316 objects degraded (0.465%); 3508/209316 objects misplaced (1.676%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:59:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:03 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 973/209316 objects degraded (0.465%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:03 smithi134 ceph-mon[90085]: pgmap v1350: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 973/209316 objects degraded (0.465%); 3508/209316 objects misplaced (1.676%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:59:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:03 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 973/209316 objects degraded (0.465%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:04.039 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:04.068 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:04.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:04.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:04.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:04.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:04.318 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:04.345 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:04.446 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:04.474 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:04.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:04.576 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:04.600 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:04.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:04.675 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T09:59:04.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:04.730 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:04.831 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:04.858 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:04.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:04 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/370520099' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:04 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:04.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:59:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:59:04] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-09-21T09:59:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:04 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/370520099' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:04 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:04.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:04.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:04.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:04 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/370520099' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:04 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:05.088 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:05.109 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:05.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:05.211 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:05.232 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:05.335 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:05.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.357 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:05.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:05.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:05.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:05.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:05.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:05.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:05.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:05.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:05.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:05.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:05.845 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:05.879 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:05.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:05 smithi017 ceph-mon[102913]: pgmap v1351: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 973/205353 objects degraded (0.474%); 2631/205353 objects misplaced (1.281%); 5.5 MiB/s, 88 objects/s recovering 2024-09-21T09:59:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:05 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:59:05.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:05 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:05.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:05 smithi017 ceph-mon[102913]: osdmap e1113: 8 total, 8 up, 8 in 2024-09-21T09:59:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:05 smithi017 ceph-mon[105151]: pgmap v1351: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 973/205353 objects degraded (0.474%); 2631/205353 objects misplaced (1.281%); 5.5 MiB/s, 88 objects/s recovering 2024-09-21T09:59:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:05 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:59:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:05 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:05 smithi017 ceph-mon[105151]: osdmap e1113: 8 total, 8 up, 8 in 2024-09-21T09:59:05.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:05 smithi134 ceph-mon[90085]: pgmap v1351: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 973/205353 objects degraded (0.474%); 2631/205353 objects misplaced (1.281%); 5.5 MiB/s, 88 objects/s recovering 2024-09-21T09:59:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:05 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:59:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:05 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:05 smithi134 ceph-mon[90085]: osdmap e1113: 8 total, 8 up, 8 in 2024-09-21T09:59:06.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:06.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:06.045 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T09:59:06.045 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:59:06.117 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:06.150 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:06.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:06.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:06.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.276 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:06.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:06.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:06.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:06.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:06.511 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:06.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:06.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:06.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:06.672 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:06.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:06.773 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:06.804 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:06.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:06.889 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T09:59:06.905 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:06.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:06 smithi017 ceph-mon[102913]: 1.0 scrub starts 2024-09-21T09:59:06.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:06 smithi017 ceph-mon[102913]: osdmap e1114: 8 total, 8 up, 8 in 2024-09-21T09:59:06.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:06 smithi017 ceph-mon[105151]: 1.0 scrub starts 2024-09-21T09:59:06.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:06 smithi017 ceph-mon[105151]: osdmap e1114: 8 total, 8 up, 8 in 2024-09-21T09:59:06.949 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:59:06.950 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:59:06.950 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T09:59:06.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:59:06.980 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:06.993 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:06 smithi134 ceph-mon[90085]: 1.0 scrub starts 2024-09-21T09:59:06.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:06 smithi134 ceph-mon[90085]: osdmap e1114: 8 total, 8 up, 8 in 2024-09-21T09:59:07.081 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.114 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:07.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:07.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:07.231 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:59:07.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:07.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:07.368 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:07.397 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:07.499 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:07.527 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:07.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:07.653 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:07.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:07.755 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:07.787 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:07.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:07.844 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:07 smithi134 ceph-mon[90085]: pgmap v1354: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 973/197574 objects degraded (0.492%); 2631/197574 objects misplaced (1.332%); 3.4 MiB/s, 54 objects/s recovering 2024-09-21T09:59:07.844 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:07 smithi134 ceph-mon[90085]: 1.4 scrub starts 2024-09-21T09:59:07.844 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:07 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2119505110' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:07.844 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:07 smithi134 ceph-mon[90085]: 1.5 scrub starts 2024-09-21T09:59:07.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:07.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:07 smithi017 ceph-mon[102913]: pgmap v1354: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 973/197574 objects degraded (0.492%); 2631/197574 objects misplaced (1.332%); 3.4 MiB/s, 54 objects/s recovering 2024-09-21T09:59:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:07 smithi017 ceph-mon[102913]: 1.4 scrub starts 2024-09-21T09:59:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:07 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2119505110' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:07 smithi017 ceph-mon[102913]: 1.5 scrub starts 2024-09-21T09:59:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:07 smithi017 ceph-mon[105151]: pgmap v1354: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 955 op/s; 973/197574 objects degraded (0.492%); 2631/197574 objects misplaced (1.332%); 3.4 MiB/s, 54 objects/s recovering 2024-09-21T09:59:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:07 smithi017 ceph-mon[105151]: 1.4 scrub starts 2024-09-21T09:59:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:07 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2119505110' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:07 smithi017 ceph-mon[105151]: 1.5 scrub starts 2024-09-21T09:59:08.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:08.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:08.175 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:08.209 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:08.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:08.221 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:59:08.221 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:59:08.221 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:59:08.222 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:59:08.222 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:59:08.222 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:59:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:59:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:59:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:59:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:59:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:59:08.223 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:59:08.223 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:59:08.223 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:59:08.223 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:59:08.223 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:59:08.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:08.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:08.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:08.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:08.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:08.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:08.570 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:08.601 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:08.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:08.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:08.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:08.824 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:08 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 973/197574 objects degraded (0.492%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:08.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:08.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:08.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:08 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 973/197574 objects degraded (0.492%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:08.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:08 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 973/197574 objects degraded (0.492%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:08.992 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:09.026 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:09.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:09.128 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:09.129 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:59:09.129 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T09:59:09.129 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T09:59:09.129 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T09:59:09.129 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:59:09.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:09.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:09.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:09.295 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:09.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:09.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:09.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:09.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:09.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:09.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:09.678 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:09.704 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:09.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:09 smithi134 ceph-mon[90085]: pgmap v1355: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 973/194772 objects degraded (0.500%); 1691/194772 objects misplaced (0.868%); 6.5 MiB/s, 104 objects/s recovering 2024-09-21T09:59:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:09 smithi134 ceph-mon[90085]: 1.a scrub starts 2024-09-21T09:59:09.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:09.839 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:09.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:09.850 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T09:59:09.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:09 smithi017 ceph-mon[102913]: pgmap v1355: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 973/194772 objects degraded (0.500%); 1691/194772 objects misplaced (0.868%); 6.5 MiB/s, 104 objects/s recovering 2024-09-21T09:59:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:09 smithi017 ceph-mon[102913]: 1.a scrub starts 2024-09-21T09:59:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:09 smithi017 ceph-mon[105151]: pgmap v1355: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 973/194772 objects degraded (0.500%); 1691/194772 objects misplaced (0.868%); 6.5 MiB/s, 104 objects/s recovering 2024-09-21T09:59:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:09 smithi017 ceph-mon[105151]: 1.a scrub starts 2024-09-21T09:59:09.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:09.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:10.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:10.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.131 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:10.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:10.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:10.252 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:59:10.276 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:10.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:10.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:10.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:10.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:59:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:59:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:10.525 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:10.558 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:10.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:10.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:10.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:10.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:10.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:10.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:10.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:10.957 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:10.987 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:10.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:11.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:11.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:11.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:11.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:11.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:11.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:11.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:11 smithi017 ceph-mon[102913]: pgmap v1356: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 973/193326 objects degraded (0.503%); 1691/193326 objects misplaced (0.875%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T09:59:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:11 smithi017 ceph-mon[105151]: pgmap v1356: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 973/193326 objects degraded (0.503%); 1691/193326 objects misplaced (0.875%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T09:59:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:11 smithi134 ceph-mon[90085]: pgmap v1356: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 973/193326 objects degraded (0.503%); 1691/193326 objects misplaced (0.875%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T09:59:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:11.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:11.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:11.555 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:11.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:11.656 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:11.659 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:11.684 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:11.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:11.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:11.813 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:11.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:11.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:11.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:11.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:12.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:12.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:12.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:12.207 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:59:12.207 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45173.0:60912 6.3 e1085/1078)", 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.399084+0000", 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 125.804119586, 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.096088522999999995, 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.208 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399084+0000", 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399084+0000", 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399082+0000", 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.209 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399113+0000", 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0623999999999997e-05 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399133+0000", 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0061e-05 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399139+0000", 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8789999999999997e-06 2024-09-21T09:59:12.210 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.495153+0000", 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.096013182000000002 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.495172+0000", 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9661e-05 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.211 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90845 6.a 6:5de4ee99:::benchmark_data_smithi017_149547_object90844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.571935+0000", 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 93.631268024999997, 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.085221939, 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90845 2024-09-21T09:59:12.212 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.571935+0000", 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.571935+0000", 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.213 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.571934+0000", 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572128+0000", 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019428100000000001 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572139+0000", 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1455e-05 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.214 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572144+0000", 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8890000000000001e-06 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572202+0000", 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8139e-05 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572213+0000", 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0562e-05 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.215 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.640648+0000", 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.068435024 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642661+0000", 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002012715 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642814+0000", 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015295599999999999 2024-09-21T09:59:12.216 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651043+0000", 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0082295990000000006 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651085+0000", 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1270000000000003e-05 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657094+0000", 2024-09-21T09:59:12.217 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.006009258 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657129+0000", 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5426999999999998e-05 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657157+0000", 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8014999999999999e-05 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.218 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90864 6.a 6:5a2b4d81:::benchmark_data_smithi017_149547_object90863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.594822+0000", 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 93.608381766999997, 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0624268729999999, 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90864 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.219 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594822+0000", 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594822+0000", 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594820+0000", 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.220 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594917+0000", 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6347000000000004e-05 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594928+0000", 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1263999999999999e-05 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.221 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594934+0000", 2024-09-21T09:59:12.222 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9699999999999996e-06 2024-09-21T09:59:12.222 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.222 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.222 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.222 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594987+0000", 2024-09-21T09:59:12.222 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3377000000000001e-05 2024-09-21T09:59:12.222 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594997+0000", 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0078000000000001e-05 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.645551+0000", 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.050553455 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.645683+0000", 2024-09-21T09:59:12.223 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132762 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.645832+0000", 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000148992 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651118+0000", 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0052851829999999997 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.224 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651160+0000", 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2794999999999999e-05 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657187+0000", 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0060262620000000001 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657217+0000", 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0439000000000001e-05 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.225 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657248+0000", 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1368999999999997e-05 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90868 6.a 6:506b43e1:::benchmark_data_smithi017_149547_object90867:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.596312+0000", 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 93.606891611999998, 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.066231537, 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.226 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90868 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596312+0000", 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596312+0000", 2024-09-21T09:59:12.227 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596311+0000", 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596386+0000", 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4889999999999996e-05 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.228 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596397+0000", 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1117999999999999e-05 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596402+0000", 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0420000000000002e-06 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596441+0000", 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8637000000000003e-05 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T09:59:12.229 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596447+0000", 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6919999999999996e-06 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.648577+0000", 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0521300920000001 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.648714+0000", 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013687000000000001 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.230 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.648850+0000", 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013567900000000001 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653519+0000", 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0046685540000000001 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653542+0000", 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3515000000000001e-05 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.231 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.662408+0000", 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0088662189999999998 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.662484+0000", 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5351000000000004e-05 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.662543+0000", 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9657000000000002e-05 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.232 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90889 6.a 6:58dc9b35:::benchmark_data_smithi017_149547_object90888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.649533+0000", 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 93.553670298, 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0149460320000001, 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90889 2024-09-21T09:59:12.233 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649533+0000", 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649533+0000", 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.234 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649532+0000", 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649557+0000", 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.5724e-05 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649570+0000", 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3212e-05 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.235 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649576+0000", 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.993e-06 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649633+0000", 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6147999999999998e-05 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649640+0000", 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2370000000000003e-06 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.236 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651180+0000", 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.00154027 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651306+0000", 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012564299999999999 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651449+0000", 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000143218 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.237 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653646+0000", 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021974989999999999 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653979+0000", 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00033292499999999997 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.664396+0000", 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010416262000000001 2024-09-21T09:59:12.238 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.664443+0000", 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7518999999999997e-05 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.664479+0000", 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5933000000000003e-05 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.239 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45173.0:90890 6.3 e1101/1086 6:c2ec370d:::benchmark_data_smithi017_149547_object90889:head v 1101'5132, mlcod=1100'5130)", 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:39.548490+0000", 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 92.654713164, 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090342275, 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.548490+0000", 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.240 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.548490+0000", 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.548490+0000", 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.548515+0000", 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.5077000000000001e-05 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.241 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.633977+0000", 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.085461703999999999 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.633985+0000", 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3410000000000003e-06 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.634040+0000", 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4664999999999999e-05 2024-09-21T09:59:12.242 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.634061+0000", 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1041999999999999e-05 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.638797+0000", 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0047363680000000003 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.638832+0000", 2024-09-21T09:59:12.243 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5349000000000003e-05 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:124603 6.a 6:5cfb9b85:::benchmark_data_smithi017_149547_object5524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:12.772944+0000", 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 59.430258930999997, 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.037996799999999997, 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.244 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 124603 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.772944+0000", 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.772944+0000", 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.245 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.772943+0000", 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.772949+0000", 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6189999999999999e-06 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.246 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.772960+0000", 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0597999999999999e-05 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.772964+0000", 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9690000000000002e-06 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.773006+0000", 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1582999999999998e-05 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.247 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.773106+0000", 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010014300000000001 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.773167+0000", 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1085000000000002e-05 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.779444+0000", 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0062768679999999997 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.248 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.779478+0000", 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4214000000000001e-05 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.810846+0000", 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.03136771 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.810903+0000", 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6728999999999998e-05 2024-09-21T09:59:12.249 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.810941+0000", 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8444e-05 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:124604 6.a 6:5dd3bce6:::benchmark_data_smithi017_149547_object5525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:12.773484+0000", 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 59.429719798999997, 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.037597063, 2024-09-21T09:59:12.250 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 124604 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.773484+0000", 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.251 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.773484+0000", 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.773483+0000", 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.773487+0000", 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8559999999999998e-06 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.252 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.773500+0000", 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2330999999999999e-05 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.773505+0000", 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4229999999999996e-06 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.776739+0000", 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032333309999999999 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.253 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.776863+0000", 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000124249 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.776953+0000", 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.9829999999999999e-05 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.779535+0000", 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0025823069999999998 2024-09-21T09:59:12.254 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.780051+0000", 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00051559999999999996 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.810990+0000", 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.030939465999999999 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.811046+0000", 2024-09-21T09:59:12.255 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6326000000000002e-05 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.811081+0000", 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4319000000000001e-05 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:127753 6.a 6:56277f43:::benchmark_data_smithi017_149547_object8674:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:16.117633+0000", 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 56.085570195999999, 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.038599228999999999, 2024-09-21T09:59:12.256 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 127753 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.117633+0000", 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.257 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.117633+0000", 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.117632+0000", 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.117637+0000", 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4620000000000004e-06 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.258 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.117649+0000", 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2197e-05 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.117655+0000", 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4099999999999999e-06 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.118077+0000", 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00042235800000000001 2024-09-21T09:59:12.259 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.118187+0000", 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000109719 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.118292+0000", 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000104552 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.122065+0000", 2024-09-21T09:59:12.260 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0037735099999999999 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.122147+0000", 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1997999999999997e-05 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.156137+0000", 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.033989520000000002 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.261 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.156197+0000", 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0108000000000001e-05 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:16.156232+0000", 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5775000000000002e-05 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.262 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.264 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:160524 6.a 6:58faa9e4:::benchmark_data_smithi017_149547_object41445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-21T09:59:12.264 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:51.271864+0000", 2024-09-21T09:59:12.264 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 20.931339099999999, 2024-09-21T09:59:12.264 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.051360630999999997, 2024-09-21T09:59:12.264 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.264 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 160524 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.271864+0000", 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.265 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.271864+0000", 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.271864+0000", 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.271867+0000", 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.427e-06 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.266 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.271880+0000", 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3095000000000001e-05 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.271885+0000", 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1960000000000001e-06 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.276552+0000", 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0046674459999999996 2024-09-21T09:59:12.267 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.276684+0000", 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013194699999999999 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.276881+0000", 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000197352 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.284050+0000", 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0071689730000000004 2024-09-21T09:59:12.268 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.286314+0000", 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0022642669999999999 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.323131+0000", 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.036816728 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.269 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.323188+0000", 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6813e-05 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.323225+0000", 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6887000000000001e-05 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:160525 6.a 6:5f3da79a:::benchmark_data_smithi017_149547_object41446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:51.278784+0000", 2024-09-21T09:59:12.270 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 20.924419138000001, 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.044583291999999997, 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 160525 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.278784+0000", 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.271 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.278784+0000", 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.278783+0000", 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.278789+0000", 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6170000000000003e-06 2024-09-21T09:59:12.272 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.278804+0000", 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5051999999999999e-05 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.278811+0000", 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5899999999999996e-06 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.280574+0000", 2024-09-21T09:59:12.273 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0017631579999999999 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.280898+0000", 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00032423899999999998 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.281296+0000", 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00039813799999999997 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.274 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.286260+0000", 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0049631559999999998 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.286741+0000", 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00048135099999999998 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.323283+0000", 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.036541767000000003 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.275 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.323330+0000", 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6774999999999999e-05 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.323368+0000", 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8028999999999998e-05 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:160529 6.a 6:56d8a967:::benchmark_data_smithi017_149547_object41450:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-21T09:59:12.276 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:51.283074+0000", 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 20.920128905999999, 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.040414512, 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 160529 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.283074+0000", 2024-09-21T09:59:12.277 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.283074+0000", 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.283073+0000", 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.278 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.283080+0000", 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2019999999999998e-06 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.283097+0000", 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6841e-05 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.283104+0000", 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.759e-06 2024-09-21T09:59:12.279 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.284068+0000", 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00096451100000000004 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.284145+0000", 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6748000000000004e-05 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.284215+0000", 2024-09-21T09:59:12.280 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9999999999999994e-05 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.286347+0000", 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021320950000000001 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.286864+0000", 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00051681000000000003 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.281 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.323409+0000", 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.036545701 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.323449+0000", 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9882999999999997e-05 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:51.323489+0000", 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9613999999999997e-05 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.282 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:174911 6.a 6:5f07c25e:::benchmark_data_smithi017_149547_object55832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:07.969173+0000", 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 4.2340300339999999, 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.057985110999999999, 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 174911 2024-09-21T09:59:12.283 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.969173+0000", 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.969173+0000", 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.969172+0000", 2024-09-21T09:59:12.284 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.969178+0000", 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9599999999999997e-06 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.969188+0000", 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0001e-05 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.285 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.969194+0000", 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1870000000000002e-06 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.969230+0000", 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5463999999999998e-05 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.969315+0000", 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5556000000000006e-05 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.286 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.969402+0000", 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.6281999999999998e-05 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.972339+0000", 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029370099999999999 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.972474+0000", 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000135599 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.287 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.027087+0000", 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.054612382000000001 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.027120+0000", 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3080999999999997e-05 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.027158+0000", 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8717000000000002e-05 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.288 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:174918 6.a 6:541a778a:::benchmark_data_smithi017_149547_object55839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:07.974413+0000", 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 4.2287902629999996, 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.052861009, 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 174918 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.289 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.974413+0000", 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.974413+0000", 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.974412+0000", 2024-09-21T09:59:12.290 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.974420+0000", 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3089999999999996e-06 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.974433+0000", 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2798e-05 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.291 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.974438+0000", 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5770000000000001e-06 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.974478+0000", 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9529e-05 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.974570+0000", 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.2122999999999999e-05 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.292 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.974657+0000", 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7318999999999995e-05 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.977656+0000", 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029989690000000002 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.978281+0000", 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00062451599999999998 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.293 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.027195+0000", 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.048914098000000003 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.027236+0000", 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1415000000000002e-05 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.027274+0000", 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7710999999999999e-05 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.294 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:174927 6.a 6:568172c5:::benchmark_data_smithi017_149547_object55848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:07.983585+0000", 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 4.2196184399999996, 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.047299310999999997, 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.295 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 174927 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.983585+0000", 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.983585+0000", 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.296 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.983583+0000", 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.983590+0000", 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7710000000000001e-06 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.983604+0000", 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3822000000000001e-05 2024-09-21T09:59:12.297 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.983610+0000", 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2400000000000004e-06 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.983655+0000", 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4948999999999999e-05 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.983782+0000", 2024-09-21T09:59:12.298 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012713500000000001 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.983916+0000", 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000133645 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.986645+0000", 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0027291809999999998 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.299 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:07.986849+0000", 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000203847 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.030777+0000", 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.043928455999999998 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.030841+0000", 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3707999999999994e-05 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.300 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.030884+0000", 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3299e-05 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175289 6.a 6:5b89aa44:::benchmark_data_smithi017_149547_object56210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.472307+0000", 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 3.7308966460000001, 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.097797054999999994, 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.301 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175289 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472307+0000", 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.302 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472307+0000", 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472305+0000", 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472312+0000", 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8179999999999998e-06 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.303 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472323+0000", 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1287999999999999e-05 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472328+0000", 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3959999999999999e-06 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472359+0000", 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1241999999999997e-05 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.304 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472433+0000", 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3633999999999993e-05 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472498+0000", 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5798999999999998e-05 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.479754+0000", 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0072554810000000003 2024-09-21T09:59:12.305 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.479766+0000", 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2462e-05 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570021+0000", 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090255143999999995 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570060+0000", 2024-09-21T09:59:12.306 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8509e-05 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570104+0000", 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3881000000000002e-05 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175291 6.a 6:57b1c613:::benchmark_data_smithi017_149547_object56212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.473105+0000", 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 3.730098645, 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.097093223000000006, 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175291 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473105+0000", 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.308 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473105+0000", 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473104+0000", 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473108+0000", 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4000000000000002e-06 2024-09-21T09:59:12.309 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473120+0000", 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1508000000000001e-05 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473125+0000", 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3129999999999998e-06 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.310 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.475454+0000", 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002329224 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.475579+0000", 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000124721 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.475713+0000", 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000133452 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.311 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.479795+0000", 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0040822089999999998 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.479840+0000", 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4749999999999997e-05 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570143+0000", 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090303226 2024-09-21T09:59:12.312 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570175+0000", 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2264999999999997e-05 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570198+0000", 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2884e-05 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.313 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175300 6.a 6:5ad5117d:::benchmark_data_smithi017_149547_object56221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.480743+0000", 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 3.7224607189999999, 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.104015855, 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175300 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.314 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480743+0000", 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480743+0000", 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480741+0000", 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.315 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480747+0000", 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.913e-06 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480758+0000", 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0512e-05 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480763+0000", 2024-09-21T09:59:12.316 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7430000000000002e-06 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480798+0000", 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5318000000000003e-05 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480906+0000", 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010820899999999999 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480981+0000", 2024-09-21T09:59:12.317 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5024000000000005e-05 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.484208+0000", 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032268499999999999 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.484654+0000", 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00044641 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.318 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584669+0000", 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.100014744 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584717+0000", 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7843000000000003e-05 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584759+0000", 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1505000000000003e-05 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.319 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175307 6.a 6:54170271:::benchmark_data_smithi017_149547_object56228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.487486+0000", 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 3.7157169809999999, 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.097409080999999995, 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175307 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.320 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487486+0000", 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487486+0000", 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487485+0000", 2024-09-21T09:59:12.321 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487494+0000", 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6320000000000003e-06 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487510+0000", 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5277e-05 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.322 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487516+0000", 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0020000000000003e-06 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487567+0000", 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1190000000000003e-05 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487696+0000", 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012929400000000001 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.323 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487831+0000", 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013468900000000001 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.490741+0000", 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029106179999999998 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.491449+0000", 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00070706499999999999 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.324 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584809+0000", 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.093360925999999997 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584850+0000", 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0460000000000002e-05 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584895+0000", 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5578999999999998e-05 2024-09-21T09:59:12.325 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175325 6.a 6:59999bd4:::benchmark_data_smithi017_149547_object56246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.505520+0000", 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 3.6976829929999999, 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.079513039999999993, 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:12.326 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175325 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.505520+0000", 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.505520+0000", 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:12.327 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.505519+0000", 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.505528+0000", 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8019999999999998e-06 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.505543+0000", 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5301000000000001e-05 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.328 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.505548+0000", 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5110000000000003e-06 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.505599+0000", 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0544999999999998e-05 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.505714+0000", 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000114821 2024-09-21T09:59:12.329 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.505790+0000", 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6055000000000003e-05 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.508478+0000", 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002687788 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.508855+0000", 2024-09-21T09:59:12.330 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00037713599999999998 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584937+0000", 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.076082330000000004 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584993+0000", 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5915999999999998e-05 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T09:59:12.331 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.585033+0000", 2024-09-21T09:59:12.332 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0388e-05 2024-09-21T09:59:12.332 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.332 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.332 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.332 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:12.332 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:12.333 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:59:12.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:12.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:12.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:12.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:12 smithi017 ceph-mon[102913]: 1.3 scrub starts 2024-09-21T09:59:12.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:12 smithi017 ceph-mon[105151]: 1.3 scrub starts 2024-09-21T09:59:12.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:12.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:12 smithi134 ceph-mon[90085]: 1.3 scrub starts 2024-09-21T09:59:12.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:12.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:12.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:12.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:12.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:12.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:12.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:12.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:13.015 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:13.043 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:13.123 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T09:59:13.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:13.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:59:13.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:13.290 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:13 smithi017 ceph-mon[102913]: pgmap v1357: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 973/182988 objects degraded (0.532%); 1327/182988 objects misplaced (0.725%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T09:59:13.291 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:13 smithi017 ceph-mon[105151]: pgmap v1357: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 973/182988 objects degraded (0.532%); 1327/182988 objects misplaced (0.725%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T09:59:13.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:13.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.352 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:13.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:13.453 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:13.464 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:59:13.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:13.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:13.493 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:13 smithi134 ceph-mon[90085]: pgmap v1357: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 973/182988 objects degraded (0.532%); 1327/182988 objects misplaced (0.725%); 4.5 MiB/s, 72 objects/s recovering 2024-09-21T09:59:13.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:13.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T09:59:13.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:13.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:13.744 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:13.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.771 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:13.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:13.858 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:13.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:13.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:13.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:14.052 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:14.076 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:14.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:14.201 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:14.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:14.233 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T09:59:14.237 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T09:59:14.303 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:14.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:14.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:14.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:14.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:14.571 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:14.599 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:14.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:14.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:14 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 973/182988 objects degraded (0.532%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:14.685 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:14 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 973/182988 objects degraded (0.532%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:14.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:14.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.737 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:14.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:14.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:14 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 973/182988 objects degraded (0.532%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:14.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:14.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:14.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:59:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:59:14] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.51.0" 2024-09-21T09:59:14.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:15.015 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:15.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:15.068 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1358,"stamp":"2024-09-21T09:59:14.052122+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3919386597,"num_objects":59823,"num_object_clones":0,"num_object_copies":179469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":973,"num_objects_misplaced":1327,"num_objects_unfound":0,"num_objects_dirty":59823,"num_whiteouts":0,"num_read":307,"num_read_kb":760,"num_write":138988,"num_write_kb":5094716,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10247,"num_bytes_recovered":672006688,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":193213,"ondisk_log_size":193213,"up":327,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":338,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14884452,"kb_used_data":12958948,"kb_used_omap":122,"kb_used_meta":1924869,"kb_avail":735076764,"statfs":{"total":767960285184,"available":752718606336,"internally_reserved":0,"allocated":13269962752,"data_stored":13251982479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125317,"internal_metadata":1971066491},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,3,6,2,0,11],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-565444608,"num_objects":-8628,"num_object_clones":0,"num_object_copies":-25884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1304,"num_objects_unfound":0,"num_objects_dirty":-8628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8628,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":581,"num_bytes_recovered":38076416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.385886"},"pg_stats":[{"pgid":"6.16","version":"1114'7421","reported_seq":7548,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.616173+0000","last_change":"2024-09-21T09:57:11.311033+0000","last_active":"2024-09-21T09:59:10.616173+0000","last_peered":"2024-09-21T09:59:10.616173+0000","last_clean":"2024-09-21T09:59:10.616173+0000","last_became_active":"2024-09-21T09:57:07.444641+0000","last_became_peered":"2024-09-21T09:57:07.444641+0000","last_unstale":"2024-09-21T09:59:10.616173+0000","last_undegraded":"2024-09-21T09:59:10.616173+0000","last_fullsized":"2024-09-21T09:59:10.616173+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'4023","last_scrub_stamp":"2024-09-21T09:57:11.310699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:11.310699+0000","objects_scrubbed":2106,"log_size":7421,"log_dups_size":0,"ondisk_log_size":7421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:52.271305+0000","scrub_duration":650,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117768192,"num_objects":1797,"num_object_clones":0,"num_object_copies":5391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3413,"num_write_kb":99328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1563,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.682390+0000","last_change":"2024-09-21T09:46:47.114206+0000","last_active":"2024-09-21T09:59:05.682390+0000","last_peered":"2024-09-21T09:59:05.682390+0000","last_clean":"2024-09-21T09:59:05.682390+0000","last_became_active":"2024-09-21T09:46:47.113819+0000","last_became_peered":"2024-09-21T09:46:47.113819+0000","last_unstale":"2024-09-21T09:59:05.682390+0000","last_undegraded":"2024-09-21T09:59:05.682390+0000","last_fullsized":"2024-09-21T09:59:05.682390+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:07.629544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1039,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679019+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T09:59:04.679019+0000","last_peered":"2024-09-21T09:59:04.679019+0000","last_clean":"2024-09-21T09:59:04.679019+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T09:59:04.679019+0000","last_undegraded":"2024-09-21T09:59:04.679019+0000","last_fullsized":"2024-09-21T09:59:04.679019+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.17","version":"1114'7546","reported_seq":7718,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.595756+0000","last_change":"2024-09-21T09:58:39.037687+0000","last_active":"2024-09-21T09:59:10.595756+0000","last_peered":"2024-09-21T09:59:10.595756+0000","last_clean":"2024-09-21T09:59:10.595756+0000","last_became_active":"2024-09-21T09:57:39.544853+0000","last_became_peered":"2024-09-21T09:57:39.544853+0000","last_unstale":"2024-09-21T09:59:10.595756+0000","last_undegraded":"2024-09-21T09:59:10.595756+0000","last_fullsized":"2024-09-21T09:59:10.595756+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'4025","last_scrub_stamp":"2024-09-21T09:57:10.848092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:10.848092+0000","objects_scrubbed":2113,"log_size":7546,"log_dups_size":0,"ondisk_log_size":7546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:42:29.235117+0000","scrub_duration":0.575755185,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128122880,"num_objects":1955,"num_object_clones":0,"num_object_copies":5865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3530,"num_write_kb":107904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":547,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679527+0000","last_change":"2024-09-21T09:57:39.541887+0000","last_active":"2024-09-21T09:59:04.679527+0000","last_peered":"2024-09-21T09:59:04.679527+0000","last_clean":"2024-09-21T09:59:04.679527+0000","last_became_active":"2024-09-21T09:57:39.541598+0000","last_became_peered":"2024-09-21T09:57:39.541598+0000","last_unstale":"2024-09-21T09:59:04.679527+0000","last_undegraded":"2024-09-21T09:59:04.679527+0000","last_fullsized":"2024-09-21T09:59:04.679527+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:41.615525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1647,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684080+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T09:59:05.684080+0000","last_peered":"2024-09-21T09:59:05.684080+0000","last_clean":"2024-09-21T09:59:05.684080+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T09:59:05.684080+0000","last_undegraded":"2024-09-21T09:59:05.684080+0000","last_fullsized":"2024-09-21T09:59:05.684080+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.14","version":"1114'7464","reported_seq":16793,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:11.561964+0000","last_change":"2024-09-21T09:59:05.689580+0000","last_active":"2024-09-21T09:59:11.561964+0000","last_peered":"2024-09-21T09:59:11.561964+0000","last_clean":"2024-09-21T09:59:11.561964+0000","last_became_active":"2024-09-21T09:59:05.672682+0000","last_became_peered":"2024-09-21T09:59:05.672682+0000","last_unstale":"2024-09-21T09:59:11.561964+0000","last_undegraded":"2024-09-21T09:59:11.561964+0000","last_fullsized":"2024-09-21T09:59:11.561964+0000","mapping_epoch":1113,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":5,"last_scrub":"1088'4137","last_scrub_stamp":"2024-09-21T09:57:16.338288+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:16.338288+0000","objects_scrubbed":2242,"log_size":7464,"log_dups_size":0,"ondisk_log_size":7464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:22:38.737280+0000","scrub_duration":308,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118816768,"num_objects":1813,"num_object_clones":0,"num_object_copies":5439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3327,"num_write_kb":92736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2297,"num_bytes_recovered":150536192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1648,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684369+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T09:59:05.684369+0000","last_peered":"2024-09-21T09:59:05.684369+0000","last_clean":"2024-09-21T09:59:05.684369+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T09:59:05.684369+0000","last_undegraded":"2024-09-21T09:59:05.684369+0000","last_fullsized":"2024-09-21T09:59:05.684369+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1020,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679498+0000","last_change":"2024-09-21T09:36:43.113059+0000","last_active":"2024-09-21T09:59:04.679498+0000","last_peered":"2024-09-21T09:59:04.679498+0000","last_clean":"2024-09-21T09:59:04.679498+0000","last_became_active":"2024-09-21T09:36:43.112769+0000","last_became_peered":"2024-09-21T09:36:43.112769+0000","last_unstale":"2024-09-21T09:59:04.679498+0000","last_undegraded":"2024-09-21T09:59:04.679498+0000","last_fullsized":"2024-09-21T09:59:04.679498+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:24.550699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.15","version":"1114'7521","reported_seq":7675,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.794987+0000","last_change":"2024-09-21T09:57:15.230133+0000","last_active":"2024-09-21T09:59:10.794987+0000","last_peered":"2024-09-21T09:59:10.794987+0000","last_clean":"2024-09-21T09:59:10.794987+0000","last_became_active":"2024-09-21T09:57:07.439001+0000","last_became_peered":"2024-09-21T09:57:07.439001+0000","last_unstale":"2024-09-21T09:59:10.794987+0000","last_undegraded":"2024-09-21T09:59:10.794987+0000","last_fullsized":"2024-09-21T09:59:10.794987+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1088'4200","last_scrub_stamp":"2024-09-21T09:57:15.230067+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:15.230067+0000","objects_scrubbed":2263,"log_size":7521,"log_dups_size":0,"ondisk_log_size":7521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:22.500492+0000","scrub_duration":0.27387100399999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123076608,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3321,"num_write_kb":93952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1563,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.682823+0000","last_change":"2024-09-21T09:46:47.117580+0000","last_active":"2024-09-21T09:59:05.682823+0000","last_peered":"2024-09-21T09:59:05.682823+0000","last_clean":"2024-09-21T09:59:05.682823+0000","last_became_active":"2024-09-21T09:46:47.113030+0000","last_became_peered":"2024-09-21T09:46:47.113030+0000","last_unstale":"2024-09-21T09:59:05.682823+0000","last_undegraded":"2024-09-21T09:59:05.682823+0000","last_fullsized":"2024-09-21T09:59:05.682823+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:28.220180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":1046,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.675442+0000","last_change":"2024-09-21T09:45:05.080014+0000","last_active":"2024-09-21T09:59:04.675442+0000","last_peered":"2024-09-21T09:59:04.675442+0000","last_clean":"2024-09-21T09:59:04.675442+0000","last_became_active":"2024-09-21T09:45:05.079618+0000","last_became_peered":"2024-09-21T09:45:05.079618+0000","last_unstale":"2024-09-21T09:59:04.675442+0000","last_undegraded":"2024-09-21T09:59:04.675442+0000","last_fullsized":"2024-09-21T09:59:04.675442+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:42.160292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1114'11062","reported_seq":11113,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.898923+0000","last_change":"2024-09-21T09:57:39.645463+0000","last_active":"2024-09-21T09:59:10.898923+0000","last_peered":"2024-09-21T09:59:10.898923+0000","last_clean":"2024-09-21T09:59:10.898923+0000","last_became_active":"2024-09-21T09:57:39.634848+0000","last_became_peered":"2024-09-21T09:57:39.634848+0000","last_unstale":"2024-09-21T09:59:10.898923+0000","last_undegraded":"2024-09-21T09:59:10.898923+0000","last_fullsized":"2024-09-21T09:59:10.898923+0000","mapping_epoch":1100,"log_start":"1087'4390","ondisk_log_start":"1087'4390","created":1078,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":6672,"log_dups_size":0,"ondisk_log_size":6672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:51:28.212787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243531776,"num_objects":3716,"num_object_clones":0,"num_object_copies":11148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11062,"num_write_kb":472896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1038,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679084+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T09:59:04.679084+0000","last_peered":"2024-09-21T09:59:04.679084+0000","last_clean":"2024-09-21T09:59:04.679084+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T09:59:04.679084+0000","last_undegraded":"2024-09-21T09:59:04.679084+0000","last_fullsized":"2024-09-21T09:59:04.679084+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":26,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678508+0000","last_change":"2024-09-21T09:57:39.636116+0000","last_active":"2024-09-21T09:59:04.678508+0000","last_peered":"2024-09-21T09:59:04.678508+0000","last_clean":"2024-09-21T09:59:04.678508+0000","last_became_active":"2024-09-21T09:57:39.635802+0000","last_became_peered":"2024-09-21T09:57:39.635802+0000","last_unstale":"2024-09-21T09:59:04.678508+0000","last_undegraded":"2024-09-21T09:59:04.678508+0000","last_fullsized":"2024-09-21T09:59:04.678508+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:43:40.045268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.b","version":"1114'11076","reported_seq":11167,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.568937+0000","last_change":"2024-09-21T09:56:26.652941+0000","last_active":"2024-09-21T09:59:10.568937+0000","last_peered":"2024-09-21T09:59:10.568937+0000","last_clean":"2024-09-21T09:59:10.568937+0000","last_became_active":"2024-09-21T09:56:26.652635+0000","last_became_peered":"2024-09-21T09:56:26.652635+0000","last_unstale":"2024-09-21T09:59:10.568937+0000","last_undegraded":"2024-09-21T09:59:10.568937+0000","last_fullsized":"2024-09-21T09:59:10.568937+0000","mapping_epoch":1078,"log_start":"1098'5000","ondisk_log_start":"1098'5000","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":6076,"log_dups_size":0,"ondisk_log_size":6076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:00.304863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250740736,"num_objects":3826,"num_object_clones":0,"num_object_copies":11478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11076,"num_write_kb":476864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1039,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679749+0000","last_change":"2024-09-21T09:45:39.143190+0000","last_active":"2024-09-21T09:59:04.679749+0000","last_peered":"2024-09-21T09:59:04.679749+0000","last_clean":"2024-09-21T09:59:04.679749+0000","last_became_active":"2024-09-21T09:45:39.142826+0000","last_became_peered":"2024-09-21T09:45:39.142826+0000","last_unstale":"2024-09-21T09:59:04.679749+0000","last_undegraded":"2024-09-21T09:59:04.679749+0000","last_fullsized":"2024-09-21T09:59:04.679749+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:45.070945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1072,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678202+0000","last_change":"2024-09-21T09:46:47.119247+0000","last_active":"2024-09-21T09:59:04.678202+0000","last_peered":"2024-09-21T09:59:04.678202+0000","last_clean":"2024-09-21T09:59:04.678202+0000","last_became_active":"2024-09-21T09:46:47.118980+0000","last_became_peered":"2024-09-21T09:46:47.118980+0000","last_unstale":"2024-09-21T09:59:04.678202+0000","last_undegraded":"2024-09-21T09:59:04.678202+0000","last_fullsized":"2024-09-21T09:59:04.678202+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.140253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1114'7580","reported_seq":7738,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.756448+0000","last_change":"2024-09-21T09:57:14.741073+0000","last_active":"2024-09-21T09:59:10.756448+0000","last_peered":"2024-09-21T09:59:10.756448+0000","last_clean":"2024-09-21T09:59:10.756448+0000","last_became_active":"2024-09-21T09:57:07.438175+0000","last_became_peered":"2024-09-21T09:57:07.438175+0000","last_unstale":"2024-09-21T09:59:10.756448+0000","last_undegraded":"2024-09-21T09:59:10.756448+0000","last_fullsized":"2024-09-21T09:59:10.756448+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1088'4206","last_scrub_stamp":"2024-09-21T09:57:14.741002+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:14.741002+0000","objects_scrubbed":2270,"log_size":7580,"log_dups_size":0,"ondisk_log_size":7580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:30.124559+0000","scrub_duration":0.80346662300000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126222336,"num_objects":1926,"num_object_clones":0,"num_object_copies":5778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3390,"num_write_kb":97472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":1021,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679385+0000","last_change":"2024-09-21T09:36:43.113880+0000","last_active":"2024-09-21T09:59:04.679385+0000","last_peered":"2024-09-21T09:59:04.679385+0000","last_clean":"2024-09-21T09:59:04.679385+0000","last_became_active":"2024-09-21T09:36:43.113561+0000","last_became_peered":"2024-09-21T09:36:43.113561+0000","last_unstale":"2024-09-21T09:59:04.679385+0000","last_undegraded":"2024-09-21T09:59:04.679385+0000","last_fullsized":"2024-09-21T09:59:04.679385+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:38.190155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1047,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.676972+0000","last_change":"2024-09-21T09:46:13.260744+0000","last_active":"2024-09-21T09:59:04.676972+0000","last_peered":"2024-09-21T09:59:04.676972+0000","last_clean":"2024-09-21T09:59:04.676972+0000","last_became_active":"2024-09-21T09:46:13.260443+0000","last_became_peered":"2024-09-21T09:46:13.260443+0000","last_unstale":"2024-09-21T09:59:04.676972+0000","last_undegraded":"2024-09-21T09:59:04.676972+0000","last_fullsized":"2024-09-21T09:59:04.676972+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:22.699110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1114'7521","reported_seq":7647,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:13.276233+0000","last_change":"2024-09-21T09:57:09.604301+0000","last_active":"2024-09-21T09:59:13.276233+0000","last_peered":"2024-09-21T09:59:13.276233+0000","last_clean":"2024-09-21T09:59:13.276233+0000","last_became_active":"2024-09-21T09:57:07.402981+0000","last_became_peered":"2024-09-21T09:57:07.402981+0000","last_unstale":"2024-09-21T09:59:13.276233+0000","last_undegraded":"2024-09-21T09:59:13.276233+0000","last_fullsized":"2024-09-21T09:59:13.276233+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'3957","last_scrub_stamp":"2024-09-21T09:57:09.604140+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:09.604140+0000","objects_scrubbed":2037,"log_size":7521,"log_dups_size":0,"ondisk_log_size":7521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:54.970082+0000","scrub_duration":518,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116326400,"num_objects":1775,"num_object_clones":0,"num_object_copies":5325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3578,"num_write_kb":106112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":562,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.682470+0000","last_change":"2024-09-21T09:57:39.641367+0000","last_active":"2024-09-21T09:59:05.682470+0000","last_peered":"2024-09-21T09:59:05.682470+0000","last_clean":"2024-09-21T09:59:05.682470+0000","last_became_active":"2024-09-21T09:57:39.641057+0000","last_became_peered":"2024-09-21T09:57:39.641057+0000","last_unstale":"2024-09-21T09:59:05.682470+0000","last_undegraded":"2024-09-21T09:59:05.682470+0000","last_fullsized":"2024-09-21T09:59:05.682470+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:22.196149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1649,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684662+0000","last_change":"2024-09-21T09:45:39.138289+0000","last_active":"2024-09-21T09:59:05.684662+0000","last_peered":"2024-09-21T09:59:05.684662+0000","last_clean":"2024-09-21T09:59:05.684662+0000","last_became_active":"2024-09-21T09:45:39.137902+0000","last_became_peered":"2024-09-21T09:45:39.137902+0000","last_unstale":"2024-09-21T09:59:05.684662+0000","last_undegraded":"2024-09-21T09:59:05.684662+0000","last_fullsized":"2024-09-21T09:59:05.684662+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:40:16.383906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1164,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679376+0000","last_change":"2024-09-21T09:58:27.541830+0000","last_active":"2024-09-21T09:59:04.679376+0000","last_peered":"2024-09-21T09:59:04.679376+0000","last_clean":"2024-09-21T09:59:04.679376+0000","last_became_active":"2024-09-21T09:58:27.541389+0000","last_became_peered":"2024-09-21T09:58:27.541389+0000","last_unstale":"2024-09-21T09:59:04.679376+0000","last_undegraded":"2024-09-21T09:59:04.679376+0000","last_fullsized":"2024-09-21T09:59:04.679376+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.e","version":"1114'11017","reported_seq":11061,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.928055+0000","last_change":"2024-09-21T09:56:26.650365+0000","last_active":"2024-09-21T09:59:10.928055+0000","last_peered":"2024-09-21T09:59:10.928055+0000","last_clean":"2024-09-21T09:59:10.928055+0000","last_became_active":"2024-09-21T09:56:26.649684+0000","last_became_peered":"2024-09-21T09:56:26.649684+0000","last_unstale":"2024-09-21T09:59:10.928055+0000","last_undegraded":"2024-09-21T09:59:10.928055+0000","last_fullsized":"2024-09-21T09:59:10.928055+0000","mapping_epoch":1078,"log_start":"1087'3800","ondisk_log_start":"1087'3800","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":7217,"log_dups_size":0,"ondisk_log_size":7217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:04.800486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242024448,"num_objects":3693,"num_object_clones":0,"num_object_copies":11079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11017,"num_write_kb":470720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1573,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.683407+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T09:59:05.683407+0000","last_peered":"2024-09-21T09:59:05.683407+0000","last_clean":"2024-09-21T09:59:05.683407+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T09:59:05.683407+0000","last_undegraded":"2024-09-21T09:59:05.683407+0000","last_fullsized":"2024-09-21T09:59:05.683407+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1055,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678450+0000","last_change":"2024-09-21T09:46:47.114293+0000","last_active":"2024-09-21T09:59:04.678450+0000","last_peered":"2024-09-21T09:59:04.678450+0000","last_clean":"2024-09-21T09:59:04.678450+0000","last_became_active":"2024-09-21T09:46:47.113993+0000","last_became_peered":"2024-09-21T09:46:47.113993+0000","last_unstale":"2024-09-21T09:59:04.678450+0000","last_undegraded":"2024-09-21T09:59:04.678450+0000","last_fullsized":"2024-09-21T09:59:04.678450+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:45:30.941931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":1164,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.677866+0000","last_change":"2024-09-21T09:58:27.535768+0000","last_active":"2024-09-21T09:59:04.677866+0000","last_peered":"2024-09-21T09:59:04.677866+0000","last_clean":"2024-09-21T09:59:04.677866+0000","last_became_active":"2024-09-21T09:58:27.535428+0000","last_became_peered":"2024-09-21T09:58:27.535428+0000","last_unstale":"2024-09-21T09:59:04.677866+0000","last_undegraded":"2024-09-21T09:59:04.677866+0000","last_fullsized":"2024-09-21T09:59:04.677866+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.f","version":"1114'11209","reported_seq":11299,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:11.540618+0000","last_change":"2024-09-21T09:56:26.651596+0000","last_active":"2024-09-21T09:59:11.540618+0000","last_peered":"2024-09-21T09:59:11.540618+0000","last_clean":"2024-09-21T09:59:11.540618+0000","last_became_active":"2024-09-21T09:56:26.651284+0000","last_became_peered":"2024-09-21T09:56:26.651284+0000","last_unstale":"2024-09-21T09:59:11.540618+0000","last_undegraded":"2024-09-21T09:59:11.540618+0000","last_fullsized":"2024-09-21T09:59:11.540618+0000","mapping_epoch":1078,"log_start":"1083'2600","ondisk_log_start":"1083'2600","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":8609,"log_dups_size":0,"ondisk_log_size":8609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:19:05.103239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244121600,"num_objects":3725,"num_object_clones":0,"num_object_copies":11175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11209,"num_write_kb":477888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1038,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.677109+0000","last_change":"2024-09-21T09:45:39.127186+0000","last_active":"2024-09-21T09:59:04.677109+0000","last_peered":"2024-09-21T09:59:04.677109+0000","last_clean":"2024-09-21T09:59:04.677109+0000","last_became_active":"2024-09-21T09:45:39.126940+0000","last_became_peered":"2024-09-21T09:45:39.126940+0000","last_unstale":"2024-09-21T09:59:04.677109+0000","last_undegraded":"2024-09-21T09:59:04.677109+0000","last_fullsized":"2024-09-21T09:59:04.677109+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:52.205703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1072,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678830+0000","last_change":"2024-09-21T09:46:13.259814+0000","last_active":"2024-09-21T09:59:04.678830+0000","last_peered":"2024-09-21T09:59:04.678830+0000","last_clean":"2024-09-21T09:59:04.678830+0000","last_became_active":"2024-09-21T09:46:13.259527+0000","last_became_peered":"2024-09-21T09:46:13.259527+0000","last_unstale":"2024-09-21T09:59:04.678830+0000","last_undegraded":"2024-09-21T09:59:04.678830+0000","last_fullsized":"2024-09-21T09:59:04.678830+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:00.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1114'11170","reported_seq":11261,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.608724+0000","last_change":"2024-09-21T09:56:27.225575+0000","last_active":"2024-09-21T09:59:10.608724+0000","last_peered":"2024-09-21T09:59:10.608724+0000","last_clean":"2024-09-21T09:59:10.608724+0000","last_became_active":"2024-09-21T09:56:27.225236+0000","last_became_peered":"2024-09-21T09:56:27.225236+0000","last_unstale":"2024-09-21T09:59:10.608724+0000","last_undegraded":"2024-09-21T09:59:10.608724+0000","last_fullsized":"2024-09-21T09:59:10.608724+0000","mapping_epoch":1078,"log_start":"1098'5100","ondisk_log_start":"1098'5100","created":1078,"last_epoch_clean":1079,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":6070,"log_dups_size":0,"ondisk_log_size":6070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:03.692657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254017536,"num_objects":3876,"num_object_clones":0,"num_object_copies":11628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11170,"num_write_kb":481472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1056,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678280+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T09:59:04.678280+0000","last_peered":"2024-09-21T09:59:04.678280+0000","last_clean":"2024-09-21T09:59:04.678280+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T09:59:04.678280+0000","last_undegraded":"2024-09-21T09:59:04.678280+0000","last_fullsized":"2024-09-21T09:59:04.678280+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1055,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678967+0000","last_change":"2024-09-21T09:45:39.143302+0000","last_active":"2024-09-21T09:59:04.678967+0000","last_peered":"2024-09-21T09:59:04.678967+0000","last_clean":"2024-09-21T09:59:04.678967+0000","last_became_active":"2024-09-21T09:45:39.142964+0000","last_became_peered":"2024-09-21T09:45:39.142964+0000","last_unstale":"2024-09-21T09:59:04.678967+0000","last_undegraded":"2024-09-21T09:59:04.678967+0000","last_fullsized":"2024-09-21T09:59:04.678967+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:54.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":1167,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:08.484380+0000","last_change":"2024-09-21T09:59:08.484380+0000","last_active":"2024-09-21T09:59:08.484380+0000","last_peered":"2024-09-21T09:59:08.484380+0000","last_clean":"2024-09-21T09:59:08.484380+0000","last_became_active":"2024-09-21T09:58:27.537559+0000","last_became_peered":"2024-09-21T09:58:27.537559+0000","last_unstale":"2024-09-21T09:59:08.484380+0000","last_undegraded":"2024-09-21T09:59:08.484380+0000","last_fullsized":"2024-09-21T09:59:08.484380+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":69,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.d","version":"1114'10911","reported_seq":10977,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:12.355592+0000","last_change":"2024-09-21T09:57:39.553065+0000","last_active":"2024-09-21T09:59:12.355592+0000","last_peered":"2024-09-21T09:59:12.355592+0000","last_clean":"2024-09-21T09:59:12.355592+0000","last_became_active":"2024-09-21T09:57:39.545966+0000","last_became_peered":"2024-09-21T09:57:39.545966+0000","last_unstale":"2024-09-21T09:59:12.355592+0000","last_undegraded":"2024-09-21T09:59:12.355592+0000","last_fullsized":"2024-09-21T09:59:12.355592+0000","mapping_epoch":1100,"log_start":"1084'3400","ondisk_log_start":"1084'3400","created":1078,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:54.066959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243793944,"num_objects":3721,"num_object_clones":0,"num_object_copies":11163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3721,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10911,"num_write_kb":468161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1564,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.681831+0000","last_change":"2024-09-21T09:46:47.121643+0000","last_active":"2024-09-21T09:59:05.681831+0000","last_peered":"2024-09-21T09:59:05.681831+0000","last_clean":"2024-09-21T09:59:05.681831+0000","last_became_active":"2024-09-21T09:46:47.120528+0000","last_became_peered":"2024-09-21T09:46:47.120528+0000","last_unstale":"2024-09-21T09:59:05.681831+0000","last_undegraded":"2024-09-21T09:59:05.681831+0000","last_fullsized":"2024-09-21T09:59:05.681831+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:07.989377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1611,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.708755+0000","last_change":"2024-09-21T09:46:47.121159+0000","last_active":"2024-09-21T09:59:05.708755+0000","last_peered":"2024-09-21T09:59:05.708755+0000","last_clean":"2024-09-21T09:59:05.708755+0000","last_became_active":"2024-09-21T09:46:47.120829+0000","last_became_peered":"2024-09-21T09:46:47.120829+0000","last_unstale":"2024-09-21T09:59:05.708755+0000","last_undegraded":"2024-09-21T09:59:05.708755+0000","last_fullsized":"2024-09-21T09:59:05.708755+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:43.393840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1114'7508","reported_seq":7675,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.545913+0000","last_change":"2024-09-21T09:57:39.547586+0000","last_active":"2024-09-21T09:59:10.545913+0000","last_peered":"2024-09-21T09:59:10.545913+0000","last_clean":"2024-09-21T09:59:10.545913+0000","last_became_active":"2024-09-21T09:57:39.544701+0000","last_became_peered":"2024-09-21T09:57:39.544701+0000","last_unstale":"2024-09-21T09:59:10.545913+0000","last_undegraded":"2024-09-21T09:59:10.545913+0000","last_fullsized":"2024-09-21T09:59:10.545913+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'3934","last_scrub_stamp":"2024-09-21T09:57:09.713484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:09.713484+0000","objects_scrubbed":1997,"log_size":7508,"log_dups_size":0,"ondisk_log_size":7508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:10:42.424723+0000","scrub_duration":0.46009026200000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125763584,"num_objects":1919,"num_object_clones":0,"num_object_copies":5757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3580,"num_write_kb":112064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1168,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.497853+0000","last_change":"2024-09-21T09:59:05.497834+0000","last_active":"2024-09-21T09:59:05.497853+0000","last_peered":"2024-09-21T09:59:05.497853+0000","last_clean":"2024-09-21T09:59:05.497853+0000","last_became_active":"2024-09-21T09:58:27.516518+0000","last_became_peered":"2024-09-21T09:58:27.516518+0000","last_unstale":"2024-09-21T09:59:05.497853+0000","last_undegraded":"2024-09-21T09:59:05.497853+0000","last_fullsized":"2024-09-21T09:59:05.497853+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":69,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":49,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.675647+0000","last_change":"2024-09-21T09:57:39.541461+0000","last_active":"2024-09-21T09:59:05.675647+0000","last_peered":"2024-09-21T09:59:05.675647+0000","last_clean":"2024-09-21T09:59:05.675647+0000","last_became_active":"2024-09-21T09:57:39.540927+0000","last_became_peered":"2024-09-21T09:57:39.540927+0000","last_unstale":"2024-09-21T09:59:05.675647+0000","last_undegraded":"2024-09-21T09:59:05.675647+0000","last_fullsized":"2024-09-21T09:59:05.675647+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:14.815783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1546,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.681895+0000","last_change":"2024-09-21T09:46:47.116045+0000","last_active":"2024-09-21T09:59:05.681895+0000","last_peered":"2024-09-21T09:59:05.681895+0000","last_clean":"2024-09-21T09:59:05.681895+0000","last_became_active":"2024-09-21T09:46:47.115526+0000","last_became_peered":"2024-09-21T09:46:47.115526+0000","last_unstale":"2024-09-21T09:59:05.681895+0000","last_undegraded":"2024-09-21T09:59:05.681895+0000","last_fullsized":"2024-09-21T09:59:05.681895+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:43:35.293486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1020,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.675978+0000","last_change":"2024-09-21T09:36:43.389033+0000","last_active":"2024-09-21T09:59:04.675978+0000","last_peered":"2024-09-21T09:59:04.675978+0000","last_clean":"2024-09-21T09:59:04.675978+0000","last_became_active":"2024-09-21T09:36:43.388719+0000","last_became_peered":"2024-09-21T09:36:43.388719+0000","last_unstale":"2024-09-21T09:59:04.675978+0000","last_undegraded":"2024-09-21T09:59:04.675978+0000","last_fullsized":"2024-09-21T09:59:04.675978+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:45.333835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":1167,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.518566+0000","last_change":"2024-09-21T09:59:10.518566+0000","last_active":"2024-09-21T09:59:10.518566+0000","last_peered":"2024-09-21T09:59:10.518566+0000","last_clean":"2024-09-21T09:59:10.518566+0000","last_became_active":"2024-09-21T09:58:27.536668+0000","last_became_peered":"2024-09-21T09:58:27.536668+0000","last_unstale":"2024-09-21T09:59:10.518566+0000","last_undegraded":"2024-09-21T09:59:10.518566+0000","last_fullsized":"2024-09-21T09:59:10.518566+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"1114'7470","reported_seq":7596,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:12.317871+0000","last_change":"2024-09-21T09:57:12.585329+0000","last_active":"2024-09-21T09:59:12.317871+0000","last_peered":"2024-09-21T09:59:12.317871+0000","last_clean":"2024-09-21T09:59:12.317871+0000","last_became_active":"2024-09-21T09:57:07.409230+0000","last_became_peered":"2024-09-21T09:57:07.409230+0000","last_unstale":"2024-09-21T09:59:12.317871+0000","last_undegraded":"2024-09-21T09:59:12.317871+0000","last_fullsized":"2024-09-21T09:59:12.317871+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4057","last_scrub_stamp":"2024-09-21T09:57:12.582965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:12.582965+0000","objects_scrubbed":2117,"log_size":7470,"log_dups_size":0,"ondisk_log_size":7470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:10.909855+0000","scrub_duration":679,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118226944,"num_objects":1804,"num_object_clones":0,"num_object_copies":5412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3425,"num_write_kb":99584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1066,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679683+0000","last_change":"2024-09-21T09:45:39.129555+0000","last_active":"2024-09-21T09:59:04.679683+0000","last_peered":"2024-09-21T09:59:04.679683+0000","last_clean":"2024-09-21T09:59:04.679683+0000","last_became_active":"2024-09-21T09:45:39.129205+0000","last_became_peered":"2024-09-21T09:45:39.129205+0000","last_unstale":"2024-09-21T09:59:04.679683+0000","last_undegraded":"2024-09-21T09:59:04.679683+0000","last_fullsized":"2024-09-21T09:59:04.679683+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:07.229126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1647,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684079+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T09:59:05.684079+0000","last_peered":"2024-09-21T09:59:05.684079+0000","last_clean":"2024-09-21T09:59:05.684079+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T09:59:05.684079+0000","last_undegraded":"2024-09-21T09:59:05.684079+0000","last_fullsized":"2024-09-21T09:59:05.684079+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1660,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684279+0000","last_change":"2024-09-21T09:46:13.273573+0000","last_active":"2024-09-21T09:59:05.684279+0000","last_peered":"2024-09-21T09:59:05.684279+0000","last_clean":"2024-09-21T09:59:05.684279+0000","last_became_active":"2024-09-21T09:46:13.273297+0000","last_became_peered":"2024-09-21T09:46:13.273297+0000","last_unstale":"2024-09-21T09:59:05.684279+0000","last_undegraded":"2024-09-21T09:59:05.684279+0000","last_fullsized":"2024-09-21T09:59:05.684279+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:38.957745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":1164,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678736+0000","last_change":"2024-09-21T09:58:27.538846+0000","last_active":"2024-09-21T09:59:04.678736+0000","last_peered":"2024-09-21T09:59:04.678736+0000","last_clean":"2024-09-21T09:59:04.678736+0000","last_became_active":"2024-09-21T09:58:27.538178+0000","last_became_peered":"2024-09-21T09:58:27.538178+0000","last_unstale":"2024-09-21T09:59:04.678736+0000","last_undegraded":"2024-09-21T09:59:04.678736+0000","last_fullsized":"2024-09-21T09:59:04.678736+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.5","version":"1114'7572","reported_seq":7725,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.935899+0000","last_change":"2024-09-21T09:57:13.591883+0000","last_active":"2024-09-21T09:59:10.935899+0000","last_peered":"2024-09-21T09:59:10.935899+0000","last_clean":"2024-09-21T09:59:10.935899+0000","last_became_active":"2024-09-21T09:57:07.409458+0000","last_became_peered":"2024-09-21T09:57:07.409458+0000","last_unstale":"2024-09-21T09:59:10.935899+0000","last_undegraded":"2024-09-21T09:59:10.935899+0000","last_fullsized":"2024-09-21T09:59:10.935899+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4144","last_scrub_stamp":"2024-09-21T09:57:13.591791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:13.591791+0000","objects_scrubbed":2205,"log_size":7572,"log_dups_size":0,"ondisk_log_size":7572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:57:10.786770+0000","scrub_duration":0.62276720100000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126156800,"num_objects":1925,"num_object_clones":0,"num_object_copies":5775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3436,"num_write_kb":100992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1622,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.668013+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T09:59:05.668013+0000","last_peered":"2024-09-21T09:59:05.668013+0000","last_clean":"2024-09-21T09:59:05.668013+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T09:59:05.668013+0000","last_undegraded":"2024-09-21T09:59:05.668013+0000","last_fullsized":"2024-09-21T09:59:05.668013+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1612,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.708038+0000","last_change":"2024-09-21T09:46:13.255342+0000","last_active":"2024-09-21T09:59:05.708038+0000","last_peered":"2024-09-21T09:59:05.708038+0000","last_clean":"2024-09-21T09:59:05.708038+0000","last_became_active":"2024-09-21T09:46:13.255014+0000","last_became_peered":"2024-09-21T09:46:13.255014+0000","last_unstale":"2024-09-21T09:59:05.708038+0000","last_undegraded":"2024-09-21T09:59:05.708038+0000","last_fullsized":"2024-09-21T09:59:05.708038+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:53.012266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1623,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.708802+0000","last_change":"2024-09-21T09:45:39.134465+0000","last_active":"2024-09-21T09:59:05.708802+0000","last_peered":"2024-09-21T09:59:05.708802+0000","last_clean":"2024-09-21T09:59:05.708802+0000","last_became_active":"2024-09-21T09:45:39.129692+0000","last_became_peered":"2024-09-21T09:45:39.129692+0000","last_unstale":"2024-09-21T09:59:05.708802+0000","last_undegraded":"2024-09-21T09:59:05.708802+0000","last_fullsized":"2024-09-21T09:59:05.708802+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:14.288180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":1163,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679293+0000","last_change":"2024-09-21T09:58:27.541442+0000","last_active":"2024-09-21T09:59:04.679293+0000","last_peered":"2024-09-21T09:59:04.679293+0000","last_clean":"2024-09-21T09:59:04.679293+0000","last_became_active":"2024-09-21T09:58:27.540958+0000","last_became_peered":"2024-09-21T09:58:27.540958+0000","last_unstale":"2024-09-21T09:59:04.679293+0000","last_undegraded":"2024-09-21T09:59:04.679293+0000","last_fullsized":"2024-09-21T09:59:04.679293+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.6","version":"1114'7446","reported_seq":7572,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.586151+0000","last_change":"2024-09-21T09:57:10.193508+0000","last_active":"2024-09-21T09:59:10.586151+0000","last_peered":"2024-09-21T09:59:10.586151+0000","last_clean":"2024-09-21T09:59:10.586151+0000","last_became_active":"2024-09-21T09:57:07.410893+0000","last_became_peered":"2024-09-21T09:57:07.410893+0000","last_unstale":"2024-09-21T09:59:10.586151+0000","last_undegraded":"2024-09-21T09:59:10.586151+0000","last_fullsized":"2024-09-21T09:59:10.586151+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'3968","last_scrub_stamp":"2024-09-21T09:57:10.193203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:10.193203+0000","objects_scrubbed":2031,"log_size":7446,"log_dups_size":0,"ondisk_log_size":7446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:06:17.441883+0000","scrub_duration":533,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122945536,"num_objects":1876,"num_object_clones":0,"num_object_copies":5628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3493,"num_write_kb":106816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":55,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.708253+0000","last_change":"2024-09-21T09:57:39.646219+0000","last_active":"2024-09-21T09:59:05.708253+0000","last_peered":"2024-09-21T09:59:05.708253+0000","last_clean":"2024-09-21T09:59:05.708253+0000","last_became_active":"2024-09-21T09:57:39.645883+0000","last_became_peered":"2024-09-21T09:57:39.645883+0000","last_unstale":"2024-09-21T09:59:05.708253+0000","last_undegraded":"2024-09-21T09:59:05.708253+0000","last_fullsized":"2024-09-21T09:59:05.708253+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:07.561254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1056,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678409+0000","last_change":"2024-09-21T09:45:05.097518+0000","last_active":"2024-09-21T09:59:04.678409+0000","last_peered":"2024-09-21T09:59:04.678409+0000","last_clean":"2024-09-21T09:59:04.678409+0000","last_became_active":"2024-09-21T09:45:05.096918+0000","last_became_peered":"2024-09-21T09:45:05.096918+0000","last_unstale":"2024-09-21T09:59:04.678409+0000","last_undegraded":"2024-09-21T09:59:04.678409+0000","last_fullsized":"2024-09-21T09:59:04.678409+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:38:13.781847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1622,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.708747+0000","last_change":"2024-09-21T09:46:47.113687+0000","last_active":"2024-09-21T09:59:05.708747+0000","last_peered":"2024-09-21T09:59:05.708747+0000","last_clean":"2024-09-21T09:59:05.708747+0000","last_became_active":"2024-09-21T09:46:47.113235+0000","last_became_peered":"2024-09-21T09:46:47.113235+0000","last_unstale":"2024-09-21T09:59:05.708747+0000","last_undegraded":"2024-09-21T09:59:05.708747+0000","last_fullsized":"2024-09-21T09:59:05.708747+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:50.851616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":1164,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679196+0000","last_change":"2024-09-21T09:58:27.537589+0000","last_active":"2024-09-21T09:59:04.679196+0000","last_peered":"2024-09-21T09:59:04.679196+0000","last_clean":"2024-09-21T09:59:04.679196+0000","last_became_active":"2024-09-21T09:58:27.536882+0000","last_became_peered":"2024-09-21T09:58:27.536882+0000","last_unstale":"2024-09-21T09:59:04.679196+0000","last_undegraded":"2024-09-21T09:59:04.679196+0000","last_fullsized":"2024-09-21T09:59:04.679196+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.0","version":"1114'7517","reported_seq":7642,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:13.225022+0000","last_change":"2024-09-21T09:57:12.654933+0000","last_active":"2024-09-21T09:59:13.225022+0000","last_peered":"2024-09-21T09:59:13.225022+0000","last_clean":"2024-09-21T09:59:13.225022+0000","last_became_active":"2024-09-21T09:57:07.435577+0000","last_became_peered":"2024-09-21T09:57:07.435577+0000","last_unstale":"2024-09-21T09:59:13.225022+0000","last_undegraded":"2024-09-21T09:59:13.225022+0000","last_fullsized":"2024-09-21T09:59:13.225022+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4081","last_scrub_stamp":"2024-09-21T09:57:12.654694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:12.654694+0000","objects_scrubbed":2087,"log_size":7517,"log_dups_size":0,"ondisk_log_size":7517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:02.588703+0000","scrub_duration":664,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118554624,"num_objects":1809,"num_object_clones":0,"num_object_copies":5427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3448,"num_write_kb":101440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":578,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.682690+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T09:59:05.682690+0000","last_peered":"2024-09-21T09:59:05.682690+0000","last_clean":"2024-09-21T09:59:05.682690+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T09:59:05.682690+0000","last_undegraded":"2024-09-21T09:59:05.682690+0000","last_fullsized":"2024-09-21T09:59:05.682690+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1636,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.708819+0000","last_change":"2024-09-21T09:45:39.132441+0000","last_active":"2024-09-21T09:59:05.708819+0000","last_peered":"2024-09-21T09:59:05.708819+0000","last_clean":"2024-09-21T09:59:05.708819+0000","last_became_active":"2024-09-21T09:45:39.129666+0000","last_became_peered":"2024-09-21T09:45:39.129666+0000","last_unstale":"2024-09-21T09:59:05.708819+0000","last_undegraded":"2024-09-21T09:59:05.708819+0000","last_fullsized":"2024-09-21T09:59:05.708819+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:34.450160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1575,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.670707+0000","last_change":"2024-09-21T09:46:13.258244+0000","last_active":"2024-09-21T09:59:05.670707+0000","last_peered":"2024-09-21T09:59:05.670707+0000","last_clean":"2024-09-21T09:59:05.670707+0000","last_became_active":"2024-09-21T09:46:13.257820+0000","last_became_peered":"2024-09-21T09:46:13.257820+0000","last_unstale":"2024-09-21T09:59:05.670707+0000","last_undegraded":"2024-09-21T09:59:05.670707+0000","last_fullsized":"2024-09-21T09:59:05.670707+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:04.618304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":1164,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679598+0000","last_change":"2024-09-21T09:58:27.538117+0000","last_active":"2024-09-21T09:59:04.679598+0000","last_peered":"2024-09-21T09:59:04.679598+0000","last_clean":"2024-09-21T09:59:04.679598+0000","last_became_active":"2024-09-21T09:58:27.537832+0000","last_became_peered":"2024-09-21T09:58:27.537832+0000","last_unstale":"2024-09-21T09:59:04.679598+0000","last_undegraded":"2024-09-21T09:59:04.679598+0000","last_fullsized":"2024-09-21T09:59:04.679598+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.1","version":"1114'7371","reported_seq":7497,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:12.331053+0000","last_change":"2024-09-21T09:57:09.426376+0000","last_active":"2024-09-21T09:59:12.331053+0000","last_peered":"2024-09-21T09:59:12.331053+0000","last_clean":"2024-09-21T09:59:12.331053+0000","last_became_active":"2024-09-21T09:57:07.399441+0000","last_became_peered":"2024-09-21T09:57:07.399441+0000","last_unstale":"2024-09-21T09:59:12.331053+0000","last_undegraded":"2024-09-21T09:59:12.331053+0000","last_fullsized":"2024-09-21T09:59:12.331053+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'3855","last_scrub_stamp":"2024-09-21T09:57:09.426224+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:09.426224+0000","objects_scrubbed":1962,"log_size":7371,"log_dups_size":0,"ondisk_log_size":7371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:08:05.610483+0000","scrub_duration":531,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119406592,"num_objects":1822,"num_object_clones":0,"num_object_copies":5466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3530,"num_write_kb":108480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1612,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.708912+0000","last_change":"2024-09-21T09:46:13.260666+0000","last_active":"2024-09-21T09:59:05.708912+0000","last_peered":"2024-09-21T09:59:05.708912+0000","last_clean":"2024-09-21T09:59:05.708912+0000","last_became_active":"2024-09-21T09:46:13.260324+0000","last_became_peered":"2024-09-21T09:46:13.260324+0000","last_unstale":"2024-09-21T09:59:05.708912+0000","last_undegraded":"2024-09-21T09:59:05.708912+0000","last_fullsized":"2024-09-21T09:59:05.708912+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:25.140797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1049,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679462+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T09:59:04.679462+0000","last_peered":"2024-09-21T09:59:04.679462+0000","last_clean":"2024-09-21T09:59:04.679462+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T09:59:04.679462+0000","last_undegraded":"2024-09-21T09:59:04.679462+0000","last_fullsized":"2024-09-21T09:59:04.679462+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1638,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684682+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T09:59:05.684682+0000","last_peered":"2024-09-21T09:59:05.684682+0000","last_clean":"2024-09-21T09:59:05.684682+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T09:59:05.684682+0000","last_undegraded":"2024-09-21T09:59:05.684682+0000","last_fullsized":"2024-09-21T09:59:05.684682+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":1167,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:07.443833+0000","last_change":"2024-09-21T09:59:07.443833+0000","last_active":"2024-09-21T09:59:07.443833+0000","last_peered":"2024-09-21T09:59:07.443833+0000","last_clean":"2024-09-21T09:59:07.443833+0000","last_became_active":"2024-09-21T09:58:27.537461+0000","last_became_peered":"2024-09-21T09:58:27.537461+0000","last_unstale":"2024-09-21T09:59:07.443833+0000","last_undegraded":"2024-09-21T09:59:07.443833+0000","last_fullsized":"2024-09-21T09:59:07.443833+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.2","version":"1114'7589","reported_seq":7745,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.914719+0000","last_change":"2024-09-21T09:57:12.078248+0000","last_active":"2024-09-21T09:59:10.914719+0000","last_peered":"2024-09-21T09:59:10.914719+0000","last_clean":"2024-09-21T09:59:10.914719+0000","last_became_active":"2024-09-21T09:57:07.419410+0000","last_became_peered":"2024-09-21T09:57:07.419410+0000","last_unstale":"2024-09-21T09:59:10.914719+0000","last_undegraded":"2024-09-21T09:59:10.914719+0000","last_fullsized":"2024-09-21T09:59:10.914719+0000","mapping_epoch":1078,"log_start":"0'0","ondisk_log_start":"0'0","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4085","last_scrub_stamp":"2024-09-21T09:57:12.078156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:12.078156+0000","objects_scrubbed":2168,"log_size":7589,"log_dups_size":0,"ondisk_log_size":7589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:31:56.497563+0000","scrub_duration":0.59593628200000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124125184,"num_objects":1894,"num_object_clones":0,"num_object_copies":5682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3514,"num_write_kb":103680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1573,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.682979+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T09:59:05.682979+0000","last_peered":"2024-09-21T09:59:05.682979+0000","last_clean":"2024-09-21T09:59:05.682979+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T09:59:05.682979+0000","last_undegraded":"2024-09-21T09:59:05.682979+0000","last_fullsized":"2024-09-21T09:59:05.682979+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1610,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.709106+0000","last_change":"2024-09-21T09:45:39.130168+0000","last_active":"2024-09-21T09:59:05.709106+0000","last_peered":"2024-09-21T09:59:05.709106+0000","last_clean":"2024-09-21T09:59:05.709106+0000","last_became_active":"2024-09-21T09:45:39.129498+0000","last_became_peered":"2024-09-21T09:45:39.129498+0000","last_unstale":"2024-09-21T09:59:05.709106+0000","last_undegraded":"2024-09-21T09:59:05.709106+0000","last_fullsized":"2024-09-21T09:59:05.709106+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:58.310876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1571,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.681580+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T09:59:05.681580+0000","last_peered":"2024-09-21T09:59:05.681580+0000","last_clean":"2024-09-21T09:59:05.681580+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T09:59:05.681580+0000","last_undegraded":"2024-09-21T09:59:05.681580+0000","last_fullsized":"2024-09-21T09:59:05.681580+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":1167,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:06.484178+0000","last_change":"2024-09-21T09:59:06.484178+0000","last_active":"2024-09-21T09:59:06.484178+0000","last_peered":"2024-09-21T09:59:06.484178+0000","last_clean":"2024-09-21T09:59:06.484178+0000","last_became_active":"2024-09-21T09:58:27.538855+0000","last_became_peered":"2024-09-21T09:58:27.538855+0000","last_unstale":"2024-09-21T09:59:06.484178+0000","last_undegraded":"2024-09-21T09:59:06.484178+0000","last_fullsized":"2024-09-21T09:59:06.484178+0000","mapping_epoch":1107,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1108,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":28,"num_read_kb":69,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.3","version":"1114'7852","reported_seq":8003,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.884311+0000","last_change":"2024-09-21T09:57:11.657896+0000","last_active":"2024-09-21T09:59:10.884311+0000","last_peered":"2024-09-21T09:59:10.884311+0000","last_clean":"2024-09-21T09:59:10.884311+0000","last_became_active":"2024-09-21T09:57:07.419558+0000","last_became_peered":"2024-09-21T09:57:07.419558+0000","last_unstale":"2024-09-21T09:59:10.884311+0000","last_undegraded":"2024-09-21T09:59:10.884311+0000","last_fullsized":"2024-09-21T09:59:10.884311+0000","mapping_epoch":1078,"log_start":"1082'541","ondisk_log_start":"1082'541","created":1078,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"1087'4365","last_scrub_stamp":"2024-09-21T09:57:11.657796+0000","last_deep_scrub":"1087'4365","last_deep_scrub_stamp":"2024-09-21T09:57:11.657796+0000","last_clean_scrub_stamp":"2024-09-21T09:57:11.657796+0000","objects_scrubbed":2040,"log_size":7311,"log_dups_size":0,"ondisk_log_size":7311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":3,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:08.924721+0000","scrub_duration":2.8729851470000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129826816,"num_objects":1981,"num_object_clones":0,"num_object_copies":5943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3557,"num_write_kb":111936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1651,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684418+0000","last_change":"2024-09-21T09:45:39.140762+0000","last_active":"2024-09-21T09:59:05.684418+0000","last_peered":"2024-09-21T09:59:05.684418+0000","last_clean":"2024-09-21T09:59:05.684418+0000","last_became_active":"2024-09-21T09:45:39.140402+0000","last_became_peered":"2024-09-21T09:45:39.140402+0000","last_unstale":"2024-09-21T09:59:05.684418+0000","last_undegraded":"2024-09-21T09:59:05.684418+0000","last_fullsized":"2024-09-21T09:59:05.684418+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:06.545428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1050,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.675795+0000","last_change":"2024-09-21T09:46:13.260656+0000","last_active":"2024-09-21T09:59:04.675795+0000","last_peered":"2024-09-21T09:59:04.675795+0000","last_clean":"2024-09-21T09:59:04.675795+0000","last_became_active":"2024-09-21T09:46:13.260368+0000","last_became_peered":"2024-09-21T09:46:13.260368+0000","last_unstale":"2024-09-21T09:59:04.675795+0000","last_undegraded":"2024-09-21T09:59:04.675795+0000","last_fullsized":"2024-09-21T09:59:04.675795+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:22.059036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1038,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678071+0000","last_change":"2024-09-21T09:36:43.135736+0000","last_active":"2024-09-21T09:59:04.678071+0000","last_peered":"2024-09-21T09:59:04.678071+0000","last_clean":"2024-09-21T09:59:04.678071+0000","last_became_active":"2024-09-21T09:36:43.135429+0000","last_became_peered":"2024-09-21T09:36:43.135429+0000","last_unstale":"2024-09-21T09:59:04.678071+0000","last_undegraded":"2024-09-21T09:59:04.678071+0000","last_fullsized":"2024-09-21T09:59:04.678071+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:25.828479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.11","version":"1114'7350","reported_seq":16470,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.591995+0000","last_change":"2024-09-21T09:58:40.575317+0000","last_active":"2024-09-21T09:59:10.591995+0000","last_peered":"2024-09-21T09:59:10.591995+0000","last_clean":"2024-09-21T09:59:10.591995+0000","last_became_active":"2024-09-21T09:58:40.563057+0000","last_became_peered":"2024-09-21T09:58:40.563057+0000","last_unstale":"2024-09-21T09:59:10.591995+0000","last_undegraded":"2024-09-21T09:59:10.591995+0000","last_fullsized":"2024-09-21T09:59:10.591995+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'3973","last_scrub_stamp":"2024-09-21T09:57:11.468690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:11.468690+0000","objects_scrubbed":2101,"log_size":7350,"log_dups_size":0,"ondisk_log_size":7350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:44.199161+0000","scrub_duration":570,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122355712,"num_objects":1867,"num_object_clones":0,"num_object_copies":5601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3388,"num_write_kb":100928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2246,"num_bytes_recovered":147193856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1055,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678258+0000","last_change":"2024-09-21T09:45:39.129538+0000","last_active":"2024-09-21T09:59:04.678258+0000","last_peered":"2024-09-21T09:59:04.678258+0000","last_clean":"2024-09-21T09:59:04.678258+0000","last_became_active":"2024-09-21T09:45:39.129198+0000","last_became_peered":"2024-09-21T09:45:39.129198+0000","last_unstale":"2024-09-21T09:59:04.678258+0000","last_undegraded":"2024-09-21T09:59:04.678258+0000","last_fullsized":"2024-09-21T09:59:04.678258+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.798277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1061,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678535+0000","last_change":"2024-09-21T09:36:43.118209+0000","last_active":"2024-09-21T09:59:04.678535+0000","last_peered":"2024-09-21T09:59:04.678535+0000","last_clean":"2024-09-21T09:59:04.678535+0000","last_became_active":"2024-09-21T09:36:43.117953+0000","last_became_peered":"2024-09-21T09:36:43.117953+0000","last_unstale":"2024-09-21T09:59:04.678535+0000","last_undegraded":"2024-09-21T09:59:04.678535+0000","last_fullsized":"2024-09-21T09:59:04.678535+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:47.500027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.10","version":"1114'7652","reported_seq":14677,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:13.354711+0000","last_change":"2024-09-21T09:57:41.648481+0000","last_active":"2024-09-21T09:59:13.354711+0000","last_peered":"2024-09-21T09:59:13.354711+0000","last_clean":"2024-09-21T09:59:13.354711+0000","last_became_active":"2024-09-21T09:57:41.645466+0000","last_became_peered":"2024-09-21T09:57:41.645466+0000","last_unstale":"2024-09-21T09:59:13.354711+0000","last_undegraded":"2024-09-21T09:59:13.354711+0000","last_fullsized":"2024-09-21T09:59:13.354711+0000","mapping_epoch":1102,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1103,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":1099,"log_size":7652,"log_dups_size":0,"ondisk_log_size":7652,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115212288,"num_objects":1758,"num_object_clones":0,"num_object_copies":5274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5730,"num_write_kb":239616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2302,"num_bytes_recovered":150863872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1610,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.708357+0000","last_change":"2024-09-21T09:46:13.254014+0000","last_active":"2024-09-21T09:59:05.708357+0000","last_peered":"2024-09-21T09:59:05.708357+0000","last_clean":"2024-09-21T09:59:05.708357+0000","last_became_active":"2024-09-21T09:46:13.253673+0000","last_became_peered":"2024-09-21T09:46:13.253673+0000","last_unstale":"2024-09-21T09:59:05.708357+0000","last_undegraded":"2024-09-21T09:59:05.708357+0000","last_fullsized":"2024-09-21T09:59:05.708357+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:02.784613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1040,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679190+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T09:59:04.679190+0000","last_peered":"2024-09-21T09:59:04.679190+0000","last_clean":"2024-09-21T09:59:04.679190+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T09:59:04.679190+0000","last_undegraded":"2024-09-21T09:59:04.679190+0000","last_fullsized":"2024-09-21T09:59:04.679190+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.13","version":"1114'7495","reported_seq":11065,"reported_epoch":1114,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:11.556930+0000","last_change":"2024-09-21T09:58:39.049875+0000","last_active":"2024-09-21T09:59:11.556930+0000","last_peered":"2024-09-21T09:59:11.556930+0000","last_clean":"2024-09-21T09:57:19.631866+0000","last_became_active":"2024-09-21T09:57:22.552789+0000","last_became_peered":"2024-09-21T09:57:22.552789+0000","last_unstale":"2024-09-21T09:59:11.556930+0000","last_undegraded":"2024-09-21T09:57:22.550233+0000","last_fullsized":"2024-09-21T09:57:22.463332+0000","mapping_epoch":1095,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.628809+0000","objects_scrubbed":0,"log_size":7495,"log_dups_size":0,"ondisk_log_size":7495,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123076608,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":973,"num_objects_misplaced":1327,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5596,"num_write_kb":239168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1164,"num_bytes_recovered":76283904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[2,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6","objects":728},{"shards":"2,6,7","objects":177}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":484,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679664+0000","last_change":"2024-09-21T09:57:39.544714+0000","last_active":"2024-09-21T09:59:04.679664+0000","last_peered":"2024-09-21T09:59:04.679664+0000","last_clean":"2024-09-21T09:59:04.679664+0000","last_became_active":"2024-09-21T09:57:39.544459+0000","last_became_peered":"2024-09-21T09:57:39.544459+0000","last_unstale":"2024-09-21T09:59:04.679664+0000","last_undegraded":"2024-09-21T09:59:04.679664+0000","last_fullsized":"2024-09-21T09:59:04.679664+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:21.280135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1546,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.681571+0000","last_change":"2024-09-21T09:46:47.122643+0000","last_active":"2024-09-21T09:59:05.681571+0000","last_peered":"2024-09-21T09:59:05.681571+0000","last_clean":"2024-09-21T09:59:05.681571+0000","last_became_active":"2024-09-21T09:46:47.122364+0000","last_became_peered":"2024-09-21T09:46:47.122364+0000","last_unstale":"2024-09-21T09:59:05.681571+0000","last_undegraded":"2024-09-21T09:59:05.681571+0000","last_fullsized":"2024-09-21T09:59:05.681571+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:05:29.222333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.12","version":"1114'7482","reported_seq":12123,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.889224+0000","last_change":"2024-09-21T09:57:40.561013+0000","last_active":"2024-09-21T09:59:10.889224+0000","last_peered":"2024-09-21T09:59:10.889224+0000","last_clean":"2024-09-21T09:59:10.889224+0000","last_became_active":"2024-09-21T09:57:40.544862+0000","last_became_peered":"2024-09-21T09:57:40.544862+0000","last_unstale":"2024-09-21T09:59:10.889224+0000","last_undegraded":"2024-09-21T09:59:10.889224+0000","last_fullsized":"2024-09-21T09:59:10.889224+0000","mapping_epoch":1101,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1102,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'4093","last_scrub_stamp":"2024-09-21T09:57:12.883129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:12.883129+0000","objects_scrubbed":2133,"log_size":7482,"log_dups_size":0,"ondisk_log_size":7482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:27.825610+0000","scrub_duration":0.369013857,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121962496,"num_objects":1861,"num_object_clones":0,"num_object_copies":5583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3390,"num_write_kb":99776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2235,"num_bytes_recovered":146472960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1646,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684556+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T09:59:05.684556+0000","last_peered":"2024-09-21T09:59:05.684556+0000","last_clean":"2024-09-21T09:59:05.684556+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T09:59:05.684556+0000","last_undegraded":"2024-09-21T09:59:05.684556+0000","last_fullsized":"2024-09-21T09:59:05.684556+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1563,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.681489+0000","last_change":"2024-09-21T09:46:47.121360+0000","last_active":"2024-09-21T09:59:05.681489+0000","last_peered":"2024-09-21T09:59:05.681489+0000","last_clean":"2024-09-21T09:59:05.681489+0000","last_became_active":"2024-09-21T09:46:47.120494+0000","last_became_peered":"2024-09-21T09:46:47.120494+0000","last_unstale":"2024-09-21T09:59:05.681489+0000","last_undegraded":"2024-09-21T09:59:05.681489+0000","last_fullsized":"2024-09-21T09:59:05.681489+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:45.862984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1047,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.675754+0000","last_change":"2024-09-21T09:46:13.260037+0000","last_active":"2024-09-21T09:59:04.675754+0000","last_peered":"2024-09-21T09:59:04.675754+0000","last_clean":"2024-09-21T09:59:04.675754+0000","last_became_active":"2024-09-21T09:46:13.259690+0000","last_became_peered":"2024-09-21T09:46:13.259690+0000","last_unstale":"2024-09-21T09:59:04.675754+0000","last_undegraded":"2024-09-21T09:59:04.675754+0000","last_fullsized":"2024-09-21T09:59:04.675754+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:57.466762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1581,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.681736+0000","last_change":"2024-09-21T09:46:47.120830+0000","last_active":"2024-09-21T09:59:05.681736+0000","last_peered":"2024-09-21T09:59:05.681736+0000","last_clean":"2024-09-21T09:59:05.681736+0000","last_became_active":"2024-09-21T09:46:47.118419+0000","last_became_peered":"2024-09-21T09:46:47.118419+0000","last_unstale":"2024-09-21T09:59:05.681736+0000","last_undegraded":"2024-09-21T09:59:05.681736+0000","last_fullsized":"2024-09-21T09:59:05.681736+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:13.789806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1650,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684328+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T09:59:05.684328+0000","last_peered":"2024-09-21T09:59:05.684328+0000","last_clean":"2024-09-21T09:59:05.684328+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T09:59:05.684328+0000","last_undegraded":"2024-09-21T09:59:05.684328+0000","last_fullsized":"2024-09-21T09:59:05.684328+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1611,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.708464+0000","last_change":"2024-09-21T09:46:47.121724+0000","last_active":"2024-09-21T09:59:05.708464+0000","last_peered":"2024-09-21T09:59:05.708464+0000","last_clean":"2024-09-21T09:59:05.708464+0000","last_became_active":"2024-09-21T09:46:47.121406+0000","last_became_peered":"2024-09-21T09:46:47.121406+0000","last_unstale":"2024-09-21T09:59:05.708464+0000","last_undegraded":"2024-09-21T09:59:05.708464+0000","last_fullsized":"2024-09-21T09:59:05.708464+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:55.493704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1562,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.682516+0000","last_change":"2024-09-21T09:46:47.113184+0000","last_active":"2024-09-21T09:59:05.682516+0000","last_peered":"2024-09-21T09:59:05.682516+0000","last_clean":"2024-09-21T09:59:05.682516+0000","last_became_active":"2024-09-21T09:46:47.112857+0000","last_became_peered":"2024-09-21T09:46:47.112857+0000","last_unstale":"2024-09-21T09:59:05.682516+0000","last_undegraded":"2024-09-21T09:59:05.682516+0000","last_fullsized":"2024-09-21T09:59:05.682516+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:46.783986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1074,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679152+0000","last_change":"2024-09-21T09:46:13.262555+0000","last_active":"2024-09-21T09:59:04.679152+0000","last_peered":"2024-09-21T09:59:04.679152+0000","last_clean":"2024-09-21T09:59:04.679152+0000","last_became_active":"2024-09-21T09:46:13.261474+0000","last_became_peered":"2024-09-21T09:46:13.261474+0000","last_unstale":"2024-09-21T09:59:04.679152+0000","last_undegraded":"2024-09-21T09:59:04.679152+0000","last_fullsized":"2024-09-21T09:59:04.679152+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:44.865988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1601,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.671463+0000","last_change":"2024-09-21T09:46:47.114406+0000","last_active":"2024-09-21T09:59:05.671463+0000","last_peered":"2024-09-21T09:59:05.671463+0000","last_clean":"2024-09-21T09:59:05.671463+0000","last_became_active":"2024-09-21T09:46:47.113986+0000","last_became_peered":"2024-09-21T09:46:47.113986+0000","last_unstale":"2024-09-21T09:59:05.671463+0000","last_undegraded":"2024-09-21T09:59:05.671463+0000","last_fullsized":"2024-09-21T09:59:05.671463+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:32.292201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1639,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684755+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T09:59:05.684755+0000","last_peered":"2024-09-21T09:59:05.684755+0000","last_clean":"2024-09-21T09:59:05.684755+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T09:59:05.684755+0000","last_undegraded":"2024-09-21T09:59:05.684755+0000","last_fullsized":"2024-09-21T09:59:05.684755+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.19","version":"1114'7517","reported_seq":7642,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:13.254741+0000","last_change":"2024-09-21T09:57:11.595394+0000","last_active":"2024-09-21T09:59:13.254741+0000","last_peered":"2024-09-21T09:59:13.254741+0000","last_clean":"2024-09-21T09:59:13.254741+0000","last_became_active":"2024-09-21T09:57:07.422449+0000","last_became_peered":"2024-09-21T09:57:07.422449+0000","last_unstale":"2024-09-21T09:59:13.254741+0000","last_undegraded":"2024-09-21T09:59:13.254741+0000","last_fullsized":"2024-09-21T09:59:13.254741+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1087'4025","last_scrub_stamp":"2024-09-21T09:57:11.595114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:11.595114+0000","objects_scrubbed":2093,"log_size":7517,"log_dups_size":0,"ondisk_log_size":7517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:49.720861+0000","scrub_duration":566,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116981760,"num_objects":1785,"num_object_clones":0,"num_object_copies":5355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3504,"num_write_kb":102272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1072,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678186+0000","last_change":"2024-09-21T09:46:47.114188+0000","last_active":"2024-09-21T09:59:04.678186+0000","last_peered":"2024-09-21T09:59:04.678186+0000","last_clean":"2024-09-21T09:59:04.678186+0000","last_became_active":"2024-09-21T09:46:47.113859+0000","last_became_peered":"2024-09-21T09:46:47.113859+0000","last_unstale":"2024-09-21T09:59:04.678186+0000","last_undegraded":"2024-09-21T09:59:04.678186+0000","last_fullsized":"2024-09-21T09:59:04.678186+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:22.670368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":36,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678344+0000","last_change":"2024-09-21T09:57:39.543638+0000","last_active":"2024-09-21T09:59:04.678344+0000","last_peered":"2024-09-21T09:59:04.678344+0000","last_clean":"2024-09-21T09:59:04.678344+0000","last_became_active":"2024-09-21T09:57:39.543224+0000","last_became_peered":"2024-09-21T09:57:39.543224+0000","last_unstale":"2024-09-21T09:59:04.678344+0000","last_undegraded":"2024-09-21T09:59:04.678344+0000","last_fullsized":"2024-09-21T09:59:04.678344+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:06.898662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"1114'7567","reported_seq":7771,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:10.925790+0000","last_change":"2024-09-21T09:57:58.103549+0000","last_active":"2024-09-21T09:59:10.925790+0000","last_peered":"2024-09-21T09:59:10.925790+0000","last_clean":"2024-09-21T09:59:10.925790+0000","last_became_active":"2024-09-21T09:57:07.454647+0000","last_became_peered":"2024-09-21T09:57:07.454647+0000","last_unstale":"2024-09-21T09:59:10.925790+0000","last_undegraded":"2024-09-21T09:59:10.925790+0000","last_fullsized":"2024-09-21T09:59:10.925790+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":5,"last_scrub":"1104'5399","last_scrub_stamp":"2024-09-21T09:57:58.103403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.628809+0000","last_clean_scrub_stamp":"2024-09-21T09:57:58.103403+0000","objects_scrubbed":3448,"log_size":7567,"log_dups_size":0,"ondisk_log_size":7567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:15.110839+0000","scrub_duration":1.1907735719999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125632512,"num_objects":1917,"num_object_clones":0,"num_object_copies":5751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2181,"num_write_kb":20800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1072,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.679066+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T09:59:04.679066+0000","last_peered":"2024-09-21T09:59:04.679066+0000","last_clean":"2024-09-21T09:59:04.679066+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T09:59:04.679066+0000","last_undegraded":"2024-09-21T09:59:04.679066+0000","last_fullsized":"2024-09-21T09:59:04.679066+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":36,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.678647+0000","last_change":"2024-09-21T09:57:39.541586+0000","last_active":"2024-09-21T09:59:04.678647+0000","last_peered":"2024-09-21T09:59:04.678647+0000","last_clean":"2024-09-21T09:59:04.678647+0000","last_became_active":"2024-09-21T09:57:39.541229+0000","last_became_peered":"2024-09-21T09:57:39.541229+0000","last_unstale":"2024-09-21T09:59:04.678647+0000","last_undegraded":"2024-09-21T09:59:04.678647+0000","last_fullsized":"2024-09-21T09:59:04.678647+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:20.209058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1573,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.682014+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T09:59:05.682014+0000","last_peered":"2024-09-21T09:59:05.682014+0000","last_clean":"2024-09-21T09:59:05.682014+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T09:59:05.682014+0000","last_undegraded":"2024-09-21T09:59:05.682014+0000","last_fullsized":"2024-09-21T09:59:05.682014+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1559,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.682220+0000","last_change":"2024-09-21T09:46:47.115788+0000","last_active":"2024-09-21T09:59:05.682220+0000","last_peered":"2024-09-21T09:59:05.682220+0000","last_clean":"2024-09-21T09:59:05.682220+0000","last_became_active":"2024-09-21T09:46:47.114864+0000","last_became_peered":"2024-09-21T09:46:47.114864+0000","last_unstale":"2024-09-21T09:59:05.682220+0000","last_undegraded":"2024-09-21T09:59:05.682220+0000","last_fullsized":"2024-09-21T09:59:05.682220+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:00.034944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1055,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:59:04.677839+0000","last_change":"2024-09-21T09:45:39.143064+0000","last_active":"2024-09-21T09:59:04.677839+0000","last_peered":"2024-09-21T09:59:04.677839+0000","last_clean":"2024-09-21T09:59:04.677839+0000","last_became_active":"2024-09-21T09:45:39.142699+0000","last_became_peered":"2024-09-21T09:45:39.142699+0000","last_unstale":"2024-09-21T09:59:04.677839+0000","last_undegraded":"2024-09-21T09:59:04.677839+0000","last_fullsized":"2024-09-21T09:59:04.677839+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:14.684721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1654,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:59:05.684644+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T09:59:05.684644+0000","last_peered":"2024-09-21T09:59:05.684644+0000","last_clean":"2024-09-21T09:59:05.684644+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T09:59:05.684644+0000","last_undegraded":"2024-09-21T09:59:05.684644+0000","last_fullsized":"2024-09-21T09:59:05.684644+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":3918790680,"num_objects":59797,"num_object_clones":0,"num_object_copies":179391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":973,"num_objects_misplaced":1327,"num_objects_unfound":0,"num_objects_dirty":59797,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138776,"num_write_kb":5093057,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10245,"num_bytes_recovered":671416320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13149876224,"data_stored":13149864008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":192055,"ondisk_log_size":192055,"up":78,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":306,"num_read_kb":759,"num_write":149,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397800,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2041916,"kb_used_data":1736244,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91703236,"statfs":{"total":95995035648,"available":93904113664,"internally_reserved":0,"allocated":1777913856,"data_stored":1775653658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594029,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1501684,"kb_used_data":1200108,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92243468,"statfs":{"total":95995035648,"available":94457311232,"internally_reserved":0,"allocated":1228910592,"data_stored":1226658451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790258,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1795564,"kb_used_data":1485732,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91949588,"statfs":{"total":95995035648,"available":94156378112,"internally_reserved":0,"allocated":1521389568,"data_stored":1519140925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986485,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1636404,"kb_used_data":1326508,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92108748,"statfs":{"total":95995035648,"available":94319357952,"internally_reserved":0,"allocated":1358344192,"data_stored":1356091564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999127,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2833120,"kb_used_data":2620696,"kb_used_omap":25,"kb_used_meta":212262,"kb_avail":90912032,"statfs":{"total":95995035648,"available":93093920768,"internally_reserved":0,"allocated":2683592704,"data_stored":2681355768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":217357255},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,11],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111070,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2274660,"kb_used_data":2089756,"kb_used_omap":31,"kb_used_meta":184672,"kb_avail":91470492,"statfs":{"total":95995035648,"available":93665783808,"internally_reserved":0,"allocated":2139910144,"data_stored":2137669634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":189104360},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318223013,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1488420,"kb_used_data":1335772,"kb_used_omap":26,"kb_used_meta":152613,"kb_avail":92256732,"statfs":{"total":95995035648,"available":94470893568,"internally_reserved":0,"allocated":1367830528,"data_stored":1365589457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":156275724},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289334956,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1312684,"kb_used_data":1164132,"kb_used_omap":31,"kb_used_meta":148512,"kb_avail":92432468,"statfs":{"total":95995035648,"available":94650847232,"internally_reserved":0,"allocated":1192071168,"data_stored":1189823022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":152076384},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1177489408,"data_stored":1177485336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1359351808,"data_stored":1359347736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2128478208,"data_stored":2128478208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2667839488,"data_stored":2667839488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1347354624,"data_stored":1347354624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1503137792,"data_stored":1503133720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1209401344,"data_stored":1209401344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1756823552,"data_stored":1756823552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:59:15.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 11 2024-09-21T09:59:15.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:15.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:15.271 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:15.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:15.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:15.397 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:15.425 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:15.427 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:181981 6.18 6:1fd1a18d:::benchmark_data_smithi017_149547_object62902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:15.414493+0000", 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.010822547, 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010855197, 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:15.428 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:15.429 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:15.429 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 181981 2024-09-21T09:59:15.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.429 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:15.429 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.429 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:15.430 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.414493+0000", 2024-09-21T09:59:15.430 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:15.430 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.430 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.430 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:15.430 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.414493+0000", 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.414492+0000", 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.414495+0000", 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.27e-06 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.431 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.414505+0000", 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.9559999999999992e-06 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.414510+0000", 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.33e-06 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.414559+0000", 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8942e-05 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.432 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.414846+0000", 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00028772100000000001 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.415319+0000", 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00047218999999999999 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.419658+0000", 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0043398849999999999 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.433 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.422880+0000", 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032210339999999998 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:181990 6.18 6:1c3de451:::benchmark_data_smithi017_149547_object62911:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:15.424172+0000", 2024-09-21T09:59:15.434 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.001143383, 2024-09-21T09:59:15.435 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0012646070000000001, 2024-09-21T09:59:15.435 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T09:59:15.435 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:59:15.435 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T09:59:15.435 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T09:59:15.435 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:15.435 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 181990 2024-09-21T09:59:15.435 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.435 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T09:59:15.436 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.436 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T09:59:15.436 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.424172+0000", 2024-09-21T09:59:15.436 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:15.436 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.436 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.436 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T09:59:15.436 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.424172+0000", 2024-09-21T09:59:15.436 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T09:59:15.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.437 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.437 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T09:59:15.437 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.424171+0000", 2024-09-21T09:59:15.437 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:15.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.437 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.437 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T09:59:15.438 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.424176+0000", 2024-09-21T09:59:15.438 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.135e-06 2024-09-21T09:59:15.438 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.438 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.438 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T09:59:15.438 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.424187+0000", 2024-09-21T09:59:15.438 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1453e-05 2024-09-21T09:59:15.438 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.438 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.439 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T09:59:15.439 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.424193+0000", 2024-09-21T09:59:15.439 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9359999999999996e-06 2024-09-21T09:59:15.439 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.439 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.439 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T09:59:15.439 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.424256+0000", 2024-09-21T09:59:15.439 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3252999999999999e-05 2024-09-21T09:59:15.440 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.440 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.440 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T09:59:15.440 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.424379+0000", 2024-09-21T09:59:15.440 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012270799999999999 2024-09-21T09:59:15.440 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T09:59:15.440 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T09:59:15.440 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:59:15.440 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:15.424880+0000", 2024-09-21T09:59:15.441 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00050083699999999996 2024-09-21T09:59:15.441 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:15.441 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T09:59:15.441 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:15.441 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T09:59:15.441 INFO:teuthology.orchestra.run.smithi134.stdout: ], 2024-09-21T09:59:15.441 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 2 2024-09-21T09:59:15.441 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:59:15.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:15.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:15.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:15.652 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:15.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:15 smithi017 ceph-mon[102913]: 1.6 scrub starts 2024-09-21T09:59:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:15 smithi017 ceph-mon[102913]: pgmap v1358: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 973/179469 objects degraded (0.542%); 1327/179469 objects misplaced (0.739%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T09:59:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:15 smithi017 ceph-mon[102913]: from='client.45275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:15 smithi017 ceph-mon[105151]: 1.6 scrub starts 2024-09-21T09:59:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:15 smithi017 ceph-mon[105151]: pgmap v1358: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 973/179469 objects degraded (0.542%); 1327/179469 objects misplaced (0.739%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T09:59:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:15 smithi017 ceph-mon[105151]: from='client.45275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:15 smithi134 ceph-mon[90085]: 1.6 scrub starts 2024-09-21T09:59:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:15 smithi134 ceph-mon[90085]: pgmap v1358: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 973/179469 objects degraded (0.542%); 1327/179469 objects misplaced (0.739%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T09:59:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:15 smithi134 ceph-mon[90085]: from='client.45275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:15.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:15.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:15.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:15.934 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:15.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.967 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:15.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:16.068 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:16.097 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:16.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:16.173 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T09:59:16.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:16.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:16.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:16.341 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:16.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:16.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:16.510 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:59:16.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:16.573 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:16.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:16.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:16.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:16.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:16.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:16.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:16.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:16.962 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:16.992 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:16.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:17.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:17.098 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:17.116 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:17.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:17.217 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:17.243 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:17.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:17.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:17.353 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:17 smithi017 ceph-mon[102913]: pgmap v1359: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 999 op/s; 973/170760 objects degraded (0.570%); 1327/170760 objects misplaced (0.777%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T09:59:17.353 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/901517389' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:17.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:17 smithi017 ceph-mon[105151]: pgmap v1359: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 999 op/s; 973/170760 objects degraded (0.570%); 1327/170760 objects misplaced (0.777%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T09:59:17.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/901517389' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:17.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:17.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:17.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:17.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:17.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:17.662 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:17.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:17 smithi134 ceph-mon[90085]: pgmap v1359: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 999 op/s; 973/170760 objects degraded (0.570%); 1327/170760 objects misplaced (0.777%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T09:59:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/901517389' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:17.764 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:17.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:17.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:17.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:17.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:59:17.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:59:17.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:59:17.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:59:17.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:59:17.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:59:17.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:59:17.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:59:17.937 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:59:17.937 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:59:17.937 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:59:17.937 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:59:17.937 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:59:17.937 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:59:17.937 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:59:17.937 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:59:17.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:17.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:18.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:18.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:18.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:18.194 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:18.223 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:18.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:18.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:18.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:18.463 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:18.487 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T09:59:18.487 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:59:18.496 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:18.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:18.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:18.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:18.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:18.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:18.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:18.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:18.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:18.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:19.028 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:19.055 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:19.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:19.157 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:19.196 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T09:59:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.202 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:19.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:19.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:19.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:19.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:19.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:19.485 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:19.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:19 smithi134 ceph-mon[90085]: pgmap v1360: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 966/167313 objects degraded (0.577%); 966/167313 objects misplaced (0.577%); 3.9 MiB/s, 63 objects/s recovering 2024-09-21T09:59:19.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:19 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 966/167313 objects degraded (0.577%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:19.557 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:59:19.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:19 smithi017 ceph-mon[102913]: pgmap v1360: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 966/167313 objects degraded (0.577%); 966/167313 objects misplaced (0.577%); 3.9 MiB/s, 63 objects/s recovering 2024-09-21T09:59:19.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:19 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 966/167313 objects degraded (0.577%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:19.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:19 smithi017 ceph-mon[105151]: pgmap v1360: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 966/167313 objects degraded (0.577%); 966/167313 objects misplaced (0.577%); 3.9 MiB/s, 63 objects/s recovering 2024-09-21T09:59:19.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:19 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 966/167313 objects degraded (0.577%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:19.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:19.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:19.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:19.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:19.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:19.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:19.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:20.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:20.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:20.164 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:20.193 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:20.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:20.295 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:20.320 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:20.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:20.336 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:59:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:59:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:20.345 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T09:59:20.421 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:20.450 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:20.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:20.553 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:20.580 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:20.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:20.682 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:20 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3438401743' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:20 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:20 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3438401743' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:20 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:20.711 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:20.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:20.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:20 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3438401743' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:20.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:20 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:20.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:20.842 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:20.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:20.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:20.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:21.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:21.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:21.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:21.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:21.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:21.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:21.363 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:21.394 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:21.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:21.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:59:21.472 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:21 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T09:59:21.472 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:21 smithi134 ceph-mon[90085]: pgmap v1361: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 966/165510 objects degraded (0.584%); 966/165510 objects misplaced (0.584%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:59:21.472 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:21 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:21.472 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:21 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:59:21.472 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:21 smithi134 ceph-mon[90085]: osdmap e1115: 8 total, 8 up, 8 in 2024-09-21T09:59:21.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:21.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.526 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:21.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:21.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:21.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T09:59:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[102913]: pgmap v1361: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 966/165510 objects degraded (0.584%); 966/165510 objects misplaced (0.584%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:59:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:59:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[102913]: osdmap e1115: 8 total, 8 up, 8 in 2024-09-21T09:59:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T09:59:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[105151]: pgmap v1361: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 966/165510 objects degraded (0.584%); 966/165510 objects misplaced (0.584%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:59:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:59:21.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:21 smithi017 ceph-mon[105151]: osdmap e1115: 8 total, 8 up, 8 in 2024-09-21T09:59:21.758 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T09:59:21.758 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:59:21.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:21.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:21.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:21.894 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:21.924 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:21.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:22.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:22.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:22.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:22.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:22.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:22.314 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:22.342 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:22.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:22.469 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:22.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:22.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T09:59:22.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:22.591 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:22.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:22.761 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:22.786 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:22.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:22.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:22.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:22.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:23.060 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:23.090 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:23.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:23.191 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:23.217 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:23.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:23.357 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 11 2024-09-21T09:59:23.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:23.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:23.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:23.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:23.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:23.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:23.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:23.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:23 smithi017 ceph-mon[102913]: pgmap v1363: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 766/154218 objects degraded (0.497%); 766/154218 objects misplaced (0.497%); 1.1 MiB/s, 38 objects/s recovering 2024-09-21T09:59:23.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:23 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2307017057' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T09:59:23.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:23 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T09:59:23.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:23 smithi017 ceph-mon[105151]: pgmap v1363: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 766/154218 objects degraded (0.497%); 766/154218 objects misplaced (0.497%); 1.1 MiB/s, 38 objects/s recovering 2024-09-21T09:59:23.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:23 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2307017057' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T09:59:23.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:23 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T09:59:23.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:23 smithi134 ceph-mon[90085]: pgmap v1363: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 766/154218 objects degraded (0.497%); 766/154218 objects misplaced (0.497%); 1.1 MiB/s, 38 objects/s recovering 2024-09-21T09:59:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:23 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2307017057' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T09:59:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:23 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T09:59:23.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:23.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:23.892 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:23.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:23.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:24.023 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:24.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:24.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:24.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T09:59:24.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:24.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:24.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:24.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:24.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:24.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:24.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:24.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:24.610 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:24.638 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:24.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:24.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:24 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 766/154218 objects degraded (0.497%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:24.792 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:24 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T09:59:24.792 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:24 smithi017 ceph-mon[102913]: osdmap e1116: 8 total, 8 up, 8 in 2024-09-21T09:59:24.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:24 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 766/154218 objects degraded (0.497%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:24.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:24 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T09:59:24.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:24 smithi017 ceph-mon[105151]: osdmap e1116: 8 total, 8 up, 8 in 2024-09-21T09:59:24.793 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:24.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:24.794 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:24 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 766/154218 objects degraded (0.497%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:24.794 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:24 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T09:59:24.794 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:24 smithi134 ceph-mon[90085]: osdmap e1116: 8 total, 8 up, 8 in 2024-09-21T09:59:24.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:24.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:24.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:24.931 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:59:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:59:24] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.51.0" 2024-09-21T09:59:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:24.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:25.074 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:25.101 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:25.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:25.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:25.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:25.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:25.343 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:25.371 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:25.473 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:25.500 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:25.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:25.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:25.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:25.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:25 smithi017 ceph-mon[102913]: pgmap v1365: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 766/150522 objects degraded (0.509%); 766/150522 objects misplaced (0.509%); 1.4 MiB/s, 47 objects/s recovering 2024-09-21T09:59:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:25.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:25 smithi017 ceph-mon[105151]: pgmap v1365: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 766/150522 objects degraded (0.509%); 766/150522 objects misplaced (0.509%); 1.4 MiB/s, 47 objects/s recovering 2024-09-21T09:59:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:25 smithi134 ceph-mon[90085]: pgmap v1365: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 766/150522 objects degraded (0.509%); 766/150522 objects misplaced (0.509%); 1.4 MiB/s, 47 objects/s recovering 2024-09-21T09:59:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:25.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:25.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:25.892 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:25.924 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:25.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:26.026 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:26.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:26.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:26.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:26.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:26.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:26.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:59:26.222 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:59:26.261 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:59:26.261 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:59:26.282 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:26.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:26.305 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:59:26.305 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:59:26.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:26.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:26.353 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:59:26.353 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:59:26.389 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:59:26.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:59:26.420 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:59:26.420 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:59:26.450 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:26.457 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:59:26.457 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:59:26.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:26.490 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:59:26.491 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:59:26.580 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:26.605 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:26.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:26 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4105843612' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:26.690 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:26 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4105843612' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:26.707 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:26.734 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:26.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:26 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4105843612' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:26.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:26.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:26.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:26.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:27.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:27.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:27.117 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:27.144 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:27.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:27.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:27.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:27.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:27.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:27.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:27.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:27.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:27.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:27.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:27.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:27 smithi017 ceph-mon[102913]: pgmap v1366: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 766/141246 objects degraded (0.542%); 766/141246 objects misplaced (0.542%); 0 B/s, 24 objects/s recovering 2024-09-21T09:59:27.603 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:27 smithi017 ceph-mon[105151]: pgmap v1366: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 766/141246 objects degraded (0.542%); 766/141246 objects misplaced (0.542%); 0 B/s, 24 objects/s recovering 2024-09-21T09:59:27.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:27.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:27.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:27 smithi134 ceph-mon[90085]: pgmap v1366: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 766/141246 objects degraded (0.542%); 766/141246 objects misplaced (0.542%); 0 B/s, 24 objects/s recovering 2024-09-21T09:59:27.826 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:27.859 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:27.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:27.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:28.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:28.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:28.109 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:28.137 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:28.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:28.240 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:28.271 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:28.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:28.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:28.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:28.523 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:28.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.557 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:28.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:28.561 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:28 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 766/141246 objects degraded (0.542%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:28.562 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:28 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 766/141246 objects degraded (0.542%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:28.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:28.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:28.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:28.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:28 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 766/141246 objects degraded (0.542%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:28.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:28.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:28.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:28.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:28.993 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:29.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:29.101 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:59:29.101 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:59:29.101 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:59:29.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd out 6 2024-09-21T09:59:29.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:29.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:29.239 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:29.267 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:29.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:29.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:29.438 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T09:59:29.438 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T09:59:29.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.438 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:195894 6.0 6:0542d829:::benchmark_data_smithi017_149547_object76815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1116)", 2024-09-21T09:59:29.438 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:29.428005+0000", 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0080685510000000002, 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081087039999999996, 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 195894 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428005+0000", 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428005+0000", 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428007+0000", 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9599999999999999e-06 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428013+0000", 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9569999999999999e-06 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428031+0000", 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8034999999999999e-05 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428040+0000", 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7949999999999996e-06 2024-09-21T09:59:29.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.430646+0000", 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026063089999999998 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.431098+0000", 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00045169299999999997 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:59:29.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.431956+0000", 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00085800299999999998 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:195901 6.19 6:9e7bdcb8:::benchmark_data_smithi017_149547_object76822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1116)", 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:29.428079+0000", 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0079948390000000001, 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081029420000000001, 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T09:59:29.443 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 195901 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428079+0000", 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T09:59:29.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428079+0000", 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428080+0000", 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9400000000000005e-07 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428083+0000", 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0419999999999998e-06 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428091+0000", 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6240000000000008e-06 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.428097+0000", 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3130000000000004e-06 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.433454+0000", 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053570019999999996 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.433569+0000", 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011416700000000001 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,7", 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:29.434069+0000", 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00050080800000000005 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 2 2024-09-21T09:59:29.447 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T09:59:29.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:29.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:29.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:29 smithi017 ceph-mon[102913]: pgmap v1367: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 547/137556 objects degraded (0.398%); 547/137556 objects misplaced (0.398%); 0 B/s, 52 objects/s recovering 2024-09-21T09:59:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:29 smithi017 ceph-mon[105151]: pgmap v1367: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 547/137556 objects degraded (0.398%); 547/137556 objects misplaced (0.398%); 0 B/s, 52 objects/s recovering 2024-09-21T09:59:29.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:29.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:29 smithi134 ceph-mon[90085]: pgmap v1367: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 547/137556 objects degraded (0.398%); 547/137556 objects misplaced (0.398%); 0 B/s, 52 objects/s recovering 2024-09-21T09:59:29.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:29.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:29.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:29.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:30.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:30.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:30.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:30.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:30.273 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.304 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:30.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:30.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:30.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:30.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:30.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:59:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:59:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:30.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:30.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:30.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:30.704 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:30.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.725 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:30.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:30.826 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:30.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.864 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:30.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:30.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:31.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:31.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:31.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:31.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:31.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:31.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:59:31.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:31.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:31.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:31.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:31 smithi017 ceph-mon[102913]: pgmap v1368: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 547/135714 objects degraded (0.403%); 547/135714 objects misplaced (0.403%); 0 B/s, 43 objects/s recovering 2024-09-21T09:59:31.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:31 smithi017 ceph-mon[105151]: pgmap v1368: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 547/135714 objects degraded (0.403%); 547/135714 objects misplaced (0.403%); 0 B/s, 43 objects/s recovering 2024-09-21T09:59:31.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:31.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:31.558 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:31.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.587 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:31.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:31.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:31.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:31.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:31 smithi134 ceph-mon[90085]: pgmap v1368: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 547/135714 objects degraded (0.403%); 547/135714 objects misplaced (0.403%); 0 B/s, 43 objects/s recovering 2024-09-21T09:59:31.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:31.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.860 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:31.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:31.961 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:31.992 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:31.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:32.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:32.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:32.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:32.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:32.255 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:32.357 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:32.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:32.490 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:32.519 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:32.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:32.621 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:32.657 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:32.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:32.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:32.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:32.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:32.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:33.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:33.106 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:33.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:33.208 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:33.240 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:33.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:33.342 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:33.378 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:33.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:33.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:33.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:33.619 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:33.640 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:33 smithi017 ceph-mon[105151]: pgmap v1369: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 319/124401 objects degraded (0.256%); 319/124401 objects misplaced (0.256%); 0 B/s, 44 objects/s recovering 2024-09-21T09:59:33.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 319/124401 objects degraded (0.256%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:33 smithi017 ceph-mon[102913]: pgmap v1369: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 319/124401 objects degraded (0.256%); 319/124401 objects misplaced (0.256%); 0 B/s, 44 objects/s recovering 2024-09-21T09:59:33.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 319/124401 objects degraded (0.256%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:33.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:33.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:33 smithi134 ceph-mon[90085]: pgmap v1369: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 319/124401 objects degraded (0.256%); 319/124401 objects misplaced (0.256%); 0 B/s, 44 objects/s recovering 2024-09-21T09:59:33.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 319/124401 objects degraded (0.256%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:33.767 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:33.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:33.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:33.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:33.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:33.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:34.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:34.136 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:34.164 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:34.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:34.266 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:34.291 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:34.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:34.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:34.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:34.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:34.534 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:34.560 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:34.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:34.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:34.705 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:34.807 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:34.833 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:34.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:34.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:59:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:59:34] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-09-21T09:59:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:34.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:35.072 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:35.093 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:35.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:35.194 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:35.223 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:35.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:35.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:35.464 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:35.488 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:35.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:35.544 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T09:59:35.590 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:35.621 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:35.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:35 smithi017 ceph-mon[102913]: pgmap v1370: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 319/120813 objects degraded (0.264%); 319/120813 objects misplaced (0.264%); 0 B/s, 41 objects/s recovering 2024-09-21T09:59:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2030018940' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:35 smithi017 ceph-mon[105151]: pgmap v1370: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 319/120813 objects degraded (0.264%); 319/120813 objects misplaced (0.264%); 0 B/s, 41 objects/s recovering 2024-09-21T09:59:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2030018940' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:35.723 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:35 smithi134 ceph-mon[90085]: pgmap v1370: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 319/120813 objects degraded (0.264%); 319/120813 objects misplaced (0.264%); 0 B/s, 41 objects/s recovering 2024-09-21T09:59:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2030018940' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:35.761 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:35.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:35.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:35.892 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:35.994 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:36.024 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:36.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:36.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:36.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:36.264 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.299 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:36.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:36.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T09:59:36.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:36.415 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:36.485 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:36.586 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:36.623 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:36.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:36 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:36 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2030018940' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:36 smithi017 ceph-mon[102913]: osdmap e1117: 8 total, 8 up, 8 in 2024-09-21T09:59:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:36 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:36 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2030018940' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:36 smithi017 ceph-mon[105151]: osdmap e1117: 8 total, 8 up, 8 in 2024-09-21T09:59:36.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:36 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:36 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2030018940' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:36 smithi134 ceph-mon[90085]: osdmap e1117: 8 total, 8 up, 8 in 2024-09-21T09:59:36.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:36.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:36.869 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:36.902 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:36.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:36.939 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:37.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:37.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:37.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:37.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:37.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:37.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:37.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:37.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:37.483 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:37.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:37.565 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:37 smithi017 ceph-mon[102913]: pgmap v1372: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 319/111834 objects degraded (0.285%); 319/111834 objects misplaced (0.285%); 0 B/s, 44 objects/s recovering 2024-09-21T09:59:37.566 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:37 smithi017 ceph-mon[105151]: pgmap v1372: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 319/111834 objects degraded (0.285%); 319/111834 objects misplaced (0.285%); 0 B/s, 44 objects/s recovering 2024-09-21T09:59:37.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:37.612 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:37.713 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:37.740 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:37.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:37.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:37 smithi134 ceph-mon[90085]: pgmap v1372: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s; 319/111834 objects degraded (0.285%); 319/111834 objects misplaced (0.285%); 0 B/s, 44 objects/s recovering 2024-09-21T09:59:37.842 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:37.873 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:37.975 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:37.998 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:37.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:38.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:38.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:38.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:38.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:38.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:38.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:38.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:38.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:38.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:38.533 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:38.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.560 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:38.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:38.661 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:38.684 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:38.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 319/111834 objects degraded (0.285%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:38 smithi134 ceph-mon[90085]: osdmap e1118: 8 total, 8 up, 8 in 2024-09-21T09:59:38.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:38.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:38.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:38.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 319/111834 objects degraded (0.285%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:38.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:38 smithi017 ceph-mon[102913]: osdmap e1118: 8 total, 8 up, 8 in 2024-09-21T09:59:38.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 319/111834 objects degraded (0.285%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T09:59:38.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:38 smithi017 ceph-mon[105151]: osdmap e1118: 8 total, 8 up, 8 in 2024-09-21T09:59:38.899 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:38.910 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:38.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:38.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:38.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T09:59:39.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:39.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:39.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:39.157 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:39.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:39.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:39.281 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:39.302 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:39.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:39.428 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:39.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:39.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:39.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:39.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:39.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:39 smithi017 ceph-mon[102913]: pgmap v1373: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 84/108246 objects degraded (0.078%); 84/108246 objects misplaced (0.078%); 0 B/s, 46 objects/s recovering 2024-09-21T09:59:39.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:39 smithi017 ceph-mon[102913]: osdmap e1119: 8 total, 8 up, 8 in 2024-09-21T09:59:39.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:39 smithi017 ceph-mon[105151]: pgmap v1373: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 84/108246 objects degraded (0.078%); 84/108246 objects misplaced (0.078%); 0 B/s, 46 objects/s recovering 2024-09-21T09:59:39.628 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:39 smithi017 ceph-mon[105151]: osdmap e1119: 8 total, 8 up, 8 in 2024-09-21T09:59:39.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:39.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:39 smithi134 ceph-mon[90085]: pgmap v1373: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 84/108246 objects degraded (0.078%); 84/108246 objects misplaced (0.078%); 0 B/s, 46 objects/s recovering 2024-09-21T09:59:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:39 smithi134 ceph-mon[90085]: osdmap e1119: 8 total, 8 up, 8 in 2024-09-21T09:59:39.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:39.839 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:39.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:39.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:39.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:40.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:40.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:40.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:40.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:40.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:40.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:40.461 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:59:40.461 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:59:40.461 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:59:40.461 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:59:40.461 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:59:40.461 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:59:40.462 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:59:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:59:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:59:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:40.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:40.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:40.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:40.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:40.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:40.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/845617460' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/845617460' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:40.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:40.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:40.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:40.936 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:40.967 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:40.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/845617460' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:41.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:41.106 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:41.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:41.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:41.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:41.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:41.399 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:41.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:41.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:41.524 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:41.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:41.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:41.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:41.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:41 smithi017 ceph-mon[102913]: pgmap v1376: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 84/106440 objects degraded (0.079%); 84/106440 objects misplaced (0.079%); 0 B/s, 39 objects/s recovering 2024-09-21T09:59:41.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:41 smithi017 ceph-mon[105151]: pgmap v1376: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 84/106440 objects degraded (0.079%); 84/106440 objects misplaced (0.079%); 0 B/s, 39 objects/s recovering 2024-09-21T09:59:41.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:41.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:41.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:41.922 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:41.949 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:41 smithi134 ceph-mon[90085]: pgmap v1376: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 84/106440 objects degraded (0.079%); 84/106440 objects misplaced (0.079%); 0 B/s, 39 objects/s recovering 2024-09-21T09:59:42.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:42.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:42.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:42.199 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:42.222 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:42.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:42.349 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:42.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:42.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:42.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:42.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:42.515 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.6. 2024-09-21T09:59:42.587 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T09:59:42.587 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:59:42.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:42.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:42.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:42.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:42.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:42.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:42.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:42.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:42.891 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-21T09:59:42.891 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T09:59:42.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2252002735' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T09:59:42.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T09:59:42.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[105151]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T09:59:42.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T09:59:42.492+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1119 definitely_dead 0 2024-09-21T09:59:42.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-21T09:59:42.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T09:59:42.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2252002735' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T09:59:42.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T09:59:42.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:42 smithi017 ceph-mon[102913]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T09:59:42.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-21T09:59:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T09:59:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:42 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2252002735' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T09:59:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:42 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T09:59:42.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:42 smithi134 ceph-mon[90085]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T09:59:42.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:42.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:43.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:43.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:43.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:43.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:43.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:43.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:43.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T09:59:43.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:43.383 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:43.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:43.482 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:43.485 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:43.507 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:43.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:43.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[102913]: pgmap v1377: 109 pgs: 109 active+clean; 2.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 0 B/s, 48 objects/s recovering 2024-09-21T09:59:43.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 84/106440 objects degraded (0.079%), 1 pg degraded, 1 pg undersized) 2024-09-21T09:59:43.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-21T09:59:43.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T09:59:43.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T09:59:43.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[102913]: osdmap e1120: 8 total, 8 up, 7 in 2024-09-21T09:59:43.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[102913]: osdmap e1121: 8 total, 8 up, 7 in 2024-09-21T09:59:43.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[105151]: pgmap v1377: 109 pgs: 109 active+clean; 2.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 0 B/s, 48 objects/s recovering 2024-09-21T09:59:43.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 84/106440 objects degraded (0.079%), 1 pg degraded, 1 pg undersized) 2024-09-21T09:59:43.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-21T09:59:43.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T09:59:43.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T09:59:43.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[105151]: osdmap e1120: 8 total, 8 up, 7 in 2024-09-21T09:59:43.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:43.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:43.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:43.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:43.756 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:43.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:43.778 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:43 smithi017 ceph-mon[105151]: osdmap e1121: 8 total, 8 up, 7 in 2024-09-21T09:59:43.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:43.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:43.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:43.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:43 smithi134 ceph-mon[90085]: pgmap v1377: 109 pgs: 109 active+clean; 2.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 0 B/s, 48 objects/s recovering 2024-09-21T09:59:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:43 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 84/106440 objects degraded (0.079%), 1 pg degraded, 1 pg undersized) 2024-09-21T09:59:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-21T09:59:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T09:59:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:43 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T09:59:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:43 smithi134 ceph-mon[90085]: osdmap e1120: 8 total, 8 up, 7 in 2024-09-21T09:59:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:43 smithi134 ceph-mon[90085]: osdmap e1121: 8 total, 8 up, 7 in 2024-09-21T09:59:44.008 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:44.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:44.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:44.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:44.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:44.234 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:44.257 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:44.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:44.359 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:44.392 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:44.493 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:44.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:59:44.525 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:44.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:44.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.653 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:44.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:44.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:44.807 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:44.909 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:44.937 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:44.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:45.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:45.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:45.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:45.107 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:59:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:59:44] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.51.0" 2024-09-21T09:59:45.177 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:45.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:45.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:45.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:45.366 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:45.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[105151]: pgmap v1380: 109 pgs: 1 remapped+peering, 108 active+clean; 1.9 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 0 B/s, 15 objects/s recovering 2024-09-21T09:59:45.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[105151]: 6.13 starting backfill to osd.2 from (0'0,1117'8347] MIN to 1119'8454 2024-09-21T09:59:45.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[105151]: 6.c starting backfill to osd.7 from (0'0,0'0] MAX to 1119'13131 2024-09-21T09:59:45.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[105151]: 6.a starting backfill to osd.2 from (0'0,0'0] MAX to 1119'12922 2024-09-21T09:59:45.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[105151]: osdmap e1122: 8 total, 8 up, 7 in 2024-09-21T09:59:45.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[105151]: 6.15 starting backfill to osd.4 from (0'0,0'0] MAX to 1119'8479 2024-09-21T09:59:45.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[105151]: 6.5 starting backfill to osd.2 from (0'0,0'0] MAX to 1119'8534 2024-09-21T09:59:45.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[105151]: 6.3 starting backfill to osd.1 from (0'0,0'0] MAX to 1119'8849 2024-09-21T09:59:45.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[105151]: 6.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1119'8479 2024-09-21T09:59:45.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[102913]: pgmap v1380: 109 pgs: 1 remapped+peering, 108 active+clean; 1.9 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 0 B/s, 15 objects/s recovering 2024-09-21T09:59:45.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[102913]: 6.13 starting backfill to osd.2 from (0'0,1117'8347] MIN to 1119'8454 2024-09-21T09:59:45.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[102913]: 6.c starting backfill to osd.7 from (0'0,0'0] MAX to 1119'13131 2024-09-21T09:59:45.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[102913]: 6.a starting backfill to osd.2 from (0'0,0'0] MAX to 1119'12922 2024-09-21T09:59:45.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[102913]: osdmap e1122: 8 total, 8 up, 7 in 2024-09-21T09:59:45.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[102913]: 6.15 starting backfill to osd.4 from (0'0,0'0] MAX to 1119'8479 2024-09-21T09:59:45.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[102913]: 6.5 starting backfill to osd.2 from (0'0,0'0] MAX to 1119'8534 2024-09-21T09:59:45.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[102913]: 6.3 starting backfill to osd.1 from (0'0,0'0] MAX to 1119'8849 2024-09-21T09:59:45.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:45 smithi017 ceph-mon[102913]: 6.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1119'8479 2024-09-21T09:59:45.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:45.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:45.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:45.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:45.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:45.667 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:45.701 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:45 smithi134 ceph-mon[90085]: pgmap v1380: 109 pgs: 1 remapped+peering, 108 active+clean; 1.9 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 0 B/s, 15 objects/s recovering 2024-09-21T09:59:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:45 smithi134 ceph-mon[90085]: 6.13 starting backfill to osd.2 from (0'0,1117'8347] MIN to 1119'8454 2024-09-21T09:59:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:45 smithi134 ceph-mon[90085]: 6.c starting backfill to osd.7 from (0'0,0'0] MAX to 1119'13131 2024-09-21T09:59:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:45 smithi134 ceph-mon[90085]: 6.a starting backfill to osd.2 from (0'0,0'0] MAX to 1119'12922 2024-09-21T09:59:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:45 smithi134 ceph-mon[90085]: osdmap e1122: 8 total, 8 up, 7 in 2024-09-21T09:59:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:45 smithi134 ceph-mon[90085]: 6.15 starting backfill to osd.4 from (0'0,0'0] MAX to 1119'8479 2024-09-21T09:59:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:45 smithi134 ceph-mon[90085]: 6.5 starting backfill to osd.2 from (0'0,0'0] MAX to 1119'8534 2024-09-21T09:59:45.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:45 smithi134 ceph-mon[90085]: 6.3 starting backfill to osd.1 from (0'0,0'0] MAX to 1119'8849 2024-09-21T09:59:45.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:45 smithi134 ceph-mon[90085]: 6.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1119'8479 2024-09-21T09:59:45.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:45.830 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:45.931 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:45.959 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:45.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:46.062 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:46.088 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:46.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:46.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.232 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:46.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:46.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:46.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:46.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:46.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:46.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:46.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:46.557 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:46.622 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:46.646 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:46.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:46.748 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:46.777 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:46.879 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:46.900 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:46.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:47.001 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:47.032 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:47.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:47.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:47.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:47.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:47.277 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:47.303 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:47.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:47.360 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T09:59:47.405 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:47.427 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:47.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:47.558 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:47.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:47.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:47.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:47 smithi017 ceph-mon[102913]: pgmap v1382: 109 pgs: 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 103 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 981 op/s; 6163/88770 objects misplaced (6.943%); 1.9 MiB/s, 44 objects/s recovering 2024-09-21T09:59:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:47 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4252784900' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:59:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:47 smithi017 ceph-mon[105151]: pgmap v1382: 109 pgs: 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 103 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 981 op/s; 6163/88770 objects misplaced (6.943%); 1.9 MiB/s, 44 objects/s recovering 2024-09-21T09:59:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:47 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4252784900' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:59:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:47 smithi134 ceph-mon[90085]: pgmap v1382: 109 pgs: 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 103 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 981 op/s; 6163/88770 objects misplaced (6.943%); 1.9 MiB/s, 44 objects/s recovering 2024-09-21T09:59:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:47 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4252784900' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:59:47.800 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:47.827 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:47.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:47.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:47.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:47.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:48.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:48.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:48.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:48.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:48.250 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:48.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:48.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:48.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:48.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:48.494 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:48.525 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:48.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:48.566 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:48 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:48.566 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:48 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4252784900' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:59:48.566 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:48 smithi017 ceph-mon[102913]: osdmap e1123: 8 total, 8 up, 7 in 2024-09-21T09:59:48.566 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:48 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:48.567 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:48 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4252784900' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:59:48.567 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:48 smithi017 ceph-mon[105151]: osdmap e1123: 8 total, 8 up, 7 in 2024-09-21T09:59:48.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:48.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:48.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:48 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:48 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4252784900' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:59:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:48 smithi134 ceph-mon[90085]: osdmap e1123: 8 total, 8 up, 7 in 2024-09-21T09:59:48.769 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:48.799 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:48.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:48.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:48.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:48.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:49.025 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:49.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:49.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:49.189 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:59:49.190 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:59:49.190 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-09-21T09:59:49.190 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T09:59:49.190 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T09:59:49.190 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-09-21T09:59:49.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:49.301 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:49.339 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:49.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:49.441 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:49.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.482 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:49.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:49.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:49 smithi134 ceph-mon[90085]: 1.5 scrub starts 2024-09-21T09:59:49.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:49 smithi134 ceph-mon[90085]: 1.5 scrub ok 2024-09-21T09:59:49.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:49 smithi134 ceph-mon[90085]: pgmap v1384: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 102 active+clean; 1.8 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 2/86916 objects degraded (0.002%); 6944/86916 objects misplaced (7.989%); 3.5 MiB/s, 57 objects/s recovering 2024-09-21T09:59:49.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:49 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 2/86916 objects degraded (0.002%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:59:49.582 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:59:49.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:49.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:49.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:49 smithi017 ceph-mon[102913]: 1.5 scrub starts 2024-09-21T09:59:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:49 smithi017 ceph-mon[102913]: 1.5 scrub ok 2024-09-21T09:59:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:49 smithi017 ceph-mon[102913]: pgmap v1384: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 102 active+clean; 1.8 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 2/86916 objects degraded (0.002%); 6944/86916 objects misplaced (7.989%); 3.5 MiB/s, 57 objects/s recovering 2024-09-21T09:59:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:49 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 2/86916 objects degraded (0.002%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:59:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:49 smithi017 ceph-mon[105151]: 1.5 scrub starts 2024-09-21T09:59:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:49 smithi017 ceph-mon[105151]: 1.5 scrub ok 2024-09-21T09:59:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:49 smithi017 ceph-mon[105151]: pgmap v1384: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 102 active+clean; 1.8 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 2/86916 objects degraded (0.002%); 6944/86916 objects misplaced (7.989%); 3.5 MiB/s, 57 objects/s recovering 2024-09-21T09:59:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:49 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 2/86916 objects degraded (0.002%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:59:49.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:49.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:49.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:49.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:49.901 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:50.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:50.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:50.144 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:50.172 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:50.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:50.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:50.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:50.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:50.394 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T09:59:50.394 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:59:50.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:50.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 09:59:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T09:59:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:50.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:50.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T09:59:50.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:50.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:50.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:50.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:50.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:50.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.944 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:50.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:51.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:51.073 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:51.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T09:59:51.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:51.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:51.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:51.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:51.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:51 smithi134 ceph-mon[90085]: pgmap v1385: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+remapped+backfilling, 102 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 2/84912 objects degraded (0.002%); 8589/84912 objects misplaced (10.115%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T09:59:51.516 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:51.539 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T09:59:51.539 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T09:59:51.539 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T09:59:51.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:51.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:51 smithi017 ceph-mon[102913]: pgmap v1385: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+remapped+backfilling, 102 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 2/84912 objects degraded (0.002%); 8589/84912 objects misplaced (10.115%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T09:59:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:51 smithi017 ceph-mon[105151]: pgmap v1385: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+remapped+backfilling, 102 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 2/84912 objects degraded (0.002%); 8589/84912 objects misplaced (10.115%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T09:59:51.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:51.682 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:51.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:51.758 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:51.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:51.841 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T09:59:51.842 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:59:51.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:51.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:51.880 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T09:59:51.880 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:59:51.918 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T09:59:51.918 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:59:51.946 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:51.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T09:59:51.959 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:59:51.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:51.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:51.988 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T09:59:51.988 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:59:52.021 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T09:59:52.021 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:59:52.055 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T09:59:52.056 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:59:52.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:52.089 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:52.091 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T09:59:52.091 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:59:52.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:52.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:52.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:52.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:52.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:52.351 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:52.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.396 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:52.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:52.497 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.530 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:52.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:52.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:52.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:52.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:52 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4276494994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:52 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:52.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:52 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4276494994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:52.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:52 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:52 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4276494994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:52 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:52.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:52.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:52.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:52.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:52.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:52.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:53.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:53.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:53.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:53.153 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:53.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:53.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:53.279 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:53.303 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:53.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:53.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:53.431 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:53.533 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:53.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.565 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:53.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:53.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:53 smithi017 ceph-mon[102913]: pgmap v1386: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 102 active+clean; 1.5 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 2/75156 objects degraded (0.003%); 7059/75156 objects misplaced (9.392%); 7.0 MiB/s, 112 objects/s recovering 2024-09-21T09:59:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:53 smithi017 ceph-mon[102913]: osdmap e1124: 8 total, 8 up, 7 in 2024-09-21T09:59:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:53 smithi017 ceph-mon[105151]: pgmap v1386: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 102 active+clean; 1.5 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 2/75156 objects degraded (0.003%); 7059/75156 objects misplaced (9.392%); 7.0 MiB/s, 112 objects/s recovering 2024-09-21T09:59:53.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:53 smithi017 ceph-mon[105151]: osdmap e1124: 8 total, 8 up, 7 in 2024-09-21T09:59:53.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:53.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:53 smithi134 ceph-mon[90085]: pgmap v1386: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 102 active+clean; 1.5 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 2/75156 objects degraded (0.003%); 7059/75156 objects misplaced (9.392%); 7.0 MiB/s, 112 objects/s recovering 2024-09-21T09:59:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:53 smithi134 ceph-mon[90085]: osdmap e1124: 8 total, 8 up, 7 in 2024-09-21T09:59:53.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:53.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:53.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:53.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:54.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:54.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:54.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:54.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:54.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:54.231 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:54.260 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:54.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:54.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:54.383 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:54.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:54.484 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:54.514 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:54.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:54.614 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:54 smithi134 ceph-mon[90085]: osdmap e1125: 8 total, 8 up, 7 in 2024-09-21T09:59:54.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:54.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:54 smithi017 ceph-mon[102913]: osdmap e1125: 8 total, 8 up, 7 in 2024-09-21T09:59:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:54 smithi017 ceph-mon[105151]: osdmap e1125: 8 total, 8 up, 7 in 2024-09-21T09:59:54.762 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:54.790 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:54.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:54.891 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:54.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:54.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:55.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:55.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:55.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:55.158 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:55.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:55.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:55.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 09:59:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:09:59:54] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.51.0" 2024-09-21T09:59:55.280 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:55.302 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:55.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:55.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:55.425 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:55.527 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:55.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:55.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:55.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:55.694 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:55.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:55 smithi017 ceph-mon[102913]: pgmap v1389: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 102 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 836 op/s; 2/71139 objects degraded (0.003%); 6567/71139 objects misplaced (9.231%); 8.8 MiB/s, 141 objects/s recovering 2024-09-21T09:59:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:55 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 2/71139 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:59:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:55 smithi017 ceph-mon[102913]: osdmap e1126: 8 total, 8 up, 7 in 2024-09-21T09:59:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:55.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:55 smithi017 ceph-mon[105151]: pgmap v1389: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 102 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 836 op/s; 2/71139 objects degraded (0.003%); 6567/71139 objects misplaced (9.231%); 8.8 MiB/s, 141 objects/s recovering 2024-09-21T09:59:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:55 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 2/71139 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:59:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:55 smithi017 ceph-mon[105151]: osdmap e1126: 8 total, 8 up, 7 in 2024-09-21T09:59:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:55 smithi134 ceph-mon[90085]: pgmap v1389: 109 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 102 active+clean; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 836 op/s; 2/71139 objects degraded (0.003%); 6567/71139 objects misplaced (9.231%); 8.8 MiB/s, 141 objects/s recovering 2024-09-21T09:59:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:55 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 2/71139 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:59:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:55 smithi134 ceph-mon[90085]: osdmap e1126: 8 total, 8 up, 7 in 2024-09-21T09:59:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:55.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:55.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:55.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:55.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:55.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:55.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:56.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:56.099 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:56.200 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:56.225 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:56.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:56.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:56.370 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:56.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:56.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:56.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:56.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:56.612 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:56.638 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:56 smithi134 ceph-mon[90085]: osdmap e1127: 8 total, 8 up, 7 in 2024-09-21T09:59:56.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:56.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:56 smithi017 ceph-mon[102913]: osdmap e1127: 8 total, 8 up, 7 in 2024-09-21T09:59:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:56 smithi017 ceph-mon[105151]: osdmap e1127: 8 total, 8 up, 7 in 2024-09-21T09:59:56.754 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:56.782 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:56.883 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:56.910 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:56.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:57.012 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:57.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.060 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:57.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:57.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:57.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:57.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:57.291 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:59:57.291 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:59:57.291 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-09-21T09:59:57.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd in 6 2024-09-21T09:59:57.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:57.332 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:57.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:57.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:57.584 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:57.608 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:57.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:57 smithi017 ceph-mon[102913]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 2/64845 objects degraded (0.003%); 4548/64845 objects misplaced (7.014%); 16 MiB/s, 259 objects/s recovering 2024-09-21T09:59:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:57 smithi017 ceph-mon[105151]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 2/64845 objects degraded (0.003%); 4548/64845 objects misplaced (7.014%); 16 MiB/s, 259 objects/s recovering 2024-09-21T09:59:57.710 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:57.732 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:57.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:57 smithi134 ceph-mon[90085]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 2/64845 objects degraded (0.003%); 4548/64845 objects misplaced (7.014%); 16 MiB/s, 259 objects/s recovering 2024-09-21T09:59:57.834 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:57.868 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:57.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:57.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:58.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:58.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:58.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:58.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:58.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:58.254 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:58.286 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:58.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:58.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:58.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:58.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:58.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:58.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:58.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:58.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:58.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:58.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:58.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:58.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:58.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:58.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:58.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:59.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:59.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:59.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:59.210 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:59.235 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:59.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:59.336 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:59.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.365 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:59.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:59 smithi017 ceph-mon[102913]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 2/62319 objects degraded (0.003%); 4153/62319 objects misplaced (6.664%); 16 MiB/s, 253 objects/s recovering 2024-09-21T09:59:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 09:59:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:59 smithi017 ceph-mon[105151]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 2/62319 objects degraded (0.003%); 4153/62319 objects misplaced (6.664%); 16 MiB/s, 253 objects/s recovering 2024-09-21T09:59:59.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:59.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 09:59:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:59.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:59.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:59.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:59.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:59.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:59.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:59.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:59 smithi134 ceph-mon[90085]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 2/62319 objects degraded (0.003%); 4153/62319 objects misplaced (6.664%); 16 MiB/s, 253 objects/s recovering 2024-09-21T09:59:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 09:59:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T09:59:59.749 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:59.775 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T09:59:59.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:59.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:59.919 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T09:59:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:00.022 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:00.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:00.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:00.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:00.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:00.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:00.306 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:00.338 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:00.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:00.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:00.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:00 smithi134 ceph-mon[90085]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 2/62319 objects degraded (0.003%), 1 pg degraded 2024-09-21T10:00:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:00 smithi134 ceph-mon[90085]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-09-21T10:00:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:00 smithi134 ceph-mon[90085]: [WRN] PG_DEGRADED: Degraded data redundancy: 2/62319 objects degraded (0.003%), 1 pg degraded 2024-09-21T10:00:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:00 smithi134 ceph-mon[90085]: pg 6.1 is active+recovering+undersized+degraded+remapped, acting [1,2] 2024-09-21T10:00:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:00:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:00:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:00.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:00 smithi017 ceph-mon[102913]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 2/62319 objects degraded (0.003%), 1 pg degraded 2024-09-21T10:00:00.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:00 smithi017 ceph-mon[102913]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-09-21T10:00:00.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:00 smithi017 ceph-mon[102913]: [WRN] PG_DEGRADED: Degraded data redundancy: 2/62319 objects degraded (0.003%), 1 pg degraded 2024-09-21T10:00:00.528 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:00 smithi017 ceph-mon[102913]: pg 6.1 is active+recovering+undersized+degraded+remapped, acting [1,2] 2024-09-21T10:00:00.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:00 smithi017 ceph-mon[105151]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 2/62319 objects degraded (0.003%), 1 pg degraded 2024-09-21T10:00:00.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:00 smithi017 ceph-mon[105151]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-09-21T10:00:00.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:00 smithi017 ceph-mon[105151]: [WRN] PG_DEGRADED: Degraded data redundancy: 2/62319 objects degraded (0.003%), 1 pg degraded 2024-09-21T10:00:00.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:00 smithi017 ceph-mon[105151]: pg 6.1 is active+recovering+undersized+degraded+remapped, acting [1,2] 2024-09-21T10:00:00.592 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:00.616 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:00.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:00.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:00.774 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:00.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:00.876 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:00.902 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:00.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:01.004 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:01.030 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:01.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:01.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:01.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:01.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:01.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:01.298 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:00:01.301 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:01.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:01.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:01.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:01.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:01.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:01.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:01.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:01 smithi017 ceph-mon[102913]: pgmap v1394: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 2/60045 objects degraded (0.003%); 3917/60045 objects misplaced (6.523%); 11 MiB/s, 180 objects/s recovering 2024-09-21T10:00:01.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:01 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 2/60045 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:00:01.629 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:01 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3667736292' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:01.629 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:01 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:01.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:01 smithi017 ceph-mon[105151]: pgmap v1394: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 2/60045 objects degraded (0.003%); 3917/60045 objects misplaced (6.523%); 11 MiB/s, 180 objects/s recovering 2024-09-21T10:00:01.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:01 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 2/60045 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:00:01.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:01 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3667736292' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:01.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:01 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:01.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:01.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:00:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:01 smithi134 ceph-mon[90085]: pgmap v1394: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 2/60045 objects degraded (0.003%); 3917/60045 objects misplaced (6.523%); 11 MiB/s, 180 objects/s recovering 2024-09-21T10:00:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:01 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 2/60045 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:00:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:01 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3667736292' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:01 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:01.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:01.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:01.857 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:01.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:01.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:01.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:02.018 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:02.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:02.120 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:02.148 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:02.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:02.249 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:02.272 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:02.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:02.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:02.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:02.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:02.510 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:02.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:02.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:02.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:02.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:02.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:02 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:00:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:02 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:02 smithi017 ceph-mon[102913]: osdmap e1128: 8 total, 8 up, 7 in 2024-09-21T10:00:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:02 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:00:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:02 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:02.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:02 smithi017 ceph-mon[105151]: osdmap e1128: 8 total, 8 up, 7 in 2024-09-21T10:00:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:02 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:00:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:02 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:02 smithi134 ceph-mon[90085]: osdmap e1128: 8 total, 8 up, 7 in 2024-09-21T10:00:02.779 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:02.808 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:02.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:02.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:02.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:03.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:03.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:03.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:03.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:03.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:03.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:03.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:03.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:03.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:03.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:03.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:03.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:03.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:03.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:03 smithi017 ceph-mon[102913]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.0 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 898 op/s; 2/50532 objects degraded (0.004%); 3317/50532 objects misplaced (6.564%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:00:03.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:03 smithi017 ceph-mon[105151]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.0 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 898 op/s; 2/50532 objects degraded (0.004%); 3317/50532 objects misplaced (6.564%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:00:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:03 smithi134 ceph-mon[90085]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 1.0 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 898 op/s; 2/50532 objects degraded (0.004%); 3317/50532 objects misplaced (6.564%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:00:03.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:03.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:03.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:03.848 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:03.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:03.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:03.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:04.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:04.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:04.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:04.132 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:00:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:00:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:225002 6.d 6:b85d5567:::benchmark_data_smithi017_149547_object105923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1128)", 2024-09-21T10:00:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:04.118165+0000", 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0091832100000000007, 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0092155729999999995, 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 225002 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:00:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.118165+0000", 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.118165+0000", 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:00:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.118167+0000", 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1440000000000001e-06 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.118174+0000", 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2590000000000003e-06 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.118193+0000", 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8495000000000002e-05 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.118204+0000", 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0677e-05 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.118269+0000", 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5122000000000004e-05 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.118478+0000", 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020969399999999999 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.118619+0000", 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140784 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.122657+0000", 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040375929999999999 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.122672+0000", 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4808e-05 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:00:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:224981 6.d 6:b4cf6b04:::benchmark_data_smithi017_149547_object105902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1128)", 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:04.085742+0000", 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.041606073, 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.041725659999999998, 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:00:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 224981 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.085742+0000", 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.085742+0000", 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:00:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.085742+0000", 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1199999999999998e-07 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.085745+0000", 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2570000000000002e-06 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.085751+0000", 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9900000000000002e-06 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.085756+0000", 2024-09-21T10:00:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6380000000000001e-06 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.101841+0000", 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016084848999999998 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.102260+0000", 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00041844799999999998 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.102643+0000", 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038285000000000001 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.107107+0000", 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044645689999999998 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:04.107547+0000", 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00044021400000000002 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:00:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:00:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-21T10:00:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 2 2024-09-21T10:00:04.143 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:00:04.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:04.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:04.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:04.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:04.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:04.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:04.514 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:04.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:04.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:04.643 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:04.671 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:04.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:04.772 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:04.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:04.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:04.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:04.927 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:04.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:04.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:00:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:00:04] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.51.0" 2024-09-21T10:00:05.029 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:05.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.063 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:05.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:05.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:05.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:05.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:05.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:05.340 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:05.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:05.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:05.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:05.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:05.587 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:05.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:05 smithi017 ceph-mon[102913]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 961 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 961 op/s; 2/46182 objects degraded (0.004%); 2866/46182 objects misplaced (6.206%); 9.8 MiB/s, 174 objects/s recovering 2024-09-21T10:00:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:05 smithi017 ceph-mon[105151]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 961 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 961 op/s; 2/46182 objects degraded (0.004%); 2866/46182 objects misplaced (6.206%); 9.8 MiB/s, 174 objects/s recovering 2024-09-21T10:00:05.733 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:05.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:05 smithi134 ceph-mon[90085]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 961 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 961 op/s; 2/46182 objects degraded (0.004%); 2866/46182 objects misplaced (6.206%); 9.8 MiB/s, 174 objects/s recovering 2024-09-21T10:00:05.771 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:05.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:05.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:00:05.873 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:05.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.907 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:05.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:06.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:06.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:06.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:06.154 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:06.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:06.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:06.277 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:06.309 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:06.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:06.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:06.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:06.547 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:06.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.580 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:06.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:06.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:06 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 2/37599 objects degraded (0.005%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:00:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:06 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 2/37599 objects degraded (0.005%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:00:06.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:06.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:06 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 2/37599 objects degraded (0.005%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:00:06.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:06.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:06.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:06.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:07.006 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:07.108 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:07.134 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:07.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:07.236 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:07.266 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:07.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:07.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:07.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:07.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:07.513 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:07.544 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:07.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:07.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:07.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:07 smithi017 ceph-mon[102913]: pgmap v1398: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 782 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 907 op/s; 2/37599 objects degraded (0.005%); 2260/37599 objects misplaced (6.011%); 3.4 MiB/s, 69 objects/s recovering 2024-09-21T10:00:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:07 smithi017 ceph-mon[105151]: pgmap v1398: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 782 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 907 op/s; 2/37599 objects degraded (0.005%); 2260/37599 objects misplaced (6.011%); 3.4 MiB/s, 69 objects/s recovering 2024-09-21T10:00:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:07 smithi134 ceph-mon[90085]: pgmap v1398: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 782 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 907 op/s; 2/37599 objects degraded (0.005%); 2260/37599 objects misplaced (6.011%); 3.4 MiB/s, 69 objects/s recovering 2024-09-21T10:00:07.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:07.817 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:07.919 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:07.944 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:08.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:08.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:08.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:00:08.185 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:08.212 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:08.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:08.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:08.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:08.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:08.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:08.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:08.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:08.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:08.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:08.781 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:08.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.811 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:08.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:08.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:08.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:08.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:09.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:09.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:09.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:09.205 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:09.232 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:09.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:09.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:09.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:09.481 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:09.514 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:09.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:09.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:09 smithi017 ceph-mon[102913]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 716 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 929 op/s; 2/34401 objects degraded (0.006%); 2260/34401 objects misplaced (6.570%); 998 KiB/s, 64 objects/s recovering 2024-09-21T10:00:09.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:09 smithi017 ceph-mon[105151]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 716 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 929 op/s; 2/34401 objects degraded (0.006%); 2260/34401 objects misplaced (6.570%); 998 KiB/s, 64 objects/s recovering 2024-09-21T10:00:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:09.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:09.688 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:09.704 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:09 smithi134 ceph-mon[90085]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 716 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 929 op/s; 2/34401 objects degraded (0.006%); 2260/34401 objects misplaced (6.570%); 998 KiB/s, 64 objects/s recovering 2024-09-21T10:00:09.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:09.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:09.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:09.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:09.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:10.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:10.103 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:10.128 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:10.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:10.230 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:10.255 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:10.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:10.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:10.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:10.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:00:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=cleanup t=2024-09-21T10:00:10.298163349Z level=info msg="Completed cleanup jobs" duration=12.617968ms 2024-09-21T10:00:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:00:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:00:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:10.514 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:10.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:10.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:10.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:10.679 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:10.780 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:10.810 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:10.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:10.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:10.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:10.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:11.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:11.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:11.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:11.220 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:11.245 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:11.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:11.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:11.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:11.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:11 smithi017 ceph-mon[102913]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 661 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 941 op/s; 2/31782 objects degraded (0.006%); 1970/31782 objects misplaced (6.198%); 998 KiB/s, 64 objects/s recovering 2024-09-21T10:00:11.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:11.459 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:11.459 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:11 smithi017 ceph-mon[105151]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 661 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 941 op/s; 2/31782 objects degraded (0.006%); 1970/31782 objects misplaced (6.198%); 998 KiB/s, 64 objects/s recovering 2024-09-21T10:00:11.460 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:11.460 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:11.475 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.511 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:11.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:11.612 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:11.636 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:11.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:11 smithi134 ceph-mon[90085]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 661 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 941 op/s; 2/31782 objects degraded (0.006%); 1970/31782 objects misplaced (6.198%); 998 KiB/s, 64 objects/s recovering 2024-09-21T10:00:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:11.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:11.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:11.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:11.916 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:12.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:12.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:12.180 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:12.206 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:12.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:12.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:12.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:12.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:12.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:12.480 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:12.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:12.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:12.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:12.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:12.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:12 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 2/21813 objects degraded (0.009%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:00:12.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:12 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 2/21813 objects degraded (0.009%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:00:12.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:12 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 2/21813 objects degraded (0.009%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:00:12.762 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:12.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:12.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:12.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:12.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:13.004 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:13.028 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:13.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:13.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:13.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:13.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:13.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:13.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:13.420 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:13.445 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:13.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:13.547 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:13.575 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:13.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:13.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:13 smithi017 ceph-mon[102913]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 453 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 925 op/s; 2/21813 objects degraded (0.009%); 1366/21813 objects misplaced (6.262%); 926 KiB/s, 60 objects/s recovering 2024-09-21T10:00:13.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:13 smithi017 ceph-mon[105151]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 453 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 925 op/s; 2/21813 objects degraded (0.009%); 1366/21813 objects misplaced (6.262%); 926 KiB/s, 60 objects/s recovering 2024-09-21T10:00:13.677 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:13.699 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:13 smithi134 ceph-mon[90085]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 453 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 925 op/s; 2/21813 objects degraded (0.009%); 1366/21813 objects misplaced (6.262%); 926 KiB/s, 60 objects/s recovering 2024-09-21T10:00:13.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:13.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:13.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:13.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:13.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:13.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:14.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:14.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:14.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:14.207 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:14.233 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:14.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:14.335 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.366 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:14.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:14.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:14.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:14.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:14.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:14.698 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:14.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:14.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:14.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:14.910 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:14.924 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:00:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:00:14] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.51.0" 2024-09-21T10:00:14.932 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:14.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:14.989 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:00:14.990 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:00:14.990 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:00:14.990 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:00:14.990 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:00:15.034 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:15.059 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:15.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:15.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:15.196 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:15.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:15.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:15 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:00:15.153+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1128 definitely_dead 0 2024-09-21T10:00:15.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:15.333 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.6. 2024-09-21T10:00:15.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:15.443 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:15.464 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:15.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:15.566 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:15.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:15.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:15.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:15 smithi017 ceph-mon[102913]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 364 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 915 op/s; 2/17544 objects degraded (0.011%); 1077/17544 objects misplaced (6.139%); 832 KiB/s, 86 objects/s recovering 2024-09-21T10:00:15.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:15 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3182602252' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:00:15.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:15 smithi017 ceph-mon[105151]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 364 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 915 op/s; 2/17544 objects degraded (0.011%); 1077/17544 objects misplaced (6.139%); 832 KiB/s, 86 objects/s recovering 2024-09-21T10:00:15.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:15 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3182602252' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:00:15.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:15.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:15 smithi134 ceph-mon[90085]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 104 active+clean; 364 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 915 op/s; 2/17544 objects degraded (0.011%); 1077/17544 objects misplaced (6.139%); 832 KiB/s, 86 objects/s recovering 2024-09-21T10:00:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:15 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3182602252' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:00:15.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:15.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:15.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:15.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:15.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:15.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:16.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:16.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:16.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:16.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:00:16.186 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:16.208 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:16.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:16.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:16.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:16.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:16.339 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:16 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3182602252' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:00:16.339 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:16 smithi017 ceph-mon[105151]: osdmap e1129: 8 total, 8 up, 8 in 2024-09-21T10:00:16.339 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:16 smithi017 ceph-mon[105151]: 1.8 scrub starts 2024-09-21T10:00:16.339 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:16 smithi017 ceph-mon[105151]: 1.8 scrub ok 2024-09-21T10:00:16.435 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:16.456 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:16.557 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:16.579 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:16.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:16.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:16 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3182602252' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:00:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:16 smithi017 ceph-mon[102913]: osdmap e1129: 8 total, 8 up, 8 in 2024-09-21T10:00:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:16 smithi017 ceph-mon[102913]: 1.8 scrub starts 2024-09-21T10:00:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:16 smithi017 ceph-mon[102913]: 1.8 scrub ok 2024-09-21T10:00:16.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:16.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:16.716 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:16 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3182602252' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:00:16.716 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:16 smithi134 ceph-mon[90085]: osdmap e1129: 8 total, 8 up, 8 in 2024-09-21T10:00:16.717 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:16 smithi134 ceph-mon[90085]: 1.8 scrub starts 2024-09-21T10:00:16.717 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:16 smithi134 ceph-mon[90085]: 1.8 scrub ok 2024-09-21T10:00:16.805 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:16.816 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-09-21T10:00:16.826 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:16.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:16.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:16.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:17.065 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:17.088 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:17.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:17.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:17.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:00:17.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:00:17.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:00:17.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:00:17.301 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:00:17.301 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:00:17.301 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:00:17.301 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:00:17.301 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:00:17.301 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:00:17.301 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:00:17.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:00:17.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:17.317 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:00:17.318 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:00:17.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:00:17.334 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:00:17.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:17.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:17.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:17.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:17.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:17.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:17.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:17.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:17.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:17 smithi017 ceph-mon[102913]: pgmap v1404: 109 pgs: 2 remapped+peering, 29 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 217 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 2/10479 objects degraded (0.019%); 273/10479 objects misplaced (2.605%); 0 B/s, 72 objects/s recovering 2024-09-21T10:00:17.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:17 smithi017 ceph-mon[102913]: osdmap e1130: 8 total, 8 up, 8 in 2024-09-21T10:00:17.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/654107784' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:17.679 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:17 smithi017 ceph-mon[105151]: pgmap v1404: 109 pgs: 2 remapped+peering, 29 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 217 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 2/10479 objects degraded (0.019%); 273/10479 objects misplaced (2.605%); 0 B/s, 72 objects/s recovering 2024-09-21T10:00:17.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:17 smithi017 ceph-mon[105151]: osdmap e1130: 8 total, 8 up, 8 in 2024-09-21T10:00:17.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/654107784' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:17.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:17 smithi134 ceph-mon[90085]: pgmap v1404: 109 pgs: 2 remapped+peering, 29 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 217 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 2/10479 objects degraded (0.019%); 273/10479 objects misplaced (2.605%); 0 B/s, 72 objects/s recovering 2024-09-21T10:00:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:17 smithi134 ceph-mon[90085]: osdmap e1130: 8 total, 8 up, 8 in 2024-09-21T10:00:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/654107784' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:17.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:17.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:17.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:17.896 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:17.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:17.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:18.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:18.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:18.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:18.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:18.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:18.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:18.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:18.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:18.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:18.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:18.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:18.582 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:18.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:18.684 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[102913]: 1.2 scrub starts 2024-09-21T10:00:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[102913]: 1.2 scrub ok 2024-09-21T10:00:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[102913]: 6.3 starting backfill to osd.6 from (1083'2585,1123'9098] MIN to 1128'9754 2024-09-21T10:00:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[102913]: 6.15 starting backfill to osd.6 from (1083'1757,1125'8734] MIN to 1128'9325 2024-09-21T10:00:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[102913]: osdmap e1131: 8 total, 8 up, 8 in 2024-09-21T10:00:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[105151]: 1.2 scrub starts 2024-09-21T10:00:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[105151]: 1.2 scrub ok 2024-09-21T10:00:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[105151]: 6.3 starting backfill to osd.6 from (1083'2585,1123'9098] MIN to 1128'9754 2024-09-21T10:00:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[105151]: 6.15 starting backfill to osd.6 from (1083'1757,1125'8734] MIN to 1128'9325 2024-09-21T10:00:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:18 smithi017 ceph-mon[105151]: osdmap e1131: 8 total, 8 up, 8 in 2024-09-21T10:00:18.720 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:18 smithi134 ceph-mon[90085]: 1.2 scrub starts 2024-09-21T10:00:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:18 smithi134 ceph-mon[90085]: 1.2 scrub ok 2024-09-21T10:00:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:18 smithi134 ceph-mon[90085]: 6.3 starting backfill to osd.6 from (1083'2585,1123'9098] MIN to 1128'9754 2024-09-21T10:00:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:18 smithi134 ceph-mon[90085]: 6.15 starting backfill to osd.6 from (1083'1757,1125'8734] MIN to 1128'9325 2024-09-21T10:00:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:18 smithi134 ceph-mon[90085]: osdmap e1131: 8 total, 8 up, 8 in 2024-09-21T10:00:18.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:18.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:18.967 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:18.993 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:18.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:19.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:19.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:19.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:19.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:19.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:19.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:19.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:19.426 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:19.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:19.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:19.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:19.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:19.668 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:19.689 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:19.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[102913]: pgmap v1407: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 29 peering, 1 active+remapped+backfill_wait, 76 active+clean; 178 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 273/8607 objects misplaced (3.172%); 0 B/s, 154 objects/s recovering 2024-09-21T10:00:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/10479 objects degraded (0.019%), 1 pg degraded) 2024-09-21T10:00:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:00:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[102913]: 6.10 scrub starts 2024-09-21T10:00:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[102913]: 6.10 scrub ok 2024-09-21T10:00:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[105151]: pgmap v1407: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 29 peering, 1 active+remapped+backfill_wait, 76 active+clean; 178 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 273/8607 objects misplaced (3.172%); 0 B/s, 154 objects/s recovering 2024-09-21T10:00:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/10479 objects degraded (0.019%), 1 pg degraded) 2024-09-21T10:00:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:00:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[105151]: 6.10 scrub starts 2024-09-21T10:00:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:19 smithi017 ceph-mon[105151]: 6.10 scrub ok 2024-09-21T10:00:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:19 smithi134 ceph-mon[90085]: pgmap v1407: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 29 peering, 1 active+remapped+backfill_wait, 76 active+clean; 178 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 273/8607 objects misplaced (3.172%); 0 B/s, 154 objects/s recovering 2024-09-21T10:00:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:19 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/10479 objects degraded (0.019%), 1 pg degraded) 2024-09-21T10:00:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:19 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:00:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:19 smithi134 ceph-mon[90085]: 6.10 scrub starts 2024-09-21T10:00:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:19 smithi134 ceph-mon[90085]: 6.10 scrub ok 2024-09-21T10:00:19.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:19.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:19.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:19.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 119076 objects 2024-09-21T10:00:19.920 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :132.982 2024-09-21T10:00:19.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:19.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:19.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:20.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:20.078 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:20.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:20.180 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:20.201 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:20.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:20.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:20.323 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:20.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:20.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:20.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:20.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:00:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:00:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:20.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:20.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:20.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:20.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:20.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:20.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:20.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:20.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:20.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:20.925 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:20.947 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:20.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:21.048 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:21.069 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:21.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:21.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:21.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:21.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:21.297 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:21.317 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:21.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:21.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:21.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[105151]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 29 peering, 1 active+remapped+backfill_wait, 76 active+clean; 159 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 273/7698 objects misplaced (3.546%) 2024-09-21T10:00:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[105151]: 1.6 scrub starts 2024-09-21T10:00:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[105151]: 1.6 scrub ok 2024-09-21T10:00:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[105151]: 6.13 scrub starts 2024-09-21T10:00:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[105151]: 6.13 scrub ok 2024-09-21T10:00:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[102913]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 29 peering, 1 active+remapped+backfill_wait, 76 active+clean; 159 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 273/7698 objects misplaced (3.546%) 2024-09-21T10:00:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[102913]: 1.6 scrub starts 2024-09-21T10:00:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[102913]: 1.6 scrub ok 2024-09-21T10:00:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[102913]: 6.13 scrub starts 2024-09-21T10:00:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:21 smithi017 ceph-mon[102913]: 6.13 scrub ok 2024-09-21T10:00:21.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:21.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:21.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:21.690 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:21 smithi134 ceph-mon[90085]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 29 peering, 1 active+remapped+backfill_wait, 76 active+clean; 159 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 273/7698 objects misplaced (3.546%) 2024-09-21T10:00:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:21 smithi134 ceph-mon[90085]: 1.6 scrub starts 2024-09-21T10:00:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:21 smithi134 ceph-mon[90085]: 1.6 scrub ok 2024-09-21T10:00:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:21 smithi134 ceph-mon[90085]: 6.13 scrub starts 2024-09-21T10:00:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:21 smithi134 ceph-mon[90085]: 6.13 scrub ok 2024-09-21T10:00:21.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:21.814 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:21.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:21.816 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:00:21.816 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:00:21.817 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-21T10:00:21.817 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:00:21.817 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:00:21.817 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-21T10:00:21.817 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:21.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:21.938 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:22.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:22.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:22.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:22.103 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:00:22.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:22.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:22.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:22.288 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:22.310 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:22.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:22.360 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:00:22.360 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[102913]: 1.3 scrub starts 2024-09-21T10:00:22.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[102913]: 1.3 scrub ok 2024-09-21T10:00:22.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[102913]: osdmap e1132: 8 total, 8 up, 8 in 2024-09-21T10:00:22.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1957871625' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:00:22.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:00:22.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[105151]: 1.3 scrub starts 2024-09-21T10:00:22.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[105151]: 1.3 scrub ok 2024-09-21T10:00:22.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[105151]: osdmap e1132: 8 total, 8 up, 8 in 2024-09-21T10:00:22.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1957871625' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:00:22.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:22 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:00:22.412 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:22.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.434 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:22.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:22.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:22 smithi134 ceph-mon[90085]: 1.3 scrub starts 2024-09-21T10:00:22.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:22 smithi134 ceph-mon[90085]: 1.3 scrub ok 2024-09-21T10:00:22.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:22 smithi134 ceph-mon[90085]: osdmap e1132: 8 total, 8 up, 8 in 2024-09-21T10:00:22.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:22 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1957871625' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:00:22.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:22 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:00:22.535 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:22.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:22.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:22.653 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-09-21T10:00:22.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-09-21T10:00:22.660 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:22.681 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:22.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:22.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:22.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:22.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:22.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:00:22.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:23.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:23.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:23.154 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:23.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:23.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:23.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:23.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:23.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[102913]: pgmap v1410: 109 pgs: 2 active+clean+remapped, 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 105 active+clean; 32 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:23.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:23.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1957871625' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:00:23.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:00:23.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[102913]: osdmap e1133: 8 total, 8 up, 8 in 2024-09-21T10:00:23.401 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:23.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:23.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:23.525 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:23.546 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:23.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:23.622 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:23 smithi134 ceph-mon[90085]: pgmap v1410: 109 pgs: 2 active+clean+remapped, 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 105 active+clean; 32 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:23.622 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:23 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:23.623 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:23 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1957871625' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:00:23.623 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:23 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:00:23.623 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:23 smithi134 ceph-mon[90085]: osdmap e1133: 8 total, 8 up, 8 in 2024-09-21T10:00:23.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:23.670 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:23.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[105151]: pgmap v1410: 109 pgs: 2 active+clean+remapped, 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 105 active+clean; 32 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1957871625' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:00:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:00:23.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:23 smithi017 ceph-mon[105151]: osdmap e1133: 8 total, 8 up, 8 in 2024-09-21T10:00:23.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:23.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:23.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:23.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:24.028 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:24.050 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:24.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:24.152 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:24.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:24.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:24.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:24.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:24.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:24.394 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:00:24.394 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "" 2024-09-21T10:00:24.394 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:00:24.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:24.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:24.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:24.534 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:24.556 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:24.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:24.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:24 smithi017 ceph-mon[102913]: osdmap e1134: 8 total, 8 up, 8 in 2024-09-21T10:00:24.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:24 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:00:24.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:24 smithi017 ceph-mon[105151]: osdmap e1134: 8 total, 8 up, 8 in 2024-09-21T10:00:24.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:24 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:00:24.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:24.696 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:24 smithi134 ceph-mon[90085]: osdmap e1134: 8 total, 8 up, 8 in 2024-09-21T10:00:24.696 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:24 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:00:24.782 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:24.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.805 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:24.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:24.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:24.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:24.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:00:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:00:24] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.51.0" 2024-09-21T10:00:25.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:25.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:25.154 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:25.175 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:25.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:25.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:25.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:25.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:25.401 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:25.403 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:25.403 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:25.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:25.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:25.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:25.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:25.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:25.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[105151]: pgmap v1413: 109 pgs: 2 active+clean+remapped, 1 active+clean+scrubbing, 106 active+clean; 576 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:25.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:00:25.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[105151]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 1131'9218 2024-09-21T10:00:25.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[105151]: osdmap e1135: 8 total, 8 up, 8 in 2024-09-21T10:00:25.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:25.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:25.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[102913]: pgmap v1413: 109 pgs: 2 active+clean+remapped, 1 active+clean+scrubbing, 106 active+clean; 576 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:25.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:00:25.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[102913]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 1131'9218 2024-09-21T10:00:25.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[102913]: osdmap e1135: 8 total, 8 up, 8 in 2024-09-21T10:00:25.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:25.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:25.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:25.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:25 smithi134 ceph-mon[90085]: pgmap v1413: 109 pgs: 2 active+clean+remapped, 1 active+clean+scrubbing, 106 active+clean; 576 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:00:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:25 smithi134 ceph-mon[90085]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 1131'9218 2024-09-21T10:00:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:25 smithi134 ceph-mon[90085]: osdmap e1135: 8 total, 8 up, 8 in 2024-09-21T10:00:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:25.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:25.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:25.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:25.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:00:25.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:25.930 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:25.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:26.032 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:26.058 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:26.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:26.160 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:26.185 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:26.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:26.226 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:00:26.226 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:00:26.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:26.311 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:26.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:26.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:26.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:26.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:26.546 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:26.568 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:26.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:26.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:26.692 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:26.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:26 smithi017 ceph-mon[102913]: osdmap e1136: 8 total, 8 up, 8 in 2024-09-21T10:00:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:26 smithi017 ceph-mon[102913]: osdmap e1137: 8 total, 8 up, 8 in 2024-09-21T10:00:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:26 smithi017 ceph-mon[105151]: osdmap e1136: 8 total, 8 up, 8 in 2024-09-21T10:00:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:26 smithi017 ceph-mon[105151]: osdmap e1137: 8 total, 8 up, 8 in 2024-09-21T10:00:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:26 smithi134 ceph-mon[90085]: osdmap e1136: 8 total, 8 up, 8 in 2024-09-21T10:00:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:26 smithi134 ceph-mon[90085]: osdmap e1137: 8 total, 8 up, 8 in 2024-09-21T10:00:26.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:26.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:26.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:26.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:00:26.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:26.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:27.047 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:27.069 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:27.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:27.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:27.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:27.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:27.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:27.323 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:27.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: pgmap v1416: 109 pgs: 1 remapped+peering, 108 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 0 B/s, 128 objects/s recovering 2024-09-21T10:00:27.492 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 1131'9484 2024-09-21T10:00:27.492 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: 6.19 starting backfill to osd.1 from (0'0,0'0] MAX to 1131'9302 2024-09-21T10:00:27.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: 6.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1131'9501 2024-09-21T10:00:27.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 1131'9484 2024-09-21T10:00:27.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: 6.19 starting backfill to osd.3 from (0'0,0'0] MAX to 1131'9302 2024-09-21T10:00:27.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: 6.19 starting backfill to osd.5 from (0'0,0'0] MAX to 1131'9302 2024-09-21T10:00:27.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:27.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:27.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: pgmap v1416: 109 pgs: 1 remapped+peering, 108 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 0 B/s, 128 objects/s recovering 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 1131'9484 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: 6.19 starting backfill to osd.1 from (0'0,0'0] MAX to 1131'9302 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: 6.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1131'9501 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 1131'9484 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: 6.19 starting backfill to osd.3 from (0'0,0'0] MAX to 1131'9302 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: 6.19 starting backfill to osd.5 from (0'0,0'0] MAX to 1131'9302 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:27.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:27.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:27.517 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:27.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:27.618 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:27.640 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:27.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: pgmap v1416: 109 pgs: 1 remapped+peering, 108 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 0 B/s, 128 objects/s recovering 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 1131'9484 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: 6.19 starting backfill to osd.1 from (0'0,0'0] MAX to 1131'9302 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: 6.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1131'9501 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 1131'9484 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: 6.19 starting backfill to osd.3 from (0'0,0'0] MAX to 1131'9302 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: 6.19 starting backfill to osd.5 from (0'0,0'0] MAX to 1131'9302 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:27.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:27.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:27.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:27.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:28.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:28.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:28.139 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:28.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:28.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:28.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:28.365 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:28.390 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:28.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:28.492 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:28.515 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:28.617 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:28.639 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:28.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:28 smithi017 ceph-mon[102913]: osdmap e1138: 8 total, 8 up, 8 in 2024-09-21T10:00:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:28 smithi017 ceph-mon[105151]: osdmap e1138: 8 total, 8 up, 8 in 2024-09-21T10:00:28.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:28 smithi134 ceph-mon[90085]: osdmap e1138: 8 total, 8 up, 8 in 2024-09-21T10:00:28.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:28.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:28.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:28.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:28.951 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:00:28.951 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:00:28.961 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:00:28.961 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:00:28.977 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:00:28.977 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:00:28.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:28.994 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:00:28.994 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:00:29.011 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:00:29.011 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:00:29.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:29.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:29.026 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:00:29.027 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:00:29.043 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:00:29.043 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:00:29.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:00:29.059 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:00:29.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:29.138 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:29.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:29.240 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:29.264 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:29.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:29.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:29.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:29.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:29.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:29.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:29.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:29.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:29.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:29 smithi017 ceph-mon[105151]: pgmap v1419: 109 pgs: 1 peering, 1 activating+remapped, 1 remapped+peering, 106 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:29 smithi017 ceph-mon[105151]: osdmap e1139: 8 total, 8 up, 8 in 2024-09-21T10:00:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:29 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3530826794' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:29 smithi017 ceph-mon[102913]: pgmap v1419: 109 pgs: 1 peering, 1 activating+remapped, 1 remapped+peering, 106 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:29 smithi017 ceph-mon[102913]: osdmap e1139: 8 total, 8 up, 8 in 2024-09-21T10:00:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:29 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3530826794' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:29.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:29 smithi134 ceph-mon[90085]: pgmap v1419: 109 pgs: 1 peering, 1 activating+remapped, 1 remapped+peering, 106 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:29 smithi134 ceph-mon[90085]: osdmap e1139: 8 total, 8 up, 8 in 2024-09-21T10:00:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:29 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3530826794' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:29.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:29.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:29.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.895 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:29.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:29.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:30.020 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:30.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:30.036 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:00:30.036 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:00:30.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:30.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:30.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:30.248 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:30.274 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:30.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:30.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:30.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:30.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:00:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:00:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:30.501 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:30.527 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:30.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:30.628 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:30.654 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:30.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:30.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:30.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:30.882 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:30.908 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:30.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:31.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:31.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:31.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:31.135 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:31.161 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:31.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:31.262 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:31.286 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:31.388 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.414 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:31.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:31.515 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:31.538 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:31.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:31.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:31.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:31 smithi017 ceph-mon[102913]: pgmap v1421: 109 pgs: 1 peering, 1 remapped+peering, 107 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:31 smithi017 ceph-mon[105151]: pgmap v1421: 109 pgs: 1 peering, 1 remapped+peering, 107 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:31.701 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:31 smithi134 ceph-mon[90085]: pgmap v1421: 109 pgs: 1 peering, 1 remapped+peering, 107 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:31.766 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.791 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:31.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:31.892 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:31.916 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:32.017 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:32.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:32.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:32.143 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:32.167 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:32.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:32.269 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:32.294 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:32.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:32.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:32.421 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:32.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:32.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:32.545 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:32.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:32.619 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_1' removed 2024-09-21T10:00:32.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:32 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1039388779' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:00:32.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:32 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:00:32.620 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:32 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1039388779' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:00:32.620 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:32 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:00:32.647 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.675 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:32.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:32 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1039388779' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:00:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:32 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:00:32.775 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:32.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:32.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:32.827 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:32.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:32.919 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:33.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:33.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:33.145 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:33.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:33.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:33.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:33.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:33.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:33.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:33.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:33 smithi017 ceph-mon[105151]: pgmap v1422: 109 pgs: 1 peering, 108 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:33 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:00:33.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:33 smithi017 ceph-mon[105151]: osdmap e1140: 8 total, 8 up, 8 in 2024-09-21T10:00:33.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:33 smithi017 ceph-mon[102913]: pgmap v1422: 109 pgs: 1 peering, 108 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:33 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:00:33.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:33 smithi017 ceph-mon[102913]: osdmap e1140: 8 total, 8 up, 8 in 2024-09-21T10:00:33.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:00:33.517 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:33.538 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:33.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:33.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:33.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:33.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:33 smithi134 ceph-mon[90085]: pgmap v1422: 109 pgs: 1 peering, 108 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:33 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:00:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:33 smithi134 ceph-mon[90085]: osdmap e1140: 8 total, 8 up, 8 in 2024-09-21T10:00:33.766 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:33.788 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:33.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:33.890 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:33.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:34.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:34.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:34.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:34.138 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:34.160 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:34.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:34.262 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:34.284 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:34.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:34.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:34.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:34.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:34.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:34.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:34.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:34.637 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:34.660 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:34.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:34.731 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:00:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:00:34] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.51.0" 2024-09-21T10:00:34.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:34.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:34.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:34.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:34.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:34.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:35.010 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:35.031 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:35.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:35.133 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:35.155 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:35.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:35.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:35.354 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:00:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:00:35.355 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-21T10:00:35.355 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:00:35.380 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:35.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.403 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:35.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:35.443 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:00:35.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:35.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:35.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:35.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:35.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:35 smithi017 ceph-mon[105151]: pgmap v1424: 83 pgs: 83 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/657148192' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:35 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:35 smithi017 ceph-mon[102913]: pgmap v1424: 83 pgs: 83 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:35.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/657148192' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:35.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:35 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:35 smithi134 ceph-mon[90085]: pgmap v1424: 83 pgs: 83 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/657148192' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:35 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:35.752 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:35.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:35.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:35.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:35.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:36.000 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:36.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.023 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:36.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:36.124 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:36.145 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:36.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:36.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:36.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:36.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:36.373 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:36.394 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:36.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:36.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:36.516 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:36.618 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:36.639 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:36.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:36 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:36 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:36 smithi017 ceph-mon[102913]: osdmap e1141: 8 total, 8 up, 8 in 2024-09-21T10:00:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:36 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:36 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:36 smithi017 ceph-mon[105151]: osdmap e1141: 8 total, 8 up, 8 in 2024-09-21T10:00:36.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:36 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:36 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:36 smithi134 ceph-mon[90085]: osdmap e1141: 8 total, 8 up, 8 in 2024-09-21T10:00:36.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:36.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:36.775 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:00:36.775 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:00:36.776 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-09-21T10:00:36.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_2 16 2024-09-21T10:00:36.834 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:36.872 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:36.893 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:36.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:36.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:37.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:37.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:37.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:37.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:37.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:37.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:37.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:37.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:37.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:00:37.367 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:37.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:37.490 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:37.511 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:37.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:37.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:37 smithi017 ceph-mon[102913]: pgmap v1426: 83 pgs: 83 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:37.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:37 smithi017 ceph-mon[105151]: pgmap v1426: 83 pgs: 83 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:37.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:37.636 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:37.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:37.737 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:37 smithi134 ceph-mon[90085]: pgmap v1426: 83 pgs: 83 active+clean; 576 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:37.759 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:37.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:37.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:37.883 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:37.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:37.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:38.007 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:38.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:38.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:38.116 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:38.140 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:38.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:38.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:38.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:38.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:00:38.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:38.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:38.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:38.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:38.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:38.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:38.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:38.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:38.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:38.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:38.768 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:38.870 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:38.895 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:38.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:38.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:39.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:39.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:39.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:39.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:39.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:39.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:39.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:39.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:39.394 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:39.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:39.521 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:39.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:39.540 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:39 smithi017 ceph-mon[102913]: pgmap v1427: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:39.541 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:39 smithi017 ceph-mon[105151]: pgmap v1427: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:39.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:39.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:39.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:39 smithi134 ceph-mon[90085]: pgmap v1427: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:39.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:39.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:39.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:39.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:39.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:39.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:39.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:40.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:40.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:40.119 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:40.144 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:40.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:40.175 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:00:40.175 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:00:40.185 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:00:40.185 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:00:40.204 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:00:40.204 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:00:40.220 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:00:40.221 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:00:40.237 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:00:40.237 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:00:40.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:40.253 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:00:40.253 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:00:40.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:40.270 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:00:40.270 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:00:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:40.286 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:00:40.286 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:00:40.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:40.397 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:40.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3553288129' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:00:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:00:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:40.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:40.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:40.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:40.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:40.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3553288129' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:40.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3553288129' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:40.747 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:40.772 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:40.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:40.898 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:41.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:41.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:41.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:41.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:41.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:41.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:41.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:41.276 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:41.369 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:00:41.369 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:00:41.376 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:41.378 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:41.403 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:41.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:41.505 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:41.530 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:41.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:41.656 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:41.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:41 smithi017 ceph-mon[102913]: pgmap v1428: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:41.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:41 smithi017 ceph-mon[105151]: pgmap v1428: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:41 smithi134 ceph-mon[90085]: pgmap v1428: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:41.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:41.784 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:41.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:41.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:41.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:42.010 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:42.035 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:42.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:42.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:42.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:42.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:42.287 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:42.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:00:42.388 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:42.413 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:42.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:42.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:42.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:42.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:42.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:42.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:42.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:42.732 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:42.763 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:42.788 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:42.890 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:42.914 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:42.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:43.016 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:43.040 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:43.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:43.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:43.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:00:43.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:43.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:43.267 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:43.292 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:43.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:43.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:43.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:43.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:43.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:43.545 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:43.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:43.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:43.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:43.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:43 smithi017 ceph-mon[102913]: pgmap v1429: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:43 smithi017 ceph-mon[105151]: pgmap v1429: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:43 smithi134 ceph-mon[90085]: pgmap v1429: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:43.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:43.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:43.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:43.895 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:43.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:43.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:43.991 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:44.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:44.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:44.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:44.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:44.272 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:44.297 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:44.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:44.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:44.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:44.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:44.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:44.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:44.645 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:44.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:44.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:44.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:44.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:44.803 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:44.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:44.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:44.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:44.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:45.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:45.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:45.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:45.075 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:00:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:00:44] "GET /metrics HTTP/1.1" 200 34145 "" "Prometheus/2.51.0" 2024-09-21T10:00:45.153 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:45.178 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:45.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:45.280 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:45.305 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:45.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:45.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:45.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:45.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:45.461 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_2' created 2024-09-21T10:00:45.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:45.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:45.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:45.657 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:45.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:45.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:45 smithi017 ceph-mon[102913]: pgmap v1430: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:45 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2482471425' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:00:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:45 smithi017 ceph-mon[105151]: pgmap v1430: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:45 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2482471425' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:00:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:45 smithi134 ceph-mon[90085]: pgmap v1430: 83 pgs: 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:45 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2482471425' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:00:45.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:45.810 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:45.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:45.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:45.936 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:45.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:46.037 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:46.061 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:46.163 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:46.187 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:46.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:46.288 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:46.310 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:46.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:46.412 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:46.434 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:46.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:46.466 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:00:46.535 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:46.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:46.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:46.660 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:46.682 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2482471425' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-21T10:00:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:46 smithi134 ceph-mon[90085]: osdmap e1142: 8 total, 8 up, 8 in 2024-09-21T10:00:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4185956725' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:00:46.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:46.806 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:46.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:46.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2482471425' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-21T10:00:46.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:46 smithi017 ceph-mon[102913]: osdmap e1142: 8 total, 8 up, 8 in 2024-09-21T10:00:46.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4185956725' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:00:46.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2482471425' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-21T10:00:46.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:46 smithi017 ceph-mon[105151]: osdmap e1142: 8 total, 8 up, 8 in 2024-09-21T10:00:46.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4185956725' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:00:46.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:46.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:46.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:47.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:47.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:47.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:47.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:47.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:47.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:47.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:47.306 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:47.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:47.407 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:47.430 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:47.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:47.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:47.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:47.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[102913]: pgmap v1432: 99 pgs: 8 creating+peering, 8 unknown, 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[102913]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4185956725' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[102913]: osdmap e1143: 8 total, 8 up, 8 in 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[102913]: 1.7 scrub ok 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[105151]: pgmap v1432: 99 pgs: 8 creating+peering, 8 unknown, 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[105151]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:47.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4185956725' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:00:47.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[105151]: osdmap e1143: 8 total, 8 up, 8 in 2024-09-21T10:00:47.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:00:47.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:47 smithi017 ceph-mon[105151]: 1.7 scrub ok 2024-09-21T10:00:47.658 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:47.680 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:47.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:47 smithi134 ceph-mon[90085]: pgmap v1432: 99 pgs: 8 creating+peering, 8 unknown, 83 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:47 smithi134 ceph-mon[90085]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:00:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:47 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:47 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4185956725' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:00:47.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:47 smithi134 ceph-mon[90085]: osdmap e1143: 8 total, 8 up, 8 in 2024-09-21T10:00:47.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:47 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:00:47.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:47 smithi134 ceph-mon[90085]: 1.7 scrub ok 2024-09-21T10:00:47.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:47.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:47.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:47.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:47.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:47.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:48.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-09-21T10:00:48.030 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:48.052 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:48.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:48.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:48.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:48.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:48.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:48.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:48.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:48.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:48.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:48.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:48.649 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:48.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:48 smithi017 ceph-mon[102913]: 1.9 scrub starts 2024-09-21T10:00:48.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:48 smithi017 ceph-mon[102913]: 1.9 scrub ok 2024-09-21T10:00:48.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:48 smithi017 ceph-mon[102913]: osdmap e1144: 8 total, 8 up, 8 in 2024-09-21T10:00:48.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:48 smithi017 ceph-mon[105151]: 1.9 scrub starts 2024-09-21T10:00:48.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:48 smithi017 ceph-mon[105151]: 1.9 scrub ok 2024-09-21T10:00:48.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:48 smithi017 ceph-mon[105151]: osdmap e1144: 8 total, 8 up, 8 in 2024-09-21T10:00:48.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.674 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:48.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:48 smithi134 ceph-mon[90085]: 1.9 scrub starts 2024-09-21T10:00:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:48 smithi134 ceph-mon[90085]: 1.9 scrub ok 2024-09-21T10:00:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:48 smithi134 ceph-mon[90085]: osdmap e1144: 8 total, 8 up, 8 in 2024-09-21T10:00:48.775 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:48.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:48.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:48.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:48.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:49.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:49.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:49.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:49.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:49.168 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:00:49.168 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:00:49.171 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:49.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:49.273 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:49.300 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:49.402 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:49.427 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:49.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:49.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:49.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:49.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:49.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:49.679 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:49.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:49 smithi017 ceph-mon[102913]: pgmap v1435: 99 pgs: 8 creating+peering, 4 unknown, 87 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:49 smithi017 ceph-mon[105151]: pgmap v1435: 99 pgs: 8 creating+peering, 4 unknown, 87 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:49 smithi134 ceph-mon[90085]: pgmap v1435: 99 pgs: 8 creating+peering, 4 unknown, 87 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:49.782 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:49.808 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:49.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:49.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:00:49.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:49.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:49.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:50.035 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:50.061 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:50.163 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:50.187 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:50.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:50.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:50.314 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:50.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:50.395 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:50.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:50.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:50.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:50.485 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:00:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:00:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:50.542 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:50.568 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:50.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:50.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:50 smithi017 ceph-mon[102913]: 1.0 scrub starts 2024-09-21T10:00:50.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:50 smithi017 ceph-mon[102913]: 1.0 scrub ok 2024-09-21T10:00:50.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:50 smithi017 ceph-mon[105151]: 1.0 scrub starts 2024-09-21T10:00:50.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:50 smithi017 ceph-mon[105151]: 1.0 scrub ok 2024-09-21T10:00:50.670 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:50.695 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:50.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:50.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:00:50.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:50 smithi134 ceph-mon[90085]: 1.0 scrub starts 2024-09-21T10:00:50.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:50 smithi134 ceph-mon[90085]: 1.0 scrub ok 2024-09-21T10:00:50.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:50.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:50.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:50.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:50.943 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:50.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:51.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:51.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:51.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:51.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:51.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:51.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:51.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:51.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:51.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:51.405 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:00:51.405 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:00:51.405 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:00:51.406 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:00:51.406 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:00:51.406 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:00:51.406 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:00:51.406 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:00:51.406 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:00:51.406 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:00:51.406 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:00:51.406 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:00:51.407 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:00:51.407 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:00:51.407 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:00:51.407 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:00:51.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:51.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:51.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:51.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:51.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:51.667 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:51.693 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:51.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:51 smithi134 ceph-mon[90085]: pgmap v1436: 99 pgs: 8 creating+peering, 91 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:51 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/137429337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:51.766 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:51 smithi017 ceph-mon[105151]: pgmap v1436: 99 pgs: 8 creating+peering, 91 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:51.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:51 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/137429337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:51.767 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:51 smithi017 ceph-mon[102913]: pgmap v1436: 99 pgs: 8 creating+peering, 91 active+clean; 576 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:51.767 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:51 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/137429337' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:51.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:51.820 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:51.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:51.921 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:51.946 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:52.048 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:52.057 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:52.073 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:52.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:52.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:52.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:52.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:52.314 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:52.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:52.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:52.428 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:52.453 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:52.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:52.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:52.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:52.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:52.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:52.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:52.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:52.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:52.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:52.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:52.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:53.051 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:53.076 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:53.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:53.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:53.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:53.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:53.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:53.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:53.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:53.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:53.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:53.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:53.684 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:53.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.711 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:53.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:53 smithi134 ceph-mon[90085]: pgmap v1437: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:53.762 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:53 smithi017 ceph-mon[102913]: pgmap v1437: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:53.762 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:53 smithi017 ceph-mon[105151]: pgmap v1437: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:53.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:53.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:53.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:53.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:53.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:53.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:54.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:54.087 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:54.189 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:54.214 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:54.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:54.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:54.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:54.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:54.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:54.565 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:54.590 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:54.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:54.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:54.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:54.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:54.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:54.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:54.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:54.940 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:54.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:00:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:00:54] "GET /metrics HTTP/1.1" 200 34145 "" "Prometheus/2.51.0" 2024-09-21T10:00:54.966 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:54.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:55.068 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:55.093 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:55.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:55.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:55.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:55.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:55.341 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:55.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:55.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:55.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:55.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:55.567 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:55.592 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:55.607 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:00:55.607 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:00:55.607 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-21T10:00:55.607 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:00:55.694 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:55 smithi017 ceph-mon[102913]: pgmap v1438: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:55.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:55 smithi017 ceph-mon[105151]: pgmap v1438: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:55.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:55.720 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:55.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:55 smithi134 ceph-mon[90085]: pgmap v1438: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:00:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:55.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:55.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:55.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:55.947 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:55.972 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:55.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:56.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:56.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:56.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:56.199 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:56.224 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:56.326 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:56.351 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:56.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:56.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:00:56.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:56.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:56.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:56.579 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:56.604 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:56.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:56.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:00:56.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:56.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:56.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:56.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:56.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:56.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:56.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:57.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:57.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:57.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:57.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:57.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:57.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:57.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:57.353 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:57.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:57.364 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:57.368 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:57.455 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:57.481 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:57.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:57.493 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-09-21T10:00:57.493 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:00:57.582 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:57.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:57.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:57.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:57.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:57.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:57.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:57 smithi017 ceph-mon[102913]: pgmap v1439: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:57.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1135182385' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:00:57.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1444265211' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:57.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:57 smithi017 ceph-mon[105151]: pgmap v1439: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:57.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1135182385' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:00:57.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1444265211' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:57.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:57.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:57.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:57.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:57.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:57.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:57 smithi134 ceph-mon[90085]: pgmap v1439: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1135182385' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:00:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1444265211' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:58.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:58.107 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:58.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:58.208 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:58.234 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:58.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:58.336 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:58.361 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:58.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:58.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:58.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:58.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:58.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:58.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:58 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:00:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:58 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1135182385' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:00:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:58 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1444265211' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:58.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:58 smithi017 ceph-mon[102913]: osdmap e1145: 8 total, 8 up, 8 in 2024-09-21T10:00:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:58 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:00:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:58 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1135182385' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:00:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:58 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1444265211' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:58 smithi017 ceph-mon[105151]: osdmap e1145: 8 total, 8 up, 8 in 2024-09-21T10:00:58.712 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:58.739 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:58.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:58.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:58.865 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:58.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:58.967 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:58.992 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:58.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:58 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:00:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:58 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1135182385' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:00:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:58 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1444265211' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:58 smithi134 ceph-mon[90085]: osdmap e1145: 8 total, 8 up, 8 in 2024-09-21T10:00:59.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:59.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:59.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:59.220 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:59.245 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:59.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:59.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:59.372 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:59.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:59.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:59.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:59.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:59.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:59 smithi017 ceph-mon[102913]: pgmap v1441: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:59 smithi017 ceph-mon[102913]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:00:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:00:59 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:00:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:59 smithi017 ceph-mon[105151]: pgmap v1441: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:59 smithi017 ceph-mon[105151]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:00:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:00:59 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:00:59.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:59.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:00:59.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:59.848 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-09-21T10:00:59.848 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:59.849 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2024-09-21T10:00:59.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:00:59.851 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:00:59.851 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:00:59.875 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:00:59.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:59.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-21T10:00:59.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:00:59.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_180590 2024-09-21T10:00:59.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:00:59.900 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:00:59.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:59 smithi134 ceph-mon[90085]: pgmap v1441: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:00:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:59 smithi134 ceph-mon[90085]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:00:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:00:59 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:01:00.006 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:00.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:00.108 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:00.137 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:00.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:00.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:01:00.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:00.311 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:00.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:00.413 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:00.440 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:00.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:01:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:01:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:00.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:00.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:00.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:00.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:00.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.732 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:00.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:00.833 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:00.856 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:00.899 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 2927 2911 181.924 181.938 0.00260323 0.00545443 2024-09-21T10:01:00.957 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:00.987 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:00.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:01.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:01.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:01.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:01.231 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:01.263 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:01.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:01.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:01.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:01.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:01 smithi134 ceph-mon[90085]: pgmap v1442: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:01:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:01 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:01.502 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:01.531 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:01.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:01.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:01.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:01 smithi017 ceph-mon[102913]: pgmap v1442: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:01:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:01 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:01 smithi017 ceph-mon[105151]: pgmap v1442: 99 pgs: 99 active+clean; 419 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:01:01.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:01 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:01.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:01.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:01.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:01.900 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 6201 6185 193.262 204.625 0.00374346 0.0051638 2024-09-21T10:01:01.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:01.946 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:01.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:02.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:02.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:02.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:02.188 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:02.217 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:02.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:02.319 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:02.349 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:02.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:02.438 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:02.451 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:02.477 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:02.579 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:02.608 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:02.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:02.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:02.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:02.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:02.902 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 9182 9166 190.814 186.312 0.00822634 0.00523266 2024-09-21T10:01:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:02.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:03.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:03.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:03.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:03.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:03.158 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:01:03.159 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:01:03.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:03.198 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:01:03.199 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:01:03.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:03.239 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:01:03.240 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:01:03.275 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:01:03.275 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:01:03.275 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:01:03.275 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:01:03.275 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:01:03.279 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:01:03.279 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:01:03.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:03.310 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:01:03.311 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:01:03.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:03.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:03.343 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:01:03.344 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:01:03.375 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:01:03.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:01:03.406 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:01:03.407 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:01:03.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:03.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:03.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:03.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:03.630 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:03.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:03 smithi017 ceph-mon[102913]: pgmap v1443: 99 pgs: 99 active+clean; 140 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s wr, 222 op/s 2024-09-21T10:01:03.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:03 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2519361821' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:03 smithi017 ceph-mon[105151]: pgmap v1443: 99 pgs: 99 active+clean; 140 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s wr, 222 op/s 2024-09-21T10:01:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:03 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2519361821' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:03.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:03 smithi134 ceph-mon[90085]: pgmap v1443: 99 pgs: 99 active+clean; 140 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s wr, 222 op/s 2024-09-21T10:01:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:03 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2519361821' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.774 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:03.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:03.875 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:03.902 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 11915 11899 185.812 170.812 0.00502698 0.00537456 2024-09-21T10:01:03.906 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:03.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:04.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:04.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:04.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:04.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:04.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:04.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:04.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:04.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:04.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:04.434 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:04.462 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:04.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:04.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:04.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:04.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:04.706 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:04.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:04.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:04.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:04.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:04.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:04.902 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 14727 14711 183.798 175.75 0.00357811 0.00543418 2024-09-21T10:01:04.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:01:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:01:04] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.51.0" 2024-09-21T10:01:04.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:04.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:01:05.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:05.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:05.121 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:05.147 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:05.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:05.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:05.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:05.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:05.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:05.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:05.577 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:05.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:05 smithi017 ceph-mon[102913]: pgmap v1444: 99 pgs: 99 active+clean; 372 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s wr, 593 op/s 2024-09-21T10:01:05.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:05 smithi017 ceph-mon[105151]: pgmap v1444: 99 pgs: 99 active+clean; 372 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s wr, 593 op/s 2024-09-21T10:01:05.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:05.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:05 smithi134 ceph-mon[90085]: pgmap v1444: 99 pgs: 99 active+clean; 372 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s wr, 593 op/s 2024-09-21T10:01:05.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:05.850 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:05.902 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 17128 17112 178.175 150.062 0.00354254 0.00560608 2024-09-21T10:01:05.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:05.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:05.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:06.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:06.127 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:06.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:06.228 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:06.257 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:06.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:06.261 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:06.277 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:06.358 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:06.387 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:06.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:06.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:06.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:06.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:01:06.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:06.663 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:06.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:06.765 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:06.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:06.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:06.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:06.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 19398 19382 172.987 141.875 0.00632925 0.00577309 2024-09-21T10:01:06.943 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:06.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:07.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:07.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.090 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:07.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:07.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:07.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:07.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:07.335 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:07.364 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:07.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:07.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:07.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:07.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:07.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:07.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:07.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:07 smithi134 ceph-mon[90085]: pgmap v1445: 99 pgs: 99 active+clean; 815 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-21T10:01:07.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:07.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:07.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:07.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:07.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 21675 21659 169.14 142.312 0.00744572 0.00590579 2024-09-21T10:01:07.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:07.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:07 smithi017 ceph-mon[102913]: pgmap v1445: 99 pgs: 99 active+clean; 815 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-21T10:01:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:07 smithi017 ceph-mon[105151]: pgmap v1445: 99 pgs: 99 active+clean; 815 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-21T10:01:08.023 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:08.046 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:08.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:08.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:08.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:08.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:08.290 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:08.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.312 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:08.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:08.413 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:08.441 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:08.543 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:08.572 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:08.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:08.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:08.701 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:08.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:08.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:08.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:08.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:08.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 23724 23708 164.576 128.062 0.00255913 0.0060712 2024-09-21T10:01:08.947 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:08.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:08.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:09.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:09.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:09.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:09.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:09.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:09.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:09.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:09.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:09.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:09.516 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:09.540 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:09.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:09.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:09.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:09 smithi017 ceph-mon[102913]: pgmap v1446: 99 pgs: 99 active+clean; 1021 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-21T10:01:09.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:09 smithi017 ceph-mon[105151]: pgmap v1446: 99 pgs: 99 active+clean; 1021 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-21T10:01:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:09 smithi134 ceph-mon[90085]: pgmap v1446: 99 pgs: 99 active+clean; 1021 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-21T10:01:09.786 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:09.813 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:09.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:09.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 25817 25801 161.199 130.812 0.00735061 0.00619721 2024-09-21T10:01:09.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:09.926 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:01:09.929 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:01:09.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:09.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:10.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:10.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:10.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:10.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:10.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:10.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:10.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:10.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:10.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:10.477 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:01:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:01:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:10.504 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:10.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:10.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:10.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:10.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:10.748 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:10.778 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:10.880 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:10.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 27986 27970 158.868 135.562 0.0168271 0.00628835 2024-09-21T10:01:10.908 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:10.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:11.010 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:11.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:11.143 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:11.172 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:11.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:11.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:11.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:11.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:11.425 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:11 smithi017 ceph-mon[102913]: from='client.45392 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:01:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:11 smithi017 ceph-mon[102913]: pgmap v1447: 99 pgs: 99 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:01:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:11 smithi017 ceph-mon[105151]: from='client.45392 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:01:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:11 smithi017 ceph-mon[105151]: pgmap v1447: 99 pgs: 99 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:01:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:11.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:11.453 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:11.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:11.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:11.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.601 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:11.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:11.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:11.728 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:11 smithi134 ceph-mon[90085]: from='client.45392 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:01:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:11 smithi134 ceph-mon[90085]: pgmap v1447: 99 pgs: 99 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-09-21T10:01:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:11.830 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:11.854 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:11.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 29738 29722 154.754 109.5 0.00272771 0.00645605 2024-09-21T10:01:11.956 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:11.984 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:12.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:12.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:12.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:12.250 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:12.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:12.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:12.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:12.477 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:12.503 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:12.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:12.605 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:12.626 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:12.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:12.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:12.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:12.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:12.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:12.904 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 31310 31294 150.408 98.25 0.0093103 0.00663852 2024-09-21T10:01:12.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:13.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:13.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:13.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:13.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:13.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:13.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:13.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:13.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.318 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:13.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:13.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:13.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:13.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:13.562 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:13.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:13.584 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:13 smithi017 ceph-mon[102913]: pgmap v1448: 99 pgs: 99 active+clean; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2024-09-21T10:01:13.584 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:13 smithi017 ceph-mon[105151]: pgmap v1448: 99 pgs: 99 active+clean; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2024-09-21T10:01:13.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:13.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:13.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:13.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:13 smithi134 ceph-mon[90085]: pgmap v1448: 99 pgs: 99 active+clean; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2024-09-21T10:01:13.829 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:13.858 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:13.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:13.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 32949 32933 146.981 102.438 0.00535947 0.00679797 2024-09-21T10:01:13.960 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:13.986 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:14.034 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1446,"stamp":"2024-09-21T10:01:08.100760+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1070959806,"num_objects":16361,"num_object_clones":0,"num_object_copies":49083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16361,"num_whiteouts":0,"num_read":80,"num_read_kb":204,"num_write":16439,"num_write_kb":1045915,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":498355,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":17493,"ondisk_log_size":17493,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5240504,"kb_used_data":3130360,"kb_used_omap":122,"kb_used_meta":2109957,"kb_avail":744720712,"statfs":{"total":767960285184,"available":762594009088,"internally_reserved":0,"allocated":3205488640,"data_stored":3186382159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":125317,"internal_metadata":2160596603},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,4,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1070530560,"num_objects":16335,"num_object_clones":0,"num_object_copies":49005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16335,"num_write_kb":1045440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.605987"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1626,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752383+0000","last_change":"2024-09-21T09:46:47.114206+0000","last_active":"2024-09-21T10:00:57.752383+0000","last_peered":"2024-09-21T10:00:57.752383+0000","last_clean":"2024-09-21T10:00:57.752383+0000","last_became_active":"2024-09-21T09:46:47.113819+0000","last_became_peered":"2024-09-21T09:46:47.113819+0000","last_unstale":"2024-09-21T10:00:57.752383+0000","last_undegraded":"2024-09-21T10:00:57.752383+0000","last_fullsized":"2024-09-21T10:00:57.752383+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:07.629544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1070,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451857+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T10:00:48.451857+0000","last_peered":"2024-09-21T10:00:48.451857+0000","last_clean":"2024-09-21T10:00:48.451857+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T10:00:48.451857+0000","last_undegraded":"2024-09-21T10:00:48.451857+0000","last_fullsized":"2024-09-21T10:00:48.451857+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":578,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451522+0000","last_change":"2024-09-21T09:57:39.541887+0000","last_active":"2024-09-21T10:00:48.451522+0000","last_peered":"2024-09-21T10:00:48.451522+0000","last_clean":"2024-09-21T10:00:48.451522+0000","last_became_active":"2024-09-21T09:57:39.541598+0000","last_became_peered":"2024-09-21T09:57:39.541598+0000","last_unstale":"2024-09-21T10:00:48.451522+0000","last_undegraded":"2024-09-21T10:00:48.451522+0000","last_fullsized":"2024-09-21T10:00:48.451522+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:41.615525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1709,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496274+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T10:00:57.496274+0000","last_peered":"2024-09-21T10:00:57.496274+0000","last_clean":"2024-09-21T10:00:57.496274+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T10:00:57.496274+0000","last_undegraded":"2024-09-21T10:00:57.496274+0000","last_fullsized":"2024-09-21T10:00:57.496274+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1711,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496724+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T10:00:57.496724+0000","last_peered":"2024-09-21T10:00:57.496724+0000","last_clean":"2024-09-21T10:00:57.496724+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T10:00:57.496724+0000","last_undegraded":"2024-09-21T10:00:57.496724+0000","last_fullsized":"2024-09-21T10:00:57.496724+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1071,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450319+0000","last_change":"2024-09-21T10:00:16.358888+0000","last_active":"2024-09-21T10:00:48.450319+0000","last_peered":"2024-09-21T10:00:48.450319+0000","last_clean":"2024-09-21T10:00:48.450319+0000","last_became_active":"2024-09-21T10:00:16.357219+0000","last_became_peered":"2024-09-21T10:00:16.357219+0000","last_unstale":"2024-09-21T10:00:48.450319+0000","last_undegraded":"2024-09-21T10:00:48.450319+0000","last_fullsized":"2024-09-21T10:00:48.450319+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:13.887798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1625,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752393+0000","last_change":"2024-09-21T09:46:47.117580+0000","last_active":"2024-09-21T10:00:57.752393+0000","last_peered":"2024-09-21T10:00:57.752393+0000","last_clean":"2024-09-21T10:00:57.752393+0000","last_became_active":"2024-09-21T09:46:47.113030+0000","last_became_peered":"2024-09-21T09:46:47.113030+0000","last_unstale":"2024-09-21T10:00:57.752393+0000","last_undegraded":"2024-09-21T10:00:57.752393+0000","last_fullsized":"2024-09-21T10:00:57.752393+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:28.220180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":34,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:47.468745+0000","last_change":"2024-09-21T10:00:16.377280+0000","last_active":"2024-09-21T10:00:47.468745+0000","last_peered":"2024-09-21T10:00:47.468745+0000","last_clean":"2024-09-21T10:00:47.468745+0000","last_became_active":"2024-09-21T10:00:16.353074+0000","last_became_peered":"2024-09-21T10:00:16.353074+0000","last_unstale":"2024-09-21T10:00:47.468745+0000","last_undegraded":"2024-09-21T10:00:47.468745+0000","last_fullsized":"2024-09-21T10:00:47.468745+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:35.382228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1145'1320","reported_seq":1337,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:07.496285+0000","last_change":"2024-09-21T10:00:46.472975+0000","last_active":"2024-09-21T10:01:07.496285+0000","last_peered":"2024-09-21T10:01:07.496285+0000","last_clean":"2024-09-21T10:01:07.496285+0000","last_became_active":"2024-09-21T10:00:46.472669+0000","last_became_peered":"2024-09-21T10:00:46.472669+0000","last_unstale":"2024-09-21T10:01:07.496285+0000","last_undegraded":"2024-09-21T10:01:07.496285+0000","last_fullsized":"2024-09-21T10:01:07.496285+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1320,"log_dups_size":0,"ondisk_log_size":1320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:37.256403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86507520,"num_objects":1320,"num_object_clones":0,"num_object_copies":3960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1320,"num_write_kb":84480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1069,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451573+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T10:00:48.451573+0000","last_peered":"2024-09-21T10:00:48.451573+0000","last_clean":"2024-09-21T10:00:48.451573+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T10:00:48.451573+0000","last_undegraded":"2024-09-21T10:00:48.451573+0000","last_fullsized":"2024-09-21T10:00:48.451573+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450760+0000","last_change":"2024-09-21T10:00:16.357541+0000","last_active":"2024-09-21T10:00:48.450760+0000","last_peered":"2024-09-21T10:00:48.450760+0000","last_clean":"2024-09-21T10:00:48.450760+0000","last_became_active":"2024-09-21T10:00:16.357209+0000","last_became_peered":"2024-09-21T10:00:16.357209+0000","last_unstale":"2024-09-21T10:00:48.450760+0000","last_undegraded":"2024-09-21T10:00:48.450760+0000","last_fullsized":"2024-09-21T10:00:48.450760+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:20.732470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"1145'1076","reported_seq":1093,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:05.934949+0000","last_change":"2024-09-21T10:00:46.477601+0000","last_active":"2024-09-21T10:01:05.934949+0000","last_peered":"2024-09-21T10:01:05.934949+0000","last_clean":"2024-09-21T10:01:05.934949+0000","last_became_active":"2024-09-21T10:00:46.477208+0000","last_became_peered":"2024-09-21T10:00:46.477208+0000","last_unstale":"2024-09-21T10:01:05.934949+0000","last_undegraded":"2024-09-21T10:01:05.934949+0000","last_fullsized":"2024-09-21T10:01:05.934949+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1076,"log_dups_size":0,"ondisk_log_size":1076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:45:36.792046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70516736,"num_objects":1076,"num_object_clones":0,"num_object_copies":3228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1076,"num_write_kb":68864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1070,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451898+0000","last_change":"2024-09-21T09:45:39.143190+0000","last_active":"2024-09-21T10:00:48.451898+0000","last_peered":"2024-09-21T10:00:48.451898+0000","last_clean":"2024-09-21T10:00:48.451898+0000","last_became_active":"2024-09-21T09:45:39.142826+0000","last_became_peered":"2024-09-21T09:45:39.142826+0000","last_unstale":"2024-09-21T10:00:48.451898+0000","last_undegraded":"2024-09-21T10:00:48.451898+0000","last_fullsized":"2024-09-21T10:00:48.451898+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:45.070945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1103,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450920+0000","last_change":"2024-09-21T09:46:47.119247+0000","last_active":"2024-09-21T10:00:48.450920+0000","last_peered":"2024-09-21T10:00:48.450920+0000","last_clean":"2024-09-21T10:00:48.450920+0000","last_became_active":"2024-09-21T09:46:47.118980+0000","last_became_peered":"2024-09-21T09:46:47.118980+0000","last_unstale":"2024-09-21T10:00:48.450920+0000","last_undegraded":"2024-09-21T10:00:48.450920+0000","last_fullsized":"2024-09-21T10:00:48.450920+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:55:18.140253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1145'627","reported_seq":644,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:03.519935+0000","last_change":"2024-09-21T10:00:46.482429+0000","last_active":"2024-09-21T10:01:03.519935+0000","last_peered":"2024-09-21T10:01:03.519935+0000","last_clean":"2024-09-21T10:01:03.519935+0000","last_became_active":"2024-09-21T10:00:46.482135+0000","last_became_peered":"2024-09-21T10:00:46.482135+0000","last_unstale":"2024-09-21T10:01:03.519935+0000","last_undegraded":"2024-09-21T10:01:03.519935+0000","last_fullsized":"2024-09-21T10:01:03.519935+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":627,"log_dups_size":0,"ondisk_log_size":627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:10.213360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41091072,"num_objects":627,"num_object_clones":0,"num_object_copies":1881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":627,"num_write_kb":40128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":1080,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450748+0000","last_change":"2024-09-21T10:00:16.373327+0000","last_active":"2024-09-21T10:00:48.450748+0000","last_peered":"2024-09-21T10:00:48.450748+0000","last_clean":"2024-09-21T10:00:48.450748+0000","last_became_active":"2024-09-21T10:00:16.360853+0000","last_became_peered":"2024-09-21T10:00:16.360853+0000","last_unstale":"2024-09-21T10:00:48.450748+0000","last_undegraded":"2024-09-21T10:00:48.450748+0000","last_fullsized":"2024-09-21T10:00:48.450748+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:18.160233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":24,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:47.468947+0000","last_change":"2024-09-21T10:00:16.356120+0000","last_active":"2024-09-21T10:00:47.468947+0000","last_peered":"2024-09-21T10:00:47.468947+0000","last_clean":"2024-09-21T10:00:47.468947+0000","last_became_active":"2024-09-21T10:00:16.354837+0000","last_became_peered":"2024-09-21T10:00:16.354837+0000","last_unstale":"2024-09-21T10:00:47.468947+0000","last_undegraded":"2024-09-21T10:00:47.468947+0000","last_fullsized":"2024-09-21T10:00:47.468947+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:41:42.635967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1145'1216","reported_seq":1233,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:06.703385+0000","last_change":"2024-09-21T10:00:46.477546+0000","last_active":"2024-09-21T10:01:06.703385+0000","last_peered":"2024-09-21T10:01:06.703385+0000","last_clean":"2024-09-21T10:01:06.703385+0000","last_became_active":"2024-09-21T10:00:46.477219+0000","last_became_peered":"2024-09-21T10:00:46.477219+0000","last_unstale":"2024-09-21T10:01:06.703385+0000","last_undegraded":"2024-09-21T10:01:06.703385+0000","last_fullsized":"2024-09-21T10:01:06.703385+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1216,"log_dups_size":0,"ondisk_log_size":1216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:53:28.312113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79691776,"num_objects":1216,"num_object_clones":0,"num_object_copies":3648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1216,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":639,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752529+0000","last_change":"2024-09-21T10:00:16.360708+0000","last_active":"2024-09-21T10:00:57.752529+0000","last_peered":"2024-09-21T10:00:57.752529+0000","last_clean":"2024-09-21T10:00:57.752529+0000","last_became_active":"2024-09-21T10:00:16.360416+0000","last_became_peered":"2024-09-21T10:00:16.360416+0000","last_unstale":"2024-09-21T10:00:57.752529+0000","last_undegraded":"2024-09-21T10:00:57.752529+0000","last_fullsized":"2024-09-21T10:00:57.752529+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:59.090215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1712,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496038+0000","last_change":"2024-09-21T09:45:39.138289+0000","last_active":"2024-09-21T10:00:57.496038+0000","last_peered":"2024-09-21T10:00:57.496038+0000","last_clean":"2024-09-21T10:00:57.496038+0000","last_became_active":"2024-09-21T09:45:39.137902+0000","last_became_peered":"2024-09-21T09:45:39.137902+0000","last_unstale":"2024-09-21T10:00:57.496038+0000","last_undegraded":"2024-09-21T10:00:57.496038+0000","last_fullsized":"2024-09-21T10:00:57.496038+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:40:16.383906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1145'670","reported_seq":686,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:03.520533+0000","last_change":"2024-09-21T10:00:46.479962+0000","last_active":"2024-09-21T10:01:03.520533+0000","last_peered":"2024-09-21T10:01:03.520533+0000","last_clean":"2024-09-21T10:01:03.520533+0000","last_became_active":"2024-09-21T10:00:46.479648+0000","last_became_peered":"2024-09-21T10:00:46.479648+0000","last_unstale":"2024-09-21T10:01:03.520533+0000","last_undegraded":"2024-09-21T10:01:03.520533+0000","last_fullsized":"2024-09-21T10:01:03.520533+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":670,"log_dups_size":0,"ondisk_log_size":670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:16.350920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43909120,"num_objects":670,"num_object_clones":0,"num_object_copies":2010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":670,"num_write_kb":42880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1215,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451499+0000","last_change":"2024-09-21T10:00:47.119852+0000","last_active":"2024-09-21T10:00:48.451499+0000","last_peered":"2024-09-21T10:00:48.451499+0000","last_clean":"2024-09-21T10:00:48.451499+0000","last_became_active":"2024-09-21T09:59:43.374051+0000","last_became_peered":"2024-09-21T09:59:43.374051+0000","last_unstale":"2024-09-21T10:00:48.451499+0000","last_undegraded":"2024-09-21T10:00:48.451499+0000","last_fullsized":"2024-09-21T10:00:48.451499+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:47.119793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:47.119793+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:23.087965+0000","scrub_duration":0.0013414659999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1636,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752142+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T10:00:57.752142+0000","last_peered":"2024-09-21T10:00:57.752142+0000","last_clean":"2024-09-21T10:00:57.752142+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T10:00:57.752142+0000","last_undegraded":"2024-09-21T10:00:57.752142+0000","last_fullsized":"2024-09-21T10:00:57.752142+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1106,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450562+0000","last_change":"2024-09-21T10:00:16.358954+0000","last_active":"2024-09-21T10:00:48.450562+0000","last_peered":"2024-09-21T10:00:48.450562+0000","last_clean":"2024-09-21T10:00:48.450562+0000","last_became_active":"2024-09-21T10:00:16.358416+0000","last_became_peered":"2024-09-21T10:00:16.358416+0000","last_unstale":"2024-09-21T10:00:48.450562+0000","last_undegraded":"2024-09-21T10:00:48.450562+0000","last_fullsized":"2024-09-21T10:00:48.450562+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:46.916076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1145'1239","reported_seq":1256,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:06.698683+0000","last_change":"2024-09-21T10:00:46.481001+0000","last_active":"2024-09-21T10:01:06.698683+0000","last_peered":"2024-09-21T10:01:06.698683+0000","last_clean":"2024-09-21T10:01:06.698683+0000","last_became_active":"2024-09-21T10:00:46.480689+0000","last_became_peered":"2024-09-21T10:00:46.480689+0000","last_unstale":"2024-09-21T10:01:06.698683+0000","last_undegraded":"2024-09-21T10:01:06.698683+0000","last_fullsized":"2024-09-21T10:01:06.698683+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1239,"log_dups_size":0,"ondisk_log_size":1239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:39.782292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81199104,"num_objects":1239,"num_object_clones":0,"num_object_copies":3717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1239,"num_write_kb":79296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":78,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.492987+0000","last_change":"2024-09-21T10:00:15.524812+0000","last_active":"2024-09-21T10:00:57.492987+0000","last_peered":"2024-09-21T10:00:57.492987+0000","last_clean":"2024-09-21T10:00:57.492987+0000","last_became_active":"2024-09-21T09:59:43.361751+0000","last_became_peered":"2024-09-21T09:59:43.361751+0000","last_unstale":"2024-09-21T10:00:57.492987+0000","last_undegraded":"2024-09-21T10:00:57.492987+0000","last_fullsized":"2024-09-21T10:00:57.492987+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.306777+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":24,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:47.468792+0000","last_change":"2024-09-21T10:00:16.358625+0000","last_active":"2024-09-21T10:00:47.468792+0000","last_peered":"2024-09-21T10:00:47.468792+0000","last_clean":"2024-09-21T10:00:47.468792+0000","last_became_active":"2024-09-21T10:00:16.355466+0000","last_became_peered":"2024-09-21T10:00:16.355466+0000","last_unstale":"2024-09-21T10:00:47.468792+0000","last_undegraded":"2024-09-21T10:00:47.468792+0000","last_fullsized":"2024-09-21T10:00:47.468792+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:29.368490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1103,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451294+0000","last_change":"2024-09-21T09:46:13.259814+0000","last_active":"2024-09-21T10:00:48.451294+0000","last_peered":"2024-09-21T10:00:48.451294+0000","last_clean":"2024-09-21T10:00:48.451294+0000","last_became_active":"2024-09-21T09:46:13.259527+0000","last_became_peered":"2024-09-21T09:46:13.259527+0000","last_unstale":"2024-09-21T10:00:48.451294+0000","last_undegraded":"2024-09-21T10:00:48.451294+0000","last_fullsized":"2024-09-21T10:00:48.451294+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:56:00.943552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1145'1050","reported_seq":1061,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:05.923051+0000","last_change":"2024-09-21T10:00:46.479388+0000","last_active":"2024-09-21T10:01:05.923051+0000","last_peered":"2024-09-21T10:01:05.923051+0000","last_clean":"2024-09-21T10:01:05.923051+0000","last_became_active":"2024-09-21T10:00:46.479095+0000","last_became_peered":"2024-09-21T10:00:46.479095+0000","last_unstale":"2024-09-21T10:01:05.923051+0000","last_undegraded":"2024-09-21T10:01:05.923051+0000","last_fullsized":"2024-09-21T10:01:05.923051+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1050,"log_dups_size":0,"ondisk_log_size":1050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:35.393978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68812800,"num_objects":1050,"num_object_clones":0,"num_object_copies":3150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1050,"num_write_kb":67200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1087,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450994+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T10:00:48.450994+0000","last_peered":"2024-09-21T10:00:48.450994+0000","last_clean":"2024-09-21T10:00:48.450994+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T10:00:48.450994+0000","last_undegraded":"2024-09-21T10:00:48.450994+0000","last_fullsized":"2024-09-21T10:00:48.450994+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1106,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451166+0000","last_change":"2024-09-21T10:00:16.356380+0000","last_active":"2024-09-21T10:00:48.451166+0000","last_peered":"2024-09-21T10:00:48.451166+0000","last_clean":"2024-09-21T10:00:48.451166+0000","last_became_active":"2024-09-21T10:00:16.355807+0000","last_became_peered":"2024-09-21T10:00:16.355807+0000","last_unstale":"2024-09-21T10:00:48.451166+0000","last_undegraded":"2024-09-21T10:00:48.451166+0000","last_fullsized":"2024-09-21T10:00:48.451166+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:13:46.080132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1145'1112","reported_seq":1123,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:05.911780+0000","last_change":"2024-09-21T10:00:46.477205+0000","last_active":"2024-09-21T10:01:05.911780+0000","last_peered":"2024-09-21T10:01:05.911780+0000","last_clean":"2024-09-21T10:01:05.911780+0000","last_became_active":"2024-09-21T10:00:46.476844+0000","last_became_peered":"2024-09-21T10:00:46.476844+0000","last_unstale":"2024-09-21T10:01:05.911780+0000","last_undegraded":"2024-09-21T10:01:05.911780+0000","last_fullsized":"2024-09-21T10:01:05.911780+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1112,"log_dups_size":0,"ondisk_log_size":1112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:29:19.716166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72876032,"num_objects":1112,"num_object_clones":0,"num_object_copies":3336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1112,"num_write_kb":71168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":67,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.493110+0000","last_change":"2024-09-21T10:00:16.351253+0000","last_active":"2024-09-21T10:00:57.493110+0000","last_peered":"2024-09-21T10:00:57.493110+0000","last_clean":"2024-09-21T10:00:57.493110+0000","last_became_active":"2024-09-21T10:00:16.351010+0000","last_became_peered":"2024-09-21T10:00:16.351010+0000","last_unstale":"2024-09-21T10:00:57.493110+0000","last_undegraded":"2024-09-21T10:00:57.493110+0000","last_fullsized":"2024-09-21T10:00:57.493110+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:01:25.606839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1627,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751612+0000","last_change":"2024-09-21T09:46:47.121643+0000","last_active":"2024-09-21T10:00:57.751612+0000","last_peered":"2024-09-21T10:00:57.751612+0000","last_clean":"2024-09-21T10:00:57.751612+0000","last_became_active":"2024-09-21T09:46:47.120528+0000","last_became_peered":"2024-09-21T09:46:47.120528+0000","last_unstale":"2024-09-21T10:00:57.751612+0000","last_undegraded":"2024-09-21T10:00:57.751612+0000","last_fullsized":"2024-09-21T10:00:57.751612+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:07.989377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1672,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752150+0000","last_change":"2024-09-21T09:46:47.121159+0000","last_active":"2024-09-21T10:00:57.752150+0000","last_peered":"2024-09-21T10:00:57.752150+0000","last_clean":"2024-09-21T10:00:57.752150+0000","last_became_active":"2024-09-21T09:46:47.120829+0000","last_became_peered":"2024-09-21T09:46:47.120829+0000","last_unstale":"2024-09-21T10:00:57.752150+0000","last_undegraded":"2024-09-21T10:00:57.752150+0000","last_fullsized":"2024-09-21T10:00:57.752150+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:43.393840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1145'662","reported_seq":679,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:03.495676+0000","last_change":"2024-09-21T10:00:46.477718+0000","last_active":"2024-09-21T10:01:03.495676+0000","last_peered":"2024-09-21T10:01:03.495676+0000","last_clean":"2024-09-21T10:01:03.495676+0000","last_became_active":"2024-09-21T10:00:46.477468+0000","last_became_peered":"2024-09-21T10:00:46.477468+0000","last_unstale":"2024-09-21T10:01:03.495676+0000","last_undegraded":"2024-09-21T10:01:03.495676+0000","last_fullsized":"2024-09-21T10:01:03.495676+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":662,"log_dups_size":0,"ondisk_log_size":662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:38:48.242889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43384832,"num_objects":662,"num_object_clones":0,"num_object_copies":1986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":662,"num_write_kb":42368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1229,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:49.116229+0000","last_change":"2024-09-21T10:00:49.116174+0000","last_active":"2024-09-21T10:00:49.116229+0000","last_peered":"2024-09-21T10:00:49.116229+0000","last_clean":"2024-09-21T10:00:49.116229+0000","last_became_active":"2024-09-21T10:00:16.360430+0000","last_became_peered":"2024-09-21T10:00:16.360430+0000","last_unstale":"2024-09-21T10:00:49.116229+0000","last_undegraded":"2024-09-21T10:00:49.116229+0000","last_fullsized":"2024-09-21T10:00:49.116229+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:49.116103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:49.116103+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:29.874911+0000","scrub_duration":0.0020386419999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":112,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.493313+0000","last_change":"2024-09-21T09:57:39.541461+0000","last_active":"2024-09-21T10:00:57.493313+0000","last_peered":"2024-09-21T10:00:57.493313+0000","last_clean":"2024-09-21T10:00:57.493313+0000","last_became_active":"2024-09-21T09:57:39.540927+0000","last_became_peered":"2024-09-21T09:57:39.540927+0000","last_unstale":"2024-09-21T10:00:57.493313+0000","last_undegraded":"2024-09-21T10:00:57.493313+0000","last_fullsized":"2024-09-21T10:00:57.493313+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:14.815783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1623,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752563+0000","last_change":"2024-09-21T10:00:16.362131+0000","last_active":"2024-09-21T10:00:57.752563+0000","last_peered":"2024-09-21T10:00:57.752563+0000","last_clean":"2024-09-21T10:00:57.752563+0000","last_became_active":"2024-09-21T10:00:16.361722+0000","last_became_peered":"2024-09-21T10:00:16.361722+0000","last_unstale":"2024-09-21T10:00:57.752563+0000","last_undegraded":"2024-09-21T10:00:57.752563+0000","last_fullsized":"2024-09-21T10:00:57.752563+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:38.717426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":24,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:47.468991+0000","last_change":"2024-09-21T10:00:16.361540+0000","last_active":"2024-09-21T10:00:47.468991+0000","last_peered":"2024-09-21T10:00:47.468991+0000","last_clean":"2024-09-21T10:00:47.468991+0000","last_became_active":"2024-09-21T10:00:16.361086+0000","last_became_peered":"2024-09-21T10:00:16.361086+0000","last_unstale":"2024-09-21T10:00:47.468991+0000","last_undegraded":"2024-09-21T10:00:47.468991+0000","last_fullsized":"2024-09-21T10:00:47.468991+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:14.645896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1145'1084","reported_seq":1095,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:05.977266+0000","last_change":"2024-09-21T10:00:46.479931+0000","last_active":"2024-09-21T10:01:05.977266+0000","last_peered":"2024-09-21T10:01:05.977266+0000","last_clean":"2024-09-21T10:01:05.977266+0000","last_became_active":"2024-09-21T10:00:46.479653+0000","last_became_peered":"2024-09-21T10:00:46.479653+0000","last_unstale":"2024-09-21T10:01:05.977266+0000","last_undegraded":"2024-09-21T10:01:05.977266+0000","last_fullsized":"2024-09-21T10:01:05.977266+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1084,"log_dups_size":0,"ondisk_log_size":1084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.219791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71041024,"num_objects":1084,"num_object_clones":0,"num_object_copies":3252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1084,"num_write_kb":69376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":74,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751592+0000","last_change":"2024-09-21T10:00:21.317893+0000","last_active":"2024-09-21T10:00:57.751592+0000","last_peered":"2024-09-21T10:00:57.751592+0000","last_clean":"2024-09-21T10:00:57.751592+0000","last_became_active":"2024-09-21T10:00:16.353337+0000","last_became_peered":"2024-09-21T10:00:16.353337+0000","last_unstale":"2024-09-21T10:00:57.751592+0000","last_undegraded":"2024-09-21T10:00:57.751592+0000","last_fullsized":"2024-09-21T10:00:57.751592+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:21.317838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:21.317838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:32.244155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1097,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451081+0000","last_change":"2024-09-21T09:45:39.129555+0000","last_active":"2024-09-21T10:00:48.451081+0000","last_peered":"2024-09-21T10:00:48.451081+0000","last_clean":"2024-09-21T10:00:48.451081+0000","last_became_active":"2024-09-21T09:45:39.129205+0000","last_became_peered":"2024-09-21T09:45:39.129205+0000","last_unstale":"2024-09-21T10:00:48.451081+0000","last_undegraded":"2024-09-21T10:00:48.451081+0000","last_fullsized":"2024-09-21T10:00:48.451081+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:07.229126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1710,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496197+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T10:00:57.496197+0000","last_peered":"2024-09-21T10:00:57.496197+0000","last_clean":"2024-09-21T10:00:57.496197+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T10:00:57.496197+0000","last_undegraded":"2024-09-21T10:00:57.496197+0000","last_fullsized":"2024-09-21T10:00:57.496197+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1723,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496521+0000","last_change":"2024-09-21T09:46:13.273573+0000","last_active":"2024-09-21T10:00:57.496521+0000","last_peered":"2024-09-21T10:00:57.496521+0000","last_clean":"2024-09-21T10:00:57.496521+0000","last_became_active":"2024-09-21T09:46:13.273297+0000","last_became_peered":"2024-09-21T09:46:13.273297+0000","last_unstale":"2024-09-21T10:00:57.496521+0000","last_undegraded":"2024-09-21T10:00:57.496521+0000","last_fullsized":"2024-09-21T10:00:57.496521+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:38.957745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"1145'703","reported_seq":720,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:03.513982+0000","last_change":"2024-09-21T10:00:46.482391+0000","last_active":"2024-09-21T10:01:03.513982+0000","last_peered":"2024-09-21T10:01:03.513982+0000","last_clean":"2024-09-21T10:01:03.513982+0000","last_became_active":"2024-09-21T10:00:46.482101+0000","last_became_peered":"2024-09-21T10:00:46.482101+0000","last_unstale":"2024-09-21T10:01:03.513982+0000","last_undegraded":"2024-09-21T10:01:03.513982+0000","last_fullsized":"2024-09-21T10:01:03.513982+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":703,"log_dups_size":0,"ondisk_log_size":703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:23.794281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46071808,"num_objects":703,"num_object_clones":0,"num_object_copies":2109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":703,"num_write_kb":44992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":1215,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451011+0000","last_change":"2024-09-21T10:00:16.913196+0000","last_active":"2024-09-21T10:00:48.451011+0000","last_peered":"2024-09-21T10:00:48.451011+0000","last_clean":"2024-09-21T10:00:48.451011+0000","last_became_active":"2024-09-21T09:59:43.370454+0000","last_became_peered":"2024-09-21T09:59:43.370454+0000","last_unstale":"2024-09-21T10:00:48.451011+0000","last_undegraded":"2024-09-21T10:00:48.451011+0000","last_fullsized":"2024-09-21T10:00:48.451011+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:16.913146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:16.913146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:23:33.788563+0000","scrub_duration":0.001342315,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1685,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.493074+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:00:57.493074+0000","last_peered":"2024-09-21T10:00:57.493074+0000","last_clean":"2024-09-21T10:00:57.493074+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:00:57.493074+0000","last_undegraded":"2024-09-21T10:00:57.493074+0000","last_fullsized":"2024-09-21T10:00:57.493074+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1688,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751560+0000","last_change":"2024-09-21T10:00:16.348231+0000","last_active":"2024-09-21T10:00:57.751560+0000","last_peered":"2024-09-21T10:00:57.751560+0000","last_clean":"2024-09-21T10:00:57.751560+0000","last_became_active":"2024-09-21T10:00:16.347884+0000","last_became_peered":"2024-09-21T10:00:16.347884+0000","last_unstale":"2024-09-21T10:00:57.751560+0000","last_undegraded":"2024-09-21T10:00:57.751560+0000","last_fullsized":"2024-09-21T10:00:57.751560+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:55:37.886835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1685,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752005+0000","last_change":"2024-09-21T09:45:39.134465+0000","last_active":"2024-09-21T10:00:57.752005+0000","last_peered":"2024-09-21T10:00:57.752005+0000","last_clean":"2024-09-21T10:00:57.752005+0000","last_became_active":"2024-09-21T09:45:39.129692+0000","last_became_peered":"2024-09-21T09:45:39.129692+0000","last_unstale":"2024-09-21T10:00:57.752005+0000","last_undegraded":"2024-09-21T10:00:57.752005+0000","last_fullsized":"2024-09-21T10:00:57.752005+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:14.288180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1145'1083","reported_seq":1094,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:05.922099+0000","last_change":"2024-09-21T10:00:46.481072+0000","last_active":"2024-09-21T10:01:05.922099+0000","last_peered":"2024-09-21T10:01:05.922099+0000","last_clean":"2024-09-21T10:01:05.922099+0000","last_became_active":"2024-09-21T10:00:46.480845+0000","last_became_peered":"2024-09-21T10:00:46.480845+0000","last_unstale":"2024-09-21T10:01:05.922099+0000","last_undegraded":"2024-09-21T10:01:05.922099+0000","last_fullsized":"2024-09-21T10:01:05.922099+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1083,"log_dups_size":0,"ondisk_log_size":1083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:36.616907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70975488,"num_objects":1083,"num_object_clones":0,"num_object_copies":3249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1083,"num_write_kb":69312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":406,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496054+0000","last_change":"2024-09-21T09:59:43.371893+0000","last_active":"2024-09-21T10:00:57.496054+0000","last_peered":"2024-09-21T10:00:57.496054+0000","last_clean":"2024-09-21T10:00:57.496054+0000","last_became_active":"2024-09-21T09:59:43.371384+0000","last_became_peered":"2024-09-21T09:59:43.371384+0000","last_unstale":"2024-09-21T10:00:57.496054+0000","last_undegraded":"2024-09-21T10:00:57.496054+0000","last_fullsized":"2024-09-21T10:00:57.496054+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:01:23.358993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":139,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751384+0000","last_change":"2024-09-21T10:00:16.379375+0000","last_active":"2024-09-21T10:00:57.751384+0000","last_peered":"2024-09-21T10:00:57.751384+0000","last_clean":"2024-09-21T10:00:57.751384+0000","last_became_active":"2024-09-21T10:00:16.361346+0000","last_became_peered":"2024-09-21T10:00:16.361346+0000","last_unstale":"2024-09-21T10:00:57.751384+0000","last_undegraded":"2024-09-21T10:00:57.751384+0000","last_fullsized":"2024-09-21T10:00:57.751384+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:57.681531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1107,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450310+0000","last_change":"2024-09-21T10:00:16.356367+0000","last_active":"2024-09-21T10:00:48.450310+0000","last_peered":"2024-09-21T10:00:48.450310+0000","last_clean":"2024-09-21T10:00:48.450310+0000","last_became_active":"2024-09-21T10:00:16.356079+0000","last_became_peered":"2024-09-21T10:00:16.356079+0000","last_unstale":"2024-09-21T10:00:48.450310+0000","last_undegraded":"2024-09-21T10:00:48.450310+0000","last_fullsized":"2024-09-21T10:00:48.450310+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:12.934640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1684,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751764+0000","last_change":"2024-09-21T09:46:47.113687+0000","last_active":"2024-09-21T10:00:57.751764+0000","last_peered":"2024-09-21T10:00:57.751764+0000","last_clean":"2024-09-21T10:00:57.751764+0000","last_became_active":"2024-09-21T09:46:47.113235+0000","last_became_peered":"2024-09-21T09:46:47.113235+0000","last_unstale":"2024-09-21T10:00:57.751764+0000","last_undegraded":"2024-09-21T10:00:57.751764+0000","last_fullsized":"2024-09-21T10:00:57.751764+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:14:50.851616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1145'1127","reported_seq":1138,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:05.972187+0000","last_change":"2024-09-21T10:00:46.477490+0000","last_active":"2024-09-21T10:01:05.972187+0000","last_peered":"2024-09-21T10:01:05.972187+0000","last_clean":"2024-09-21T10:01:05.972187+0000","last_became_active":"2024-09-21T10:00:46.477214+0000","last_became_peered":"2024-09-21T10:00:46.477214+0000","last_unstale":"2024-09-21T10:01:05.972187+0000","last_undegraded":"2024-09-21T10:01:05.972187+0000","last_fullsized":"2024-09-21T10:01:05.972187+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1127,"log_dups_size":0,"ondisk_log_size":1127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:26:41.632086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73859072,"num_objects":1127,"num_object_clones":0,"num_object_copies":3381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1127,"num_write_kb":72128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":34,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:47.469224+0000","last_change":"2024-09-21T10:00:46.494180+0000","last_active":"2024-09-21T10:00:47.469224+0000","last_peered":"2024-09-21T10:00:47.469224+0000","last_clean":"2024-09-21T10:00:47.469224+0000","last_became_active":"2024-09-21T10:00:16.355386+0000","last_became_peered":"2024-09-21T10:00:16.355386+0000","last_unstale":"2024-09-21T10:00:47.469224+0000","last_undegraded":"2024-09-21T10:00:47.469224+0000","last_fullsized":"2024-09-21T10:00:47.469224+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:46.494129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:46.494129+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:11.193593+0000","scrub_duration":0.00183863,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":641,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752375+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T10:00:57.752375+0000","last_peered":"2024-09-21T10:00:57.752375+0000","last_clean":"2024-09-21T10:00:57.752375+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T10:00:57.752375+0000","last_undegraded":"2024-09-21T10:00:57.752375+0000","last_fullsized":"2024-09-21T10:00:57.752375+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1698,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751815+0000","last_change":"2024-09-21T09:45:39.132441+0000","last_active":"2024-09-21T10:00:57.751815+0000","last_peered":"2024-09-21T10:00:57.751815+0000","last_clean":"2024-09-21T10:00:57.751815+0000","last_became_active":"2024-09-21T09:45:39.129666+0000","last_became_peered":"2024-09-21T09:45:39.129666+0000","last_unstale":"2024-09-21T10:00:57.751815+0000","last_undegraded":"2024-09-21T10:00:57.751815+0000","last_fullsized":"2024-09-21T10:00:57.751815+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:34.450160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1638,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.493233+0000","last_change":"2024-09-21T09:46:13.258244+0000","last_active":"2024-09-21T10:00:57.493233+0000","last_peered":"2024-09-21T10:00:57.493233+0000","last_clean":"2024-09-21T10:00:57.493233+0000","last_became_active":"2024-09-21T09:46:13.257820+0000","last_became_peered":"2024-09-21T09:46:13.257820+0000","last_unstale":"2024-09-21T10:00:57.493233+0000","last_undegraded":"2024-09-21T10:00:57.493233+0000","last_fullsized":"2024-09-21T10:00:57.493233+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:04.618304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.0","version":"1145'1277","reported_seq":1294,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:07.506768+0000","last_change":"2024-09-21T10:00:46.477631+0000","last_active":"2024-09-21T10:01:07.506768+0000","last_peered":"2024-09-21T10:01:07.506768+0000","last_clean":"2024-09-21T10:01:07.506768+0000","last_became_active":"2024-09-21T10:00:46.477283+0000","last_became_peered":"2024-09-21T10:00:46.477283+0000","last_unstale":"2024-09-21T10:01:07.506768+0000","last_undegraded":"2024-09-21T10:01:07.506768+0000","last_fullsized":"2024-09-21T10:01:07.506768+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1277,"log_dups_size":0,"ondisk_log_size":1277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:51:17.721334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83689472,"num_objects":1277,"num_object_clones":0,"num_object_copies":3831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1277,"num_write_kb":81728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":34,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:47.469135+0000","last_change":"2024-09-21T10:00:20.637858+0000","last_active":"2024-09-21T10:00:47.469135+0000","last_peered":"2024-09-21T10:00:47.469135+0000","last_clean":"2024-09-21T10:00:47.469135+0000","last_became_active":"2024-09-21T10:00:16.353796+0000","last_became_peered":"2024-09-21T10:00:16.353796+0000","last_unstale":"2024-09-21T10:00:47.469135+0000","last_undegraded":"2024-09-21T10:00:47.469135+0000","last_fullsized":"2024-09-21T10:00:47.469135+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:20.637806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:20.637806+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:12:16.884495+0000","scrub_duration":0.0016753919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1674,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751996+0000","last_change":"2024-09-21T09:46:13.260666+0000","last_active":"2024-09-21T10:00:57.751996+0000","last_peered":"2024-09-21T10:00:57.751996+0000","last_clean":"2024-09-21T10:00:57.751996+0000","last_became_active":"2024-09-21T09:46:13.260324+0000","last_became_peered":"2024-09-21T09:46:13.260324+0000","last_unstale":"2024-09-21T10:00:57.751996+0000","last_undegraded":"2024-09-21T10:00:57.751996+0000","last_fullsized":"2024-09-21T10:00:57.751996+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:25.140797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1080,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451910+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T10:00:48.451910+0000","last_peered":"2024-09-21T10:00:48.451910+0000","last_clean":"2024-09-21T10:00:48.451910+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T10:00:48.451910+0000","last_undegraded":"2024-09-21T10:00:48.451910+0000","last_fullsized":"2024-09-21T10:00:48.451910+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1701,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496078+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T10:00:57.496078+0000","last_peered":"2024-09-21T10:00:57.496078+0000","last_clean":"2024-09-21T10:00:57.496078+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T10:00:57.496078+0000","last_undegraded":"2024-09-21T10:00:57.496078+0000","last_fullsized":"2024-09-21T10:00:57.496078+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1145'994","reported_seq":1011,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:05.929723+0000","last_change":"2024-09-21T10:00:46.478721+0000","last_active":"2024-09-21T10:01:05.929723+0000","last_peered":"2024-09-21T10:01:05.929723+0000","last_clean":"2024-09-21T10:01:05.929723+0000","last_became_active":"2024-09-21T10:00:46.478367+0000","last_became_peered":"2024-09-21T10:00:46.478367+0000","last_unstale":"2024-09-21T10:01:05.929723+0000","last_undegraded":"2024-09-21T10:01:05.929723+0000","last_fullsized":"2024-09-21T10:01:05.929723+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":994,"log_dups_size":0,"ondisk_log_size":994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:53.365995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65142784,"num_objects":994,"num_object_clones":0,"num_object_copies":2982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":994,"num_write_kb":63616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":54,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450498+0000","last_change":"2024-09-21T10:00:16.356251+0000","last_active":"2024-09-21T10:00:48.450498+0000","last_peered":"2024-09-21T10:00:48.450498+0000","last_clean":"2024-09-21T10:00:48.450498+0000","last_became_active":"2024-09-21T10:00:16.355833+0000","last_became_peered":"2024-09-21T10:00:16.355833+0000","last_unstale":"2024-09-21T10:00:48.450498+0000","last_undegraded":"2024-09-21T10:00:48.450498+0000","last_fullsized":"2024-09-21T10:00:48.450498+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T09:59:47.767654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:59:47.767654+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:50.939198+0000","scrub_duration":0.006614371,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1636,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752626+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T10:00:57.752626+0000","last_peered":"2024-09-21T10:00:57.752626+0000","last_clean":"2024-09-21T10:00:57.752626+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T10:00:57.752626+0000","last_undegraded":"2024-09-21T10:00:57.752626+0000","last_fullsized":"2024-09-21T10:00:57.752626+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1686,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752137+0000","last_change":"2024-09-21T10:00:16.348998+0000","last_active":"2024-09-21T10:00:57.752137+0000","last_peered":"2024-09-21T10:00:57.752137+0000","last_clean":"2024-09-21T10:00:57.752137+0000","last_became_active":"2024-09-21T10:00:16.348137+0000","last_became_peered":"2024-09-21T10:00:16.348137+0000","last_unstale":"2024-09-21T10:00:57.752137+0000","last_undegraded":"2024-09-21T10:00:57.752137+0000","last_fullsized":"2024-09-21T10:00:57.752137+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:04:01.486723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1634,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751479+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T10:00:57.751479+0000","last_peered":"2024-09-21T10:00:57.751479+0000","last_clean":"2024-09-21T10:00:57.751479+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T10:00:57.751479+0000","last_undegraded":"2024-09-21T10:00:57.751479+0000","last_fullsized":"2024-09-21T10:00:57.751479+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1145'1095","reported_seq":1111,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:01:05.933388+0000","last_change":"2024-09-21T10:00:46.481230+0000","last_active":"2024-09-21T10:01:05.933388+0000","last_peered":"2024-09-21T10:01:05.933388+0000","last_clean":"2024-09-21T10:01:05.933388+0000","last_became_active":"2024-09-21T10:00:46.480791+0000","last_became_peered":"2024-09-21T10:00:46.480791+0000","last_unstale":"2024-09-21T10:01:05.933388+0000","last_undegraded":"2024-09-21T10:01:05.933388+0000","last_fullsized":"2024-09-21T10:01:05.933388+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":1095,"log_dups_size":0,"ondisk_log_size":1095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:18.514132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71761920,"num_objects":1095,"num_object_clones":0,"num_object_copies":3285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1095,"num_write_kb":70080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":66,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752325+0000","last_change":"2024-09-21T10:00:22.052497+0000","last_active":"2024-09-21T10:00:57.752325+0000","last_peered":"2024-09-21T10:00:57.752325+0000","last_clean":"2024-09-21T10:00:57.752325+0000","last_became_active":"2024-09-21T09:59:43.368157+0000","last_became_peered":"2024-09-21T09:59:43.368157+0000","last_unstale":"2024-09-21T10:00:57.752325+0000","last_undegraded":"2024-09-21T10:00:57.752325+0000","last_fullsized":"2024-09-21T10:00:57.752325+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-28T12:21:36.382264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1714,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496745+0000","last_change":"2024-09-21T09:45:39.140762+0000","last_active":"2024-09-21T10:00:57.496745+0000","last_peered":"2024-09-21T10:00:57.496745+0000","last_clean":"2024-09-21T10:00:57.496745+0000","last_became_active":"2024-09-21T09:45:39.140402+0000","last_became_peered":"2024-09-21T09:45:39.140402+0000","last_unstale":"2024-09-21T10:00:57.496745+0000","last_undegraded":"2024-09-21T10:00:57.496745+0000","last_fullsized":"2024-09-21T10:00:57.496745+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:06.545428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":24,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:47.469134+0000","last_change":"2024-09-21T10:00:16.355102+0000","last_active":"2024-09-21T10:00:47.469134+0000","last_peered":"2024-09-21T10:00:47.469134+0000","last_clean":"2024-09-21T10:00:47.469134+0000","last_became_active":"2024-09-21T10:00:16.354473+0000","last_became_peered":"2024-09-21T10:00:16.354473+0000","last_unstale":"2024-09-21T10:00:47.469134+0000","last_undegraded":"2024-09-21T10:00:47.469134+0000","last_fullsized":"2024-09-21T10:00:47.469134+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:49:35.449111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1089,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450282+0000","last_change":"2024-09-21T10:00:16.355601+0000","last_active":"2024-09-21T10:00:48.450282+0000","last_peered":"2024-09-21T10:00:48.450282+0000","last_clean":"2024-09-21T10:00:48.450282+0000","last_became_active":"2024-09-21T10:00:16.355110+0000","last_became_peered":"2024-09-21T10:00:16.355110+0000","last_unstale":"2024-09-21T10:00:48.450282+0000","last_undegraded":"2024-09-21T10:00:48.450282+0000","last_fullsized":"2024-09-21T10:00:48.450282+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:10.590252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1106,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450602+0000","last_change":"2024-09-21T10:00:16.356676+0000","last_active":"2024-09-21T10:00:48.450602+0000","last_peered":"2024-09-21T10:00:48.450602+0000","last_clean":"2024-09-21T10:00:48.450602+0000","last_became_active":"2024-09-21T10:00:16.356389+0000","last_became_peered":"2024-09-21T10:00:16.356389+0000","last_unstale":"2024-09-21T10:00:48.450602+0000","last_undegraded":"2024-09-21T10:00:48.450602+0000","last_fullsized":"2024-09-21T10:00:48.450602+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:30.110147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1132,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450463+0000","last_change":"2024-09-21T10:00:16.371426+0000","last_active":"2024-09-21T10:00:48.450463+0000","last_peered":"2024-09-21T10:00:48.450463+0000","last_clean":"2024-09-21T10:00:48.450463+0000","last_became_active":"2024-09-21T10:00:16.357486+0000","last_became_peered":"2024-09-21T10:00:16.357486+0000","last_unstale":"2024-09-21T10:00:48.450463+0000","last_undegraded":"2024-09-21T10:00:48.450463+0000","last_fullsized":"2024-09-21T10:00:48.450463+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:02.923682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1686,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751451+0000","last_change":"2024-09-21T10:00:16.353817+0000","last_active":"2024-09-21T10:00:57.751451+0000","last_peered":"2024-09-21T10:00:57.751451+0000","last_clean":"2024-09-21T10:00:57.751451+0000","last_became_active":"2024-09-21T10:00:16.353341+0000","last_became_peered":"2024-09-21T10:00:16.353341+0000","last_unstale":"2024-09-21T10:00:57.751451+0000","last_undegraded":"2024-09-21T10:00:57.751451+0000","last_fullsized":"2024-09-21T10:00:57.751451+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:28:55.175624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1071,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451604+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T10:00:48.451604+0000","last_peered":"2024-09-21T10:00:48.451604+0000","last_clean":"2024-09-21T10:00:48.451604+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T10:00:48.451604+0000","last_undegraded":"2024-09-21T10:00:48.451604+0000","last_fullsized":"2024-09-21T10:00:48.451604+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":515,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451775+0000","last_change":"2024-09-21T09:57:39.544714+0000","last_active":"2024-09-21T10:00:48.451775+0000","last_peered":"2024-09-21T10:00:48.451775+0000","last_clean":"2024-09-21T10:00:48.451775+0000","last_became_active":"2024-09-21T09:57:39.544459+0000","last_became_peered":"2024-09-21T09:57:39.544459+0000","last_unstale":"2024-09-21T10:00:48.451775+0000","last_undegraded":"2024-09-21T10:00:48.451775+0000","last_fullsized":"2024-09-21T10:00:48.451775+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:21.280135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1623,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751429+0000","last_change":"2024-09-21T10:00:16.357151+0000","last_active":"2024-09-21T10:00:57.751429+0000","last_peered":"2024-09-21T10:00:57.751429+0000","last_clean":"2024-09-21T10:00:57.751429+0000","last_became_active":"2024-09-21T10:00:16.356864+0000","last_became_peered":"2024-09-21T10:00:16.356864+0000","last_unstale":"2024-09-21T10:00:57.751429+0000","last_undegraded":"2024-09-21T10:00:57.751429+0000","last_fullsized":"2024-09-21T10:00:57.751429+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:52.851887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1709,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496883+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T10:00:57.496883+0000","last_peered":"2024-09-21T10:00:57.496883+0000","last_clean":"2024-09-21T10:00:57.496883+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T10:00:57.496883+0000","last_undegraded":"2024-09-21T10:00:57.496883+0000","last_fullsized":"2024-09-21T10:00:57.496883+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1640,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751371+0000","last_change":"2024-09-21T10:00:16.354804+0000","last_active":"2024-09-21T10:00:57.751371+0000","last_peered":"2024-09-21T10:00:57.751371+0000","last_clean":"2024-09-21T10:00:57.751371+0000","last_became_active":"2024-09-21T10:00:16.354276+0000","last_became_peered":"2024-09-21T10:00:16.354276+0000","last_unstale":"2024-09-21T10:00:57.751371+0000","last_undegraded":"2024-09-21T10:00:57.751371+0000","last_fullsized":"2024-09-21T10:00:57.751371+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:54:40.860963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":24,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:47.469212+0000","last_change":"2024-09-21T10:00:16.354885+0000","last_active":"2024-09-21T10:00:47.469212+0000","last_peered":"2024-09-21T10:00:47.469212+0000","last_clean":"2024-09-21T10:00:47.469212+0000","last_became_active":"2024-09-21T10:00:16.354001+0000","last_became_peered":"2024-09-21T10:00:16.354001+0000","last_unstale":"2024-09-21T10:00:47.469212+0000","last_undegraded":"2024-09-21T10:00:47.469212+0000","last_fullsized":"2024-09-21T10:00:47.469212+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:49.470604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1643,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751574+0000","last_change":"2024-09-21T09:46:47.120830+0000","last_active":"2024-09-21T10:00:57.751574+0000","last_peered":"2024-09-21T10:00:57.751574+0000","last_clean":"2024-09-21T10:00:57.751574+0000","last_became_active":"2024-09-21T09:46:47.118419+0000","last_became_peered":"2024-09-21T09:46:47.118419+0000","last_unstale":"2024-09-21T10:00:57.751574+0000","last_undegraded":"2024-09-21T10:00:57.751574+0000","last_fullsized":"2024-09-21T10:00:57.751574+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:13.789806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1713,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496656+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T10:00:57.496656+0000","last_peered":"2024-09-21T10:00:57.496656+0000","last_clean":"2024-09-21T10:00:57.496656+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T10:00:57.496656+0000","last_undegraded":"2024-09-21T10:00:57.496656+0000","last_fullsized":"2024-09-21T10:00:57.496656+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1673,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751785+0000","last_change":"2024-09-21T09:46:47.121724+0000","last_active":"2024-09-21T10:00:57.751785+0000","last_peered":"2024-09-21T10:00:57.751785+0000","last_clean":"2024-09-21T10:00:57.751785+0000","last_became_active":"2024-09-21T09:46:47.121406+0000","last_became_peered":"2024-09-21T09:46:47.121406+0000","last_unstale":"2024-09-21T10:00:57.751785+0000","last_undegraded":"2024-09-21T10:00:57.751785+0000","last_fullsized":"2024-09-21T10:00:57.751785+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:55.493704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1625,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752166+0000","last_change":"2024-09-21T09:46:47.113184+0000","last_active":"2024-09-21T10:00:57.752166+0000","last_peered":"2024-09-21T10:00:57.752166+0000","last_clean":"2024-09-21T10:00:57.752166+0000","last_became_active":"2024-09-21T09:46:47.112857+0000","last_became_peered":"2024-09-21T09:46:47.112857+0000","last_unstale":"2024-09-21T10:00:57.752166+0000","last_undegraded":"2024-09-21T10:00:57.752166+0000","last_fullsized":"2024-09-21T10:00:57.752166+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:46.783986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1105,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451677+0000","last_change":"2024-09-21T09:46:13.262555+0000","last_active":"2024-09-21T10:00:48.451677+0000","last_peered":"2024-09-21T10:00:48.451677+0000","last_clean":"2024-09-21T10:00:48.451677+0000","last_became_active":"2024-09-21T09:46:13.261474+0000","last_became_peered":"2024-09-21T09:46:13.261474+0000","last_unstale":"2024-09-21T10:00:48.451677+0000","last_undegraded":"2024-09-21T10:00:48.451677+0000","last_fullsized":"2024-09-21T10:00:48.451677+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:44.865988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1663,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.493304+0000","last_change":"2024-09-21T09:46:47.114406+0000","last_active":"2024-09-21T10:00:57.493304+0000","last_peered":"2024-09-21T10:00:57.493304+0000","last_clean":"2024-09-21T10:00:57.493304+0000","last_became_active":"2024-09-21T09:46:47.113986+0000","last_became_peered":"2024-09-21T09:46:47.113986+0000","last_unstale":"2024-09-21T10:00:57.493304+0000","last_undegraded":"2024-09-21T10:00:57.493304+0000","last_fullsized":"2024-09-21T10:00:57.493304+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:32.292201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1702,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496136+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T10:00:57.496136+0000","last_peered":"2024-09-21T10:00:57.496136+0000","last_clean":"2024-09-21T10:00:57.496136+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T10:00:57.496136+0000","last_undegraded":"2024-09-21T10:00:57.496136+0000","last_fullsized":"2024-09-21T10:00:57.496136+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1103,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451047+0000","last_change":"2024-09-21T09:46:47.114188+0000","last_active":"2024-09-21T10:00:48.451047+0000","last_peered":"2024-09-21T10:00:48.451047+0000","last_clean":"2024-09-21T10:00:48.451047+0000","last_became_active":"2024-09-21T09:46:47.113859+0000","last_became_peered":"2024-09-21T09:46:47.113859+0000","last_unstale":"2024-09-21T10:00:48.451047+0000","last_undegraded":"2024-09-21T10:00:48.451047+0000","last_fullsized":"2024-09-21T10:00:48.451047+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:22.670368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":67,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451044+0000","last_change":"2024-09-21T09:57:39.543638+0000","last_active":"2024-09-21T10:00:48.451044+0000","last_peered":"2024-09-21T10:00:48.451044+0000","last_clean":"2024-09-21T10:00:48.451044+0000","last_became_active":"2024-09-21T09:57:39.543224+0000","last_became_peered":"2024-09-21T09:57:39.543224+0000","last_unstale":"2024-09-21T10:00:48.451044+0000","last_undegraded":"2024-09-21T10:00:48.451044+0000","last_fullsized":"2024-09-21T10:00:48.451044+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:06.898662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1103,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.451592+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T10:00:48.451592+0000","last_peered":"2024-09-21T10:00:48.451592+0000","last_clean":"2024-09-21T10:00:48.451592+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T10:00:48.451592+0000","last_undegraded":"2024-09-21T10:00:48.451592+0000","last_fullsized":"2024-09-21T10:00:48.451592+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":95,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450365+0000","last_change":"2024-09-21T10:00:16.366351+0000","last_active":"2024-09-21T10:00:48.450365+0000","last_peered":"2024-09-21T10:00:48.450365+0000","last_clean":"2024-09-21T10:00:48.450365+0000","last_became_active":"2024-09-21T10:00:16.355385+0000","last_became_peered":"2024-09-21T10:00:16.355385+0000","last_unstale":"2024-09-21T10:00:48.450365+0000","last_undegraded":"2024-09-21T10:00:48.450365+0000","last_fullsized":"2024-09-21T10:00:48.450365+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:29.014677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1635,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.751683+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:00:57.751683+0000","last_peered":"2024-09-21T10:00:57.751683+0000","last_clean":"2024-09-21T10:00:57.751683+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:00:57.751683+0000","last_undegraded":"2024-09-21T10:00:57.751683+0000","last_fullsized":"2024-09-21T10:00:57.751683+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1636,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.752512+0000","last_change":"2024-09-21T10:00:16.361160+0000","last_active":"2024-09-21T10:00:57.752512+0000","last_peered":"2024-09-21T10:00:57.752512+0000","last_clean":"2024-09-21T10:00:57.752512+0000","last_became_active":"2024-09-21T10:00:16.360524+0000","last_became_peered":"2024-09-21T10:00:16.360524+0000","last_unstale":"2024-09-21T10:00:57.752512+0000","last_undegraded":"2024-09-21T10:00:57.752512+0000","last_fullsized":"2024-09-21T10:00:57.752512+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:52.747211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1106,"reported_epoch":1144,"state":"active+clean","last_fresh":"2024-09-21T10:00:48.450282+0000","last_change":"2024-09-21T10:00:16.355914+0000","last_active":"2024-09-21T10:00:48.450282+0000","last_peered":"2024-09-21T10:00:48.450282+0000","last_clean":"2024-09-21T10:00:48.450282+0000","last_became_active":"2024-09-21T10:00:16.355605+0000","last_became_peered":"2024-09-21T10:00:16.355605+0000","last_unstale":"2024-09-21T10:00:48.450282+0000","last_undegraded":"2024-09-21T10:00:48.450282+0000","last_fullsized":"2024-09-21T10:00:48.450282+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:20.179907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1717,"reported_epoch":1145,"state":"active+clean","last_fresh":"2024-09-21T10:00:57.496901+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T10:00:57.496901+0000","last_peered":"2024-09-21T10:00:57.496901+0000","last_clean":"2024-09-21T10:00:57.496901+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T10:00:57.496901+0000","last_undegraded":"2024-09-21T10:00:57.496901+0000","last_fullsized":"2024-09-21T10:00:57.496901+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1070530560,"num_objects":16335,"num_object_clones":0,"num_object_copies":49005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16335,"num_write_kb":1045440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3311337472,"data_stored":3311337472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16335,"ondisk_log_size":16335,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9628,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":423697,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":80,"num_read_kb":204,"num_write":41,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488689,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397823,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":664132,"kb_used_data":354300,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93081020,"statfs":{"total":95995035648,"available":95314964480,"internally_reserved":0,"allocated":362803200,"data_stored":360404251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594052,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":639356,"kb_used_data":333684,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93105796,"statfs":{"total":95995035648,"available":95340335104,"internally_reserved":0,"allocated":341692416,"data_stored":339301524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790281,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":574776,"kb_used_data":269104,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93170376,"statfs":{"total":95995035648,"available":95406465024,"internally_reserved":0,"allocated":275562496,"data_stored":273172070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986508,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":634688,"kb_used_data":329016,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93110464,"statfs":{"total":95995035648,"available":95345115136,"internally_reserved":0,"allocated":336912384,"data_stored":334517453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999150,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":939120,"kb_used_data":614632,"kb_used_omap":25,"kb_used_meta":324454,"kb_avail":92806032,"statfs":{"total":95995035648,"available":95033376768,"internally_reserved":0,"allocated":629383168,"data_stored":627000377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":332241863},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111093,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":674356,"kb_used_data":432044,"kb_used_omap":31,"kb_used_meta":242272,"kb_avail":93070796,"statfs":{"total":95995035648,"available":95304495104,"internally_reserved":0,"allocated":442413056,"data_stored":440026691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":248086760},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318223036,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":633708,"kb_used_data":471332,"kb_used_omap":26,"kb_used_meta":162341,"kb_avail":93111444,"statfs":{"total":95995035648,"available":95346118656,"internally_reserved":0,"allocated":482643968,"data_stored":480264666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":166237196},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289334978,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":480368,"kb_used_data":326248,"kb_used_omap":31,"kb_used_meta":154080,"kb_avail":93264784,"statfs":{"total":95995035648,"available":95503138816,"internally_reserved":0,"allocated":334077952,"data_stored":331695127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":157778016},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":349372416,"data_stored":349372416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":506789888,"data_stored":506789888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":470614016,"data_stored":470614016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":633274368,"data_stored":633274368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":349634560,"data_stored":349634560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":287178752,"data_stored":287178752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":359202816,"data_stored":359202816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":355270656,"data_stored":355270656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:01:14.038 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:01:14.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_2 pg_num 26 2024-09-21T10:01:14.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:14.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:14.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:14.236 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:14.261 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:14.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:14.363 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:14.389 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:14.490 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:14.520 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:14.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:14.621 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:14.630 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:01:14.630 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:01:14.647 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:14.748 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:14.775 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:14.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:14.850 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:01:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:01:14] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.51.0" 2024-09-21T10:01:14.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:14.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 34758 34742 144.719 113.062 0.0125389 0.00690356 2024-09-21T10:01:14.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:14.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:15.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:15.042 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:15.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:15.168 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:15.195 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:15.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:15.297 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:15.324 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:15.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:15.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:15.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:15 smithi017 ceph-mon[102913]: pgmap v1449: 99 pgs: 99 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-21T10:01:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:15 smithi017 ceph-mon[105151]: pgmap v1449: 99 pgs: 99 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-21T10:01:15.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:15.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:15.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:15.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:15.646 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:01:15.676 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:15 smithi134 ceph-mon[90085]: pgmap v1449: 99 pgs: 99 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-09-21T10:01:15.716 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:15.744 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:15.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:15.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:15.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:15.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:15.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 36656 36640 143.088 118.625 0.00695794 0.00698263 2024-09-21T10:01:15.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:15.989 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.024 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:16.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:16.125 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:16.152 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:16.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:16.254 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:16.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:16.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:16.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:16.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:16.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:16.536 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:16.570 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:16.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:16.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:16.723 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:16.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:16.828 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:16.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:16.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:16.904 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 38429 38413 141.189 110.812 0.00883939 0.00707716 2024-09-21T10:01:16.958 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:16.988 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:17.089 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:17.119 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:17.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:17.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:17.288 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:01:17.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:17.402 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:17.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:17 smithi017 ceph-mon[102913]: pgmap v1450: 99 pgs: 99 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-21T10:01:17.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4029091037' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:01:17.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:17 smithi017 ceph-mon[105151]: pgmap v1450: 99 pgs: 99 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-21T10:01:17.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4029091037' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:01:17.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:17.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:17.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:17.613 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:17 smithi134 ceph-mon[90085]: pgmap v1450: 99 pgs: 99 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-21T10:01:17.613 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4029091037' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:01:17.647 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:17.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:17.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:17.818 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:17.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:17.904 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 40239 40223 139.629 113.125 0.00599205 0.00715626 2024-09-21T10:01:17.914 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:01:17.914 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:01:17.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:17.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:17.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:18.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:18.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:18.201 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:18.230 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:18.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:18.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:18.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:18.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:18 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:18 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4029091037' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:01:18.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:18 smithi017 ceph-mon[102913]: osdmap e1146: 8 total, 8 up, 8 in 2024-09-21T10:01:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:18 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:18 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4029091037' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:01:18.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:18 smithi017 ceph-mon[105151]: osdmap e1146: 8 total, 8 up, 8 in 2024-09-21T10:01:18.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:18.519 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:18.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:18.603 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:18 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:18.603 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:18 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4029091037' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:01:18.603 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:18 smithi134 ceph-mon[90085]: osdmap e1146: 8 total, 8 up, 8 in 2024-09-21T10:01:18.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:18.640 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:01:18.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:18.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:18.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:18.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:18.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:18.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:18.904 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 41728 41712 137.176 93.0625 0.0135406 0.00728486 2024-09-21T10:01:18.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:18.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:19.025 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:19.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:19.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:19.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:19.167 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:19.197 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:19.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:19.298 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:19.330 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:19.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:19.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:19.456 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:19 smithi134 ceph-mon[90085]: pgmap v1452: 99 pgs: 99 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-09-21T10:01:19.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:19.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:19.576 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:19.597 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:19.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:19.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:19 smithi017 ceph-mon[102913]: pgmap v1452: 99 pgs: 99 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-09-21T10:01:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:19 smithi017 ceph-mon[105151]: pgmap v1452: 99 pgs: 99 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-09-21T10:01:19.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:19.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:19.840 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:19.869 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:19.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:19.904 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:01:19.903362+0000 min lat: 0.0012027 max lat: 0.102904 avg lat: 0.00739161 2024-09-21T10:01:19.905 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:01:19.905 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 43286 43270 135.186 97.375 0.00465192 0.00739161 2024-09-21T10:01:19.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:20.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:20.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:20.115 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:20.145 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:20.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:20.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:20.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:20.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:20.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:20.488 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:01:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:01:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:20.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:20.559 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:20.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:20.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:20.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:20.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:20.849 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:20.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:20.901 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:01:20.901 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:01:20.904 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 44992 44976 133.825 106.625 0.00909945 0.00746695 2024-09-21T10:01:20.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:20.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:21.094 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:21.123 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:21.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:21.130 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:01:21.130 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:01:21.162 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:01:21.162 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:01:21.187 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:01:21.188 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:01:21.224 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:01:21.224 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:01:21.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:21.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:21.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:21.257 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:01:21.257 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:01:21.289 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:01:21.290 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:01:21.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:01:21.322 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:01:21.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:21.352 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:01:21.353 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:01:21.396 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:21.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:21.497 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:21.527 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:21.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:21.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:21.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:21.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:21 smithi017 ceph-mon[102913]: pgmap v1453: 99 pgs: 99 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:01:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:21 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1975056132' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:21 smithi017 ceph-mon[105151]: pgmap v1453: 99 pgs: 99 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:01:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:21 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1975056132' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:21 smithi134 ceph-mon[90085]: pgmap v1453: 99 pgs: 99 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:01:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:21 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1975056132' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:21.772 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:21.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.802 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:21.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:21.903 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:21.906 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 46560 46544 132.197 98 0.0110657 0.00755915 2024-09-21T10:01:21.930 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:21.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:22.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:22.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:22.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:22.174 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:22.199 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:22.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:22.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:01:22.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:22.336 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:22.438 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:22.464 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:22.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:22.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:22.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:22.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:22.616 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:22.708 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:22.739 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:22.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:22.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:22.869 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:22.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:22.904 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 48196 48180 130.895 102.25 0.00834261 0.00763375 2024-09-21T10:01:22.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:23.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:23.117 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:23.145 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:23.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:23.246 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:23.270 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:23.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:23.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:23.398 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:23.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:23 smithi017 ceph-mon[102913]: pgmap v1454: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-21T10:01:23.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:23 smithi017 ceph-mon[105151]: pgmap v1454: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-21T10:01:23.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:23.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:23.624 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:23 smithi134 ceph-mon[90085]: pgmap v1454: 99 pgs: 99 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-21T10:01:23.628 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:23.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:23.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:23.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:23.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:23.904 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 49665 49649 129.266 91.8125 0.00254781 0.00772499 2024-09-21T10:01:23.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:23.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:24.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:24.062 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:24.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:24.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:24.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:24.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:01:24.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:24.309 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:24.320 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 7 pg_num to 26 2024-09-21T10:01:24.334 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:24.352 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1741994434' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:01:24.352 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:24 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:01:24.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:24.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:24.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:24.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:24.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1741994434' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:01:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:24 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:01:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1741994434' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:01:24.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:24 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:01:24.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:24.751 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:24.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:24.852 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:24.881 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:24.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:24.904 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 51125 51109 127.746 91.25 0.00949918 0.0078231 2024-09-21T10:01:24.983 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:25.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:25.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:01:25.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:25.112 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:01:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:01:24] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.51.0" 2024-09-21T10:01:25.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:25.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:25.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:25.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:25.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:25.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:25.431 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[102913]: pgmap v1455: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-21T10:01:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:01:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[102913]: osdmap e1147: 8 total, 8 up, 8 in 2024-09-21T10:01:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:25.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[105151]: pgmap v1455: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-21T10:01:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:01:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[105151]: osdmap e1147: 8 total, 8 up, 8 in 2024-09-21T10:01:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:25.532 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:25.562 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:25.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:25.660 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:25 smithi134 ceph-mon[90085]: pgmap v1455: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-21T10:01:25.660 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:25 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:01:25.660 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:25 smithi134 ceph-mon[90085]: osdmap e1147: 8 total, 8 up, 8 in 2024-09-21T10:01:25.660 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:25.660 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:25.662 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60917 6.2 6:45cf8ae7:::benchmark_data_smithi017_149547_object60916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.398649+0000", 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 259.25824137400002, 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13833842199999999, 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.663 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60917 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398649+0000", 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398649+0000", 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.664 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398648+0000", 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398706+0000", 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8044000000000002e-05 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398721+0000", 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5298000000000002e-05 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.665 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398727+0000", 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7679999999999997e-06 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398765+0000", 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7812999999999999e-05 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398864+0000", 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.9423e-05 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.666 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.398982+0000", 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000118461 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.401903+0000", 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029207220000000002 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.536966+0000", 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13506279700000001 2024-09-21T10:01:25.667 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.536987+0000", 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1508000000000001e-05 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60925 6.2 6.45ad44e2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.419283+0000", 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 259.23760703599999, 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.118468055, 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.668 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60925 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419283+0000", 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419283+0000", 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.669 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419282+0000", 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419334+0000", 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2682999999999997e-05 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419347+0000", 2024-09-21T10:01:25.670 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2378000000000001e-05 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.419353+0000", 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7899999999999996e-06 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537738+0000", 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.118385267 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.671 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537751+0000", 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3498e-05 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60917 6.2 6:45cf8ae7:::benchmark_data_smithi017_149547_object60916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.430482+0000", 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 259.22640802799998, 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.00240142, 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.672 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60917 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430482+0000", 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430482+0000", 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430480+0000", 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:01:25.673 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430535+0000", 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5269999999999998e-05 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430560+0000", 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.5000000000000001e-05 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430569+0000", 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7290000000000006e-06 2024-09-21T10:01:25.674 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537803+0000", 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10723382400000001 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537811+0000", 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.6440000000000006e-06 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.420962+0000", 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.88315087199999998 2024-09-21T10:01:25.675 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.421705+0000", 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00074260199999999998 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432810+0000", 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011104852 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432884+0000", 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4006999999999998e-05 2024-09-21T10:01:25.676 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60925 6.2 6:4722b5a2:::benchmark_data_smithi017_149547_object60924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.430577+0000", 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 259.226313027, 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0022010509999999, 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.677 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60925 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430577+0000", 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430577+0000", 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.678 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430577+0000", 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430619+0000", 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2605000000000003e-05 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430627+0000", 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2289999999999997e-06 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.679 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.430632+0000", 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9099999999999996e-06 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537828+0000", 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.107196871 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537833+0000", 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6090000000000003e-06 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.421894+0000", 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.88406131600000004 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.422024+0000", 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000129207 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.422186+0000", 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000161982 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.681 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.423391+0000", 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0012054660000000001 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.427355+0000", 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0039639389999999997 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432653+0000", 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0052981959999999998 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432704+0000", 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0963e-05 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.432778+0000", 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4288000000000004e-05 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:91930 6.2 6:42d23dfa:::benchmark_data_smithi017_149547_object91929:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:01:25.683 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:41.674378+0000", 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 223.982511961, 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11810723400000001, 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 91930 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674378+0000", 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.684 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674378+0000", 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674368+0000", 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999895 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674413+0000", 2024-09-21T10:01:25.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5049000000000002e-05 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674430+0000", 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7016000000000001e-05 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674437+0000", 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.669e-06 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674482+0000", 2024-09-21T10:01:25.686 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5361000000000002e-05 2024-09-21T10:01:25.687 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.687 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.687 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.687 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674592+0000", 2024-09-21T10:01:25.687 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010995 2024-09-21T10:01:25.687 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.687 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.687 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:01:25.687 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.674711+0000", 2024-09-21T10:01:25.688 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000118763 2024-09-21T10:01:25.688 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.688 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.688 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.688 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.676478+0000", 2024-09-21T10:01:25.688 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0017673910000000001 2024-09-21T10:01:25.688 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.689 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.689 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.689 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.679252+0000", 2024-09-21T10:01:25.689 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0027736190000000002 2024-09-21T10:01:25.689 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.689 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.689 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.689 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.792402+0000", 2024-09-21T10:01:25.689 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:25.690 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.113149972 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.792442+0000", 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0438000000000001e-05 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:41.792486+0000", 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3560000000000003e-05 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.691 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:92446 6.e 6:7f22a4f7:::benchmark_data_smithi017_149547_object92445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:42.167363+0000", 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 223.489527006, 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.120846588, 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 92446 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.692 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167363+0000", 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167363+0000", 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167362+0000", 2024-09-21T10:01:25.693 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167417+0000", 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5041999999999999e-05 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167430+0000", 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3580999999999999e-05 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167436+0000", 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1460000000000001e-06 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167486+0000", 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9453000000000002e-05 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167626+0000", 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013969899999999999 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.695 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.167787+0000", 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016156499999999999 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.170662+0000", 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0028746520000000001 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.170722+0000", 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0239999999999999e-05 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.696 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288106+0000", 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11738346299999999 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288164+0000", 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8634000000000001e-05 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288210+0000", 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5766999999999997e-05 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.697 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:92450 6.e 6:742c5309:::benchmark_data_smithi017_149547_object92449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:42.172337+0000", 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 223.484553113, 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11601592300000001, 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.698 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 92450 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172337+0000", 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172337+0000", 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.699 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172336+0000", 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172384+0000", 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8196999999999999e-05 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172396+0000", 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1904e-05 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.700 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172400+0000", 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3460000000000004e-06 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172445+0000", 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4499999999999997e-05 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172582+0000", 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000137176 2024-09-21T10:01:25.701 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.172735+0000", 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000152967 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.175058+0000", 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0023232909999999999 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.175223+0000", 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016430199999999999 2024-09-21T10:01:25.702 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288272+0000", 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.113049715 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288319+0000", 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6632999999999999e-05 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:42.288353+0000", 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4209999999999999e-05 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.703 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:122941 6.e 6:72cd0250:::benchmark_data_smithi017_149547_object3862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:10.576941+0000", 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 195.07994944999999, 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.155614635, 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122941 2024-09-21T10:01:25.704 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576941+0000", 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576941+0000", 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.705 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576940+0000", 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576946+0000", 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.241e-06 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576961+0000", 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4987999999999999e-05 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.706 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.576967+0000", 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4130000000000003e-06 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577049+0000", 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1457000000000003e-05 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577161+0000", 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000112606 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.707 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577280+0000", 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000119129 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.583018+0000", 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0057380390000000003 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.584726+0000", 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0017071790000000001 2024-09-21T10:01:25.708 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732474+0000", 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.14774837599999999 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732518+0000", 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3837000000000001e-05 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732556+0000", 2024-09-21T10:01:25.709 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7710000000000003e-05 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:122943 6.e 6:7dbda6e0:::benchmark_data_smithi017_149547_object3864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:10.577994+0000", 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 195.07889622900001, 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.154652334, 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.710 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122943 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577994+0000", 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577994+0000", 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.711 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577993+0000", 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.577998+0000", 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9420000000000003e-06 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.578010+0000", 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1796e-05 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.712 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.578015+0000", 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4029999999999998e-06 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580136+0000", 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021202069999999998 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580239+0000", 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000103319 2024-09-21T10:01:25.713 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580310+0000", 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0637999999999997e-05 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.584691+0000", 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0043812809999999999 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.586116+0000", 2024-09-21T10:01:25.714 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0014252010000000001 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732585+0000", 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.14646848300000001 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732618+0000", 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3581999999999998e-05 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732646+0000", 2024-09-21T10:01:25.715 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8347000000000002e-05 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:122947 6.e 6:7d385fde:::benchmark_data_smithi017_149547_object3868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:10.580733+0000", 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 195.07615748800001, 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.15199675800000001, 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.716 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122947 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580733+0000", 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580733+0000", 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.717 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580732+0000", 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580739+0000", 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3119999999999996e-06 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580750+0000", 2024-09-21T10:01:25.718 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0719e-05 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.580755+0000", 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4369999999999998e-06 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.583033+0000", 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0022775339999999999 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.719 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.583109+0000", 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5663999999999999e-05 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.583185+0000", 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6848999999999996e-05 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.584792+0000", 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001606892 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.720 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.586161+0000", 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0013685310000000001 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732671+0000", 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.14651046500000001 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732704+0000", 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2187999999999997e-05 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.721 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:10.732730+0000", 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.6088000000000002e-05 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:123433 6.e 6:7f54b96a:::benchmark_data_smithi017_149547_object4354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:11.295471+0000", 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 194.361418902, 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.150257261, 2024-09-21T10:01:25.722 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 123433 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295471+0000", 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.723 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295471+0000", 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295470+0000", 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295477+0000", 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5880000000000003e-06 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.724 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295491+0000", 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4024e-05 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295498+0000", 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9600000000000003e-06 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295546+0000", 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8374000000000001e-05 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.725 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295626+0000", 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0322999999999997e-05 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.295695+0000", 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8498999999999996e-05 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.298939+0000", 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032442619999999999 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.726 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.299749+0000", 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00081013199999999997 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.445644+0000", 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.145894317 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.445690+0000", 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5998999999999998e-05 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.727 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:11.445729+0000", 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9221999999999998e-05 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:124124 6.e 6:70f8299d:::benchmark_data_smithi017_149547_object5045:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:12.198188+0000", 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 193.45870207300001, 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10971036300000001, 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.728 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 124124 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198188+0000", 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198188+0000", 2024-09-21T10:01:25.729 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198187+0000", 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198194+0000", 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7669999999999998e-06 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.730 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198208+0000", 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4262e-05 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198214+0000", 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5579999999999997e-06 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198262+0000", 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7599000000000003e-05 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.731 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198366+0000", 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000104261 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.198479+0000", 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000112327 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.202028+0000", 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0035496740000000001 2024-09-21T10:01:25.732 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.202182+0000", 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000154203 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.307794+0000", 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.105611815 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.307841+0000", 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6826000000000002e-05 2024-09-21T10:01:25.733 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:12.307899+0000", 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7590999999999998e-05 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:150148 6.e 6:7101fe91:::benchmark_data_smithi017_149547_object31069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:39.399817+0000", 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 166.25707325499999, 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13629029300000001, 2024-09-21T10:01:25.734 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 150148 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399817+0000", 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.735 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399817+0000", 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399816+0000", 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399822+0000", 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3009999999999997e-06 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.736 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399832+0000", 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0672999999999999e-05 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399837+0000", 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2420000000000002e-06 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399873+0000", 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6791000000000001e-05 2024-09-21T10:01:25.737 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.399995+0000", 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000122125 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.400082+0000", 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.6749999999999994e-05 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.403983+0000", 2024-09-21T10:01:25.738 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003900761 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.417968+0000", 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013985205000000001 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536010+0000", 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.118041418 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536069+0000", 2024-09-21T10:01:25.739 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9500000000000003e-05 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536107+0000", 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8294999999999998e-05 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:150149 6.e 6:7280da22:::benchmark_data_smithi017_149547_object31070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:39.404830+0000", 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 166.25206075899999, 2024-09-21T10:01:25.740 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13136800500000001, 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 150149 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404830+0000", 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.741 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404830+0000", 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404828+0000", 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404837+0000", 2024-09-21T10:01:25.742 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.9789999999999999e-06 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404854+0000", 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6361000000000001e-05 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404861+0000", 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0010000000000004e-06 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.743 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404953+0000", 2024-09-21T10:01:25.744 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.2493999999999999e-05 2024-09-21T10:01:25.744 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.744 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.744 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.744 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.405123+0000", 2024-09-21T10:01:25.744 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016946999999999999 2024-09-21T10:01:25.744 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.744 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.744 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.745 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.405236+0000", 2024-09-21T10:01:25.745 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000113758 2024-09-21T10:01:25.745 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.745 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.745 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.745 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.410836+0000", 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0055995100000000003 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.418175+0000", 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0073390479999999999 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536137+0000", 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11796179599999999 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.746 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536169+0000", 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2088999999999997e-05 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536198+0000", 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8858000000000001e-05 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.747 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.749 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:150156 6.e 6:79b93165:::benchmark_data_smithi017_149547_object31077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:25.749 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:39.404992+0000", 2024-09-21T10:01:25.749 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 166.25189870099999, 2024-09-21T10:01:25.749 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13130247, 2024-09-21T10:01:25.749 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 150156 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404992+0000", 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.750 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404992+0000", 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404991+0000", 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.404996+0000", 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7090000000000001e-06 2024-09-21T10:01:25.751 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.405004+0000", 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.67e-06 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.405009+0000", 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0069999999999997e-06 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.408315+0000", 2024-09-21T10:01:25.752 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0033059320000000001 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.408396+0000", 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0297999999999996e-05 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.408472+0000", 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6533999999999996e-05 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.410988+0000", 2024-09-21T10:01:25.753 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002515485 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.418351+0000", 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0073635569999999997 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536236+0000", 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11788446599999999 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.754 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536267+0000", 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1146999999999998e-05 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536294+0000", 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7419999999999998e-05 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:150170 6.e 6:7571e895:::benchmark_data_smithi017_149547_object31091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:58:39.431315+0000", 2024-09-21T10:01:25.755 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 166.22557567499999, 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.105067644, 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 150170 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431315+0000", 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.756 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431315+0000", 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431313+0000", 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431321+0000", 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4370000000000001e-06 2024-09-21T10:01:25.757 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431333+0000", 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2401999999999999e-05 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431339+0000", 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7529999999999998e-06 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431377+0000", 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8614e-05 2024-09-21T10:01:25.758 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431469+0000", 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.2052000000000006e-05 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.431551+0000", 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1301e-05 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.434274+0000", 2024-09-21T10:01:25.759 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002722801 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.434473+0000", 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019968899999999999 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536319+0000", 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.101845463 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.760 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536354+0000", 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5369e-05 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:58:39.536382+0000", 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8260999999999999e-05 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175260 6.e 6:730073ac:::benchmark_data_smithi017_149547_object56181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.305298+0000", 2024-09-21T10:01:25.761 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 137.35159192899999, 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13994115100000001, 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175260 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.305298+0000", 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.762 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.305298+0000", 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.305297+0000", 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.305304+0000", 2024-09-21T10:01:25.763 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3000000000000004e-06 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.305317+0000", 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3407999999999999e-05 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.305322+0000", 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8609999999999997e-06 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.305361+0000", 2024-09-21T10:01:25.764 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8791999999999997e-05 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.305456+0000", 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.4666000000000005e-05 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.305534+0000", 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8466000000000004e-05 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.308287+0000", 2024-09-21T10:01:25.765 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002752492 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.308422+0000", 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013554499999999999 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.445150+0000", 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13672779400000001 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.766 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.445198+0000", 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7892000000000002e-05 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.445240+0000", 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1627e-05 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:217045 6.2 6:4789a926:::benchmark_data_smithi017_149547_object97966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1127)", 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:55.477698+0000", 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 90.179192701999995, 2024-09-21T10:01:25.767 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.126459246, 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 217045 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.477698+0000", 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.768 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.477698+0000", 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.477696+0000", 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.477707+0000", 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1537e-05 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.769 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.477726+0000", 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8587999999999999e-05 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.477734+0000", 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0809999999999994e-06 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.477785+0000", 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0662999999999999e-05 2024-09-21T10:01:25.770 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.477933+0000", 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014838399999999999 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.478433+0000", 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00049961199999999997 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.481441+0000", 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003008139 2024-09-21T10:01:25.771 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.481928+0000", 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00048702800000000003 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.604050+0000", 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.122122094 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.772 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.604115+0000", 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4925000000000006e-05 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.604157+0000", 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2243000000000002e-05 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:217375 6.e 6:71e42c90:::benchmark_data_smithi017_149547_object98296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1127)", 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:55.893222+0000", 2024-09-21T10:01:25.773 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 89.763668699999997, 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.14268144699999999, 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 217375 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.893222+0000", 2024-09-21T10:01:25.774 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.893222+0000", 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.893220+0000", 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.775 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.893228+0000", 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.2230000000000007e-06 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.893246+0000", 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8264999999999999e-05 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.893253+0000", 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.092e-06 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.776 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.893305+0000", 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2191000000000002e-05 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.893432+0000", 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000126604 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.893555+0000", 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012313499999999999 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.777 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.896182+0000", 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0026266689999999999 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.902334+0000", 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0061523639999999996 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:56.035759+0000", 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13342498699999999 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.778 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:56.035851+0000", 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1895e-05 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:56.035903+0000", 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2188999999999998e-05 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.779 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:217379 6.e 6:771dd01c:::benchmark_data_smithi017_149547_object98300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1127)", 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:55.898901+0000", 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 89.757989096000003, 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13714712400000001, 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 217379 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:25.780 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.898901+0000", 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.898901+0000", 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.898900+0000", 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.781 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.898907+0000", 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0779999999999998e-06 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.898925+0000", 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7532000000000001e-05 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.898931+0000", 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7700000000000004e-06 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.782 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.898984+0000", 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2296000000000002e-05 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.899103+0000", 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011938999999999999 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.899220+0000", 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000117064 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.783 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.902183+0000", 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029631039999999998 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:55.905851+0000", 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0036679579999999998 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:56.035940+0000", 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.130088923 2024-09-21T10:01:25.784 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:56.036013+0000", 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2534999999999996e-05 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:56.036048+0000", 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5864000000000001e-05 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.785 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:25.786 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:25.786 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:01:25.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.787 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:25.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:25.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:25.905 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 52411 52395 125.924 80.375 0.00694054 0.00793474 2024-09-21T10:01:25.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:26.031 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:26.060 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:26.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:26.162 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:26.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.190 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:26.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:26.291 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:26.320 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:26.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:26.422 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:26.444 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:26.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:01:26.546 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:26.567 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:26.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:26.670 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:26.692 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:26.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:01:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:01:26.794 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:26.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.815 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:26.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:26.905 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 53013 52997 122.653 37.625 0.153074 0.00798324 2024-09-21T10:01:26.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:26.937 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:26.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:27.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:27.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:27.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:27.168 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:27.190 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:27.252 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:27.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:27.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:27.416 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:27.445 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:27.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:27 smithi017 ceph-mon[105151]: pgmap v1457: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-21T10:01:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:01:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:27 smithi017 ceph-mon[105151]: osdmap e1148: 8 total, 8 up, 8 in 2024-09-21T10:01:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:27 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:27 smithi017 ceph-mon[102913]: pgmap v1457: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-21T10:01:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:01:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:27 smithi017 ceph-mon[102913]: osdmap e1148: 8 total, 8 up, 8 in 2024-09-21T10:01:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:27 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:27.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:27.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:27.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:27.614 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:27 smithi134 ceph-mon[90085]: pgmap v1457: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-21T10:01:27.614 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:01:27.614 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:27 smithi134 ceph-mon[90085]: osdmap e1148: 8 total, 8 up, 8 in 2024-09-21T10:01:27.614 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:27 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:27.640 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:27.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:01:27.709 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:27.757 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:27.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:27.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:27.905 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 53899 53883 120.251 55.375 0.00349345 0.00831108 2024-09-21T10:01:27.906 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:28.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:28.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:28.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:28.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:28.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:28.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:28.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:28.375 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:28 smithi134 ceph-mon[90085]: osdmap e1149: 8 total, 8 up, 8 in 2024-09-21T10:01:28.376 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:28 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:28.376 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:28 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:28.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:28.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:28.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:28.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:28.635 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:28 smithi017 ceph-mon[102913]: osdmap e1149: 8 total, 8 up, 8 in 2024-09-21T10:01:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:28 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:28.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:28 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:28 smithi017 ceph-mon[105151]: osdmap e1149: 8 total, 8 up, 8 in 2024-09-21T10:01:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:28 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:28 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:28.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:28.753 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:28.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:28.855 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:28.882 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:28.905 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 55442 55426 119.429 96.4375 0.00348344 0.00836833 2024-09-21T10:01:28.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:29.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:29.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:29.124 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:29.323 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:29.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:29.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:29.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:29.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:29.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:29.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:29.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:29.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:29.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:29.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:29.906 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 57136 57120 118.975 105.875 0.00670121 0.00839968 2024-09-21T10:01:29.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:29.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:29 smithi134 ceph-mon[90085]: pgmap v1460: 109 pgs: 1 peering, 9 unknown, 99 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T10:01:29.996 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:29 smithi017 ceph-mon[102913]: pgmap v1460: 109 pgs: 1 peering, 9 unknown, 99 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T10:01:29.996 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:29 smithi017 ceph-mon[105151]: pgmap v1460: 109 pgs: 1 peering, 9 unknown, 99 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T10:01:30.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:30.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:30.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:30.159 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:30.188 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:30.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:30.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:30.320 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:30.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:30.422 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:30.451 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:30.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:01:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:01:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:30.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:30.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:30.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:30.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:30.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:30.812 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:30 smithi134 ceph-mon[90085]: pgmap v1461: 109 pgs: 1 peering, 6 unknown, 102 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-21T10:01:30.846 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:30.875 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:30.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:30.906 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 58918 58902 118.73 111.375 0.0089795 0.00841767 2024-09-21T10:01:30.977 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.011 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:31.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:31.094 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:30 smithi017 ceph-mon[102913]: pgmap v1461: 109 pgs: 1 peering, 6 unknown, 102 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-21T10:01:31.094 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:30 smithi017 ceph-mon[105151]: pgmap v1461: 109 pgs: 1 peering, 6 unknown, 102 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-21T10:01:31.111 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:31.127 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:01:31.127 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:01:31.127 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 0 2024-09-21T10:01:31.128 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:01:31.142 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:31.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:31.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:31.271 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:31.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:31.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:31.398 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:31.450 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:01:31.450 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:01:31.488 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:01:31.488 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:01:31.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:31.527 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:01:31.528 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:01:31.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:31.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:31.570 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:01:31.571 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:01:31.601 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:01:31.602 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:01:31.634 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:01:31.634 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:01:31.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:31.667 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:01:31.667 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:01:31.693 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:31.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:31.702 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:01:31.703 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:01:31.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:31.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:31.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:31.906 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 15 60614 60599 118.334 106.062 0.00445417 0.00844593 2024-09-21T10:01:31.940 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:31 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1723039023' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:31 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1723039023' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:31.969 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:32.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:32.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:32.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:32.162 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:31 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1723039023' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:32.199 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:01:32.217 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:32.219 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:01:32.219 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:01:32.220 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 5, 7, 6] 2024-09-21T10:01:32.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd out 1 2024-09-21T10:01:32.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.275 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:32.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:32.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:32.440 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:32.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:32.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:32.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:32.703 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:32.731 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:32.788 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:01:32.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:32.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:32.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 62131 62115 117.616 94.75 0.00298741 0.00849113 2024-09-21T10:01:32.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:32 smithi134 ceph-mon[90085]: pgmap v1462: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-21T10:01:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:01:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:32 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/337722735' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:33.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:33.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:33.116 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:33.138 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:33.176 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:32 smithi017 ceph-mon[102913]: pgmap v1462: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-21T10:01:33.176 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:01:33.176 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:32 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/337722735' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:33.176 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:33.177 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:32 smithi017 ceph-mon[105151]: pgmap v1462: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-21T10:01:33.177 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:01:33.177 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:32 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/337722735' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:33.240 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:33.262 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:33.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:33.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:33.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:33.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:33.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:33.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:33.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:33.635 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:33.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:33.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:33.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:33.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:33.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:33.886 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:33.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 62373 62357 114.602 15.125 0.00177068 0.00846981 2024-09-21T10:01:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:01:33.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/337722735' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:01:33.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[102913]: osdmap e1150: 8 total, 8 up, 8 in 2024-09-21T10:01:33.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[102913]: osdmap e1151: 8 total, 8 up, 8 in 2024-09-21T10:01:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:01:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/337722735' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:01:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[105151]: osdmap e1150: 8 total, 8 up, 8 in 2024-09-21T10:01:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:33 smithi017 ceph-mon[105151]: osdmap e1151: 8 total, 8 up, 8 in 2024-09-21T10:01:33.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:34.008 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:34.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:34.110 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:34.130 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:34.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:34.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:33 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:33 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:01:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:33 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/337722735' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:01:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:33 smithi134 ceph-mon[90085]: osdmap e1150: 8 total, 8 up, 8 in 2024-09-21T10:01:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:33 smithi134 ceph-mon[90085]: osdmap e1151: 8 total, 8 up, 8 in 2024-09-21T10:01:34.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:34.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:34.347 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:34.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:34.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:34.483 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:34.504 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:34.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:34.605 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:34.626 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:34.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:34.728 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:34.749 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:34.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:34.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:34.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 62373 62357 111.327 0 - 0.00846981 2024-09-21T10:01:34.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:34 smithi017 ceph-mon[102913]: pgmap v1465: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T10:01:34.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:01:34.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:01:34.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:34 smithi017 ceph-mon[102913]: osdmap e1152: 8 total, 8 up, 8 in 2024-09-21T10:01:34.911 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:01:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:01:34] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.51.0" 2024-09-21T10:01:34.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:34 smithi017 ceph-mon[105151]: pgmap v1465: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T10:01:34.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:01:34.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:01:34.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:34 smithi017 ceph-mon[105151]: osdmap e1152: 8 total, 8 up, 8 in 2024-09-21T10:01:34.937 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:34 smithi134 ceph-mon[90085]: pgmap v1465: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-21T10:01:34.937 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:01:34.937 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:01:34.937 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:34 smithi134 ceph-mon[90085]: osdmap e1152: 8 total, 8 up, 8 in 2024-09-21T10:01:34.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:34.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:34.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:35.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:35.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:35.131 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:01:35.131 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:01:35.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:01:35.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:35.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:35.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:35.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:35.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:35.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:35.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:35.654 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:35.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:35.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:35.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:35.800 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:35.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:35.831 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:01:35.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:35.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 62959 62943 109.253 18.3125 0.00302679 0.00908824 2024-09-21T10:01:35.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:35.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:35.976 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:36.043 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:36.065 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:36.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:36.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:36.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:36.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:36.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:36.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:36.367 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:36.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:01:36.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:36.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:36.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:36 smithi017 ceph-mon[102913]: osdmap e1153: 8 total, 8 up, 8 in 2024-09-21T10:01:36.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:36 smithi017 ceph-mon[105151]: osdmap e1153: 8 total, 8 up, 8 in 2024-09-21T10:01:36.544 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:36.573 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:36.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:36.640 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:36 smithi134 ceph-mon[90085]: osdmap e1153: 8 total, 8 up, 8 in 2024-09-21T10:01:36.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:36.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:36.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:36.826 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:36.858 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:36.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 63648 63632 107.464 43.0625 0.0113352 0.0093004 2024-09-21T10:01:36.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:36.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:36.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:37.084 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:37.116 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:37.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:37.218 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:37.240 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:37.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:37.341 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:37.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:37.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:37.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:37.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:37.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:37.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:37.642 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:37.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:37.685 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:37 smithi134 ceph-mon[90085]: pgmap v1468: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 1802/187131 objects misplaced (0.963%); 752 KiB/s, 11 objects/s recovering 2024-09-21T10:01:37.685 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:37 smithi134 ceph-mon[90085]: osdmap e1154: 8 total, 8 up, 8 in 2024-09-21T10:01:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:37 smithi017 ceph-mon[105151]: pgmap v1468: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 1802/187131 objects misplaced (0.963%); 752 KiB/s, 11 objects/s recovering 2024-09-21T10:01:37.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:37 smithi017 ceph-mon[105151]: osdmap e1154: 8 total, 8 up, 8 in 2024-09-21T10:01:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:37 smithi017 ceph-mon[102913]: pgmap v1468: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 1802/187131 objects misplaced (0.963%); 752 KiB/s, 11 objects/s recovering 2024-09-21T10:01:37.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:37 smithi017 ceph-mon[102913]: osdmap e1154: 8 total, 8 up, 8 in 2024-09-21T10:01:37.745 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:37.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.773 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:37.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:37.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:37.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 64944 64928 106.767 81 0.00716144 0.00936046 2024-09-21T10:01:37.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:38.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:38.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:38.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:38.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:38.189 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:38.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:38.291 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:38.319 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:38.420 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:38.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.454 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:38.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:38.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:38.590 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:38.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:38.692 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:38.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:38.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:38.864 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:38.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:38.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 66369 66353 106.313 89.0625 0.00315118 0.00940083 2024-09-21T10:01:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:38 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:01:38.545+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1154 definitely_dead 0 2024-09-21T10:01:38.967 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:38.994 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:38.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:39.096 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:39.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:39.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:39.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.276 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:39.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:39.335 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:01:39.335 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:01:39.377 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:39.405 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:39.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:39.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[102913]: pgmap v1470: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 1802/188976 objects misplaced (0.954%); 635 KiB/s, 9 objects/s recovering 2024-09-21T10:01:39.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:39.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:39.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1798919449' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:01:39.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:01:39.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[102913]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:01:39.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[105151]: pgmap v1470: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 1802/188976 objects misplaced (0.954%); 635 KiB/s, 9 objects/s recovering 2024-09-21T10:01:39.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:39.418 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:39.418 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1798919449' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:01:39.418 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:01:39.418 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:39 smithi017 ceph-mon[105151]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:01:39.418 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.1. 2024-09-21T10:01:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:39 smithi134 ceph-mon[90085]: pgmap v1470: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 1802/188976 objects misplaced (0.954%); 635 KiB/s, 9 objects/s recovering 2024-09-21T10:01:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:39 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1798919449' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:01:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:39 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:01:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:39 smithi134 ceph-mon[90085]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:01:39.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:39.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:39.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:39.634 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:39.655 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:39.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:39.757 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:39.769 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:39.777 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:39.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:39.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:39.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:39.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:39.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:01:39.906567+0000 min lat: 0.0012027 max lat: 2.57441 avg lat: 0.00941514 2024-09-21T10:01:39.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:01:39.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 67159 67143 104.89 49.375 0.00217574 0.00941514 2024-09-21T10:01:40.004 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:40.025 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:40.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:40.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:01:40.127 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:40.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.149 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:40.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:40.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:40.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:40.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:40.326 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:40.374 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:40.395 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:40.404 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:01:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:01:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:40.497 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:40.518 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:40.619 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:40.641 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:40.662 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:40 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:01:40.662 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:40 smithi134 ceph-mon[90085]: osdmap e1155: 8 total, 8 up, 7 in 2024-09-21T10:01:40.662 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:40 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:01:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:40 smithi017 ceph-mon[102913]: osdmap e1155: 8 total, 8 up, 7 in 2024-09-21T10:01:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:40 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:01:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:40 smithi017 ceph-mon[105151]: osdmap e1155: 8 total, 8 up, 7 in 2024-09-21T10:01:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:40.743 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:40.764 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:40.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:40.866 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:40.888 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:40.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:40.907 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 67159 67143 102.332 0 - 0.00941514 2024-09-21T10:01:40.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:41.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:41.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:41.117 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:41.139 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:41.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:41.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:41.264 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:41.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:41.366 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:41.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:41.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:41.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:41.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:41.587 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:41 smithi134 ceph-mon[90085]: pgmap v1472: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 38 MiB/s wr, 610 op/s; 1802/191664 objects misplaced (0.940%); 525 KiB/s, 8 objects/s recovering 2024-09-21T10:01:41.587 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:41 smithi134 ceph-mon[90085]: osdmap e1156: 8 total, 8 up, 7 in 2024-09-21T10:01:41.587 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:41.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:41 smithi017 ceph-mon[102913]: pgmap v1472: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 38 MiB/s wr, 610 op/s; 1802/191664 objects misplaced (0.940%); 525 KiB/s, 8 objects/s recovering 2024-09-21T10:01:41.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:41 smithi017 ceph-mon[102913]: osdmap e1156: 8 total, 8 up, 7 in 2024-09-21T10:01:41.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:41.600 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:41 smithi017 ceph-mon[105151]: pgmap v1472: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 38 MiB/s wr, 610 op/s; 1802/191664 objects misplaced (0.940%); 525 KiB/s, 8 objects/s recovering 2024-09-21T10:01:41.600 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:41 smithi017 ceph-mon[105151]: osdmap e1156: 8 total, 8 up, 7 in 2024-09-21T10:01:41.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:41.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:41.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:41.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:41.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:41.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:41.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:41.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 67213 67197 99.9759 1.6875 0.00267777 0.00983688 2024-09-21T10:01:41.924 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:41.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:42.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:42.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:42.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:42.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:42.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:42.199 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:01:42.199 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:01:42.279 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:42.302 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:42.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:42.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:42.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:42.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:42 smithi134 ceph-mon[90085]: osdmap e1157: 8 total, 8 up, 7 in 2024-09-21T10:01:42.532 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:42.554 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:42.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:42.656 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:42.678 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:42.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:42 smithi017 ceph-mon[102913]: osdmap e1157: 8 total, 8 up, 7 in 2024-09-21T10:01:42.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:42 smithi017 ceph-mon[105151]: osdmap e1157: 8 total, 8 up, 7 in 2024-09-21T10:01:42.780 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:42.801 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:42.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:42.901 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:01:42.901 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:01:42.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:42.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 67213 67197 97.6508 0 - 0.00983688 2024-09-21T10:01:42.927 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:01:42.927 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:01:42.930 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:42.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:42.948 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:01:42.948 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:01:42.971 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:01:42.972 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:01:42.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:01:42.991 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:01:43.012 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:01:43.012 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:01:43.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:43.033 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:01:43.033 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:01:43.052 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:01:43.052 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:01:43.055 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:43.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:43.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:43.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:43.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:43.305 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:43.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:01:43.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:43.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:43.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:43.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:43.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:43.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:43.652 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:43.660 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:43.693 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:43.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:43 smithi017 ceph-mon[102913]: pgmap v1475: 109 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 102 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 12 KiB/s rd, 49 MiB/s wr, 795 op/s; 1604/200682 objects misplaced (0.799%); 7.0 MiB/s, 112 objects/s recovering 2024-09-21T10:01:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:43 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/941418959' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:43 smithi017 ceph-mon[105151]: pgmap v1475: 109 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 102 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 12 KiB/s rd, 49 MiB/s wr, 795 op/s; 1604/200682 objects misplaced (0.799%); 7.0 MiB/s, 112 objects/s recovering 2024-09-21T10:01:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:43 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/941418959' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:43 smithi134 ceph-mon[90085]: pgmap v1475: 109 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 102 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 12 KiB/s rd, 49 MiB/s wr, 795 op/s; 1604/200682 objects misplaced (0.799%); 7.0 MiB/s, 112 objects/s recovering 2024-09-21T10:01:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:43 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/941418959' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:43.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:43.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:43.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:43.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 67400 67384 95.6973 5.84375 0.00230888 0.0100963 2024-09-21T10:01:43.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:43.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:44.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:44.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:44.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:44.182 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:44.205 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:44.307 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:44.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:44.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:44.456 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:44.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:44.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:44.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:44.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:44 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 6112/201573 objects degraded (3.032%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:44.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:44 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 6112/201573 objects degraded (3.032%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:44.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:44.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:44 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 6112/201573 objects degraded (3.032%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:44.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:44.841 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:44.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:44.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 67400 67384 93.571 0 - 0.0100963 2024-09-21T10:01:44.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:44.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:44.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:45.058 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:01:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:01:44] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-09-21T10:01:45.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:45.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:45.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:45.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:45.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:45.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:45.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:45.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:45.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:45.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:45.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:45.588 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:45.612 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:45.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:45.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:45 smithi017 ceph-mon[102913]: pgmap v1476: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 6 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 11 KiB/s rd, 44 MiB/s wr, 706 op/s; 6112/201573 objects degraded (3.032%); 3688/201573 objects misplaced (1.830%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:01:45.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:45.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:45.630 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:45 smithi017 ceph-mon[105151]: pgmap v1476: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 6 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 11 KiB/s rd, 44 MiB/s wr, 706 op/s; 6112/201573 objects degraded (3.032%); 3688/201573 objects misplaced (1.830%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:01:45.630 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:45 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:45.630 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:45 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:45.707 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:45 smithi134 ceph-mon[90085]: pgmap v1476: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 6 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 11 KiB/s rd, 44 MiB/s wr, 706 op/s; 6112/201573 objects degraded (3.032%); 3688/201573 objects misplaced (1.830%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:01:45.707 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:45 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:45.707 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:45 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:45.714 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:45.736 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:45.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:45.828 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:01:45.829 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:01:45.829 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:01:45.829 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:01:45.829 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:01:45.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:45.866 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:45.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 67400 67384 91.537 0 - 0.0100963 2024-09-21T10:01:45.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:45.991 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:45.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:46.000 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 6] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:01:46.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 0 1 2024-09-21T10:01:46.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:46.115 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:46.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:46.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:46.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:46.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:46.312 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:46.346 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:46.369 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:46.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:46.471 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:46.493 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:46.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:46.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:46.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:46.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2197517685' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:01:46.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:01:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2197517685' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:01:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:46 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:01:46.722 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:46.743 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2197517685' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:01:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:46 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:01:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:46.798 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:01:46.846 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:46.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:01:46.869 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:46.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 67400 67384 89.5896 0 - 0.0100963 2024-09-21T10:01:46.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:46.982 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:01:47.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:47.077 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:47.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:47.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:47.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:47.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:47.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:47.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:47.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:47.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:47.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:47.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:47.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:47.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:47.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:47.638 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:47.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:47 smithi017 ceph-mon[102913]: pgmap v1477: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 10 KiB/s rd, 32 MiB/s wr, 513 op/s; 14951/201894 objects degraded (7.405%); 3785/201894 objects misplaced (1.875%); 15 MiB/s, 241 objects/s recovering 2024-09-21T10:01:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:47 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:47 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:01:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:47 smithi017 ceph-mon[102913]: osdmap e1158: 8 total, 8 up, 7 in 2024-09-21T10:01:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:47 smithi017 ceph-mon[105151]: pgmap v1477: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 10 KiB/s rd, 32 MiB/s wr, 513 op/s; 14951/201894 objects degraded (7.405%); 3785/201894 objects misplaced (1.875%); 15 MiB/s, 241 objects/s recovering 2024-09-21T10:01:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:47 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:47 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:01:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:47 smithi017 ceph-mon[105151]: osdmap e1158: 8 total, 8 up, 7 in 2024-09-21T10:01:47.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:47 smithi134 ceph-mon[90085]: pgmap v1477: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 10 KiB/s rd, 32 MiB/s wr, 513 op/s; 14951/201894 objects degraded (7.405%); 3785/201894 objects misplaced (1.875%); 15 MiB/s, 241 objects/s recovering 2024-09-21T10:01:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:47 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:47 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:01:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:47 smithi134 ceph-mon[90085]: osdmap e1158: 8 total, 8 up, 7 in 2024-09-21T10:01:47.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:47.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:47.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:47.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:47.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:47.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 67400 67384 87.7233 0 - 0.0100963 2024-09-21T10:01:47.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:48.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:48.120 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:48.143 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:48.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:48.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:48.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:48.371 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:48.393 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:48.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:48.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:48.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:48.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:48.622 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:48.644 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:48.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:48.746 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:48.770 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:48.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:48.872 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:48.895 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:48.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:48.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 67400 67384 85.9332 0 - 0.0100963 2024-09-21T10:01:48.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:49.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:49.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:49.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:49.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:49.249 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:49.273 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:49.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:49.315 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:01:49.315 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45173.0:60912 6.3 e1085/1078)", 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.399084+0000", 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 282.91254872500002, 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.096088522999999995, 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.316 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399084+0000", 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399084+0000", 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399082+0000", 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.317 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399113+0000", 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0623999999999997e-05 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399133+0000", 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0061e-05 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.399139+0000", 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8789999999999997e-06 2024-09-21T10:01:49.318 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.495153+0000", 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.096013182000000002 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.495172+0000", 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9661e-05 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.319 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90845 6.a 6:5de4ee99:::benchmark_data_smithi017_149547_object90844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.571935+0000", 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 250.73969716400001, 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.085221939, 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90845 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.320 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.571935+0000", 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.571935+0000", 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.571934+0000", 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:49.321 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572128+0000", 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019428100000000001 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572139+0000", 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1455e-05 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.322 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572144+0000", 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8890000000000001e-06 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572202+0000", 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8139e-05 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572213+0000", 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0562e-05 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.640648+0000", 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.068435024 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642661+0000", 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002012715 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642814+0000", 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015295599999999999 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.324 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651043+0000", 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0082295990000000006 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651085+0000", 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1270000000000003e-05 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657094+0000", 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.006009258 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.325 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657129+0000", 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5426999999999998e-05 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657157+0000", 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8014999999999999e-05 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.326 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90864 6.a 6:5a2b4d81:::benchmark_data_smithi017_149547_object90863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.594822+0000", 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 250.71681090600001, 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0624268729999999, 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90864 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.327 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594822+0000", 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594822+0000", 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594820+0000", 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.328 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594917+0000", 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6347000000000004e-05 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594928+0000", 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1263999999999999e-05 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.329 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594934+0000", 2024-09-21T10:01:49.330 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9699999999999996e-06 2024-09-21T10:01:49.330 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.330 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.330 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.330 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594987+0000", 2024-09-21T10:01:49.330 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3377000000000001e-05 2024-09-21T10:01:49.330 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.330 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594997+0000", 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0078000000000001e-05 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.645551+0000", 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.050553455 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.645683+0000", 2024-09-21T10:01:49.331 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132762 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.645832+0000", 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000148992 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651118+0000", 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0052851829999999997 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.332 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651160+0000", 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2794999999999999e-05 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657187+0000", 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0060262620000000001 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657217+0000", 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0439000000000001e-05 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.333 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657248+0000", 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1368999999999997e-05 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90868 6.a 6:506b43e1:::benchmark_data_smithi017_149547_object90867:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.596312+0000", 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 250.71532075100001, 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.066231537, 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.334 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90868 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596312+0000", 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596312+0000", 2024-09-21T10:01:49.335 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596311+0000", 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596386+0000", 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4889999999999996e-05 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.336 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596397+0000", 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1117999999999999e-05 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596402+0000", 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0420000000000002e-06 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596441+0000", 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8637000000000003e-05 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.337 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596447+0000", 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6919999999999996e-06 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.648577+0000", 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0521300920000001 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.648714+0000", 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013687000000000001 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.338 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.648850+0000", 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013567900000000001 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653519+0000", 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0046685540000000001 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653542+0000", 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3515000000000001e-05 2024-09-21T10:01:49.339 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.662408+0000", 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0088662189999999998 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.662484+0000", 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5351000000000004e-05 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.340 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.662543+0000", 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9657000000000002e-05 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90889 6.a 6:58dc9b35:::benchmark_data_smithi017_149547_object90888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.649533+0000", 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 250.66209943699999, 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0149460320000001, 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.341 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90889 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649533+0000", 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649533+0000", 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.342 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649532+0000", 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649557+0000", 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.5724e-05 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649570+0000", 2024-09-21T10:01:49.343 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3212e-05 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649576+0000", 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.993e-06 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649633+0000", 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6147999999999998e-05 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:01:49.344 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649640+0000", 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2370000000000003e-06 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651180+0000", 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.00154027 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651306+0000", 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012564299999999999 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.345 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651449+0000", 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000143218 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653646+0000", 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021974989999999999 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653979+0000", 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00033292499999999997 2024-09-21T10:01:49.346 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.664396+0000", 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010416262000000001 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.664443+0000", 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7518999999999997e-05 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.664479+0000", 2024-09-21T10:01:49.347 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5933000000000003e-05 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45173.0:90890 6.3 e1101/1086 6:c2ec370d:::benchmark_data_smithi017_149547_object90889:head v 1101'5132, mlcod=1100'5130)", 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:39.548490+0000", 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 249.763142303, 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090342275, 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.348 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.548490+0000", 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.548490+0000", 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.548490+0000", 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.349 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.548515+0000", 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.5077000000000001e-05 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.633977+0000", 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.085461703999999999 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.633985+0000", 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3410000000000003e-06 2024-09-21T10:01:49.350 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.634040+0000", 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4664999999999999e-05 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.634061+0000", 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1041999999999999e-05 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.638797+0000", 2024-09-21T10:01:49.351 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0047363680000000003 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.638832+0000", 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5349000000000003e-05 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175289 6.a 6:5b89aa44:::benchmark_data_smithi017_149547_object56210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.472307+0000", 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 160.839325785, 2024-09-21T10:01:49.352 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.097797054999999994, 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175289 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472307+0000", 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.353 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472307+0000", 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472305+0000", 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472312+0000", 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8179999999999998e-06 2024-09-21T10:01:49.354 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472323+0000", 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1287999999999999e-05 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472328+0000", 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3959999999999999e-06 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472359+0000", 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1241999999999997e-05 2024-09-21T10:01:49.355 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472433+0000", 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3633999999999993e-05 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.472498+0000", 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5798999999999998e-05 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.479754+0000", 2024-09-21T10:01:49.356 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0072554810000000003 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.479766+0000", 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2462e-05 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570021+0000", 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090255143999999995 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.357 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570060+0000", 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8509e-05 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570104+0000", 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3881000000000002e-05 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175291 6.a 6:57b1c613:::benchmark_data_smithi017_149547_object56212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.473105+0000", 2024-09-21T10:01:49.358 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 160.83852778400001, 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.097093223000000006, 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175291 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473105+0000", 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.359 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473105+0000", 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473104+0000", 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473108+0000", 2024-09-21T10:01:49.360 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4000000000000002e-06 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473120+0000", 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1508000000000001e-05 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.473125+0000", 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3129999999999998e-06 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.361 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.475454+0000", 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002329224 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.475579+0000", 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000124721 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.475713+0000", 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000133452 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.362 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.479795+0000", 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0040822089999999998 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.479840+0000", 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4749999999999997e-05 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570143+0000", 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090303226 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.363 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570175+0000", 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2264999999999997e-05 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.570198+0000", 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2884e-05 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.364 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175300 6.a 6:5ad5117d:::benchmark_data_smithi017_149547_object56221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.480743+0000", 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 160.83088985800001, 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.104015855, 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175300 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.365 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480743+0000", 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480743+0000", 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480741+0000", 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.366 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480747+0000", 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.913e-06 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480758+0000", 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0512e-05 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480763+0000", 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7430000000000002e-06 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.367 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480798+0000", 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5318000000000003e-05 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480906+0000", 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010820899999999999 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.480981+0000", 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5024000000000005e-05 2024-09-21T10:01:49.368 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.484208+0000", 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032268499999999999 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.484654+0000", 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00044641 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584669+0000", 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.100014744 2024-09-21T10:01:49.369 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584717+0000", 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7843000000000003e-05 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584759+0000", 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1505000000000003e-05 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.370 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.371 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.371 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:175307 6.a 6:54170271:::benchmark_data_smithi017_149547_object56228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:01:49.371 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:08.487486+0000", 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 160.82414611999999, 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.097409080999999995, 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 175307 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.372 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487486+0000", 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487486+0000", 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487485+0000", 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:49.373 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487494+0000", 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6320000000000003e-06 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487510+0000", 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5277e-05 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.374 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487516+0000", 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0020000000000003e-06 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487567+0000", 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1190000000000003e-05 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487696+0000", 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012929400000000001 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.375 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.487831+0000", 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013468900000000001 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.490741+0000", 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029106179999999998 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.491449+0000", 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00070706499999999999 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.376 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584809+0000", 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.093360925999999997 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584850+0000", 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0460000000000002e-05 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:08.584895+0000", 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5578999999999998e-05 2024-09-21T10:01:49.377 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52999 7.17 7:e8d33f0f:::benchmark_data_smithi017_180590_object52998:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345686+0000", 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 22.965946504000001, 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0320069839999999, 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.378 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52999 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345686+0000", 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345686+0000", 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.379 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345684+0000", 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346356+0000", 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000671431 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346375+0000", 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9690999999999998e-05 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.380 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346383+0000", 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0290000000000005e-06 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.411662+0000", 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.065278180000000005 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.411668+0000", 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7700000000000004e-06 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.381 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.377605+0000", 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96593649999999998 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.377693+0000", 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8140999999999995e-05 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.382 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:53013 7.17 7:e95e05b0:::benchmark_data_smithi017_180590_object53012:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.497938+0000", 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 22.813694827999999, 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.88975737499999996, 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 53013 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.383 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.497938+0000", 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.497938+0000", 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.497936+0000", 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.384 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.497993+0000", 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7674999999999999e-05 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.498010+0000", 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6167999999999998e-05 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.498016+0000", 2024-09-21T10:01:49.385 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3160000000000002e-06 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.498053+0000", 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7552999999999998e-05 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.498061+0000", 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.7780000000000007e-06 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.386 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.377719+0000", 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.87965766899999998 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.377832+0000", 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011304899999999999 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.377988+0000", 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015580800000000001 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.387 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.381672+0000", 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0036843370000000002 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.384644+0000", 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029719669999999998 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.387604+0000", 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029601950000000001 2024-09-21T10:01:49.388 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.387656+0000", 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1749999999999997e-05 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.387695+0000", 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9076999999999999e-05 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.389 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:54896 7.c 7:3723cbca:::benchmark_data_smithi017_180590_object54895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1149)", 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:28.481867+0000", 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 20.829765556000002, 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.101037537, 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 54896 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.390 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.481867+0000", 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.481867+0000", 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.481866+0000", 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.391 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.481941+0000", 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5532999999999997e-05 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.481958+0000", 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7224999999999999e-05 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.481965+0000", 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.348e-06 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.392 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.482032+0000", 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7346000000000002e-05 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.482171+0000", 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013932500000000001 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.482300+0000", 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000128794 2024-09-21T10:01:49.393 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484344+0000", 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020438040000000002 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484589+0000", 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00024507499999999998 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.582557+0000", 2024-09-21T10:01:49.394 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.097967974999999999 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.582690+0000", 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013323400000000001 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.582905+0000", 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00021419200000000001 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.395 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:54898 7.c 7:3ea68770:::benchmark_data_smithi017_180590_object54897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1149)", 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:28.484613+0000", 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 20.827019712999999, 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.098498613999999998, 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 54898 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.396 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484613+0000", 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484613+0000", 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484611+0000", 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:49.397 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484655+0000", 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3430999999999998e-05 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484669+0000", 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4608e-05 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484675+0000", 2024-09-21T10:01:49.398 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9909999999999999e-06 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484712+0000", 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6189000000000002e-05 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484853+0000", 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014167499999999999 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:01:49.399 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484997+0000", 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000143752 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.489115+0000", 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0041184389999999998 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.489156+0000", 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1035000000000003e-05 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.400 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.582951+0000", 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.093794894000000004 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.582992+0000", 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0608000000000002e-05 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.583111+0000", 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000119496 2024-09-21T10:01:49.401 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:54899 7.7 7:e541492a:::benchmark_data_smithi017_180590_object54898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1149)", 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:28.484685+0000", 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 20.826947978, 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.092490488999999995, 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.402 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 54899 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484685+0000", 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484685+0000", 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.403 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484684+0000", 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484713+0000", 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9465e-05 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484721+0000", 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1740000000000003e-06 2024-09-21T10:01:49.404 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484725+0000", 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0060000000000003e-06 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484766+0000", 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1353999999999997e-05 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484850+0000", 2024-09-21T10:01:49.405 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3832999999999995e-05 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484953+0000", 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000103568 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.486014+0000", 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0010605479999999999 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.406 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.576190+0000", 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090176150999999996 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.577087+0000", 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00089690899999999999 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.577136+0000", 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9311000000000003e-05 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.407 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.577175+0000", 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8726000000000002e-05 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:54903 7.c 7:39151915:::benchmark_data_smithi017_180590_object54902:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1149)", 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:28.484806+0000", 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 20.82682621, 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.103311844, 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.408 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 54903 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484806+0000", 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.409 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484806+0000", 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484806+0000", 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484866+0000", 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0441e-05 2024-09-21T10:01:49.410 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484881+0000", 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5003e-05 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.484885+0000", 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4209999999999997e-06 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.486537+0000", 2024-09-21T10:01:49.411 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001651439 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.486670+0000", 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000133258 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.486820+0000", 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000149946 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.412 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.489152+0000", 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0023315129999999999 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.489160+0000", 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5080000000000005e-06 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.587907+0000", 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.098746588999999996 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.413 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.587966+0000", 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9599999999999999e-05 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.588118+0000", 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015183699999999999 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.414 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:54906 7.c 7:38e7b5a4:::benchmark_data_smithi017_180590_object54905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1149)", 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:28.485846+0000", 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 20.825786707999999, 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.102413885, 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 54906 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.485846+0000", 2024-09-21T10:01:49.415 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.485846+0000", 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.485845+0000", 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.416 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.485907+0000", 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1853999999999994e-05 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.485921+0000", 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4127e-05 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.485926+0000", 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5960000000000003e-06 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.417 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.488142+0000", 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0022159359999999999 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.488246+0000", 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000103484 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.488324+0000", 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.7807999999999998e-05 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.418 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.489252+0000", 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00092840000000000002 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.489401+0000", 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014863699999999999 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.588163+0000", 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.098761858999999994 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.419 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.588227+0000", 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4764000000000005e-05 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:28.588260+0000", 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2364999999999999e-05 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:56385 7.7 7:e28ffea4:::benchmark_data_smithi017_180590_object56384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1149)", 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:29.402981+0000", 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 19.908651372000001, 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090206374000000006, 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 56385 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.402981+0000", 2024-09-21T10:01:49.421 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.402981+0000", 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.402980+0000", 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.422 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.403122+0000", 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000141917 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.403129+0000", 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.314e-06 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.403132+0000", 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0850000000000001e-06 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.423 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.403165+0000", 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3333000000000001e-05 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.403236+0000", 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1160000000000003e-05 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.403308+0000", 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1552999999999997e-05 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.424 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.404405+0000", 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001097156 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.406051+0000", 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001646408 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.493091+0000", 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.087039952000000004 2024-09-21T10:01:49.425 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.426 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.426 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.426 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.493145+0000", 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3341999999999999e-05 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.493188+0000", 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2939999999999999e-05 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.427 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:56387 7.c 7:31d2884d:::benchmark_data_smithi017_180590_object56386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1149)", 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:29.404656+0000", 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 19.906976601, 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.093469423999999995, 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 56387 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.428 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.404656+0000", 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.404656+0000", 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.404655+0000", 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:01:49.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.404774+0000", 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000119009 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.404786+0000", 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2167999999999999e-05 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.430 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.404792+0000", 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8000000000000004e-06 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.404831+0000", 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9208999999999997e-05 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.404945+0000", 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011429000000000001 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.431 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.405034+0000", 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8329000000000001e-05 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.406691+0000", 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0016573169999999999 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.406848+0000", 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015717199999999999 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.432 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.498048+0000", 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.091199752999999995 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.498092+0000", 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4254e-05 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.498125+0000", 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3265000000000001e-05 2024-09-21T10:01:49.433 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:56396 7.17 7:e959716d:::benchmark_data_smithi017_180590_object56395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1149)", 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:29.406935+0000", 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 19.904697050999999, 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090851712000000001, 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 56396 2024-09-21T10:01:49.434 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.406935+0000", 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.406935+0000", 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:01:49.435 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.406935+0000", 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.406998+0000", 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2785000000000003e-05 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.407008+0000", 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0016e-05 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.407013+0000", 2024-09-21T10:01:49.436 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0690000000000001e-06 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.407043+0000", 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0156000000000001e-05 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.407144+0000", 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010067499999999999 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.407311+0000", 2024-09-21T10:01:49.437 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016777200000000001 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.494039+0000", 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.086727423999999997 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.494066+0000", 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7362999999999999e-05 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:01:49.438 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.497703+0000", 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00363708 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.497750+0000", 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7049000000000003e-05 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:29.497787+0000", 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6924000000000001e-05 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.439 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.440 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.440 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:01:49.440 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:01:49.440 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:01:49.461 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:49.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:49.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:49.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:49.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:49.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:49.737 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:49 smithi134 ceph-mon[90085]: pgmap v1479: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 8.8 KiB/s rd, 28 MiB/s wr, 450 op/s; 14951/201969 objects degraded (7.403%); 3785/201969 objects misplaced (1.874%); 13 MiB/s, 210 objects/s recovering 2024-09-21T10:01:49.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:49.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:49.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:49.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:01:49.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:49 smithi017 ceph-mon[102913]: pgmap v1479: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 8.8 KiB/s rd, 28 MiB/s wr, 450 op/s; 14951/201969 objects degraded (7.403%); 3785/201969 objects misplaced (1.874%); 13 MiB/s, 210 objects/s recovering 2024-09-21T10:01:49.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:49 smithi017 ceph-mon[105151]: pgmap v1479: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 8.8 KiB/s rd, 28 MiB/s wr, 450 op/s; 14951/201969 objects degraded (7.403%); 3785/201969 objects misplaced (1.874%); 13 MiB/s, 210 objects/s recovering 2024-09-21T10:01:49.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 67400 67384 84.2147 0 - 0.0100963 2024-09-21T10:01:49.942 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:49.966 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:49.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:50.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:50.096 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:50.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:50.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:50.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:50.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:50.273 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:01:50.326 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:50.349 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:50.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:50.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:50.485 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:50.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:01:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:01:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:50.573 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:50.587 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:50.620 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:50.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:50.722 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:50.757 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:50.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:50.799 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:50 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 14951/202104 objects degraded (7.398%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:50.799 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:50 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:50.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:50.888 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:50.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 67897 67881 83.1726 4.4375 0.00543899 0.0120177 2024-09-21T10:01:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:50 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 14951/202104 objects degraded (7.398%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:50 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:50.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:50 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 14951/202104 objects degraded (7.398%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:50.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:50 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:50.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:51.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:51.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:51.134 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:51.164 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:51.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:51.266 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:51.293 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:51.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:51.357 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:51.374 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:51.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:51.451 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:51.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:51.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:51 smithi017 ceph-mon[102913]: pgmap v1480: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 7.0 MiB/s wr, 111 op/s; 14951/202104 objects degraded (7.398%); 2888/202104 objects misplaced (1.429%); 13 MiB/s, 215 objects/s recovering 2024-09-21T10:01:51.544 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:51 smithi017 ceph-mon[105151]: pgmap v1480: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 7.0 MiB/s wr, 111 op/s; 14951/202104 objects degraded (7.398%); 2888/202104 objects misplaced (1.429%); 13 MiB/s, 215 objects/s recovering 2024-09-21T10:01:51.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:51.585 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:51.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:51.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:51.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:51.801 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:51 smithi134 ceph-mon[90085]: pgmap v1480: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 7.0 MiB/s wr, 111 op/s; 14951/202104 objects degraded (7.398%); 2888/202104 objects misplaced (1.429%); 13 MiB/s, 215 objects/s recovering 2024-09-21T10:01:51.827 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:51.862 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:51.908 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 69107 69091 83.0274 75.625 0.00706252 0.0120357 2024-09-21T10:01:51.964 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:51.994 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:52.095 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:52.125 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:52.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:52.227 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:52.252 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:52.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:52.354 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:52.384 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:52.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:52.486 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:52.515 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:52.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:52.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:52.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:52.763 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:52.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:52.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:52.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:52.909 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 70244 70228 82.8015 71.0625 0.0112652 0.0120718 2024-09-21T10:01:52.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:53.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:53.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:53.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:53.186 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:53.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:53.326 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:53.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:53.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:53.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:53.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:53.546 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2024-09-21T10:01:53.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:53.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:53.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:53.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:53.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:53.756 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:53 smithi017 ceph-mon[102913]: pgmap v1481: 109 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.8 MiB/s wr, 93 op/s; 19114/203484 objects degraded (9.393%); 5190/203484 objects misplaced (2.551%); 24 MiB/s, 376 objects/s recovering 2024-09-21T10:01:53.757 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:53 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3989287698' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:01:53.757 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:53 smithi017 ceph-mon[105151]: pgmap v1481: 109 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.8 MiB/s wr, 93 op/s; 19114/203484 objects degraded (9.393%); 5190/203484 objects misplaced (2.551%); 24 MiB/s, 376 objects/s recovering 2024-09-21T10:01:53.757 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:53 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3989287698' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:01:53.844 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:53.875 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:53.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:53.909 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 71424 71408 82.6338 73.75 0.00450402 0.0120962 2024-09-21T10:01:53.920 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:53 smithi134 ceph-mon[90085]: pgmap v1481: 109 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 98 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.8 MiB/s wr, 93 op/s; 19114/203484 objects degraded (9.393%); 5190/203484 objects misplaced (2.551%); 24 MiB/s, 376 objects/s recovering 2024-09-21T10:01:53.920 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:53 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3989287698' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:01:53.977 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:53.998 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:53.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:54.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:54.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:54.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:54.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:54.264 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:54.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:54.366 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:54.398 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:54.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:54.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:54.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:54.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:54.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:54.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:54.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:54 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3989287698' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:01:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:54 smithi017 ceph-mon[102913]: osdmap e1159: 8 total, 8 up, 7 in 2024-09-21T10:01:54.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:54 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3989287698' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:01:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:54 smithi017 ceph-mon[105151]: osdmap e1159: 8 total, 8 up, 7 in 2024-09-21T10:01:54.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:54.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:54.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:54.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:54.909 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 72505 72489 82.3597 67.5625 0.00462156 0.0121204 2024-09-21T10:01:54.937 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:54.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:54.951 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:54 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3989287698' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:01:54.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:54 smithi134 ceph-mon[90085]: osdmap e1159: 8 total, 8 up, 7 in 2024-09-21T10:01:55.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:55.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:55.177 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:55.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:01:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:01:54] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-09-21T10:01:55.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:55.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:55.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:55.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:55.431 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:55.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.456 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:55.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:55.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:55.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:55.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:55.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:55.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:55.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:55.827 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:55.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:55.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:55.888 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:55 smithi134 ceph-mon[90085]: pgmap v1483: 109 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 98 active+clean; 4.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 12 MiB/s wr, 184 op/s; 19114/206313 objects degraded (9.265%); 4318/206313 objects misplaced (2.093%); 29 MiB/s, 457 objects/s recovering 2024-09-21T10:01:55.889 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:55 smithi134 ceph-mon[90085]: osdmap e1160: 8 total, 8 up, 7 in 2024-09-21T10:01:55.889 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:55.889 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:55.904 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:01:55.904 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:01:55.905 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 0 2024-09-21T10:01:55.905 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:01:55.909 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 73105 73089 81.5586 37.5 0.00417533 0.0122277 2024-09-21T10:01:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:55 smithi017 ceph-mon[102913]: pgmap v1483: 109 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 98 active+clean; 4.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 12 MiB/s wr, 184 op/s; 19114/206313 objects degraded (9.265%); 4318/206313 objects misplaced (2.093%); 29 MiB/s, 457 objects/s recovering 2024-09-21T10:01:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:55 smithi017 ceph-mon[102913]: osdmap e1160: 8 total, 8 up, 7 in 2024-09-21T10:01:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:55 smithi017 ceph-mon[105151]: pgmap v1483: 109 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 98 active+clean; 4.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 12 MiB/s wr, 184 op/s; 19114/206313 objects degraded (9.265%); 4318/206313 objects misplaced (2.093%); 29 MiB/s, 457 objects/s recovering 2024-09-21T10:01:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:55 smithi017 ceph-mon[105151]: osdmap e1160: 8 total, 8 up, 7 in 2024-09-21T10:01:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:01:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:55.959 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:55.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:55.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:56.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:56.108 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:56.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:56.210 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:56.232 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:56.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:56.335 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:56.357 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:56.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:56.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:56.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:56.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:56.583 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:56.602 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:01:56.611 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:56.713 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:56.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:56.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:56.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:56.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.884 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:56.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:56.909 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 73375 73359 80.4239 16.875 0.00853908 0.0124282 2024-09-21T10:01:56.932 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:56 smithi017 ceph-mon[102913]: osdmap e1161: 8 total, 8 up, 7 in 2024-09-21T10:01:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:56 smithi017 ceph-mon[105151]: osdmap e1161: 8 total, 8 up, 7 in 2024-09-21T10:01:56.985 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:56.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:56 smithi134 ceph-mon[90085]: osdmap e1161: 8 total, 8 up, 7 in 2024-09-21T10:01:57.017 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:57.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:57.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:57.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:57.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:57.258 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:57.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:57.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:57.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:57.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:57.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:57.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:57.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:57.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:57.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:57.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:57.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:57 smithi134 ceph-mon[90085]: pgmap v1486: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 99 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 49 MiB/s wr, 787 op/s; 18431/216291 objects degraded (8.521%); 4426/216291 objects misplaced (2.046%); 35 MiB/s, 556 objects/s recovering 2024-09-21T10:01:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:57 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 18431/216291 objects degraded (8.521%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:57 smithi134 ceph-mon[90085]: osdmap e1162: 8 total, 8 up, 7 in 2024-09-21T10:01:57.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:57.844 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:57.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:57.909 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 74493 74477 80.2419 69.875 0.00824029 0.0124564 2024-09-21T10:01:57.946 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:57 smithi017 ceph-mon[102913]: pgmap v1486: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 99 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 49 MiB/s wr, 787 op/s; 18431/216291 objects degraded (8.521%); 4426/216291 objects misplaced (2.046%); 35 MiB/s, 556 objects/s recovering 2024-09-21T10:01:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:57 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 18431/216291 objects degraded (8.521%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:57 smithi017 ceph-mon[102913]: osdmap e1162: 8 total, 8 up, 7 in 2024-09-21T10:01:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:57 smithi017 ceph-mon[105151]: pgmap v1486: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 99 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 49 MiB/s wr, 787 op/s; 18431/216291 objects degraded (8.521%); 4426/216291 objects misplaced (2.046%); 35 MiB/s, 556 objects/s recovering 2024-09-21T10:01:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:57 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 18431/216291 objects degraded (8.521%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:57 smithi017 ceph-mon[105151]: osdmap e1162: 8 total, 8 up, 7 in 2024-09-21T10:01:57.976 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:58.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:58.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:58.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:58.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:58.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:58.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:58.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:58.486 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:58.521 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:58.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:58.622 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:58.656 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:58.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:58.757 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:58.784 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:58.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:58.885 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:58.909 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 75575 75559 80.028 67.625 0.0124711 0.0124905 2024-09-21T10:01:58.924 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:58.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:58 smithi017 ceph-mon[102913]: 7.5 scrub starts 2024-09-21T10:01:58.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:58 smithi017 ceph-mon[105151]: 7.5 scrub starts 2024-09-21T10:01:58.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:58.933 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:58 smithi134 ceph-mon[90085]: 7.5 scrub starts 2024-09-21T10:01:59.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:59.056 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:59.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:59.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:59.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:59.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:59.278 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:01:59.278 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:01:59.279 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:01:59.279 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:01:59.279 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:01:59.297 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:59.322 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:59.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:59.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:59.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:59.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:59.566 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:59.599 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:59.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:59.700 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:59.737 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:01:59.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:59.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:01:59.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:59.910 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:01:59.908797+0000 min lat: 0.0012027 max lat: 11.1239 avg lat: 0.0125076 2024-09-21T10:01:59.910 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:01:59.910 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 76745 76729 79.9123 73.125 0.00770597 0.0125076 2024-09-21T10:01:59.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:59 smithi017 ceph-mon[102913]: pgmap v1488: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 99 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 18431/217878 objects degraded (8.459%); 4426/217878 objects misplaced (2.031%); 20 MiB/s, 315 objects/s recovering 2024-09-21T10:01:59.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:59 smithi017 ceph-mon[102913]: 7.5 scrub ok 2024-09-21T10:01:59.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:01:59 smithi017 ceph-mon[102913]: 7.15 scrub starts 2024-09-21T10:01:59.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:59 smithi017 ceph-mon[105151]: pgmap v1488: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 99 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 18431/217878 objects degraded (8.459%); 4426/217878 objects misplaced (2.031%); 20 MiB/s, 315 objects/s recovering 2024-09-21T10:01:59.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:59 smithi017 ceph-mon[105151]: 7.5 scrub ok 2024-09-21T10:01:59.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:01:59 smithi017 ceph-mon[105151]: 7.15 scrub starts 2024-09-21T10:01:59.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:59 smithi134 ceph-mon[90085]: pgmap v1488: 109 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 99 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 18431/217878 objects degraded (8.459%); 4426/217878 objects misplaced (2.031%); 20 MiB/s, 315 objects/s recovering 2024-09-21T10:01:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:59 smithi134 ceph-mon[90085]: 7.5 scrub ok 2024-09-21T10:01:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:01:59 smithi134 ceph-mon[90085]: 7.15 scrub starts 2024-09-21T10:02:00.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:00.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:00.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:00.109 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:02:00.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:00.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:00.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:00.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:00.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:00.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:00.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:00.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:00.455 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:02:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:02:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:02:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:00.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:00.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:00.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:00.661 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:00.693 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:00.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:00.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:00.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:00.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:00.910 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 15 77945 77930 79.8327 75.0625 0.0077507 0.0125205 2024-09-21T10:02:00.934 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:00 smithi134 ceph-mon[90085]: 7.15 scrub ok 2024-09-21T10:02:00.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:00 smithi017 ceph-mon[102913]: 7.15 scrub ok 2024-09-21T10:02:00.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:00 smithi017 ceph-mon[105151]: 7.15 scrub ok 2024-09-21T10:02:00.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:00.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:01.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.107 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:01.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:01.208 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:01.240 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:01.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:01.342 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:01.380 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:01.482 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:01.514 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:01.616 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:01.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:02:01.643 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:01.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:01 smithi134 ceph-mon[90085]: pgmap v1489: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 4.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 815 op/s; 16417/220998 objects degraded (7.429%); 5766/220998 objects misplaced (2.609%); 13 MiB/s, 207 objects/s recovering 2024-09-21T10:02:01.745 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:01.771 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:01.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:01.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:01.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:01 smithi017 ceph-mon[102913]: pgmap v1489: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 4.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 815 op/s; 16417/220998 objects degraded (7.429%); 5766/220998 objects misplaced (2.609%); 13 MiB/s, 207 objects/s recovering 2024-09-21T10:02:01.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:01 smithi017 ceph-mon[105151]: pgmap v1489: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 4.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 815 op/s; 16417/220998 objects degraded (7.429%); 5766/220998 objects misplaced (2.609%); 13 MiB/s, 207 objects/s recovering 2024-09-21T10:02:01.912 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 79182 79166 79.7877 77.25 0.0243893 0.0125273 2024-09-21T10:02:01.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:02.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:02.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:02.174 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:02.208 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:02.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:02.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:02.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:02.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:02.479 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:02:02.479 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:02:02.479 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:02:02.479 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:02:02.479 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.479 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60910 6.8 6:115de437:::benchmark_data_smithi017_149547_object60909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:02:02.479 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422268+0000", 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 296.052333938, 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0318208280000001, 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60910 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.480 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422268+0000", 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422268+0000", 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422265+0000", 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.481 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422345+0000", 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9598000000000006e-05 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422368+0000", 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3481000000000001e-05 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422376+0000", 2024-09-21T10:02:02.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5209999999999997e-06 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522605+0000", 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.1002286 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522613+0000", 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7090000000000008e-06 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440150+0000", 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.91753706199999996 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440785+0000", 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00063470900000000001 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.454016+0000", 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013230936 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.454089+0000", 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2690999999999999e-05 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60912 6.3 6:c077dba9:::benchmark_data_smithi017_149547_object60911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422388+0000", 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 296.05221340899999, 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0068617500000001, 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.485 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60912 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422388+0000", 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.486 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422388+0000", 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422388+0000", 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422462+0000", 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3874000000000005e-05 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.487 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422472+0000", 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.91e-06 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422476+0000", 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3610000000000003e-06 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.528785+0000", 2024-09-21T10:02:02.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10630861 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.528795+0000", 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0499000000000001e-05 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.421255+0000", 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.89245958400000003 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.422521+0000", 2024-09-21T10:02:02.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0012658389999999999 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.429183+0000", 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0066627240000000001 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.429250+0000", 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6773000000000006e-05 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.490 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60914 6.18 6:1ac7c059:::benchmark_data_smithi017_149547_object60913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422482+0000", 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 296.05211928900002, 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.044582076, 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60914 2024-09-21T10:02:02.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422482+0000", 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422482+0000", 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422482+0000", 2024-09-21T10:02:02.493 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.493 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.493 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.493 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.493 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422558+0000", 2024-09-21T10:02:02.494 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5767999999999997e-05 2024-09-21T10:02:02.494 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.494 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.494 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.494 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422567+0000", 2024-09-21T10:02:02.494 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6039999999999999e-06 2024-09-21T10:02:02.494 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.494 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422571+0000", 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9609999999999999e-06 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522630+0000", 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10005905700000001 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522635+0000", 2024-09-21T10:02:02.495 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6129999999999997e-06 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.456028+0000", 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.93339256800000003 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.456812+0000", 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00078471200000000002 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.496 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467017+0000", 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.01020501 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467065+0000", 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7205e-05 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60918 6.18 6:1b86a79a:::benchmark_data_smithi017_149547_object60917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:02:02.497 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422576+0000", 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 296.052025931, 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0446003159999999, 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60918 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422576+0000", 2024-09-21T10:02:02.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422576+0000", 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422575+0000", 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422664+0000", 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8214999999999995e-05 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422673+0000", 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5880000000000004e-06 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422677+0000", 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9360000000000001e-06 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.500 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522651+0000", 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.099973353000000001 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522655+0000", 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3050000000000003e-06 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.456857+0000", 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.93420210199999998 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.456995+0000", 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013829599999999999 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.457146+0000", 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015081799999999999 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.458255+0000", 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001108574 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.466295+0000", 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0080404480000000004 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467091+0000", 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00079628900000000005 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467119+0000", 2024-09-21T10:02:02.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7991999999999999e-05 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467176+0000", 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6798000000000001e-05 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60923 6.15 6:a9be3957:::benchmark_data_smithi017_149547_object60922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422681+0000", 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 296.05192030400002, 2024-09-21T10:02:02.504 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0276440060000001, 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60923 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422681+0000", 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.505 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422681+0000", 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422681+0000", 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422755+0000", 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3478000000000004e-05 2024-09-21T10:02:02.506 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422764+0000", 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.7510000000000003e-06 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422768+0000", 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8070000000000001e-06 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537758+0000", 2024-09-21T10:02:02.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.114989657 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537768+0000", 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8940000000000005e-06 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440664+0000", 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.90289677300000004 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:02:02.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440778+0000", 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011325299999999999 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440930+0000", 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015233299999999999 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.441879+0000", 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00094917099999999996 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.442913+0000", 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0010342750000000001 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.450243+0000", 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0073293229999999996 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.450291+0000", 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7846999999999998e-05 2024-09-21T10:02:02.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.450325+0000", 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4833999999999999e-05 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90914 6.12 6:4f1f32ba:::benchmark_data_smithi017_149547_object90913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:39.572975+0000", 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 262.90162718599998, 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0214511260000001, 2024-09-21T10:02:02.511 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90914 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.572975+0000", 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.512 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.572975+0000", 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.572967+0000", 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999928 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573027+0000", 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9933999999999999e-05 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573040+0000", 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2594e-05 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573045+0000", 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5669999999999996e-06 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573086+0000", 2024-09-21T10:02:02.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1261999999999999e-05 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573092+0000", 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0159999999999997e-06 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.547378+0000", 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97428627099999998 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.547494+0000", 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000116019 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.547596+0000", 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000102067 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.552155+0000", 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0045592899999999997 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.561091+0000", 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0089351870000000007 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.594334+0000", 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.033243237000000002 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.594385+0000", 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0929e-05 2024-09-21T10:02:02.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.594426+0000", 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0877999999999997e-05 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90932 6.12 6:4ca4244b:::benchmark_data_smithi017_149547_object90931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:39.642171+0000", 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 262.83243123900002, 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95359630500000003, 2024-09-21T10:02:02.518 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90932 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642171+0000", 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642171+0000", 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642169+0000", 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642203+0000", 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3807999999999999e-05 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642214+0000", 2024-09-21T10:02:02.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0739e-05 2024-09-21T10:02:02.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:02.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:02.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642219+0000", 2024-09-21T10:02:02.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0710000000000002e-06 2024-09-21T10:02:02.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642299+0000", 2024-09-21T10:02:02.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0338000000000002e-05 2024-09-21T10:02:02.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642308+0000", 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.6410000000000008e-06 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.548566+0000", 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.90625812500000003 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.548644+0000", 2024-09-21T10:02:02.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8574999999999999e-05 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.548730+0000", 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5718999999999997e-05 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.552167+0000", 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0034371950000000001 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.561123+0000", 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0089554820000000007 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595687+0000", 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.034564267000000003 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595733+0000", 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6477999999999998e-05 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595767+0000", 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3416e-05 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90934 6.12 6:4a514f78:::benchmark_data_smithi017_149547_object90933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:39.643035+0000", 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 262.83156652899999, 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95286870599999995, 2024-09-21T10:02:02.527 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90934 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643035+0000", 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643035+0000", 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643034+0000", 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643085+0000", 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0504000000000003e-05 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643098+0000", 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3852999999999999e-05 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643104+0000", 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5509999999999999e-06 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643148+0000", 2024-09-21T10:02:02.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4313000000000001e-05 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643157+0000", 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3610000000000001e-06 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.549525+0000", 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.90636815299999995 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:02:02.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.549638+0000", 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011333 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.549788+0000", 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000149856 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.553948+0000", 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0041600489999999999 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.561172+0000", 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0072234380000000004 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595814+0000", 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.034642138000000003 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595862+0000", 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7945999999999998e-05 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595904+0000", 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2296999999999998e-05 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:215116 6.3 6:c3180f7c:::benchmark_data_smithi017_149547_object96037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1123)", 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:52.411445+0000", 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 130.06315630500001, 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.056761965, 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.534 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215116 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411445+0000", 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411445+0000", 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411444+0000", 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411450+0000", 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.959e-06 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411459+0000", 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8279999999999992e-06 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411464+0000", 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8319999999999996e-06 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.414029+0000", 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0025651929999999999 2024-09-21T10:02:02.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.414038+0000", 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.0250000000000008e-06 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.434973+0000", 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0209354340000001 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:02:02.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.435056+0000", 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.2681000000000003e-05 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.435461+0000", 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00040495199999999998 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.443579+0000", 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0081180230000000003 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468046+0000", 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.024467073999999998 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468107+0000", 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0485000000000001e-05 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468170+0000", 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3707999999999994e-05 2024-09-21T10:02:02.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468207+0000", 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6993999999999999e-05 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:215170 6.3 6:c71c1ba7:::benchmark_data_smithi017_149547_object96091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1124)", 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:52.469683+0000", 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 130.00491828200001, 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.998898701, 2024-09-21T10:02:02.541 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215170 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469683+0000", 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469683+0000", 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469682+0000", 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469687+0000", 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7490000000000004e-06 2024-09-21T10:02:02.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469704+0000", 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6920000000000001e-05 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469712+0000", 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5580000000000003e-06 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469753+0000", 2024-09-21T10:02:02.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0760999999999998e-05 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469761+0000", 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0930000000000003e-06 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.437472+0000", 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96771112500000001 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.437540+0000", 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8157999999999994e-05 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.437787+0000", 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00024686800000000002 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.447907+0000", 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.01012044 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468254+0000", 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.020347054999999999 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468535+0000", 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00028029800000000001 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468562+0000", 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7610000000000002e-05 2024-09-21T10:02:02.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468582+0000", 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9896000000000001e-05 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:215213 6.3 6:c300d469:::benchmark_data_smithi017_149547_object96134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1124)", 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:52.526237+0000", 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 129.94836502499999, 2024-09-21T10:02:02.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.94241559200000002, 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215213 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526237+0000", 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526237+0000", 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526235+0000", 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526241+0000", 2024-09-21T10:02:02.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1870000000000002e-06 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526252+0000", 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0171e-05 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526257+0000", 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2390000000000002e-06 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526298+0000", 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0932e-05 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526306+0000", 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1620000000000008e-06 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.439549+0000", 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.91324313599999996 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.439617+0000", 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8078999999999997e-05 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.439893+0000", 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00027582399999999998 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.447915+0000", 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0080218749999999995 2024-09-21T10:02:02.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468412+0000", 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.020497517999999999 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468609+0000", 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019681599999999999 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:02:02.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468633+0000", 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3538000000000001e-05 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468652+0000", 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9638000000000001e-05 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52978 7.5 7:a7209f46:::benchmark_data_smithi017_180590_object52977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345746+0000", 2024-09-21T10:02:02.555 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 36.128856216000003, 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.014767234, 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52978 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345746+0000", 2024-09-21T10:02:02.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345746+0000", 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345743+0000", 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345789+0000", 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5448000000000003e-05 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345813+0000", 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4360999999999999e-05 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345822+0000", 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.636e-06 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407517+0000", 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061694909999999999 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407525+0000", 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1599999999999998e-06 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360423+0000", 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95289824000000001 2024-09-21T10:02:02.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360513+0000", 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.9736999999999998e-05 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52980 7.15 7:aaa003b0:::benchmark_data_smithi017_180590_object52979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345830+0000", 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 36.128771843999999, 2024-09-21T10:02:02.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.024545134, 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52980 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345830+0000", 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345830+0000", 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345829+0000", 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345883+0000", 2024-09-21T10:02:02.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3947999999999999e-05 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345891+0000", 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3499999999999999e-06 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345895+0000", 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8569999999999997e-06 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407919+0000", 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.062024397000000002 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407925+0000", 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6980000000000003e-06 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370322+0000", 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96239729100000004 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370375+0000", 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3143000000000003e-05 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52981 7.15 7:adf0cbd4:::benchmark_data_smithi017_180590_object52980:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345901+0000", 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 36.128701038999999, 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.024740942, 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.565 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52981 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345901+0000", 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345901+0000", 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345900+0000", 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345941+0000", 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1102000000000001e-05 2024-09-21T10:02:02.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345950+0000", 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8599999999999999e-06 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345955+0000", 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.707e-06 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407943+0000", 2024-09-21T10:02:02.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061987918000000003 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407947+0000", 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6040000000000003e-06 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370604+0000", 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.962656399 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370642+0000", 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7769999999999999e-05 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52985 7.5 7:a76eb8c9:::benchmark_data_smithi017_180590_object52984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345962+0000", 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 36.128639909, 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.014771651, 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.570 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52985 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345962+0000", 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345962+0000", 2024-09-21T10:02:02.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345961+0000", 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345999+0000", 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7415e-05 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.572 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346008+0000", 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.0149999999999992e-06 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346012+0000", 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1320000000000004e-06 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407966+0000", 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061954189 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407972+0000", 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9390000000000002e-06 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360688+0000", 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95271553399999998 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360733+0000", 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5875999999999999e-05 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.574 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.575 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.575 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52987 7.5 7:a671d9ea:::benchmark_data_smithi017_180590_object52986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:02.575 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346016+0000", 2024-09-21T10:02:02.575 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 36.128585411000003, 2024-09-21T10:02:02.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0147849289999999, 2024-09-21T10:02:02.575 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.575 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.575 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.577 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:02:02.577 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:02.577 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52987 2024-09-21T10:02:02.577 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346016+0000", 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346016+0000", 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346016+0000", 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346053+0000", 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6952000000000001e-05 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346061+0000", 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5599999999999994e-06 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346065+0000", 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7270000000000001e-06 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407990+0000", 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061924664999999997 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.581 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.581 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407996+0000", 2024-09-21T10:02:02.581 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7790000000000001e-06 2024-09-21T10:02:02.581 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.581 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.581 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.581 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360758+0000", 2024-09-21T10:02:02.581 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95276244799999998 2024-09-21T10:02:02.581 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360801+0000", 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3266000000000002e-05 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52995 7.15 7:ab424fbb:::benchmark_data_smithi017_180590_object52994:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346069+0000", 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 36.128532688, 2024-09-21T10:02:02.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.024628104, 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52995 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346069+0000", 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.583 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346069+0000", 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346069+0000", 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346103+0000", 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4833999999999999e-05 2024-09-21T10:02:02.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346111+0000", 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6020000000000002e-06 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346115+0000", 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4589999999999998e-06 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408012+0000", 2024-09-21T10:02:02.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061897677999999998 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408017+0000", 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5140000000000003e-06 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370663+0000", 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96264632299999997 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370697+0000", 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4124000000000001e-05 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52998 7.15 7:aba51093:::benchmark_data_smithi017_180590_object52997:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346118+0000", 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 36.128483426000003, 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0246339739999999, 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.587 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52998 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346118+0000", 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346118+0000", 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346118+0000", 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346152+0000", 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4393000000000001e-05 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346162+0000", 2024-09-21T10:02:02.589 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8209999999999997e-06 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346166+0000", 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3799999999999998e-06 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408044+0000", 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061878264000000002 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408048+0000", 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3789999999999999e-06 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370720+0000", 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96267132899999996 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370752+0000", 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2796999999999998e-05 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.591 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:53001 7.15 7:adde1279:::benchmark_data_smithi017_180590_object53000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346179+0000", 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 36.128423243, 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0246183339999999, 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 53001 2024-09-21T10:02:02.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346179+0000", 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346179+0000", 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346178+0000", 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346243+0000", 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5024000000000006e-05 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346251+0000", 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4460000000000004e-06 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346269+0000", 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8881000000000001e-05 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408064+0000", 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061794295999999999 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408068+0000", 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5619999999999997e-06 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.595 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370765+0000", 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96269701200000002 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370797+0000", 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1537e-05 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:53005 7.5 7:a05ee3e9:::benchmark_data_smithi017_180590_object53004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346275+0000", 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 36.128326747999999, 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.022219387, 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 53005 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.597 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346275+0000", 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346275+0000", 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346273+0000", 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:02.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346305+0000", 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1532999999999998e-05 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346312+0000", 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2969999999999998e-06 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:02:02.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346315+0000", 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4029999999999999e-06 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408087+0000", 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061771271000000003 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408092+0000", 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7649999999999999e-06 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360817+0000", 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95272487800000005 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360914+0000", 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6275999999999998e-05 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.361012+0000", 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8227999999999995e-05 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.601 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.361817+0000", 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000805582 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.365156+0000", 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003338597 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.368399+0000", 2024-09-21T10:02:02.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032432709999999998 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.368457+0000", 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7213999999999998e-05 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.368494+0000", 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7870000000000002e-05 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.603 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:02:02.604 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:02:02.604 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:02:02.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:02.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:02.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:02.691 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:02:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:02 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 16417/231870 objects degraded (7.080%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3643567644' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:02.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:02.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:02.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:02.912 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 80325 80309 79.6551 71.4375 0.0361479 0.0125474 2024-09-21T10:02:02.938 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:02.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:02 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 16417/231870 objects degraded (7.080%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3643567644' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:02.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:02 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 16417/231870 objects degraded (7.080%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:02.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3643567644' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:03.040 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:03.069 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:03.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:03.171 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:03.192 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:03.199 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:03.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:03.259 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:02:03.259 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:02:03.294 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:02:03.295 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:02:03.301 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:03.330 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:03.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:02:03.334 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:02:03.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:02:03.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:03.506 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:02:03.506 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:02:03.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:03.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:03.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:03.732 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:03.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:03.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:03.861 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:03.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.896 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:03.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:03.912 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 81454 81438 79.5129 70.5625 0.00543594 0.0125706 2024-09-21T10:02:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[102913]: pgmap v1490: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 32 MiB/s wr, 505 op/s; 16417/231870 objects degraded (7.080%); 5550/231870 objects misplaced (2.394%); 12 MiB/s, 193 objects/s recovering 2024-09-21T10:02:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:02:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3643567644' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[102913]: osdmap e1163: 8 total, 8 up, 7 in 2024-09-21T10:02:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1463261414' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[105151]: pgmap v1490: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 32 MiB/s wr, 505 op/s; 16417/231870 objects degraded (7.080%); 5550/231870 objects misplaced (2.394%); 12 MiB/s, 193 objects/s recovering 2024-09-21T10:02:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:02:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3643567644' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[105151]: osdmap e1163: 8 total, 8 up, 7 in 2024-09-21T10:02:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:03 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1463261414' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:03 smithi134 ceph-mon[90085]: pgmap v1490: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 32 MiB/s wr, 505 op/s; 16417/231870 objects degraded (7.080%); 5550/231870 objects misplaced (2.394%); 12 MiB/s, 193 objects/s recovering 2024-09-21T10:02:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:03 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:02:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:03 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3643567644' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:03 smithi134 ceph-mon[90085]: osdmap e1163: 8 total, 8 up, 7 in 2024-09-21T10:02:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:03 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1463261414' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:03.997 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:04.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.025 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:04.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:04.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:04.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:04.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:04.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:04.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:04.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:04.430 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:04.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.462 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:04.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:04.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:04.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:04.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:04.705 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:04.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:04.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:04.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:04.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:04.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:04.914 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 82723 82707 79.5097 79.3125 0.00794562 0.0125716 2024-09-21T10:02:04.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:04.991 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:02:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:02:04] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.51.0" 2024-09-21T10:02:05.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:05.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:05.107 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:05.138 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:05.240 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:05.271 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:05.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:05.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:05.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:05.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:05.516 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:05.550 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:05.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:05.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:05.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:05.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:05.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:02:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.836 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:05.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:05.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:05 smithi017 ceph-mon[102913]: pgmap v1492: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 4.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 16 MiB/s wr, 249 op/s; 16417/236406 objects degraded (6.944%); 5201/236406 objects misplaced (2.200%); 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:02:05.847 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:05 smithi017 ceph-mon[105151]: pgmap v1492: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 4.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 16 MiB/s wr, 249 op/s; 16417/236406 objects degraded (6.944%); 5201/236406 objects misplaced (2.200%); 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:02:05.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 84021 84005 79.5341 81.125 0.0441147 0.0125671 2024-09-21T10:02:05.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:05.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:05.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:05 smithi134 ceph-mon[90085]: pgmap v1492: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 4.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 16 MiB/s wr, 249 op/s; 16417/236406 objects degraded (6.944%); 5201/236406 objects misplaced (2.200%); 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:02:06.044 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:06.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:06.123 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:06.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:06.224 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:06.262 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:06.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:06.364 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:06.402 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:06.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:06.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:06.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:06.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:06.643 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:06.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:06.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:06.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:06.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:06.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:06.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:02:06.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 85156 85140 79.4057 70.9375 0.0101821 0.0125885 2024-09-21T10:02:06.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:06.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:06.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:07.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:07.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:07.219 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.255 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:07.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:07.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:07.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:07.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:07.493 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:07.533 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:07.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:07.635 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:07.665 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:07.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:07.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:07.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:07.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:07.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 86313 86297 79.3013 72.3125 0.0102956 0.0126051 2024-09-21T10:02:07.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:07 smithi017 ceph-mon[102913]: pgmap v1493: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 604 op/s; 16417/247689 objects degraded (6.628%); 5053/247689 objects misplaced (2.040%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T10:02:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:07 smithi017 ceph-mon[105151]: pgmap v1493: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 604 op/s; 16417/247689 objects degraded (6.628%); 5053/247689 objects misplaced (2.040%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T10:02:07.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:07.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:07 smithi134 ceph-mon[90085]: pgmap v1493: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 38 MiB/s wr, 604 op/s; 16417/247689 objects degraded (6.628%); 5053/247689 objects misplaced (2.040%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T10:02:08.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:08.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:08.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:08.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:08.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:08.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:08.351 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.380 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:08.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:08.481 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:08.513 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:08.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:08.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:08.647 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:08.702 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:02:08.748 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:08.785 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:08.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:08.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 87433 87417 79.1664 70 0.036136 0.0126261 2024-09-21T10:02:08.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.914 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:08.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:08 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2616387624' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:02:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:08 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:02:08.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:08 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 16417/247689 objects degraded (6.628%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:08 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2616387624' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:02:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:08 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:02:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:08 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 16417/247689 objects degraded (6.628%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:08 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2616387624' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:02:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:08 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:02:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:08 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 16417/247689 objects degraded (6.628%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:09.015 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:09.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:09.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:09.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:09.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.185 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:09.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:09.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:09.316 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:09.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:09.417 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:09.447 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:09.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:09.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:09.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:09.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:09.685 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:09.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:09.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:09.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:09.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:09.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:09.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 88623 88607 79.0979 74.375 0.00606583 0.0126374 2024-09-21T10:02:09.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:09 smithi017 ceph-mon[102913]: pgmap v1494: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 37 MiB/s wr, 597 op/s; 16417/249924 objects degraded (6.569%); 5053/249924 objects misplaced (2.022%); 8.7 MiB/s, 138 objects/s recovering 2024-09-21T10:02:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:09 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:02:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:09 smithi017 ceph-mon[102913]: osdmap e1164: 8 total, 8 up, 7 in 2024-09-21T10:02:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:09 smithi017 ceph-mon[105151]: pgmap v1494: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 37 MiB/s wr, 597 op/s; 16417/249924 objects degraded (6.569%); 5053/249924 objects misplaced (2.022%); 8.7 MiB/s, 138 objects/s recovering 2024-09-21T10:02:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:09 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:02:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:09 smithi017 ceph-mon[105151]: osdmap e1164: 8 total, 8 up, 7 in 2024-09-21T10:02:09.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:09 smithi134 ceph-mon[90085]: pgmap v1494: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 37 MiB/s wr, 597 op/s; 16417/249924 objects degraded (6.569%); 5053/249924 objects misplaced (2.022%); 8.7 MiB/s, 138 objects/s recovering 2024-09-21T10:02:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:09 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:02:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:09 smithi134 ceph-mon[90085]: osdmap e1164: 8 total, 8 up, 7 in 2024-09-21T10:02:10.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:10.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:10.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:10.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:10.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:10.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:10.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:10.384 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:10.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:10.486 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:02:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:02:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:10.509 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:10.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:10.610 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:10.643 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:10.745 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:10.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:10.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:10.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:10.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:10.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 89723 89707 78.9521 68.75 0.0193443 0.0126603 2024-09-21T10:02:11.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:11.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:11.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:11.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:11.189 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:11.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:11.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:11.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:11.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:11.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:11.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:11.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.592 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:11.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:11.693 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:11 smithi017 ceph-mon[102913]: pgmap v1496: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s; 16417/254484 objects degraded (6.451%); 4609/254484 objects misplaced (1.811%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:11 smithi017 ceph-mon[105151]: pgmap v1496: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s; 16417/254484 objects degraded (6.451%); 4609/254484 objects misplaced (1.811%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:11.725 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:11.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:11 smithi134 ceph-mon[90085]: pgmap v1496: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s; 16417/254484 objects degraded (6.451%); 4609/254484 objects misplaced (1.811%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:11.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:11.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:11.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 90814 90798 78.8025 68.1875 0.0237795 0.0126846 2024-09-21T10:02:11.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:12.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:12.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:12.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:12.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:12.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:12.255 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:12.278 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:12.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:12.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:12.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:12.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:12.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:12.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:12.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:12.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:12.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:12.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:12.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:12.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:12.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 91665 91649 78.4516 53.1875 0.00659989 0.0127404 2024-09-21T10:02:12.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:12.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:12.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:13.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:13.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:13.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:13.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:13.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:13.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:13.344 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:13.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:13 smithi017 ceph-mon[102913]: pgmap v1497: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 16417/267336 objects degraded (6.141%); 4467/267336 objects misplaced (1.671%); 7.2 MiB/s, 114 objects/s recovering 2024-09-21T10:02:13.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:13 smithi017 ceph-mon[105151]: pgmap v1497: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 16417/267336 objects degraded (6.141%); 4467/267336 objects misplaced (1.671%); 7.2 MiB/s, 114 objects/s recovering 2024-09-21T10:02:13.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:13.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:13.605 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:13.638 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:13.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:13.740 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:13 smithi134 ceph-mon[90085]: pgmap v1497: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 16417/267336 objects degraded (6.141%); 4467/267336 objects misplaced (1.671%); 7.2 MiB/s, 114 objects/s recovering 2024-09-21T10:02:13.767 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:13.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:13.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:13.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:13.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:13.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 92607 92591 78.187 58.875 0.0115377 0.0127844 2024-09-21T10:02:13.980 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:02:13.980 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:02:14.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:14.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:14.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:14.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:14.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:14.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:14.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:14.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:14.445 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:14.475 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:14.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:14.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:14 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 16417/267336 objects degraded (6.141%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:14.528 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:14 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 16417/267336 objects degraded (6.141%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:14.577 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:14.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:14.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:14.709 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:14.743 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:14 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 16417/267336 objects degraded (6.141%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:14.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:14.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:14.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:14.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:14.913 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 93876 93860 78.2019 79.3125 0.0107304 0.0127824 2024-09-21T10:02:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:02:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:02:14] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-09-21T10:02:14.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:15.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:02:15.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:15.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:15.135 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:15.164 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:15.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:15.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:15.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:15.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:02:15.411 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:15.422 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:15.436 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:15.438 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:15.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:15.565 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:15 smithi017 ceph-mon[105151]: pgmap v1498: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 16417/271173 objects degraded (6.054%); 3977/271173 objects misplaced (1.467%); 7.6 MiB/s, 122 objects/s recovering 2024-09-21T10:02:15.566 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:15 smithi017 ceph-mon[102913]: pgmap v1498: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 16417/271173 objects degraded (6.054%); 3977/271173 objects misplaced (1.467%); 7.6 MiB/s, 122 objects/s recovering 2024-09-21T10:02:15.577 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:15.679 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:15.709 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:15 smithi134 ceph-mon[90085]: pgmap v1498: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 16417/271173 objects degraded (6.054%); 3977/271173 objects misplaced (1.467%); 7.6 MiB/s, 122 objects/s recovering 2024-09-21T10:02:15.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:15.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:15.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:15.914 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 94928 94912 78.038 65.75 0.0157257 0.0128094 2024-09-21T10:02:15.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:15.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:15.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:16.098 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:16.133 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:16.235 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:16.266 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:16.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:16.367 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:16.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.399 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:16.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:16.445 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:16.501 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:16.528 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:16.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:16.663 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:16.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:16.765 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:16.794 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:16.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:16.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:16.914 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 96123 96107 77.9944 74.6875 0.0262885 0.0128164 2024-09-21T10:02:16.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:16.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:17.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:17.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:17.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:17.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:17.213 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:17.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:17.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:17.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:17.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:17.458 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:17.487 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:17.588 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:17.621 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:17.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:17 smithi017 ceph-mon[102913]: pgmap v1499: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 16417/281148 objects degraded (5.839%); 3789/281148 objects misplaced (1.348%); 7.9 MiB/s, 126 objects/s recovering 2024-09-21T10:02:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:17 smithi017 ceph-mon[105151]: pgmap v1499: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 16417/281148 objects degraded (5.839%); 3789/281148 objects misplaced (1.348%); 7.9 MiB/s, 126 objects/s recovering 2024-09-21T10:02:17.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:17 smithi134 ceph-mon[90085]: pgmap v1499: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 16417/281148 objects degraded (5.839%); 3789/281148 objects misplaced (1.348%); 7.9 MiB/s, 126 objects/s recovering 2024-09-21T10:02:17.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:17.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:17.859 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:17.880 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:17.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:17.914 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 97105 97089 77.7812 61.375 0.00353855 0.0128094 2024-09-21T10:02:17.981 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:18.002 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:18.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:18.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:18.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:18.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:18.250 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:18.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:18.351 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:18.385 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:18.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:18.487 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:18.519 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:18.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:18.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:18.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:18 smithi017 ceph-mon[102913]: osdmap e1165: 8 total, 8 up, 7 in 2024-09-21T10:02:18.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:18 smithi017 ceph-mon[105151]: osdmap e1165: 8 total, 8 up, 7 in 2024-09-21T10:02:18.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:18.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:18 smithi134 ceph-mon[90085]: osdmap e1165: 8 total, 8 up, 7 in 2024-09-21T10:02:18.761 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:18.790 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:18.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:18.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:18.916 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 97801 97785 77.3468 43.5 0.0203602 0.0129229 2024-09-21T10:02:18.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:19.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:19.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:19.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:19.162 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:19.198 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:19.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:19.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:19.324 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:19.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:19.426 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:19.458 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:19.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:19.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:19.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:19.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:19 smithi017 ceph-mon[102913]: pgmap v1501: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 16417/283071 objects degraded (5.800%); 3789/283071 objects misplaced (1.339%); 8.4 MiB/s, 133 objects/s recovering 2024-09-21T10:02:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:19 smithi017 ceph-mon[102913]: osdmap e1166: 8 total, 8 up, 7 in 2024-09-21T10:02:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:19 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 16417/283071 objects degraded (5.800%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:19 smithi017 ceph-mon[105151]: pgmap v1501: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 16417/283071 objects degraded (5.800%); 3789/283071 objects misplaced (1.339%); 8.4 MiB/s, 133 objects/s recovering 2024-09-21T10:02:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:19 smithi017 ceph-mon[105151]: osdmap e1166: 8 total, 8 up, 7 in 2024-09-21T10:02:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:19 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 16417/283071 objects degraded (5.800%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:19.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:19.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:19 smithi134 ceph-mon[90085]: pgmap v1501: 109 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 100 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 16417/283071 objects degraded (5.800%); 3789/283071 objects misplaced (1.339%); 8.4 MiB/s, 133 objects/s recovering 2024-09-21T10:02:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:19 smithi134 ceph-mon[90085]: osdmap e1166: 8 total, 8 up, 7 in 2024-09-21T10:02:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:19 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 16417/283071 objects degraded (5.800%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:19.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:19.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:19.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:19.915 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:02:19.913715+0000 min lat: 0.0012027 max lat: 11.1239 avg lat: 0.0129549 2024-09-21T10:02:19.915 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:02:19.915 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 98797 98781 77.158 62.25 0.0113777 0.0129549 2024-09-21T10:02:19.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:20.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:20.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:20.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:20.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:20.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:20.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:20.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:20.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:20.406 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:20.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.440 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:20.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:02:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:02:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:20.542 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:20.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:20.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:20.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:20.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:20.807 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:20.838 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:20.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:20.915 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 99879 99863 77.0402 67.625 0.029749 0.0129741 2024-09-21T10:02:20.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:20.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:20.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:21.057 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:21.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:21.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:21.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:21.258 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:21.291 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:21.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:21.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:21.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:21.543 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:21.572 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:21.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:21.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:21 smithi017 ceph-mon[102913]: pgmap v1503: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 14377/287019 objects degraded (5.009%); 5212/287019 objects misplaced (1.816%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:02:21.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:21 smithi017 ceph-mon[105151]: pgmap v1503: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 14377/287019 objects degraded (5.009%); 5212/287019 objects misplaced (1.816%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:02:21.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:21.703 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:21.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:21 smithi134 ceph-mon[90085]: pgmap v1503: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 14377/287019 objects degraded (5.009%); 5212/287019 objects misplaced (1.816%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:02:21.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:21.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:21.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:21.915 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 100934 100918 76.9048 65.9375 0.0169637 0.0129975 2024-09-21T10:02:21.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:21.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:21.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:22.083 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:22.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:22.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:22.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:22.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.264 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:22.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:22.313 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:02:22.365 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:22.395 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:22.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:22.527 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:22.629 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:22.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:22 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2004547052' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:22.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:22 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:22.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:22 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2004547052' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:22.646 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:22 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:22.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:22.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:22 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2004547052' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:22 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:22.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:22.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:22.915 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 102049 102033 76.8177 69.6875 0.00934199 0.0130133 2024-09-21T10:02:22.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:22.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:22.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:23.053 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:23.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:23.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:23.183 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:23.213 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:23.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:23.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:23.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:23 smithi017 ceph-mon[102913]: pgmap v1504: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 14377/297984 objects degraded (4.825%); 4982/297984 objects misplaced (1.672%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:02:23.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:23 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:23.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:23 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:02:23.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:23 smithi017 ceph-mon[102913]: osdmap e1167: 8 total, 8 up, 7 in 2024-09-21T10:02:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:23 smithi017 ceph-mon[105151]: pgmap v1504: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 14377/297984 objects degraded (4.825%); 4982/297984 objects misplaced (1.672%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:02:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:23 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:23 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:02:23.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:23 smithi017 ceph-mon[105151]: osdmap e1167: 8 total, 8 up, 7 in 2024-09-21T10:02:23.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:23.485 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:23.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:23.587 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:23.636 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:23.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:23.738 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:23 smithi134 ceph-mon[90085]: pgmap v1504: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 14377/297984 objects degraded (4.825%); 4982/297984 objects misplaced (1.672%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:02:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:23 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:23 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:02:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:23 smithi134 ceph-mon[90085]: osdmap e1167: 8 total, 8 up, 7 in 2024-09-21T10:02:23.762 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:23.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:23.878 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:02:23.878 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:02:23.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:23.903 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:02:23.903 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:02:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:23.915 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 103102 103086 76.6867 65.8125 0.00937426 0.0130343 2024-09-21T10:02:23.915 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:02:23.916 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:02:23.951 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:02:23.952 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:02:23.989 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:02:23.990 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:02:24.005 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:24.024 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:02:24.025 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:02:24.039 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:24.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:24.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:02:24.060 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:02:24.095 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:02:24.095 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:02:24.127 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:02:24.127 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:02:24.141 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:24.171 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:24.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:24.273 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:24.304 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:24.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:24.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:24.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:24.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:24.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:24.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:24.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:24.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:24 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 14377/297984 objects degraded (4.825%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:24.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1176207264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:24.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:24 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 14377/297984 objects degraded (4.825%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:24.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1176207264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:24.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:24.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:24 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 14377/297984 objects degraded (4.825%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1176207264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:24.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:24.866 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:24.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:24.915 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 104269 104253 76.6425 72.9375 0.0110222 0.0130426 2024-09-21T10:02:24.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:25.006 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:25.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:25.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:25.129 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:02:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:02:24] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-09-21T10:02:25.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:25.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:02:25.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:25.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:25.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:25.417 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:25.444 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:25.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:25 smithi017 ceph-mon[102913]: pgmap v1506: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s; 14377/302037 objects degraded (4.760%); 4639/302037 objects misplaced (1.536%); 11 MiB/s, 173 objects/s recovering 2024-09-21T10:02:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:25.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:25.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:25 smithi017 ceph-mon[105151]: pgmap v1506: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s; 14377/302037 objects degraded (4.760%); 4639/302037 objects misplaced (1.536%); 11 MiB/s, 173 objects/s recovering 2024-09-21T10:02:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:25.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:25.545 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:25.579 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:25.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:25.681 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:25.710 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:25.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:25 smithi134 ceph-mon[90085]: pgmap v1506: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s; 14377/302037 objects degraded (4.760%); 4639/302037 objects misplaced (1.536%); 11 MiB/s, 173 objects/s recovering 2024-09-21T10:02:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:25.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:25.842 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:25.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:25.915 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 105355 105339 76.5405 67.875 0.0099779 0.0130595 2024-09-21T10:02:25.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:25.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:26.088 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:26.115 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:26.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:26.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:26.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:26.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:26.358 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:26.387 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:26.488 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:26.496 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:26.516 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:26.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:26.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:26.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:26.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:02:26.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:26.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:26.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:26.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:26.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 106365 106349 76.3846 63.125 0.0170456 0.013085 2024-09-21T10:02:26.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:26.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:27.046 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:27.074 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:27.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:27.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:27.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:27.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:27.335 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:02:27.349 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:27.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:27.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:27.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:27.593 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:27.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.629 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:27.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:27 smithi017 ceph-mon[102913]: pgmap v1507: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 14377/312411 objects degraded (4.602%); 4478/312411 objects misplaced (1.433%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:02:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:27 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3106748147' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:02:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:27 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:02:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:27 smithi017 ceph-mon[105151]: pgmap v1507: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 14377/312411 objects degraded (4.602%); 4478/312411 objects misplaced (1.433%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:02:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:27 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3106748147' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:02:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:27 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:02:27.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:27.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:27 smithi134 ceph-mon[90085]: pgmap v1507: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.4 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 14377/312411 objects degraded (4.602%); 4478/312411 objects misplaced (1.433%); 11 MiB/s, 168 objects/s recovering 2024-09-21T10:02:27.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:27 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3106748147' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:02:27.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:27 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:02:27.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:27.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:27.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:27.900 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:27.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:27.917 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 107485 107469 76.312 70 0.021283 0.0130975 2024-09-21T10:02:28.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:28.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:28.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:28.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:02:28.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:28.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:28.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:28.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:02:28.281 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:28.308 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:28.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:28.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:28.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:28.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:28.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:28.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.606 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:28.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:28 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:02:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:28 smithi017 ceph-mon[102913]: osdmap e1168: 8 total, 8 up, 7 in 2024-09-21T10:02:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:28 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:02:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:28 smithi017 ceph-mon[105151]: osdmap e1168: 8 total, 8 up, 7 in 2024-09-21T10:02:28.707 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:28.738 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:28.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:28 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:02:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:28 smithi134 ceph-mon[90085]: osdmap e1168: 8 total, 8 up, 7 in 2024-09-21T10:02:28.840 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:28.876 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:28.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:28.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 108560 108544 76.2091 67.1875 0.00966633 0.0131156 2024-09-21T10:02:28.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:29.009 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:29.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:29.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:29.073 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:29.172 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:29.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:29.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:29.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:29.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:29.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:29 smithi017 ceph-mon[102913]: pgmap v1509: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 14377/314481 objects degraded (4.572%); 4478/314481 objects misplaced (1.424%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:02:29.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:29 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 14377/314481 objects degraded (4.572%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:29 smithi017 ceph-mon[105151]: pgmap v1509: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 14377/314481 objects degraded (4.572%); 4478/314481 objects misplaced (1.424%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:02:29.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:29 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 14377/314481 objects degraded (4.572%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:29.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:29.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:29.595 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:29.627 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:29.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:29 smithi134 ceph-mon[90085]: pgmap v1509: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 14377/314481 objects degraded (4.572%); 4478/314481 objects misplaced (1.424%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:02:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:29 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 14377/314481 objects degraded (4.572%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:29.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:29.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:29.882 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:29.908 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:29.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0132 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 109557 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 76.0701 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 45.8047 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 204.625 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1217 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 732.884 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3274 2024-09-21T10:02:29.918 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2024-09-21T10:02:29.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0131409 2024-09-21T10:02:29.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.110613 2024-09-21T10:02:29.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 11.1239 2024-09-21T10:02:29.919 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.0012027 2024-09-21T10:02:29.927 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:02:30.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:30.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:30.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:30.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:30.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:30.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:30.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:30.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:30.485 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:30.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:02:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:02:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:30.561 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:02:30.561 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:60915 6.10 6:0e7dbdda:::benchmark_data_smithi017_149547_object60914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.422484+0000", 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 323.75141381499998, 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.042944147, 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.562 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60915 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422484+0000", 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422484+0000", 2024-09-21T10:02:30.563 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422487+0000", 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.931e-06 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422536+0000", 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9316e-05 2024-09-21T10:02:30.564 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422577+0000", 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1190999999999999e-05 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422593+0000", 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5591999999999999e-05 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.565 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.499893+0000", 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077299781999999997 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.499902+0000", 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1400000000000006e-06 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.456197+0000", 2024-09-21T10:02:30.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95629477799999996 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.456846+0000", 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00064964700000000005 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.465294+0000", 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0084478999999999995 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.567 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.465428+0000", 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013386999999999999 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91404 6.10 6:0e0e7bfc:::benchmark_data_smithi017_149547_object91403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.613932+0000", 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.559965342, 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0513244660000001, 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.568 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91404 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613932+0000", 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.569 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613932+0000", 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613933+0000", 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.34e-07 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613952+0000", 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9488e-05 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.570 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613959+0000", 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5089999999999999e-06 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613964+0000", 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9520000000000002e-06 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.616064+0000", 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00209991 2024-09-21T10:02:30.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.616072+0000", 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1009999999999992e-06 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.647238+0000", 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0311661350000001 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.647363+0000", 2024-09-21T10:02:30.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012525599999999999 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.647460+0000", 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7281999999999995e-05 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650199+0000", 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002738613 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650356+0000", 2024-09-21T10:02:30.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015701499999999999 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665141+0000", 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014785400000000001 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665208+0000", 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6427000000000006e-05 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665257+0000", 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8943999999999998e-05 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91409 6.10 6:0d2601f6:::benchmark_data_smithi017_149547_object91408:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.621998+0000", 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.55190005200001, 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0433568630000001, 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.575 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91409 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.621998+0000", 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.621998+0000", 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.621999+0000", 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.933e-06 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622021+0000", 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1667000000000001e-05 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622034+0000", 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2906e-05 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622042+0000", 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2099999999999993e-06 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622143+0000", 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100886 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622154+0000", 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0513000000000001e-05 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.648650+0000", 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0264962689999999 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.648749+0000", 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9400000000000004e-05 2024-09-21T10:02:30.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.648846+0000", 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6937999999999999e-05 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650374+0000", 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015274659999999999 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651976+0000", 2024-09-21T10:02:30.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016021449999999999 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665292+0000", 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013315993999999999 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665328+0000", 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6068000000000001e-05 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665354+0000", 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6468000000000001e-05 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91437 6.10 6:0f983da6:::benchmark_data_smithi017_149547_object91436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.645165+0000", 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.528732881, 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0215859970000001, 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.582 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91437 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645165+0000", 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645165+0000", 2024-09-21T10:02:30.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645166+0000", 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.466e-06 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645190+0000", 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3915999999999999e-05 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645204+0000", 2024-09-21T10:02:30.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4166e-05 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645212+0000", 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1629999999999996e-06 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645357+0000", 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014457399999999999 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645368+0000", 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1324e-05 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.649427+0000", 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004059067 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.649545+0000", 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011762200000000001 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.649640+0000", 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5396000000000006e-05 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651930+0000", 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022892300000000002 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652014+0000", 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3967000000000004e-05 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.666638+0000", 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014624511 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.666709+0000", 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0510000000000001e-05 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.666751+0000", 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2085000000000001e-05 2024-09-21T10:02:30.588 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91468 6.10 6:0b811041:::benchmark_data_smithi017_149547_object91467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.668218+0000", 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.50567974500001, 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99924510700000002, 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:02:30.589 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91468 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668218+0000", 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668218+0000", 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668220+0000", 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.685e-06 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668247+0000", 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7639000000000001e-05 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668278+0000", 2024-09-21T10:02:30.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0616e-05 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668287+0000", 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3319999999999998e-06 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668434+0000", 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014714700000000001 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668445+0000", 2024-09-21T10:02:30.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0542e-05 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650407+0000", 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98196202300000002 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650475+0000", 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8063000000000002e-05 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650555+0000", 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0555000000000004e-05 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652024+0000", 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014685810000000001 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652062+0000", 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7450000000000002e-05 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.667339+0000", 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015277824000000001 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.667412+0000", 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2403000000000005e-05 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.667463+0000", 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1246999999999999e-05 2024-09-21T10:02:30.595 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91476 6.10 6:0f93928d:::benchmark_data_smithi017_149547_object91475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.673753+0000", 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.50014485000003, 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99824931400000005, 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:02:30.596 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91476 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673753+0000", 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673753+0000", 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673754+0000", 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5629999999999999e-06 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673782+0000", 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7636000000000001e-05 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673805+0000", 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2946999999999999e-05 2024-09-21T10:02:30.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673813+0000", 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2700000000000004e-06 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673946+0000", 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013301900000000001 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673955+0000", 2024-09-21T10:02:30.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1880000000000008e-06 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651172+0000", 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97721698300000004 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651289+0000", 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116628 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:02:30.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651376+0000", 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6612000000000004e-05 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.653572+0000", 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021968230000000001 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.653583+0000", 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0685e-05 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.671796+0000", 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018212424000000001 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.671952+0000", 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000156422 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.672002+0000", 2024-09-21T10:02:30.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0114000000000002e-05 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91478 6.10 6:083e0ee5:::benchmark_data_smithi017_149547_object91477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.675325+0000", 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.498572712, 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99692708900000004, 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.603 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91478 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675325+0000", 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675325+0000", 2024-09-21T10:02:30.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675326+0000", 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3960000000000001e-06 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675349+0000", 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3180000000000002e-05 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675367+0000", 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7591000000000001e-05 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675375+0000", 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5129999999999999e-06 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675438+0000", 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3595000000000004e-05 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675448+0000", 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5659999999999996e-06 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652093+0000", 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97664543400000003 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652161+0000", 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8285000000000001e-05 2024-09-21T10:02:30.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652243+0000", 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1036000000000001e-05 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.655074+0000", 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028315800000000002 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.655124+0000", 2024-09-21T10:02:30.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9434999999999997e-05 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.672049+0000", 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016925639999999999 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.672210+0000", 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016079700000000001 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.672252+0000", 2024-09-21T10:02:30.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2040999999999999e-05 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53003 7.4 7:22671f4e:::benchmark_data_smithi017_180590_object53002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.348891+0000", 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 63.825007004, 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004137056, 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.610 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53003 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348891+0000", 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348891+0000", 2024-09-21T10:02:30.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348891+0000", 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3099999999999997e-07 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348917+0000", 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5942e-05 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348926+0000", 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7260000000000008e-06 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348933+0000", 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6749999999999996e-06 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.351946+0000", 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030134020000000001 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.351952+0000", 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7289999999999997e-06 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.470485+0000", 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11853295 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.470493+0000", 2024-09-21T10:02:30.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3440000000000001e-06 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.352953+0000", 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.882459565 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.353028+0000", 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5092000000000005e-05 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.615 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53008 7.4 7:20f177af:::benchmark_data_smithi017_180590_object53007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.348943+0000", 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 63.824954130999998, 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004518749, 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.616 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53008 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348943+0000", 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348943+0000", 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348944+0000", 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5000000000000003e-07 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348969+0000", 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5337000000000002e-05 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348980+0000", 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.057e-05 2024-09-21T10:02:30.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348986+0000", 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0000000000000002e-06 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.352013+0000", 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003027222 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.352019+0000", 2024-09-21T10:02:30.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1879999999999999e-06 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.476795+0000", 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.124775264 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.476825+0000", 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0437999999999999e-05 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.353396+0000", 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.87657047899999996 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.353462+0000", 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6700999999999997e-05 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53011 7.6 7:60b3f60f:::benchmark_data_smithi017_180590_object53010:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.393629+0000", 2024-09-21T10:02:30.621 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 63.780268464000002, 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98845262599999995, 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53011 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.393629+0000", 2024-09-21T10:02:30.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.393629+0000", 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.393631+0000", 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1330000000000002e-06 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.393667+0000", 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5549000000000001e-05 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.415542+0000", 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021875623 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.415559+0000", 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6694999999999999e-05 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.496749+0000", 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.081189497999999999 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.496760+0000", 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1809e-05 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.354373+0000", 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.85761251199999999 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.354542+0000", 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000168722 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.354731+0000", 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018961499999999999 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.359104+0000", 2024-09-21T10:02:30.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043730399999999999 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.364073+0000", 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049688809999999996 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.381981+0000", 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017908013 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.382040+0000", 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8689e-05 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.382082+0000", 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1847000000000001e-05 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72895 7.4 7:27e3b190:::benchmark_data_smithi017_180590_object72894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.669787+0000", 2024-09-21T10:02:30.628 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.504111068, 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1010801379999999, 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72895 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669787+0000", 2024-09-21T10:02:30.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669787+0000", 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669788+0000", 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.739e-06 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669824+0000", 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5379000000000001e-05 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669842+0000", 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8621e-05 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669851+0000", 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3110000000000006e-06 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669933+0000", 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1909000000000005e-05 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669943+0000", 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0488e-05 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.599855+0000", 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92991173400000005 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.599935+0000", 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0220000000000001e-05 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.600024+0000", 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9548000000000004e-05 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602963+0000", 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029380550000000002 2024-09-21T10:02:30.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605565+0000", 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026027089999999999 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770758+0000", 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16519246000000001 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770816+0000", 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8242000000000002e-05 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770867+0000", 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0723000000000002e-05 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72907 7.4 7:254bdb82:::benchmark_data_smithi017_180590_object72906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.677271+0000", 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.496627021000002, 2024-09-21T10:02:30.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0938297560000001, 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72907 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677271+0000", 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677271+0000", 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677273+0000", 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4660000000000002e-06 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677299+0000", 2024-09-21T10:02:30.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5531999999999999e-05 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677312+0000", 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3246e-05 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677321+0000", 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7919999999999998e-06 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677392+0000", 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1397999999999996e-05 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677402+0000", 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0053e-05 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.600705+0000", 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92330275500000003 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.600807+0000", 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101969 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.600880+0000", 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3143000000000001e-05 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602968+0000", 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020885119999999998 2024-09-21T10:02:30.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605625+0000", 2024-09-21T10:02:30.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026569359999999999 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771015+0000", 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16538913199999999 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771063+0000", 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8959999999999999e-05 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771100+0000", 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6862e-05 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72934 7.4 7:21599fe0:::benchmark_data_smithi017_180590_object72933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.700107+0000", 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.473790561999998, 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0711450929999999, 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.643 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72934 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700107+0000", 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700107+0000", 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700108+0000", 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6599999999999996e-07 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700128+0000", 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0166e-05 2024-09-21T10:02:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700135+0000", 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3880000000000002e-06 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700141+0000", 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7010000000000001e-06 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700271+0000", 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130419 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700280+0000", 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0010000000000007e-06 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.601448+0000", 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90116788199999998 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.601515+0000", 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6563000000000006e-05 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.601594+0000", 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9702000000000004e-05 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.604156+0000", 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025619150000000001 2024-09-21T10:02:30.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605670+0000", 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015133569999999999 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771153+0000", 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.165483349 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771215+0000", 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1717999999999994e-05 2024-09-21T10:02:30.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771252+0000", 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7366000000000001e-05 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72952 7.4 7:21f84216:::benchmark_data_smithi017_180590_object72951:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.715253+0000", 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.458644296999999, 2024-09-21T10:02:30.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0561387069999999, 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72952 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715253+0000", 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715253+0000", 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715254+0000", 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7400000000000003e-07 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715273+0000", 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9479000000000001e-05 2024-09-21T10:02:30.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715281+0000", 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5660000000000001e-06 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715286+0000", 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3750000000000002e-06 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.716709+0000", 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001422817 2024-09-21T10:02:30.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.716719+0000", 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4709999999999996e-06 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602200+0000", 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88548115699999996 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602274+0000", 2024-09-21T10:02:30.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3770999999999996e-05 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602356+0000", 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2385000000000005e-05 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605519+0000", 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031632660000000001 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.768876+0000", 2024-09-21T10:02:30.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16335690999999999 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771315+0000", 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024384879999999999 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771359+0000", 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4230999999999997e-05 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771392+0000", 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3216999999999997e-05 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72954 7.4 7:27352eec:::benchmark_data_smithi017_180590_object72953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.718051+0000", 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.455846127999997, 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.053472309, 2024-09-21T10:02:30.657 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72954 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718051+0000", 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718051+0000", 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718053+0000", 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5340000000000001e-06 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718078+0000", 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4752999999999998e-05 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718092+0000", 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3943e-05 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718099+0000", 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5000000000000002e-06 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718163+0000", 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4077000000000003e-05 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718172+0000", 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1560000000000001e-06 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.603120+0000", 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88494770199999995 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.603220+0000", 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100022 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.603340+0000", 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120164 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605635+0000", 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022948489999999998 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770915+0000", 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16528029599999999 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771430+0000", 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051476500000000004 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771492+0000", 2024-09-21T10:02:30.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1538000000000006e-05 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771524+0000", 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.201e-05 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72980 7.4 7:266036bf:::benchmark_data_smithi017_180590_object72979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.739931+0000", 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.433966521999999, 2024-09-21T10:02:30.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.031714531, 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72980 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739931+0000", 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739931+0000", 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739932+0000", 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9100000000000003e-07 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739950+0000", 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7878000000000001e-05 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739960+0000", 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0139999999999999e-05 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739965+0000", 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1939999999999997e-06 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.740017+0000", 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1517999999999997e-05 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.740025+0000", 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2859999999999999e-06 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.604188+0000", 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.86416299100000005 2024-09-21T10:02:30.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.604288+0000", 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100313 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.604412+0000", 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012408100000000001 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770710+0000", 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.166298106 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770960+0000", 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024910999999999998 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771561+0000", 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060110899999999995 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771601+0000", 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0813999999999999e-05 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771646+0000", 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4199999999999997e-05 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72985 7.4 7:22398816:::benchmark_data_smithi017_180590_object72984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.743380+0000", 2024-09-21T10:02:30.671 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.430517248999998, 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.032025556, 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72985 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743380+0000", 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743380+0000", 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743382+0000", 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3680000000000001e-06 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743408+0000", 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6115999999999999e-05 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743418+0000", 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0105999999999999e-05 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743425+0000", 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5899999999999996e-06 2024-09-21T10:02:30.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743491+0000", 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6229000000000006e-05 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743501+0000", 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0304e-05 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.768925+0000", 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.02542402 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.769066+0000", 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140512 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.769184+0000", 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118606 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770971+0000", 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001787177 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771686+0000", 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00071413399999999995 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775288+0000", 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036027530000000002 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775354+0000", 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5534e-05 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775406+0000", 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2107000000000001e-05 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.678 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:73030 7.4 7:2535c205:::benchmark_data_smithi017_180590_object73029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.787612+0000", 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.386285835000002, 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99312697800000005, 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 73030 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787612+0000", 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787612+0000", 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787613+0000", 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.609e-06 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787638+0000", 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4691000000000001e-05 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787650+0000", 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1525999999999999e-05 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787657+0000", 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0979999999999996e-06 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787739+0000", 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2614999999999994e-05 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787749+0000", 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2350000000000005e-06 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771717+0000", 2024-09-21T10:02:30.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98396802299999997 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771839+0000", 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012282600000000001 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771997+0000", 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015736600000000001 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775456+0000", 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034597149999999999 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775500+0000", 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3065999999999997e-05 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780576+0000", 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050762309999999996 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780695+0000", 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119022 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780739+0000", 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3955000000000002e-05 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:73041 7.4 7:27218752:::benchmark_data_smithi017_180590_object73040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.685 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.797618+0000", 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.376279584999999, 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98331477199999995, 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 73041 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797618+0000", 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797618+0000", 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797620+0000", 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.576e-06 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797646+0000", 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6086000000000001e-05 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797659+0000", 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3589e-05 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797666+0000", 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0809999999999996e-06 2024-09-21T10:02:30.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.798014+0000", 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034764000000000003 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.798028+0000", 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4021e-05 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.773406+0000", 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97537753000000005 2024-09-21T10:02:30.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.773542+0000", 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013645700000000001 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.773716+0000", 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017366899999999999 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780752+0000", 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070358240000000004 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780794+0000", 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2048999999999997e-05 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780848+0000", 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3959000000000003e-05 2024-09-21T10:02:30.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780896+0000", 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8223000000000002e-05 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780933+0000", 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7067999999999998e-05 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.692 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:73045 7.4 7:2226a9f9:::benchmark_data_smithi017_180590_object73044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.805230+0000", 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 34.368667969999997, 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97914256600000005, 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 73045 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805230+0000", 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805230+0000", 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805232+0000", 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9740000000000001e-06 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805258+0000", 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6401e-05 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805272+0000", 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4212999999999999e-05 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805280+0000", 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6690000000000006e-06 2024-09-21T10:02:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.806040+0000", 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075974500000000004 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.806046+0000", 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.212e-06 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775534+0000", 2024-09-21T10:02:30.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96948825199999999 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775674+0000", 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013970700000000001 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775820+0000", 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014583400000000001 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:02:30.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780805+0000", 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049853930000000003 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.784242+0000", 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034368900000000002 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.784277+0000", 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5036999999999997e-05 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:02:30.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.784329+0000", 2024-09-21T10:02:30.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2142999999999999e-05 2024-09-21T10:02:30.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:02:30.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:02:30.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:02:30.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:30.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.784372+0000", 2024-09-21T10:02:30.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3096000000000002e-05 2024-09-21T10:02:30.700 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.700 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.700 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.701 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:02:30.701 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:02:30.701 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:02:30.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:30.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:30.973 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:30.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:30.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:31.087 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:31.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:31.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:31.223 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:31.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.259 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:31.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:31.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:31.395 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:31.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:31.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:31 smithi017 ceph-mon[102913]: pgmap v1510: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 53 MiB/s wr, 853 op/s; 14377/318468 objects degraded (4.514%); 4087/318468 objects misplaced (1.283%); 7.0 MiB/s, 111 objects/s recovering 2024-09-21T10:02:31.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:31 smithi017 ceph-mon[105151]: pgmap v1510: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 53 MiB/s wr, 853 op/s; 14377/318468 objects degraded (4.514%); 4087/318468 objects misplaced (1.283%); 7.0 MiB/s, 111 objects/s recovering 2024-09-21T10:02:31.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:31.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:31.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:31.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:31.667 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:31 smithi134 ceph-mon[90085]: pgmap v1510: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 53 MiB/s wr, 853 op/s; 14377/318468 objects degraded (4.514%); 4087/318468 objects misplaced (1.283%); 7.0 MiB/s, 111 objects/s recovering 2024-09-21T10:02:31.769 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:31.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:31.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:31.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:31.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:32.029 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:32.065 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:32.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:02:32.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:32.234 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:32.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:32.336 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:32.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.365 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:32.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:32.466 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:32.499 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:32.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:32.600 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:32.632 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:32.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:32.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:32.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:32.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:32.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:32.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:32.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:33.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:33.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:33.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:33.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:33.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:33.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:33.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:33.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:33.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:33.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:33.487 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:33.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:33.508 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:33 smithi017 ceph-mon[102913]: pgmap v1511: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 104 B/s rd, 63 MiB/s wr, 1.10k op/s; 14377/325050 objects degraded (4.423%); 3922/325050 objects misplaced (1.207%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:02:33.508 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:33 smithi017 ceph-mon[105151]: pgmap v1511: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 104 B/s rd, 63 MiB/s wr, 1.10k op/s; 14377/325050 objects degraded (4.423%); 3922/325050 objects misplaced (1.207%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:02:33.589 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:33.625 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:33.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:33.727 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:33 smithi134 ceph-mon[90085]: pgmap v1511: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 104 B/s rd, 63 MiB/s wr, 1.10k op/s; 14377/325050 objects degraded (4.423%); 3922/325050 objects misplaced (1.207%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:02:33.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:33.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:33.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:33.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:33.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:34.003 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:34.033 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:34.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:34.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:34.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:34.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:34.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:34.382 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:02:34.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:34.456 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:34.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:34.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:34.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:34.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:34 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 14377/325050 objects degraded (4.423%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3677074804' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:34.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:34 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 14377/325050 objects degraded (4.423%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:34.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3677074804' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:34.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:34.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:34.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:34 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 14377/325050 objects degraded (4.423%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3677074804' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:34.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:34.869 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:34.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:34.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:35.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:35.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:35.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:35.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:35.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:02:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:02:34] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.51.0" 2024-09-21T10:02:35.252 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.285 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:35.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:35.385 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:35.429 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:35.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:35.532 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:35.563 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:35.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:35.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:35 smithi134 ceph-mon[90085]: pgmap v1512: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 56 MiB/s wr, 1.09k op/s; 14377/322659 objects degraded (4.456%); 3423/322659 objects misplaced (1.061%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:02:35.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:35 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:35.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3677074804' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:35.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:35 smithi134 ceph-mon[90085]: osdmap e1169: 8 total, 8 up, 7 in 2024-09-21T10:02:35.665 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:35.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:35.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:35 smithi017 ceph-mon[102913]: pgmap v1512: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 56 MiB/s wr, 1.09k op/s; 14377/322659 objects degraded (4.456%); 3423/322659 objects misplaced (1.061%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:02:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:35 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3677074804' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:35 smithi017 ceph-mon[102913]: osdmap e1169: 8 total, 8 up, 7 in 2024-09-21T10:02:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:35 smithi017 ceph-mon[105151]: pgmap v1512: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 56 MiB/s wr, 1.09k op/s; 14377/322659 objects degraded (4.456%); 3423/322659 objects misplaced (1.061%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:02:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:35 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3677074804' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:35 smithi017 ceph-mon[105151]: osdmap e1169: 8 total, 8 up, 7 in 2024-09-21T10:02:35.796 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:35.821 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:35.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:35.922 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:35.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.955 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:35.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:36.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:36.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:36.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:36.193 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:36.225 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:36.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:36.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:36.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:36.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:36.465 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:36.496 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:36.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:36.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:36.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:36.721 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:36.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.752 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:36.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:36.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:36.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:36.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:36.994 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:37.024 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:37.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:37.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:37.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:37.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:37.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:37.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:37.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:37.429 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:37.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:37.530 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:37.562 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:37.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:37 smithi017 ceph-mon[102913]: pgmap v1514: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 116 B/s rd, 34 MiB/s wr, 1.18k op/s; 14377/311886 objects degraded (4.610%); 3241/311886 objects misplaced (1.039%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:02:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:37 smithi017 ceph-mon[105151]: pgmap v1514: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 116 B/s rd, 34 MiB/s wr, 1.18k op/s; 14377/311886 objects degraded (4.610%); 3241/311886 objects misplaced (1.039%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:02:37.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:37.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:37 smithi134 ceph-mon[90085]: pgmap v1514: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 116 B/s rd, 34 MiB/s wr, 1.18k op/s; 14377/311886 objects degraded (4.610%); 3241/311886 objects misplaced (1.039%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:02:37.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:37.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:37.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:37.954 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:37.980 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:37.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:38.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:38.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:38.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:38.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:38.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:38.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:38.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:38.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:38.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:38.489 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:38.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.520 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:38.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:38.621 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:38.654 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:38.755 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:38.785 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:38.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:38.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:38.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:39.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:39.043 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:39.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:39.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:39.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:39.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:39.288 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:39.319 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:39.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:39.421 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:39.442 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:39.543 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:39.571 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:39.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:39.672 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:39 smithi017 ceph-mon[102913]: pgmap v1515: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 1.11k op/s; 14377/309795 objects degraded (4.641%); 3241/309795 objects misplaced (1.046%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:02:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:39 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 14377/309795 objects degraded (4.641%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:39 smithi017 ceph-mon[105151]: pgmap v1515: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 1.11k op/s; 14377/309795 objects degraded (4.641%); 3241/309795 objects misplaced (1.046%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:39 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 14377/309795 objects degraded (4.641%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:02:39.702 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:39.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:39 smithi134 ceph-mon[90085]: pgmap v1515: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 101 active+clean; 6.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 30 MiB/s wr, 1.11k op/s; 14377/309795 objects degraded (4.641%); 3241/309795 objects misplaced (1.046%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:02:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:39 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 14377/309795 objects degraded (4.641%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:02:39.804 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:39.832 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:39.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:39.934 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:39.957 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:39.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:40.059 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:40.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:40.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:40.183 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:40.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:40.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:40.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:40.328 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:40.430 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:40.459 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:40.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:40 smithi134 ceph-mon[90085]: osdmap e1170: 8 total, 8 up, 7 in 2024-09-21T10:02:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:02:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:02:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:40.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:40.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:40.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:40 smithi017 ceph-mon[102913]: osdmap e1170: 8 total, 8 up, 7 in 2024-09-21T10:02:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:40 smithi017 ceph-mon[105151]: osdmap e1170: 8 total, 8 up, 7 in 2024-09-21T10:02:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:40.707 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:40.740 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:40.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:40.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:40.874 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:40.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:40.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:41.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:41.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:41.115 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:41.144 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:41.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:41.246 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:41.276 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:41.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:41.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:41.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:41.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:41.566 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:41.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:41.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:41.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:41 smithi017 ceph-mon[102913]: pgmap v1517: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 101 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2.5 MiB/s wr, 886 op/s; 12615/305655 objects degraded (4.127%); 4434/305655 objects misplaced (1.451%); 9.8 MiB/s, 156 objects/s recovering 2024-09-21T10:02:41.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:41 smithi017 ceph-mon[102913]: osdmap e1171: 8 total, 8 up, 7 in 2024-09-21T10:02:41.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:41 smithi017 ceph-mon[105151]: pgmap v1517: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 101 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2.5 MiB/s wr, 886 op/s; 12615/305655 objects degraded (4.127%); 4434/305655 objects misplaced (1.451%); 9.8 MiB/s, 156 objects/s recovering 2024-09-21T10:02:41.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:41 smithi017 ceph-mon[105151]: osdmap e1171: 8 total, 8 up, 7 in 2024-09-21T10:02:41.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:41.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:41 smithi134 ceph-mon[90085]: pgmap v1517: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 101 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2.5 MiB/s wr, 886 op/s; 12615/305655 objects degraded (4.127%); 4434/305655 objects misplaced (1.451%); 9.8 MiB/s, 156 objects/s recovering 2024-09-21T10:02:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:41 smithi134 ceph-mon[90085]: osdmap e1171: 8 total, 8 up, 7 in 2024-09-21T10:02:41.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:41.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:41.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:41.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:42.008 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:42.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:42.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:42.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:42.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:42.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:42.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:42.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:42.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:42.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:42.529 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:42.557 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:42.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:02:42.658 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:42.680 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:42.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:42.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:42.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:42.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:42.922 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:42.946 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:42.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:43.048 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:43.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:43.182 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:43.212 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:43.314 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:43.334 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:43.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:43.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:43.560 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:43.593 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:43.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:43.696 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:43 smithi017 ceph-mon[102913]: pgmap v1519: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 12463/294915 objects degraded (4.226%); 4210/294915 objects misplaced (1.428%); 8.5 MiB/s, 135 objects/s recovering 2024-09-21T10:02:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:43 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 12463/294915 objects degraded (4.226%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:43 smithi017 ceph-mon[105151]: pgmap v1519: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 12463/294915 objects degraded (4.226%); 4210/294915 objects misplaced (1.428%); 8.5 MiB/s, 135 objects/s recovering 2024-09-21T10:02:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:43 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 12463/294915 objects degraded (4.226%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:43.727 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:43.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:43 smithi134 ceph-mon[90085]: pgmap v1519: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 12463/294915 objects degraded (4.226%); 4210/294915 objects misplaced (1.428%); 8.5 MiB/s, 135 objects/s recovering 2024-09-21T10:02:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:43 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 12463/294915 objects degraded (4.226%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:43.829 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:43.858 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:43.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:43.960 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:43.991 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:44.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:44.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:44.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:44.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:44.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:44.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:44.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:44.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:44.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:44.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:44.634 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:44.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:44.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:44.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:44.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:44.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:44.901 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:44.932 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:44.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:45.034 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:45.061 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:45.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:45.163 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:45.191 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:45.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:45.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:02:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:02:44] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.51.0" 2024-09-21T10:02:45.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:45.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:45.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:45.428 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:45.463 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:45.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:45.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:45.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:45 smithi017 ceph-mon[102913]: pgmap v1520: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 12463/292014 objects degraded (4.268%); 3842/292014 objects misplaced (1.316%); 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:02:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:45 smithi017 ceph-mon[105151]: pgmap v1520: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 12463/292014 objects degraded (4.268%); 3842/292014 objects misplaced (1.316%); 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:02:45.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:45 smithi134 ceph-mon[90085]: pgmap v1520: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 12463/292014 objects degraded (4.268%); 3842/292014 objects misplaced (1.316%); 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:02:45.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:45.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:45.847 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:45.874 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:45.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:45.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:46.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:46.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:46.107 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:46.130 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:46.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:46.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:46.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:46.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:46.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:46.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:46.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:46.555 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:46.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:46.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/89678646' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:02:46.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:46 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:02:46.674 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:46.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/89678646' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:02:46.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:46 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:02:46.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:46.674 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:02:46.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:46.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/89678646' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:02:46.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:46 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:02:46.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:46.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:46.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:46.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:46.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:46.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:47.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:47.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:47.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:47.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:47.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:47.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:47.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:47.506 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:47.529 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:47.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:47.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:47.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:47.689 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:47 smithi134 ceph-mon[90085]: pgmap v1521: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 940 op/s; 12463/287205 objects degraded (4.339%); 3626/287205 objects misplaced (1.263%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:02:47.689 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:47 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:02:47.689 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:47 smithi134 ceph-mon[90085]: osdmap e1172: 8 total, 8 up, 7 in 2024-09-21T10:02:47.689 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:47.689 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[102913]: pgmap v1521: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 940 op/s; 12463/287205 objects degraded (4.339%); 3626/287205 objects misplaced (1.263%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:02:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:02:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[102913]: osdmap e1172: 8 total, 8 up, 7 in 2024-09-21T10:02:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[105151]: pgmap v1521: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 940 op/s; 12463/287205 objects degraded (4.339%); 3626/287205 objects misplaced (1.263%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:02:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:02:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[105151]: osdmap e1172: 8 total, 8 up, 7 in 2024-09-21T10:02:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:47.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:47.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:47.913 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:47.944 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:47.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:48.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:48.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:48.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:48.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:48.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:48.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:48.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:48.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:48.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:48.458 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:48.464 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:48.491 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:48.593 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:48.622 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:48 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 12463/287205 objects degraded (4.339%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:48.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:48 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 12463/287205 objects degraded (4.339%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:48.707 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:48 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 12463/287205 objects degraded (4.339%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:48.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:48.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.774 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:48.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:48.803 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:02:48.803 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:02:48.842 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:02:48.842 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:02:48.875 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:02:48.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:02:48.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:48.911 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:02:48.911 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:02:48.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:48.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:48.955 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:02:48.955 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:02:48.989 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:02:48.989 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:02:49.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:49.034 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:02:49.034 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:02:49.076 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:02:49.077 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:02:49.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:49.186 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:49.229 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:49.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:49.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:49.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:49.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:49.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:49.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:49.646 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:49 smithi017 ceph-mon[102913]: pgmap v1523: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 984 op/s; 12239/282018 objects degraded (4.340%); 3626/282018 objects misplaced (1.286%); 7.8 MiB/s, 124 objects/s recovering 2024-09-21T10:02:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:49 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2349059298' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:49 smithi017 ceph-mon[105151]: pgmap v1523: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 984 op/s; 12239/282018 objects degraded (4.340%); 3626/282018 objects misplaced (1.286%); 7.8 MiB/s, 124 objects/s recovering 2024-09-21T10:02:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:49 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2349059298' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:49 smithi134 ceph-mon[90085]: pgmap v1523: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 984 op/s; 12239/282018 objects degraded (4.340%); 3626/282018 objects misplaced (1.286%); 7.8 MiB/s, 124 objects/s recovering 2024-09-21T10:02:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:49 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2349059298' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:49.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:49.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:49.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:49.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:49.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:02:49.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:49.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:50.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:50.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.074 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:50.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:50.175 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:50.202 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:50.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:50.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.352 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:50.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:50.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:50.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:50.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:02:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:02:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:50.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:50.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:50.733 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:50.763 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:50.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:50.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:50.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:50.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:51.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:51.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:51.146 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:51.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:51.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:51.279 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:51.288 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:51.300 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:51.309 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:51.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:51.411 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:51.440 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:51.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:51.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:51.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:51.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:51.697 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:51 smithi017 ceph-mon[102913]: pgmap v1524: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 924 op/s; 12239/278640 objects degraded (4.392%); 3284/278640 objects misplaced (1.179%); 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:02:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:51 smithi017 ceph-mon[105151]: pgmap v1524: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 924 op/s; 12239/278640 objects degraded (4.392%); 3284/278640 objects misplaced (1.179%); 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:02:51.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:51.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:51 smithi134 ceph-mon[90085]: pgmap v1524: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 924 op/s; 12239/278640 objects degraded (4.392%); 3284/278640 objects misplaced (1.179%); 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:02:51.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:51.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:51.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:52.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:52.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:52.024 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:02:52.024 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:02:52.115 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:52.152 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:52.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:52.254 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:52.281 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:52.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:52.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:52.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:52.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:52.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:52.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:52.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:52.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:52.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:52.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:02:52.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:52.835 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:52.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:52.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:52.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:53.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:53.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:53.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:53.186 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:52 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:53.186 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:52 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:53.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:52 smithi017 ceph-mon[102913]: pgmap v1525: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 920 op/s; 11937/267294 objects degraded (4.466%); 3046/267294 objects misplaced (1.140%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:02:53.187 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:52 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:53.187 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:52 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:53.187 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:52 smithi017 ceph-mon[105151]: pgmap v1525: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 920 op/s; 11937/267294 objects degraded (4.466%); 3046/267294 objects misplaced (1.140%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:02:53.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:52 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:52 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:02:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:52 smithi134 ceph-mon[90085]: pgmap v1525: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 920 op/s; 11937/267294 objects degraded (4.466%); 3046/267294 objects misplaced (1.140%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:02:53.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:53.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:53.379 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:53.402 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:53.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:53.503 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:53.536 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:53.638 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:53.668 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:53.769 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:53.801 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:53.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:53.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:54.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:54.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:54.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:54.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:54.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:54.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:53 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 11937/267294 objects degraded (4.466%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:54.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:54.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:54.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:54.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:53 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 11937/267294 objects degraded (4.466%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:54.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:53 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 11937/267294 objects degraded (4.466%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:54.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:54.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:54.589 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:54.621 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:54.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:54.723 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:54.752 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:54.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:54.854 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:54.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.885 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:54.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:54.986 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:55.017 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:55.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:55.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:55.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:55.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:54 smithi017 ceph-mon[102913]: pgmap v1526: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 11937/263676 objects degraded (4.527%); 2657/263676 objects misplaced (1.008%); 9.0 MiB/s, 144 objects/s recovering 2024-09-21T10:02:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:54 smithi017 ceph-mon[105151]: pgmap v1526: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 11937/263676 objects degraded (4.527%); 2657/263676 objects misplaced (1.008%); 9.0 MiB/s, 144 objects/s recovering 2024-09-21T10:02:55.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:02:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:02:54] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.51.0" 2024-09-21T10:02:55.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:54 smithi134 ceph-mon[90085]: pgmap v1526: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 11937/263676 objects degraded (4.527%); 2657/263676 objects misplaced (1.008%); 9.0 MiB/s, 144 objects/s recovering 2024-09-21T10:02:55.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:55.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:55.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:55.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:55.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:55.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:55.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:55.587 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:55.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:55.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:55.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:55.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:55.867 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:55.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:55.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:56.000 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:56.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:56.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:56.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:56.223 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:56.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:56.263 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:56.293 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:56.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:56.395 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:56.427 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:56.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:56.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:56.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:56.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:56.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:56.804 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:56.840 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:56.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:56.942 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:56.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:56.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:57.087 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:57.118 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:57.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:57.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:56 smithi134 ceph-mon[90085]: pgmap v1527: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 11937/258831 objects degraded (4.612%); 2412/258831 objects misplaced (0.932%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:02:57.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:57.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:57.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:56 smithi017 ceph-mon[102913]: pgmap v1527: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 11937/258831 objects degraded (4.612%); 2412/258831 objects misplaced (0.932%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:02:57.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:56 smithi017 ceph-mon[105151]: pgmap v1527: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 11937/258831 objects degraded (4.612%); 2412/258831 objects misplaced (0.932%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:02:57.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:57.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:57.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:57.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:57.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:57.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:57.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:57.681 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:57.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:57.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:57.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:57.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:57.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:57.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:58.074 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:58.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:58.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:58.211 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:58.240 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:58.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:58.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:58.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:58.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:58.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:58.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:58.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:58.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:58.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:58.928 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:58.953 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:58.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:59.055 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:59.083 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:59.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:59.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:59.217 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:59.319 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:59.350 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:59.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:59.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:02:59.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:59.472 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:59 smithi017 ceph-mon[102913]: pgmap v1528: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 876 op/s; 11683/252894 objects degraded (4.620%); 2412/252894 objects misplaced (0.954%); 8.0 MiB/s, 127 objects/s recovering 2024-09-21T10:02:59.472 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:02:59 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 11683/252894 objects degraded (4.620%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:59.472 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:59 smithi017 ceph-mon[105151]: pgmap v1528: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 876 op/s; 11683/252894 objects degraded (4.620%); 2412/252894 objects misplaced (0.954%); 8.0 MiB/s, 127 objects/s recovering 2024-09-21T10:02:59.472 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:02:59 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 11683/252894 objects degraded (4.620%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:59.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:59.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:59 smithi134 ceph-mon[90085]: pgmap v1528: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 876 op/s; 11683/252894 objects degraded (4.620%); 2412/252894 objects misplaced (0.954%); 8.0 MiB/s, 127 objects/s recovering 2024-09-21T10:02:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:02:59 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 11683/252894 objects degraded (4.620%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:02:59.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:59.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:59.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:59.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:59.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:02:59.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:59.910 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:59.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.939 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:02:59.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:00.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:00.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:00.165 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:00.191 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:00.293 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:00.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:00.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:00.429 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:00.458 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:00.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:03:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:03:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:00.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:00.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:00.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:00.696 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:00.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:00.821 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:00.852 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:00.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:00.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:00.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:01.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:01.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:01.217 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:01.240 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:01.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:01.262 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:03:01.342 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:01.372 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:01.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:01 smithi017 ceph-mon[105151]: pgmap v1529: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 921 op/s; 11683/248829 objects degraded (4.695%); 1952/248829 objects misplaced (0.784%); 10 MiB/s, 162 objects/s recovering 2024-09-21T10:03:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:01 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2520664936' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:03:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:01 smithi017 ceph-mon[102913]: pgmap v1529: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 921 op/s; 11683/248829 objects degraded (4.695%); 1952/248829 objects misplaced (0.784%); 10 MiB/s, 162 objects/s recovering 2024-09-21T10:03:01.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:01 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2520664936' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:03:01.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:01 smithi134 ceph-mon[90085]: pgmap v1529: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 921 op/s; 11683/248829 objects degraded (4.695%); 1952/248829 objects misplaced (0.784%); 10 MiB/s, 162 objects/s recovering 2024-09-21T10:03:01.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:01 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2520664936' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:03:01.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:01.613 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:01.644 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:01.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:01.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:01.870 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:01.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.901 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:01.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:02.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:02.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:02.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:02.279 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:03:02.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:02.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2520664936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:03:02.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[102913]: osdmap e1173: 8 total, 8 up, 7 in 2024-09-21T10:03:02.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1658641314' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:03:02.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:03:02.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:02.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2520664936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:03:02.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[105151]: osdmap e1173: 8 total, 8 up, 7 in 2024-09-21T10:03:02.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1658641314' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:03:02.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:02 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:03:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:02 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2520664936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:03:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:02 smithi134 ceph-mon[90085]: osdmap e1173: 8 total, 8 up, 7 in 2024-09-21T10:03:02.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1658641314' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:03:02.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:02 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:03:02.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:02.527 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:02.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:02.673 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:02.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:02.721 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:02.775 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:02.779 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:02.800 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:02.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:02.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:02.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:03.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:03.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:03.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:03.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:03.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:03.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:03.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:03.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:03.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:03.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:03.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:03.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:03.597 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:03 smithi017 ceph-mon[102913]: pgmap v1531: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 995 op/s; 11341/237423 objects degraded (4.777%); 1694/237423 objects misplaced (0.713%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:03:03.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:03 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:03:03.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:03 smithi017 ceph-mon[102913]: osdmap e1174: 8 total, 8 up, 7 in 2024-09-21T10:03:03.598 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:03 smithi017 ceph-mon[105151]: pgmap v1531: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 995 op/s; 11341/237423 objects degraded (4.777%); 1694/237423 objects misplaced (0.713%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:03:03.598 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:03 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:03:03.598 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:03 smithi017 ceph-mon[105151]: osdmap e1174: 8 total, 8 up, 7 in 2024-09-21T10:03:03.611 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:03.635 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:03.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:03.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:03 smithi134 ceph-mon[90085]: pgmap v1531: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 995 op/s; 11341/237423 objects degraded (4.777%); 1694/237423 objects misplaced (0.713%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:03:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:03 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:03:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:03 smithi134 ceph-mon[90085]: osdmap e1174: 8 total, 8 up, 7 in 2024-09-21T10:03:03.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:03.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:03.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:03.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:03.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:03.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:04.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:04.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 7 1 2024-09-21T10:03:04.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:04.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:04.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:04.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:04.265 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:04.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:04.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:04.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:04.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:04.511 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:04.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:04.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:04.565 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:04 smithi017 ceph-mon[102913]: osdmap e1175: 8 total, 8 up, 7 in 2024-09-21T10:03:04.565 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:04 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 11341/237423 objects degraded (4.777%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:04.565 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:04 smithi017 ceph-mon[105151]: osdmap e1175: 8 total, 8 up, 7 in 2024-09-21T10:03:04.565 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:04 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 11341/237423 objects degraded (4.777%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:04.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:04.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:04.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:04.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:04 smithi134 ceph-mon[90085]: osdmap e1175: 8 total, 8 up, 7 in 2024-09-21T10:03:04.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:04 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 11341/237423 objects degraded (4.777%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:04.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:04.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:04.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:04.909 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:03:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:03:04] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.51.0" 2024-09-21T10:03:04.932 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:04.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:04.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:05.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:05.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:05.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:05.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:05.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:05.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:03:05.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:05.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:05.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:05.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:05.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:05.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:05 smithi134 ceph-mon[90085]: pgmap v1534: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 11045/234132 objects degraded (4.717%); 1498/234132 objects misplaced (0.640%); 11 MiB/s, 229 objects/s recovering 2024-09-21T10:03:05.646 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:05 smithi134 ceph-mon[90085]: osdmap e1176: 8 total, 8 up, 7 in 2024-09-21T10:03:05.646 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:05 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:05.668 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:05.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:05 smithi017 ceph-mon[105151]: pgmap v1534: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 11045/234132 objects degraded (4.717%); 1498/234132 objects misplaced (0.640%); 11 MiB/s, 229 objects/s recovering 2024-09-21T10:03:05.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:05 smithi017 ceph-mon[105151]: osdmap e1176: 8 total, 8 up, 7 in 2024-09-21T10:03:05.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:05 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:05.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:05 smithi017 ceph-mon[102913]: pgmap v1534: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 11045/234132 objects degraded (4.717%); 1498/234132 objects misplaced (0.640%); 11 MiB/s, 229 objects/s recovering 2024-09-21T10:03:05.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:05 smithi017 ceph-mon[102913]: osdmap e1176: 8 total, 8 up, 7 in 2024-09-21T10:03:05.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:05 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:05.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.693 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:05.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:05.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:05.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:05.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:05.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:05.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:05.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:06.073 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:06.111 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:06.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:06.235 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:06.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:06.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:06.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:06.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:06.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:06.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:06.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:06.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:06.767 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:06.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:06.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:06.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:06.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:07.032 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:07.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.071 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:07.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:07.172 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:07.206 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:07.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:07.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:07.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:07.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:07.447 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:07.481 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:07.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:07.552 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:07 smithi017 ceph-mon[102913]: pgmap v1536: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 11045/229728 objects degraded (4.808%); 1134/229728 objects misplaced (0.494%); 7.2 MiB/s, 175 objects/s recovering 2024-09-21T10:03:07.552 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:07 smithi017 ceph-mon[105151]: pgmap v1536: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 11045/229728 objects degraded (4.808%); 1134/229728 objects misplaced (0.494%); 7.2 MiB/s, 175 objects/s recovering 2024-09-21T10:03:07.583 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:07.612 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:07.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:07 smithi134 ceph-mon[90085]: pgmap v1536: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 11045/229728 objects degraded (4.808%); 1134/229728 objects misplaced (0.494%); 7.2 MiB/s, 175 objects/s recovering 2024-09-21T10:03:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:07.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:07.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:07.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:07.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:08.007 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:08.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:08.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:08.142 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:08.172 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:08.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:08.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:08.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:08.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:08.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:08.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:08.577 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:03:08.578 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:03:08.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:08 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:08.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:08 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:08.601 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:08.614 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:03:08.614 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:03:08.628 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:08.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:08.650 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:03:08.650 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:03:08.687 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:03:08.687 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:03:08.721 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:03:08.721 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:03:08.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:08 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:08.754 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:03:08.754 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:03:08.759 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:08.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:08.783 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:03:08.783 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:03:08.814 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:03:08.814 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:03:08.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:08.886 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:08.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:08.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:09.020 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:09.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:09.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:09.262 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:09.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:09.391 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:09.421 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:09.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:09.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:09.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.554 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:09.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:09.654 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:09.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:09.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:09 smithi017 ceph-mon[102913]: pgmap v1537: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 10787/224268 objects degraded (4.810%); 1134/224268 objects misplaced (0.506%); 7.2 MiB/s, 164 objects/s recovering 2024-09-21T10:03:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:09 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 10787/224268 objects degraded (4.810%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:09 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:09 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/767778832' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:09 smithi017 ceph-mon[105151]: pgmap v1537: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 10787/224268 objects degraded (4.810%); 1134/224268 objects misplaced (0.506%); 7.2 MiB/s, 164 objects/s recovering 2024-09-21T10:03:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:09 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 10787/224268 objects degraded (4.810%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:09 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:09 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/767778832' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:09 smithi134 ceph-mon[90085]: pgmap v1537: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 10787/224268 objects degraded (4.810%); 1134/224268 objects misplaced (0.506%); 7.2 MiB/s, 164 objects/s recovering 2024-09-21T10:03:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:09 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 10787/224268 objects degraded (4.810%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:09 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:09 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/767778832' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:09.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:09.804 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:09.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:09.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:09.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:09.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:10.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:10.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:10.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:10.187 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:10.219 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:10.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:10.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:10.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:10.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:10.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:10 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:03:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:03:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:10.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:10.601 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:10.625 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:10.633 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:10.638 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:10.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:10 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:10.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:10.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:10 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:10.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:10.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:10.776 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:10.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:10.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.919 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:10.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:11.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:11.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:11.160 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:11.188 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:11.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:11.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:11.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:11.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:11.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:11 smithi017 ceph-mon[102913]: 7.16 scrub starts 2024-09-21T10:03:11.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:11 smithi017 ceph-mon[102913]: pgmap v1538: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 10787/220680 objects degraded (4.888%); 1134/220680 objects misplaced (0.514%); 5.5 MiB/s, 125 objects/s recovering 2024-09-21T10:03:11.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:11 smithi017 ceph-mon[102913]: 7.16 scrub ok 2024-09-21T10:03:11.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:11 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:11.370 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:11 smithi017 ceph-mon[105151]: 7.16 scrub starts 2024-09-21T10:03:11.370 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:11 smithi017 ceph-mon[105151]: pgmap v1538: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 10787/220680 objects degraded (4.888%); 1134/220680 objects misplaced (0.514%); 5.5 MiB/s, 125 objects/s recovering 2024-09-21T10:03:11.370 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:11 smithi017 ceph-mon[105151]: 7.16 scrub ok 2024-09-21T10:03:11.370 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:11 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:11.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:11.447 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:03:11.447 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:03:11.447 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:03:11.447 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:03:11.448 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:03:11.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:11.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:11.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:11.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:11.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:11 smithi134 ceph-mon[90085]: 7.16 scrub starts 2024-09-21T10:03:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:11 smithi134 ceph-mon[90085]: pgmap v1538: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 10787/220680 objects degraded (4.888%); 1134/220680 objects misplaced (0.514%); 5.5 MiB/s, 125 objects/s recovering 2024-09-21T10:03:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:11 smithi134 ceph-mon[90085]: 7.16 scrub ok 2024-09-21T10:03:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:11 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:11.753 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:11.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:11.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:11.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:11.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:12.005 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:12.031 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:12.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:12.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:12.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:12.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:12.285 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:12.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:03:12.310 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:12.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:12.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:12.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:12.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:12.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:12.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:12.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:12.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:12 smithi017 ceph-mon[102913]: 7.6 scrub starts 2024-09-21T10:03:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:12 smithi017 ceph-mon[105151]: 7.6 scrub starts 2024-09-21T10:03:12.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:12.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:12 smithi134 ceph-mon[90085]: 7.6 scrub starts 2024-09-21T10:03:12.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:12.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:12.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:12.965 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:12.992 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:13.094 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:13.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:13.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:13.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:13.256 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:13.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:13.358 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:13.381 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:13.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:13.483 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:13.503 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:13.605 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:13.626 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:13.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:13 smithi017 ceph-mon[102913]: pgmap v1539: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 10501/208818 objects degraded (5.029%); 842/208818 objects misplaced (0.403%); 7.9 MiB/s, 160 objects/s recovering 2024-09-21T10:03:13.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:13 smithi017 ceph-mon[102913]: 7.6 scrub ok 2024-09-21T10:03:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:13 smithi017 ceph-mon[105151]: pgmap v1539: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 10501/208818 objects degraded (5.029%); 842/208818 objects misplaced (0.403%); 7.9 MiB/s, 160 objects/s recovering 2024-09-21T10:03:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:13 smithi017 ceph-mon[105151]: 7.6 scrub ok 2024-09-21T10:03:13.727 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:13 smithi134 ceph-mon[90085]: pgmap v1539: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 10501/208818 objects degraded (5.029%); 842/208818 objects misplaced (0.403%); 7.9 MiB/s, 160 objects/s recovering 2024-09-21T10:03:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:13 smithi134 ceph-mon[90085]: 7.6 scrub ok 2024-09-21T10:03:13.748 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:13.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:13.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:13.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:13.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:13.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:14.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:14.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:14.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:14.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:14.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:14.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:14.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:14.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:14.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:14.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:14.615 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:14 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 10501/208818 objects degraded (5.029%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:14.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:14 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 10501/208818 objects degraded (5.029%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:14.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.634 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:14.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:14.735 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:14 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 10501/208818 objects degraded (5.029%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:14.764 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:14.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:14.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:14.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:14.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:03:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:03:14] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.51.0" 2024-09-21T10:03:15.012 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:15.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:15.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:15.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:15.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:15.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:15.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:15.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:15.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:15.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:15.572 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:15.605 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:15.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:15 smithi017 ceph-mon[102913]: pgmap v1540: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 19 op/s; 10501/205395 objects degraded (5.113%); 842/205395 objects misplaced (0.410%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T10:03:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:15 smithi017 ceph-mon[105151]: pgmap v1540: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 19 op/s; 10501/205395 objects degraded (5.113%); 842/205395 objects misplaced (0.410%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T10:03:15.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:15 smithi134 ceph-mon[90085]: pgmap v1540: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 19 op/s; 10501/205395 objects degraded (5.113%); 842/205395 objects misplaced (0.410%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T10:03:15.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:15.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:15.846 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:15.873 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:15.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:15.974 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:16.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:16.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:16.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:16.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:16.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:16.247 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:16.275 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:16.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:16.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:16.521 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:16.553 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:16.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:16.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:16.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:16 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:16.656 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:16 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:16.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:16.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:16.731 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:16 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:16.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:16.812 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:16.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:16.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:16.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:16.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:17.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:17.086 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:17.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:17.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:03:17.187 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:17.216 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:17.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:17.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:17.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:17.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:17.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:17.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:17.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:17.626 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:17.659 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:17.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:17 smithi017 ceph-mon[102913]: pgmap v1541: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 10501/200784 objects degraded (5.230%); 521/200784 objects misplaced (0.259%); 6.6 MiB/s, 105 objects/s recovering 2024-09-21T10:03:17.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:17 smithi017 ceph-mon[105151]: pgmap v1541: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 10501/200784 objects degraded (5.230%); 521/200784 objects misplaced (0.259%); 6.6 MiB/s, 105 objects/s recovering 2024-09-21T10:03:17.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:17 smithi134 ceph-mon[90085]: pgmap v1541: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 10501/200784 objects degraded (5.230%); 521/200784 objects misplaced (0.259%); 6.6 MiB/s, 105 objects/s recovering 2024-09-21T10:03:17.760 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:17.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:17.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:17.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:17.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:17.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:18.046 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:18.076 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:18.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:18.177 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:18.202 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:18.304 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:18.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:18.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:18.405 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:18.431 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:18.456 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:18.558 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:18.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:18.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:18.687 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:18.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:18.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:18.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:18.849 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:18.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:18.951 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:18.981 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:18.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:19.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:19.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:19.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:19.224 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:19.255 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:19.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:19.357 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:19.381 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2024-09-21T10:03:19.381 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:19.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:19.388 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:03:19.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:19.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:19.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:19.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:19.657 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:19.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[102913]: pgmap v1542: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 10220/195294 objects degraded (5.233%); 521/195294 objects misplaced (0.267%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:03:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/792285278' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 10220/195294 objects degraded (5.233%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3451203378' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[105151]: pgmap v1542: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 10220/195294 objects degraded (5.233%); 521/195294 objects misplaced (0.267%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/792285278' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 10220/195294 objects degraded (5.233%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3451203378' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:19.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:19 smithi134 ceph-mon[90085]: pgmap v1542: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 10220/195294 objects degraded (5.233%); 521/195294 objects misplaced (0.267%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:03:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:19 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/792285278' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:03:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:19 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 10220/195294 objects degraded (5.233%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:19 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3451203378' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:19 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:19 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:19 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:19.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:19.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:19.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:19.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:19.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:19.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:20.030 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:20.060 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:20.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:20.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:20.193 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:20.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:20.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:20.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:20.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:03:20.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:03:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:03:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:20 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:03:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:20 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/792285278' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:03:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:20 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:03:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:20 smithi134 ceph-mon[90085]: osdmap e1177: 8 total, 8 up, 7 in 2024-09-21T10:03:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:20 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:20 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:20.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:20 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:20.591 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:20.617 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:03:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/792285278' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:03:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:03:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[102913]: osdmap e1177: 8 total, 8 up, 7 in 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/792285278' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[105151]: osdmap e1177: 8 total, 8 up, 7 in 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:20.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:20.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:20.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:20.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:20.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.930 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:20.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:21.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:21.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:21.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:21.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:21.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:21.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:21.323 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:21.328 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:21.355 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:21.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:21.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:21.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:21.633 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:21.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:21.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:21 smithi017 ceph-mon[102913]: pgmap v1544: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 10220/191601 objects degraded (5.334%); 521/191601 objects misplaced (0.272%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:03:21.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:21 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:21.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:21 smithi017 ceph-mon[105151]: pgmap v1544: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 10220/191601 objects degraded (5.334%); 521/191601 objects misplaced (0.272%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:03:21.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:21 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:21.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:21 smithi134 ceph-mon[90085]: pgmap v1544: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 31 op/s; 10220/191601 objects degraded (5.334%); 521/191601 objects misplaced (0.272%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:03:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:21 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:21.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:21.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:21.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:21.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:22.023 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:22.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:22.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:22.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:22.185 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:22.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:22.287 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:22.316 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:22.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:22.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:22.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:22.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:22.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:22.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:22.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:22.659 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:22.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:22.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:22 smithi017 ceph-mon[102913]: 7.1 deep-scrub starts 2024-09-21T10:03:22.703 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:22 smithi017 ceph-mon[105151]: 7.1 deep-scrub starts 2024-09-21T10:03:22.728 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:22.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:22 smithi134 ceph-mon[90085]: 7.1 deep-scrub starts 2024-09-21T10:03:22.830 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:22.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:22.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:22.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:03:22.959 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:22.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:22.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:23.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:23.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:23.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:23.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:23.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:23.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:23.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:23.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:23.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:23.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:23.644 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:23.668 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:23.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:23 smithi017 ceph-mon[102913]: pgmap v1545: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 9952/180177 objects degraded (5.523%); 182/180177 objects misplaced (0.101%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T10:03:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:23 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:23 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 9952/180177 objects degraded (5.523%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:23.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:23 smithi017 ceph-mon[102913]: osdmap e1178: 8 total, 8 up, 7 in 2024-09-21T10:03:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:23 smithi017 ceph-mon[105151]: pgmap v1545: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 9952/180177 objects degraded (5.523%); 182/180177 objects misplaced (0.101%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T10:03:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:23 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:23 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 9952/180177 objects degraded (5.523%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:23 smithi017 ceph-mon[105151]: osdmap e1178: 8 total, 8 up, 7 in 2024-09-21T10:03:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:23 smithi134 ceph-mon[90085]: pgmap v1545: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 9952/180177 objects degraded (5.523%); 182/180177 objects misplaced (0.101%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T10:03:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:23 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:23 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 9952/180177 objects degraded (5.523%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:23 smithi134 ceph-mon[90085]: osdmap e1178: 8 total, 8 up, 7 in 2024-09-21T10:03:23.770 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:23.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:23.894 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:23.915 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:24.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:24.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:24.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:24.145 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:24.167 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:24.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:24.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:24.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:24.397 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:24.422 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:24.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:24.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:24.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:24.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:24 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:24.602 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:24 smithi017 ceph-mon[102913]: osdmap e1179: 8 total, 8 up, 7 in 2024-09-21T10:03:24.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:24 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:24.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:24 smithi017 ceph-mon[105151]: osdmap e1179: 8 total, 8 up, 7 in 2024-09-21T10:03:24.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:24.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:24.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:24 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:24 smithi134 ceph-mon[90085]: osdmap e1179: 8 total, 8 up, 7 in 2024-09-21T10:03:24.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:24.838 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:24.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:24.844 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:03:24.844 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:03:24.871 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:03:24.871 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:03:24.872 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:03:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:03:24] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.51.0" 2024-09-21T10:03:24.907 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:03:24.907 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:03:24.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:24.949 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:03:24.949 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:03:24.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:03:24.984 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:03:24.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:24.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:25.019 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:03:25.019 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:03:25.052 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:03:25.053 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:03:25.087 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:03:25.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:03:25.089 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:25.118 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:25.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:25.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:25.365 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:25.397 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:25.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:25.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:25.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:25.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:25.637 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:25.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:25 smithi017 ceph-mon[102913]: pgmap v1547: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 9952/176478 objects degraded (5.639%); 182/176478 objects misplaced (0.103%); 4.8 MiB/s, 76 objects/s recovering 2024-09-21T10:03:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:25 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3573389002' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:25 smithi017 ceph-mon[105151]: pgmap v1547: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 9952/176478 objects degraded (5.639%); 182/176478 objects misplaced (0.103%); 4.8 MiB/s, 76 objects/s recovering 2024-09-21T10:03:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:25 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3573389002' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:25.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:25 smithi134 ceph-mon[90085]: pgmap v1547: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 9952/176478 objects degraded (5.639%); 182/176478 objects misplaced (0.103%); 4.8 MiB/s, 76 objects/s recovering 2024-09-21T10:03:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:25 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3573389002' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:25.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:25.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:25.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:25.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:25.933 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:25.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:25.971 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:03:25.971 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:03:26.034 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:26.059 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:26.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:26.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:26.198 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:26.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:26.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:26.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:26.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:26.429 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:26.460 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:26.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:26.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:26.599 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:26.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:26.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:26.726 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:26.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:26.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:26.869 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:26.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:26.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:27.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:27.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:27.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:27.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:27.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:27.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:27.382 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:27.416 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:27.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:27.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:27.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:27.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:27.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:27.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:27.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:27.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:27 smithi017 ceph-mon[102913]: pgmap v1549: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 950 op/s; 9025/172305 objects degraded (5.238%); 776/172305 objects misplaced (0.450%); 7.4 MiB/s, 119 objects/s recovering 2024-09-21T10:03:27.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:27 smithi017 ceph-mon[105151]: pgmap v1549: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 950 op/s; 9025/172305 objects degraded (5.238%); 776/172305 objects misplaced (0.450%); 7.4 MiB/s, 119 objects/s recovering 2024-09-21T10:03:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:27 smithi134 ceph-mon[90085]: pgmap v1549: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 950 op/s; 9025/172305 objects degraded (5.238%); 776/172305 objects misplaced (0.450%); 7.4 MiB/s, 119 objects/s recovering 2024-09-21T10:03:27.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:27.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:27.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:27.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:27.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:28.078 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:28.100 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:28.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:28.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:28.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:28.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:28.344 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:28.384 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:28.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:28.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:28.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:28.625 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:28.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:28 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 9025/172305 objects degraded (5.238%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:28 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 9025/172305 objects degraded (5.238%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:28 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 9025/172305 objects degraded (5.238%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:28.754 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:28.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:28.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:28.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:28.935 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:28.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:29.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:29.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:29.174 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:29.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.203 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:29.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:29.304 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:29.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:29.433 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:29.455 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:29.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:29.556 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:29.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:29.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:29 smithi017 ceph-mon[102913]: pgmap v1550: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 8272/167136 objects degraded (4.949%); 995/167136 objects misplaced (0.595%); 9.5 MiB/s, 152 objects/s recovering 2024-09-21T10:03:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:29 smithi017 ceph-mon[102913]: 7.1 deep-scrub starts 2024-09-21T10:03:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:29 smithi017 ceph-mon[105151]: pgmap v1550: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 8272/167136 objects degraded (4.949%); 995/167136 objects misplaced (0.595%); 9.5 MiB/s, 152 objects/s recovering 2024-09-21T10:03:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:29 smithi017 ceph-mon[105151]: 7.1 deep-scrub starts 2024-09-21T10:03:29.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:29.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:29 smithi134 ceph-mon[90085]: pgmap v1550: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 8272/167136 objects degraded (4.949%); 995/167136 objects misplaced (0.595%); 9.5 MiB/s, 152 objects/s recovering 2024-09-21T10:03:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:29 smithi134 ceph-mon[90085]: 7.1 deep-scrub starts 2024-09-21T10:03:29.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:29.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:29.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:29.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:30.096 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:30.127 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:30.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:30.250 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:30.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:30.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:30.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:30.480 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:03:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:03:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:30.504 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:30.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:30.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:30.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:30.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:30.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:30.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:30.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:30.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:30.937 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:30.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:31.038 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:31.073 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:31.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:31.174 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:31.206 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:31.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:31.308 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:31.335 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:31.437 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:31.467 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:31.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:31.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:31.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:31 smithi017 ceph-mon[102913]: pgmap v1551: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 8272/164073 objects degraded (5.042%); 995/164073 objects misplaced (0.606%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T10:03:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:31 smithi017 ceph-mon[105151]: pgmap v1551: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 8272/164073 objects degraded (5.042%); 995/164073 objects misplaced (0.606%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T10:03:31.708 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:31 smithi134 ceph-mon[90085]: pgmap v1551: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 8272/164073 objects degraded (5.042%); 995/164073 objects misplaced (0.606%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T10:03:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.745 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:31.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:31.846 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:31.879 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:31.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:03:31.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:32.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:32.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:32.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:32.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:32.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:32.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:32.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:32.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:32.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:32.558 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:32.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:32.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:32.589 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:32.663 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:32.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:32.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:32.796 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:32.824 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:32.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:32.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:32.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:32.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:33.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:33.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:33.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:33.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:33.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:03:33.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:33 smithi017 ceph-mon[102913]: pgmap v1552: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 7956/153576 objects degraded (5.180%); 545/153576 objects misplaced (0.355%); 9.1 MiB/s, 156 objects/s recovering 2024-09-21T10:03:33.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7956/153576 objects degraded (5.180%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:33.537 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:33 smithi017 ceph-mon[105151]: pgmap v1552: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 7956/153576 objects degraded (5.180%); 545/153576 objects misplaced (0.355%); 9.1 MiB/s, 156 objects/s recovering 2024-09-21T10:03:33.537 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7956/153576 objects degraded (5.180%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:33.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:33.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:33.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:33.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:33.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:33 smithi134 ceph-mon[90085]: pgmap v1552: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 7956/153576 objects degraded (5.180%); 545/153576 objects misplaced (0.355%); 9.1 MiB/s, 156 objects/s recovering 2024-09-21T10:03:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7956/153576 objects degraded (5.180%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:33.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:33.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:33.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:33.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:34.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:34.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:34.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:34.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:34.267 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.298 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:34.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:34.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:34.429 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:34.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:34.476 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:03:34.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:34.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:34.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:34.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:34.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:34.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:34 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:34.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2646695605' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:03:34.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:34 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:03:34.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:34 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:34.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2646695605' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:03:34.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:34 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:03:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:34 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2646695605' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:03:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:34 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:03:34.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:34.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:34.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.980 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:34.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:35.000 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:03:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:03:34] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.51.0" 2024-09-21T10:03:35.081 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:35.108 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:35.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:35.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:35.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:35.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:35.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:35.477 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:35.510 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:35.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:35.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:35.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:35.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:35.735 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:35 smithi134 ceph-mon[90085]: pgmap v1553: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 883 op/s; 7956/149967 objects degraded (5.305%); 545/149967 objects misplaced (0.363%); 8.0 MiB/s, 136 objects/s recovering 2024-09-21T10:03:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:35 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:03:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:35 smithi134 ceph-mon[90085]: osdmap e1180: 8 total, 8 up, 7 in 2024-09-21T10:03:35.775 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:35.876 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:35.914 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:35.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:35 smithi017 ceph-mon[102913]: pgmap v1553: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 883 op/s; 7956/149967 objects degraded (5.305%); 545/149967 objects misplaced (0.363%); 8.0 MiB/s, 136 objects/s recovering 2024-09-21T10:03:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:35 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:03:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:35 smithi017 ceph-mon[102913]: osdmap e1180: 8 total, 8 up, 7 in 2024-09-21T10:03:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:35 smithi017 ceph-mon[105151]: pgmap v1553: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 883 op/s; 7956/149967 objects degraded (5.305%); 545/149967 objects misplaced (0.363%); 8.0 MiB/s, 136 objects/s recovering 2024-09-21T10:03:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:35 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:03:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:35 smithi017 ceph-mon[105151]: osdmap e1180: 8 total, 8 up, 7 in 2024-09-21T10:03:36.015 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:36.052 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:36.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:36.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:36.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:36.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:36.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:36.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:36.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:36.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:36.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:36.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:36.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:36.652 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:03:36.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:03:36.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:36.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:36.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:36.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:37.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:37.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:37.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:37.219 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:37.242 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:37.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:37.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:37.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:37.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:37.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:37.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:37.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:37.606 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:37.639 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:37.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:37.741 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:37 smithi134 ceph-mon[90085]: pgmap v1555: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 8122/144858 objects degraded (5.607%); 219/144858 objects misplaced (0.151%); 6.9 MiB/s, 119 objects/s recovering 2024-09-21T10:03:37.768 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:37.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:37.794 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:37 smithi017 ceph-mon[102913]: pgmap v1555: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 8122/144858 objects degraded (5.607%); 219/144858 objects misplaced (0.151%); 6.9 MiB/s, 119 objects/s recovering 2024-09-21T10:03:37.795 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:37 smithi017 ceph-mon[105151]: pgmap v1555: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 8122/144858 objects degraded (5.607%); 219/144858 objects misplaced (0.151%); 6.9 MiB/s, 119 objects/s recovering 2024-09-21T10:03:37.870 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.904 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:37.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:38.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:38.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:38.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:38.184 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:38.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:38.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:38.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:38.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:38.427 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:38.463 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:38.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:38.565 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:38.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.598 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:38.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:38.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:38.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:38.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:38.748 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 8122/144858 objects degraded (5.607%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:38.748 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 8122/144858 objects degraded (5.607%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:38.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:38.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:38.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:38.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 8122/144858 objects degraded (5.607%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:39.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:39.134 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:39.160 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:39.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:39.187 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:03:39.187 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:03:39.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:39.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:39.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:39.402 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:39.432 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:39.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:39.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:39.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:39.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:39.680 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:39.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:39.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:39.764 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:03:39.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:39 smithi017 ceph-mon[102913]: pgmap v1556: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 943 op/s; 7612/138822 objects degraded (5.483%); 219/138822 objects misplaced (0.158%); 4.9 MiB/s, 120 objects/s recovering 2024-09-21T10:03:39.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:39 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3780965310' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:39.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:39 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:39.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:39 smithi017 ceph-mon[105151]: pgmap v1556: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 943 op/s; 7612/138822 objects degraded (5.483%); 219/138822 objects misplaced (0.158%); 4.9 MiB/s, 120 objects/s recovering 2024-09-21T10:03:39.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:39 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3780965310' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:39.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:39 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:39.810 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:39.840 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:39.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:39.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:39.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:39 smithi134 ceph-mon[90085]: pgmap v1556: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 943 op/s; 7612/138822 objects degraded (5.483%); 219/138822 objects misplaced (0.158%); 4.9 MiB/s, 120 objects/s recovering 2024-09-21T10:03:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:39 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3780965310' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:39 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:40.087 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:40.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:40.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:40.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:40.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:40.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:40.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:40.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:40.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:03:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:03:40.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:40.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:40.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:40.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:03:40.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:40.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:03:40.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[102913]: osdmap e1181: 8 total, 8 up, 7 in 2024-09-21T10:03:40.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:40.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:40.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:03:40.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:40.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:03:40.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[105151]: osdmap e1181: 8 total, 8 up, 7 in 2024-09-21T10:03:40.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:40.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:40.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:40.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:40.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:03:40.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:40.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:40.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:40.935 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:40.962 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:40.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:40 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:03:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:40 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:40 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:03:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:40 smithi134 ceph-mon[90085]: osdmap e1181: 8 total, 8 up, 7 in 2024-09-21T10:03:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:03:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:41.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:41.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:41.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:41.174 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:41.197 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:41.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:41.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:41.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:41.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:41.532 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:41.555 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:41.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:41.657 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:41.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:41 smithi017 ceph-mon[102913]: pgmap v1558: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 7612/135084 objects degraded (5.635%); 219/135084 objects misplaced (0.162%); 1.9 MiB/s, 72 objects/s recovering 2024-09-21T10:03:41.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:41 smithi017 ceph-mon[105151]: pgmap v1558: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 7612/135084 objects degraded (5.635%); 219/135084 objects misplaced (0.162%); 1.9 MiB/s, 72 objects/s recovering 2024-09-21T10:03:41.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:41.810 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:41.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:41.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:41.932 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:41 smithi134 ceph-mon[90085]: pgmap v1558: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 7612/135084 objects degraded (5.635%); 219/135084 objects misplaced (0.162%); 1.9 MiB/s, 72 objects/s recovering 2024-09-21T10:03:42.034 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:42.064 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:42.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:42.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:42.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:42.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:42.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:42.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:42.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:42.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:42.570 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:42.591 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:42.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:42.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:42.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:42.819 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:42.841 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:42.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:42.943 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:42.973 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:42.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:43.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:43.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:43.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:43.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:03:43.211 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:43.239 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:43.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:43.340 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:43.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.367 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:43.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:43.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:43.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:43.594 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:43.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.624 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:43.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:43.725 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:43.746 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:43.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:43.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:43.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:43.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:43 smithi017 ceph-mon[102913]: pgmap v1559: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 7197/124209 objects degraded (5.794%); 3.2 MiB/s, 151 objects/s recovering 2024-09-21T10:03:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:43 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7197/124209 objects degraded (5.794%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:43 smithi017 ceph-mon[105151]: pgmap v1559: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 7197/124209 objects degraded (5.794%); 3.2 MiB/s, 151 objects/s recovering 2024-09-21T10:03:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:43 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7197/124209 objects degraded (5.794%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:43.987 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:43 smithi134 ceph-mon[90085]: pgmap v1559: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 7197/124209 objects degraded (5.794%); 3.2 MiB/s, 151 objects/s recovering 2024-09-21T10:03:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:43 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7197/124209 objects degraded (5.794%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:44.021 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:44.123 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:44.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.151 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:44.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:44.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:44.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:44.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:44.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:44.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:44.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:44.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:44.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:44.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:44.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:44.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:44.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:44.814 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:44.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:44.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.947 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:44.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:45.048 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:45.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:45.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:45.182 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:03:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:03:44] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.51.0" 2024-09-21T10:03:45.207 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:45.309 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:45.343 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:45.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:45.445 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:45.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.475 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:45.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:45.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:45.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:45.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:45.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:45.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:45.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:45.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:45.886 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:45.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:45 smithi017 ceph-mon[102913]: pgmap v1560: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 999 op/s; 7197/120906 objects degraded (5.953%); 2.6 MiB/s, 125 objects/s recovering 2024-09-21T10:03:45.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:45 smithi017 ceph-mon[105151]: pgmap v1560: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 999 op/s; 7197/120906 objects degraded (5.953%); 2.6 MiB/s, 125 objects/s recovering 2024-09-21T10:03:45.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:45 smithi134 ceph-mon[90085]: pgmap v1560: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 999 op/s; 7197/120906 objects degraded (5.953%); 2.6 MiB/s, 125 objects/s recovering 2024-09-21T10:03:46.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:46.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:46.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:46.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:46.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:46.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:46.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:46.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:46.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:46.438 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:46.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:46.467 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:46.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:46.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:46.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:46.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:46.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:46.828 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:46.861 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:46.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:47.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:47.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:47.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:47.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:47.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:47.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:47.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:47.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:47.394 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:47.426 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:47.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:47.555 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:47.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:47.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:47 smithi017 ceph-mon[102913]: pgmap v1561: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 962 op/s; 7004/115971 objects degraded (6.039%); 1.6 MiB/s, 124 objects/s recovering 2024-09-21T10:03:47.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:47 smithi017 ceph-mon[105151]: pgmap v1561: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 962 op/s; 7004/115971 objects degraded (6.039%); 1.6 MiB/s, 124 objects/s recovering 2024-09-21T10:03:47.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:47.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:47 smithi134 ceph-mon[90085]: pgmap v1561: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 962 op/s; 7004/115971 objects degraded (6.039%); 1.6 MiB/s, 124 objects/s recovering 2024-09-21T10:03:47.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:47.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:47.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:47.938 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:47.970 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:48.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:48.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:48.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:48.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:48.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:48.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:48.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:48.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:48.506 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:48.544 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:48.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:48.646 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:48.679 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:48.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:48.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:48.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:48.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:48.880 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:48 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7004/115971 objects degraded (6.039%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:48.880 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:48 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7004/115971 objects degraded (6.039%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:48.922 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:48.953 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:48 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7004/115971 objects degraded (6.039%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:49.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:49.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:49.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:49.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:49.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:49.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:49.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:49.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:49.464 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:49.468 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:03:49.470 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:03:49.498 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:49.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:49.600 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:49.632 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:49.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:49.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:49.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:49.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:49.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:49 smithi017 ceph-mon[102913]: pgmap v1562: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 942 op/s; 6530/110520 objects degraded (5.908%); 1.0 MiB/s, 129 objects/s recovering 2024-09-21T10:03:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:49 smithi017 ceph-mon[105151]: pgmap v1562: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 942 op/s; 6530/110520 objects degraded (5.908%); 1.0 MiB/s, 129 objects/s recovering 2024-09-21T10:03:49.993 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:49 smithi134 ceph-mon[90085]: pgmap v1562: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 942 op/s; 6530/110520 objects degraded (5.908%); 1.0 MiB/s, 129 objects/s recovering 2024-09-21T10:03:50.020 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:50.122 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:50.148 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:50.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:50.250 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:50.273 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:50.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:50.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:50.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:03:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:03:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:50.499 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:50.521 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:50.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:50.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:50.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:50.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:50.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:50.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:50.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:50.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:50 smithi017 ceph-mon[102913]: from='client.35655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:03:50.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:50 smithi017 ceph-mon[102913]: osdmap e1182: 8 total, 8 up, 7 in 2024-09-21T10:03:50.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:50 smithi017 ceph-mon[105151]: from='client.35655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:03:50.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:50 smithi017 ceph-mon[105151]: osdmap e1182: 8 total, 8 up, 7 in 2024-09-21T10:03:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:50 smithi134 ceph-mon[90085]: from='client.35655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:03:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:50 smithi134 ceph-mon[90085]: osdmap e1182: 8 total, 8 up, 7 in 2024-09-21T10:03:51.039 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:51.070 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:51.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:51.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:51.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:51.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:51.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:51.340 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:51.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:51.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:51.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:51.583 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:51.615 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:51.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:51.716 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:51.748 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:51.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:51.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:51 smithi017 ceph-mon[102913]: pgmap v1564: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 953 op/s; 6530/106464 objects degraded (6.134%); 1.0 MiB/s, 130 objects/s recovering 2024-09-21T10:03:51.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:51 smithi017 ceph-mon[102913]: osdmap e1183: 8 total, 8 up, 7 in 2024-09-21T10:03:51.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:51 smithi017 ceph-mon[105151]: pgmap v1564: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 953 op/s; 6530/106464 objects degraded (6.134%); 1.0 MiB/s, 130 objects/s recovering 2024-09-21T10:03:51.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:51 smithi017 ceph-mon[105151]: osdmap e1183: 8 total, 8 up, 7 in 2024-09-21T10:03:51.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:51.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:51.991 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:51.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:51 smithi134 ceph-mon[90085]: pgmap v1564: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 2.2 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 953 op/s; 6530/106464 objects degraded (6.134%); 1.0 MiB/s, 130 objects/s recovering 2024-09-21T10:03:51.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:51 smithi134 ceph-mon[90085]: osdmap e1183: 8 total, 8 up, 7 in 2024-09-21T10:03:52.022 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:52.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:52.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:52.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:52.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:52.325 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:03:52.325 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:03:52.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:52.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:52.356 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:03:52.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:03:52.395 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:03:52.395 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:03:52.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:52.435 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:03:52.435 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:03:52.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:52.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:52.474 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:03:52.474 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:03:52.514 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:03:52.515 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:03:52.551 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:03:52.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:03:52.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:52.590 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:03:52.591 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:03:52.602 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:52.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:52.606 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:03:52.606 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:03:52.606 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:03:52.606 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:03:52.606 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:03:52.606 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:52.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:52 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1180114423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:52.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:52 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1180114423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:52.704 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:52.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.734 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:52.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:52.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:52.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:52.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:52.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:52 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1180114423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:53.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:53.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:53.125 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:53.153 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:53.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:53.182 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1562,"stamp":"2024-09-21T10:03:48.169265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2413059708,"num_objects":36840,"num_object_clones":0,"num_object_copies":110520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6530,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36840,"num_whiteouts":0,"num_read":108,"num_read_kb":273,"num_write":168332,"num_write_kb":6272624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20720,"num_bytes_recovered":1234725977,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":176877,"ondisk_log_size":176877,"up":327,"acting":322,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":329,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9884856,"kb_used_data":7376704,"kb_used_omap":95,"kb_used_meta":2507424,"kb_avail":646331208,"statfs":{"total":671965249536,"available":661843156992,"internally_reserved":0,"allocated":7553744896,"data_stored":7535888830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97681,"internal_metadata":2567602799},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-618266624,"num_objects":-9434,"num_object_clones":0,"num_object_copies":-28302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1082,"num_objects_misplaced":-219,"num_objects_unfound":0,"num_objects_dirty":-9434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9434,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1301,"num_bytes_recovered":10551296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008681"},"pg_stats":[{"pgid":"7.17","version":"1181'7376","reported_seq":7432,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:45.497226+0000","last_change":"2024-09-21T10:01:27.381508+0000","last_active":"2024-09-21T10:03:45.497226+0000","last_peered":"2024-09-21T10:03:45.497226+0000","last_clean":"2024-09-21T10:03:45.497226+0000","last_became_active":"2024-09-21T10:01:27.375972+0000","last_became_peered":"2024-09-21T10:01:27.375972+0000","last_unstale":"2024-09-21T10:03:45.497226+0000","last_undegraded":"2024-09-21T10:03:45.497226+0000","last_fullsized":"2024-09-21T10:03:45.497226+0000","mapping_epoch":1148,"log_start":"0'0","ondisk_log_start":"0'0","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7376,"log_dups_size":0,"ondisk_log_size":7376,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75726848,"num_objects":1156,"num_object_clones":0,"num_object_copies":3468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5714,"num_write_kb":219808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1706,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.522296+0000","last_change":"2024-09-21T10:01:40.433720+0000","last_active":"2024-09-21T10:03:39.522296+0000","last_peered":"2024-09-21T10:03:39.522296+0000","last_clean":"2024-09-21T10:03:39.522296+0000","last_became_active":"2024-09-21T10:01:40.433360+0000","last_became_peered":"2024-09-21T10:01:40.433360+0000","last_unstale":"2024-09-21T10:03:39.522296+0000","last_undegraded":"2024-09-21T10:03:39.522296+0000","last_fullsized":"2024-09-21T10:03:39.522296+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:08.257176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1106,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.477102+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T10:03:34.477102+0000","last_peered":"2024-09-21T10:03:34.477102+0000","last_clean":"2024-09-21T10:03:34.477102+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T10:03:34.477102+0000","last_undegraded":"2024-09-21T10:03:34.477102+0000","last_fullsized":"2024-09-21T10:03:34.477102+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.16","version":"1181'7304","reported_seq":7434,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:43.898771+0000","last_change":"2024-09-21T10:03:10.420081+0000","last_active":"2024-09-21T10:03:43.898771+0000","last_peered":"2024-09-21T10:03:43.898771+0000","last_clean":"2024-09-21T10:03:43.898771+0000","last_became_active":"2024-09-21T10:01:27.383294+0000","last_became_peered":"2024-09-21T10:01:27.383294+0000","last_unstale":"2024-09-21T10:03:43.898771+0000","last_undegraded":"2024-09-21T10:03:43.898771+0000","last_fullsized":"2024-09-21T10:03:43.898771+0000","mapping_epoch":1148,"log_start":"0'0","ondisk_log_start":"0'0","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"1176'6333","last_scrub_stamp":"2024-09-21T10:03:10.419712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:03:10.419712+0000","objects_scrubbed":2183,"log_size":7304,"log_dups_size":0,"ondisk_log_size":7304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:18.821196+0000","scrub_duration":884,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78446592,"num_objects":1197,"num_object_clones":0,"num_object_copies":3591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":986,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":614,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.477266+0000","last_change":"2024-09-21T09:57:39.541887+0000","last_active":"2024-09-21T10:03:34.477266+0000","last_peered":"2024-09-21T10:03:34.477266+0000","last_clean":"2024-09-21T10:03:34.477266+0000","last_became_active":"2024-09-21T09:57:39.541598+0000","last_became_peered":"2024-09-21T09:57:39.541598+0000","last_unstale":"2024-09-21T10:03:34.477266+0000","last_undegraded":"2024-09-21T10:03:34.477266+0000","last_fullsized":"2024-09-21T10:03:34.477266+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:41.615525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1783,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.519198+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T10:03:39.519198+0000","last_peered":"2024-09-21T10:03:39.519198+0000","last_clean":"2024-09-21T10:03:39.519198+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T10:03:39.519198+0000","last_undegraded":"2024-09-21T10:03:39.519198+0000","last_fullsized":"2024-09-21T10:03:39.519198+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.15","version":"1181'7419","reported_seq":7574,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:46.014769+0000","last_change":"2024-09-21T10:01:59.676028+0000","last_active":"2024-09-21T10:03:46.014769+0000","last_peered":"2024-09-21T10:03:46.014769+0000","last_clean":"2024-09-21T10:03:46.014769+0000","last_became_active":"2024-09-21T10:01:27.368645+0000","last_became_peered":"2024-09-21T10:01:27.368645+0000","last_unstale":"2024-09-21T10:03:46.014769+0000","last_undegraded":"2024-09-21T10:03:46.014769+0000","last_fullsized":"2024-09-21T10:03:46.014769+0000","mapping_epoch":1148,"log_start":"1145'29","ondisk_log_start":"1145'29","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"1162'4060","last_scrub_stamp":"2024-09-21T10:01:59.675861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:01:59.675861+0000","objects_scrubbed":2420,"log_size":7390,"log_dups_size":0,"ondisk_log_size":7390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:02.268701+0000","scrub_duration":0.74397808499999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72089600,"num_objects":1100,"num_object_clones":0,"num_object_copies":3300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3366,"num_write_kb":65472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1784,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.521536+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T10:03:39.521536+0000","last_peered":"2024-09-21T10:03:39.521536+0000","last_clean":"2024-09-21T10:03:39.521536+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T10:03:39.521536+0000","last_undegraded":"2024-09-21T10:03:39.521536+0000","last_fullsized":"2024-09-21T10:03:39.521536+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1107,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.476853+0000","last_change":"2024-09-21T10:00:16.358888+0000","last_active":"2024-09-21T10:03:34.476853+0000","last_peered":"2024-09-21T10:03:34.476853+0000","last_clean":"2024-09-21T10:03:34.476853+0000","last_became_active":"2024-09-21T10:00:16.357219+0000","last_became_peered":"2024-09-21T10:00:16.357219+0000","last_unstale":"2024-09-21T10:03:34.476853+0000","last_undegraded":"2024-09-21T10:03:34.476853+0000","last_fullsized":"2024-09-21T10:03:34.476853+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:13.887798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.14","version":"1181'7146","reported_seq":11286,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:43.889188+0000","last_change":"2024-09-21T10:03:21.493141+0000","last_active":"2024-09-21T10:03:43.889188+0000","last_peered":"2024-09-21T10:03:43.889188+0000","last_clean":"2024-09-21T10:03:43.889188+0000","last_became_active":"2024-09-21T10:02:18.295754+0000","last_became_peered":"2024-09-21T10:02:18.295754+0000","last_unstale":"2024-09-21T10:03:43.889188+0000","last_undegraded":"2024-09-21T10:03:43.889188+0000","last_fullsized":"2024-09-21T10:03:43.889188+0000","mapping_epoch":1165,"log_start":"0'0","ondisk_log_start":"0'0","created":1148,"last_epoch_clean":1166,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7146,"log_dups_size":0,"ondisk_log_size":7146,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-29T05:19:32.789172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83230720,"num_objects":1270,"num_object_clones":0,"num_object_copies":3810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5530,"num_write_kb":217600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2014,"num_bytes_recovered":131989504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1706,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.522858+0000","last_change":"2024-09-21T10:01:40.431450+0000","last_active":"2024-09-21T10:03:39.522858+0000","last_peered":"2024-09-21T10:03:39.522858+0000","last_clean":"2024-09-21T10:03:39.522858+0000","last_became_active":"2024-09-21T10:01:40.431140+0000","last_became_peered":"2024-09-21T10:01:40.431140+0000","last_unstale":"2024-09-21T10:03:39.522858+0000","last_undegraded":"2024-09-21T10:03:39.522858+0000","last_fullsized":"2024-09-21T10:03:39.522858+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:37.604030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":70,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479650+0000","last_change":"2024-09-21T10:00:16.377280+0000","last_active":"2024-09-21T10:03:34.479650+0000","last_peered":"2024-09-21T10:03:34.479650+0000","last_clean":"2024-09-21T10:03:34.479650+0000","last_became_active":"2024-09-21T10:00:16.353074+0000","last_became_peered":"2024-09-21T10:00:16.353074+0000","last_unstale":"2024-09-21T10:03:34.479650+0000","last_undegraded":"2024-09-21T10:03:34.479650+0000","last_fullsized":"2024-09-21T10:03:34.479650+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:35.382228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1181'11385","reported_seq":19797,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:45.470024+0000","last_change":"2024-09-21T10:03:24.437255+0000","last_active":"2024-09-21T10:03:45.470024+0000","last_peered":"2024-09-21T10:03:45.470024+0000","last_clean":"2024-09-21T10:03:45.470024+0000","last_became_active":"2024-09-21T10:03:24.413749+0000","last_became_peered":"2024-09-21T10:03:24.413749+0000","last_unstale":"2024-09-21T10:03:45.470024+0000","last_undegraded":"2024-09-21T10:03:45.470024+0000","last_fullsized":"2024-09-21T10:03:45.470024+0000","mapping_epoch":1178,"log_start":"1163'5000","ondisk_log_start":"1163'5000","created":1142,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6385,"log_dups_size":0,"ondisk_log_size":6385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:40.139577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846912,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11385,"num_write_kb":438464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4161,"num_bytes_recovered":272695296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1105,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.477382+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T10:03:34.477382+0000","last_peered":"2024-09-21T10:03:34.477382+0000","last_clean":"2024-09-21T10:03:34.477382+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T10:03:34.477382+0000","last_undegraded":"2024-09-21T10:03:34.477382+0000","last_fullsized":"2024-09-21T10:03:34.477382+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":113,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.476620+0000","last_change":"2024-09-21T10:00:16.357541+0000","last_active":"2024-09-21T10:03:34.476620+0000","last_peered":"2024-09-21T10:03:34.476620+0000","last_clean":"2024-09-21T10:03:34.476620+0000","last_became_active":"2024-09-21T10:00:16.357209+0000","last_became_peered":"2024-09-21T10:00:16.357209+0000","last_unstale":"2024-09-21T10:03:34.476620+0000","last_undegraded":"2024-09-21T10:03:34.476620+0000","last_fullsized":"2024-09-21T10:03:34.476620+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:20.732470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"1181'11618","reported_seq":11712,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:46.356352+0000","last_change":"2024-09-21T10:00:46.477601+0000","last_active":"2024-09-21T10:03:46.356352+0000","last_peered":"2024-09-21T10:03:46.356352+0000","last_clean":"2024-09-21T10:03:46.356352+0000","last_became_active":"2024-09-21T10:00:46.477208+0000","last_became_peered":"2024-09-21T10:00:46.477208+0000","last_unstale":"2024-09-21T10:03:46.356352+0000","last_undegraded":"2024-09-21T10:03:46.356352+0000","last_fullsized":"2024-09-21T10:03:46.356352+0000","mapping_epoch":1142,"log_start":"1167'6500","ondisk_log_start":"1167'6500","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5118,"log_dups_size":0,"ondisk_log_size":5118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:55.418857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147193856,"num_objects":2246,"num_object_clones":0,"num_object_copies":6738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11618,"num_write_kb":443648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1120,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.477608+0000","last_change":"2024-09-21T10:01:40.429966+0000","last_active":"2024-09-21T10:03:34.477608+0000","last_peered":"2024-09-21T10:03:34.477608+0000","last_clean":"2024-09-21T10:03:34.477608+0000","last_became_active":"2024-09-21T10:01:40.419541+0000","last_became_peered":"2024-09-21T10:01:40.419541+0000","last_unstale":"2024-09-21T10:03:34.477608+0000","last_undegraded":"2024-09-21T10:03:34.477608+0000","last_fullsized":"2024-09-21T10:03:34.477608+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:10.848843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1149,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.478857+0000","last_change":"2024-09-21T10:01:40.423234+0000","last_active":"2024-09-21T10:03:34.478857+0000","last_peered":"2024-09-21T10:03:34.478857+0000","last_clean":"2024-09-21T10:03:34.478857+0000","last_became_active":"2024-09-21T10:01:40.422854+0000","last_became_peered":"2024-09-21T10:01:40.422854+0000","last_unstale":"2024-09-21T10:03:34.478857+0000","last_undegraded":"2024-09-21T10:03:34.478857+0000","last_fullsized":"2024-09-21T10:03:34.478857+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:51.607040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1181'7138","reported_seq":11331,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:43.819343+0000","last_change":"2024-09-21T10:02:40.410442+0000","last_active":"2024-09-21T10:03:43.819343+0000","last_peered":"2024-09-21T10:03:43.819343+0000","last_clean":"2024-09-21T10:03:43.819343+0000","last_became_active":"2024-09-21T10:02:40.405438+0000","last_became_peered":"2024-09-21T10:02:40.405438+0000","last_unstale":"2024-09-21T10:03:43.819343+0000","last_undegraded":"2024-09-21T10:03:43.819343+0000","last_fullsized":"2024-09-21T10:03:43.819343+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7138,"log_dups_size":0,"ondisk_log_size":7138,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80609280,"num_objects":1230,"num_object_clones":0,"num_object_copies":3690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5534,"num_write_kb":216448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2040,"num_bytes_recovered":133693440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":1116,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.477319+0000","last_change":"2024-09-21T10:00:16.373327+0000","last_active":"2024-09-21T10:03:34.477319+0000","last_peered":"2024-09-21T10:03:34.477319+0000","last_clean":"2024-09-21T10:03:34.477319+0000","last_became_active":"2024-09-21T10:00:16.360853+0000","last_became_peered":"2024-09-21T10:00:16.360853+0000","last_unstale":"2024-09-21T10:03:34.477319+0000","last_undegraded":"2024-09-21T10:03:34.477319+0000","last_fullsized":"2024-09-21T10:03:34.477319+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:18.160233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.480150+0000","last_change":"2024-09-21T10:01:40.417926+0000","last_active":"2024-09-21T10:03:34.480150+0000","last_peered":"2024-09-21T10:03:34.480150+0000","last_clean":"2024-09-21T10:03:34.480150+0000","last_became_active":"2024-09-21T10:01:40.417540+0000","last_became_peered":"2024-09-21T10:01:40.417540+0000","last_unstale":"2024-09-21T10:03:34.480150+0000","last_undegraded":"2024-09-21T10:03:34.480150+0000","last_fullsized":"2024-09-21T10:03:34.480150+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:43.890839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1181'7396","reported_seq":7497,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:46.925510+0000","last_change":"2024-09-21T10:01:27.367802+0000","last_active":"2024-09-21T10:03:46.925510+0000","last_peered":"2024-09-21T10:03:46.925510+0000","last_clean":"2024-09-21T10:03:46.925510+0000","last_became_active":"2024-09-21T10:01:27.365655+0000","last_became_peered":"2024-09-21T10:01:27.365655+0000","last_unstale":"2024-09-21T10:03:46.925510+0000","last_undegraded":"2024-09-21T10:03:46.925510+0000","last_fullsized":"2024-09-21T10:03:46.925510+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7396,"log_dups_size":0,"ondisk_log_size":7396,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71237632,"num_objects":1087,"num_object_clones":0,"num_object_copies":3261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5733,"num_write_kb":218240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":712,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.522556+0000","last_change":"2024-09-21T10:00:16.360708+0000","last_active":"2024-09-21T10:03:39.522556+0000","last_peered":"2024-09-21T10:03:39.522556+0000","last_clean":"2024-09-21T10:03:39.522556+0000","last_became_active":"2024-09-21T10:00:16.360416+0000","last_became_peered":"2024-09-21T10:00:16.360416+0000","last_unstale":"2024-09-21T10:03:39.522556+0000","last_undegraded":"2024-09-21T10:03:39.522556+0000","last_fullsized":"2024-09-21T10:03:39.522556+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:59.090215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1792,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.518768+0000","last_change":"2024-09-21T10:01:40.425332+0000","last_active":"2024-09-21T10:03:39.518768+0000","last_peered":"2024-09-21T10:03:39.518768+0000","last_clean":"2024-09-21T10:03:39.518768+0000","last_became_active":"2024-09-21T10:01:40.425034+0000","last_became_peered":"2024-09-21T10:01:40.425034+0000","last_unstale":"2024-09-21T10:03:39.518768+0000","last_undegraded":"2024-09-21T10:03:39.518768+0000","last_fullsized":"2024-09-21T10:03:39.518768+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:57.701200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1181'11327","reported_seq":11418,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:43.889770+0000","last_change":"2024-09-21T10:00:46.479962+0000","last_active":"2024-09-21T10:03:43.889770+0000","last_peered":"2024-09-21T10:03:43.889770+0000","last_clean":"2024-09-21T10:03:43.889770+0000","last_became_active":"2024-09-21T10:00:46.479648+0000","last_became_peered":"2024-09-21T10:00:46.479648+0000","last_unstale":"2024-09-21T10:03:43.889770+0000","last_undegraded":"2024-09-21T10:03:43.889770+0000","last_fullsized":"2024-09-21T10:03:43.889770+0000","mapping_epoch":1142,"log_start":"1163'5400","ondisk_log_start":"1163'5400","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5927,"log_dups_size":0,"ondisk_log_size":5927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:22.382825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157089792,"num_objects":2397,"num_object_clones":0,"num_object_copies":7191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11327,"num_write_kb":439168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1251,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.476700+0000","last_change":"2024-09-21T10:00:47.119852+0000","last_active":"2024-09-21T10:03:34.476700+0000","last_peered":"2024-09-21T10:03:34.476700+0000","last_clean":"2024-09-21T10:03:34.476700+0000","last_became_active":"2024-09-21T09:59:43.374051+0000","last_became_peered":"2024-09-21T09:59:43.374051+0000","last_unstale":"2024-09-21T10:03:34.476700+0000","last_undegraded":"2024-09-21T10:03:34.476700+0000","last_fullsized":"2024-09-21T10:03:34.476700+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:47.119793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:47.119793+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:23.087965+0000","scrub_duration":0.0013414659999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1709,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.530462+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T10:03:39.530462+0000","last_peered":"2024-09-21T10:03:39.530462+0000","last_clean":"2024-09-21T10:03:39.530462+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T10:03:39.530462+0000","last_undegraded":"2024-09-21T10:03:39.530462+0000","last_fullsized":"2024-09-21T10:03:39.530462+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1142,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479166+0000","last_change":"2024-09-21T10:00:16.358954+0000","last_active":"2024-09-21T10:03:34.479166+0000","last_peered":"2024-09-21T10:03:34.479166+0000","last_clean":"2024-09-21T10:03:34.479166+0000","last_became_active":"2024-09-21T10:00:16.358416+0000","last_became_peered":"2024-09-21T10:00:16.358416+0000","last_unstale":"2024-09-21T10:03:34.479166+0000","last_undegraded":"2024-09-21T10:03:34.479166+0000","last_fullsized":"2024-09-21T10:03:34.479166+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:46.916076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1181'11599","reported_seq":11693,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:46.925567+0000","last_change":"2024-09-21T10:00:46.481001+0000","last_active":"2024-09-21T10:03:46.925567+0000","last_peered":"2024-09-21T10:03:46.925567+0000","last_clean":"2024-09-21T10:03:46.925567+0000","last_became_active":"2024-09-21T10:00:46.480689+0000","last_became_peered":"2024-09-21T10:00:46.480689+0000","last_unstale":"2024-09-21T10:03:46.925567+0000","last_undegraded":"2024-09-21T10:03:46.925567+0000","last_fullsized":"2024-09-21T10:03:46.925567+0000","mapping_epoch":1142,"log_start":"1153'4000","ondisk_log_start":"1153'4000","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7599,"log_dups_size":0,"ondisk_log_size":7599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:56.487842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141754368,"num_objects":2163,"num_object_clones":0,"num_object_copies":6489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11599,"num_write_kb":440384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":151,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.514186+0000","last_change":"2024-09-21T10:00:15.524812+0000","last_active":"2024-09-21T10:03:39.514186+0000","last_peered":"2024-09-21T10:03:39.514186+0000","last_clean":"2024-09-21T10:03:39.514186+0000","last_became_active":"2024-09-21T09:59:43.361751+0000","last_became_peered":"2024-09-21T09:59:43.361751+0000","last_unstale":"2024-09-21T10:03:39.514186+0000","last_undegraded":"2024-09-21T10:03:39.514186+0000","last_fullsized":"2024-09-21T10:03:39.514186+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.306777+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.480007+0000","last_change":"2024-09-21T10:01:40.415754+0000","last_active":"2024-09-21T10:03:34.480007+0000","last_peered":"2024-09-21T10:03:34.480007+0000","last_clean":"2024-09-21T10:03:34.480007+0000","last_became_active":"2024-09-21T10:01:40.415428+0000","last_became_peered":"2024-09-21T10:01:40.415428+0000","last_unstale":"2024-09-21T10:03:34.480007+0000","last_undegraded":"2024-09-21T10:03:34.480007+0000","last_fullsized":"2024-09-21T10:03:34.480007+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:53.323411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1149,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479283+0000","last_change":"2024-09-21T10:01:40.414164+0000","last_active":"2024-09-21T10:03:34.479283+0000","last_peered":"2024-09-21T10:03:34.479283+0000","last_clean":"2024-09-21T10:03:34.479283+0000","last_became_active":"2024-09-21T10:01:40.413838+0000","last_became_peered":"2024-09-21T10:01:40.413838+0000","last_unstale":"2024-09-21T10:03:34.479283+0000","last_undegraded":"2024-09-21T10:03:34.479283+0000","last_fullsized":"2024-09-21T10:03:34.479283+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:13:24.380350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1181'11397","reported_seq":14625,"reported_epoch":1181,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:03:45.499361+0000","last_change":"2024-09-21T10:03:22.658284+0000","last_active":"2024-09-21T10:03:45.499361+0000","last_peered":"2024-09-21T10:03:45.499361+0000","last_clean":"2024-09-21T10:01:39.398694+0000","last_became_active":"2024-09-21T10:01:41.614592+0000","last_became_peered":"2024-09-21T10:01:41.614592+0000","last_unstale":"2024-09-21T10:03:45.499361+0000","last_undegraded":"2024-09-21T10:01:50.421725+0000","last_fullsized":"2024-09-21T10:01:41.406829+0000","mapping_epoch":1156,"log_start":"1162'5000","ondisk_log_start":"1162'5000","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6397,"log_dups_size":0,"ondisk_log_size":6397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:44:35.847340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150601752,"num_objects":2299,"num_object_clones":0,"num_object_copies":6897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2612,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2299,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11397,"num_write_kb":438209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1583,"num_bytes_recovered":88932352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,6","objects":-313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1123,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.478939+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T10:03:34.478939+0000","last_peered":"2024-09-21T10:03:34.478939+0000","last_clean":"2024-09-21T10:03:34.478939+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T10:03:34.478939+0000","last_undegraded":"2024-09-21T10:03:34.478939+0000","last_fullsized":"2024-09-21T10:03:34.478939+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1152,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479351+0000","last_change":"2024-09-21T10:01:40.424536+0000","last_active":"2024-09-21T10:03:34.479351+0000","last_peered":"2024-09-21T10:03:34.479351+0000","last_clean":"2024-09-21T10:03:34.479351+0000","last_became_active":"2024-09-21T10:01:40.424229+0000","last_became_peered":"2024-09-21T10:01:40.424229+0000","last_unstale":"2024-09-21T10:03:34.479351+0000","last_undegraded":"2024-09-21T10:03:34.479351+0000","last_fullsized":"2024-09-21T10:03:34.479351+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:10.771540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1181'11303","reported_seq":11350,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:45.510186+0000","last_change":"2024-09-21T10:00:46.477205+0000","last_active":"2024-09-21T10:03:45.510186+0000","last_peered":"2024-09-21T10:03:45.510186+0000","last_clean":"2024-09-21T10:03:45.510186+0000","last_became_active":"2024-09-21T10:00:46.476844+0000","last_became_peered":"2024-09-21T10:00:46.476844+0000","last_unstale":"2024-09-21T10:03:45.510186+0000","last_undegraded":"2024-09-21T10:03:45.510186+0000","last_fullsized":"2024-09-21T10:03:45.510186+0000","mapping_epoch":1142,"log_start":"1162'4900","ondisk_log_start":"1162'4900","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:29:19.716166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151060480,"num_objects":2305,"num_object_clones":0,"num_object_copies":6915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11303,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":140,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.513851+0000","last_change":"2024-09-21T10:00:16.351253+0000","last_active":"2024-09-21T10:03:39.513851+0000","last_peered":"2024-09-21T10:03:39.513851+0000","last_clean":"2024-09-21T10:03:39.513851+0000","last_became_active":"2024-09-21T10:00:16.351010+0000","last_became_peered":"2024-09-21T10:00:16.351010+0000","last_unstale":"2024-09-21T10:03:39.513851+0000","last_undegraded":"2024-09-21T10:03:39.513851+0000","last_fullsized":"2024-09-21T10:03:39.513851+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1707,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.521938+0000","last_change":"2024-09-21T10:01:40.428561+0000","last_active":"2024-09-21T10:03:39.521938+0000","last_peered":"2024-09-21T10:03:39.521938+0000","last_clean":"2024-09-21T10:03:39.521938+0000","last_became_active":"2024-09-21T10:01:40.428250+0000","last_became_peered":"2024-09-21T10:01:40.428250+0000","last_unstale":"2024-09-21T10:03:39.521938+0000","last_undegraded":"2024-09-21T10:03:39.521938+0000","last_fullsized":"2024-09-21T10:03:39.521938+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:58.781370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":612,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.522955+0000","last_change":"2024-09-21T10:01:40.423834+0000","last_active":"2024-09-21T10:03:39.522955+0000","last_peered":"2024-09-21T10:03:39.522955+0000","last_clean":"2024-09-21T10:03:39.522955+0000","last_became_active":"2024-09-21T10:01:40.423506+0000","last_became_peered":"2024-09-21T10:01:40.423506+0000","last_unstale":"2024-09-21T10:03:39.522955+0000","last_undegraded":"2024-09-21T10:03:39.522955+0000","last_fullsized":"2024-09-21T10:03:39.522955+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:11.358742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.6","version":"1181'7353","reported_seq":7483,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:43.890295+0000","last_change":"2024-09-21T10:03:12.185190+0000","last_active":"2024-09-21T10:03:43.890295+0000","last_peered":"2024-09-21T10:03:43.890295+0000","last_clean":"2024-09-21T10:03:43.890295+0000","last_became_active":"2024-09-21T10:01:27.352341+0000","last_became_peered":"2024-09-21T10:01:27.352341+0000","last_unstale":"2024-09-21T10:03:43.890295+0000","last_undegraded":"2024-09-21T10:03:43.890295+0000","last_fullsized":"2024-09-21T10:03:43.890295+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1176'6402","last_scrub_stamp":"2024-09-21T10:03:12.185040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:03:12.185040+0000","objects_scrubbed":2137,"log_size":7353,"log_dups_size":0,"ondisk_log_size":7353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:51.219397+0000","scrub_duration":683,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77201408,"num_objects":1178,"num_object_clones":0,"num_object_copies":3534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":959,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1265,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.476949+0000","last_change":"2024-09-21T10:00:49.116174+0000","last_active":"2024-09-21T10:03:34.476949+0000","last_peered":"2024-09-21T10:03:34.476949+0000","last_clean":"2024-09-21T10:03:34.476949+0000","last_became_active":"2024-09-21T10:00:16.360430+0000","last_became_peered":"2024-09-21T10:00:16.360430+0000","last_unstale":"2024-09-21T10:03:34.476949+0000","last_undegraded":"2024-09-21T10:03:34.476949+0000","last_fullsized":"2024-09-21T10:03:34.476949+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:49.116103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:49.116103+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:29.874911+0000","scrub_duration":0.0020386419999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":185,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.517041+0000","last_change":"2024-09-21T09:57:39.541461+0000","last_active":"2024-09-21T10:03:39.517041+0000","last_peered":"2024-09-21T10:03:39.517041+0000","last_clean":"2024-09-21T10:03:39.517041+0000","last_became_active":"2024-09-21T09:57:39.540927+0000","last_became_peered":"2024-09-21T09:57:39.540927+0000","last_unstale":"2024-09-21T10:03:39.517041+0000","last_undegraded":"2024-09-21T10:03:39.517041+0000","last_fullsized":"2024-09-21T10:03:39.517041+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:14.815783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1697,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.525271+0000","last_change":"2024-09-21T10:00:16.362131+0000","last_active":"2024-09-21T10:03:39.525271+0000","last_peered":"2024-09-21T10:03:39.525271+0000","last_clean":"2024-09-21T10:03:39.525271+0000","last_became_active":"2024-09-21T10:00:16.361722+0000","last_became_peered":"2024-09-21T10:00:16.361722+0000","last_unstale":"2024-09-21T10:03:39.525271+0000","last_undegraded":"2024-09-21T10:03:39.525271+0000","last_fullsized":"2024-09-21T10:03:39.525271+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:38.717426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.482015+0000","last_change":"2024-09-21T10:00:16.361540+0000","last_active":"2024-09-21T10:03:34.482015+0000","last_peered":"2024-09-21T10:03:34.482015+0000","last_clean":"2024-09-21T10:03:34.482015+0000","last_became_active":"2024-09-21T10:00:16.361086+0000","last_became_peered":"2024-09-21T10:00:16.361086+0000","last_unstale":"2024-09-21T10:03:34.482015+0000","last_undegraded":"2024-09-21T10:03:34.482015+0000","last_fullsized":"2024-09-21T10:03:34.482015+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:14.645896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1181'7293","reported_seq":7444,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:45.885752+0000","last_change":"2024-09-21T10:01:58.715049+0000","last_active":"2024-09-21T10:03:45.885752+0000","last_peered":"2024-09-21T10:03:45.885752+0000","last_clean":"2024-09-21T10:03:45.885752+0000","last_became_active":"2024-09-21T10:01:27.359228+0000","last_became_peered":"2024-09-21T10:01:27.359228+0000","last_unstale":"2024-09-21T10:03:45.885752+0000","last_undegraded":"2024-09-21T10:03:45.885752+0000","last_fullsized":"2024-09-21T10:03:45.885752+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1162'3998","last_scrub_stamp":"2024-09-21T10:01:58.714971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:01:58.714971+0000","objects_scrubbed":2286,"log_size":7293,"log_dups_size":0,"ondisk_log_size":7293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:21.752075+0000","scrub_duration":0.73731484599999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73400320,"num_objects":1120,"num_object_clones":0,"num_object_copies":3360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3308,"num_write_kb":68544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":65,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.482801+0000","last_change":"2024-09-21T10:01:40.412152+0000","last_active":"2024-09-21T10:03:34.482801+0000","last_peered":"2024-09-21T10:03:34.482801+0000","last_clean":"2024-09-21T10:03:34.482801+0000","last_became_active":"2024-09-21T10:01:40.411508+0000","last_became_peered":"2024-09-21T10:01:40.411508+0000","last_unstale":"2024-09-21T10:03:34.482801+0000","last_undegraded":"2024-09-21T10:03:34.482801+0000","last_fullsized":"2024-09-21T10:03:34.482801+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:21.317838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:21.317838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1143,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.476851+0000","last_change":"2024-09-21T10:01:40.428393+0000","last_active":"2024-09-21T10:03:34.476851+0000","last_peered":"2024-09-21T10:03:34.476851+0000","last_clean":"2024-09-21T10:03:34.476851+0000","last_became_active":"2024-09-21T10:01:40.428204+0000","last_became_peered":"2024-09-21T10:01:40.428204+0000","last_unstale":"2024-09-21T10:03:34.476851+0000","last_undegraded":"2024-09-21T10:03:34.476851+0000","last_fullsized":"2024-09-21T10:03:34.476851+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:23:02.136035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1783,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.521549+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T10:03:39.521549+0000","last_peered":"2024-09-21T10:03:39.521549+0000","last_clean":"2024-09-21T10:03:39.521549+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T10:03:39.521549+0000","last_undegraded":"2024-09-21T10:03:39.521549+0000","last_fullsized":"2024-09-21T10:03:39.521549+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1803,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.521298+0000","last_change":"2024-09-21T10:01:40.421887+0000","last_active":"2024-09-21T10:03:39.521298+0000","last_peered":"2024-09-21T10:03:39.521298+0000","last_clean":"2024-09-21T10:03:39.521298+0000","last_became_active":"2024-09-21T10:01:40.420730+0000","last_became_peered":"2024-09-21T10:01:40.420730+0000","last_unstale":"2024-09-21T10:03:39.521298+0000","last_undegraded":"2024-09-21T10:03:39.521298+0000","last_fullsized":"2024-09-21T10:03:39.521298+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:51.504051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"1181'7180","reported_seq":11463,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:43.846832+0000","last_change":"2024-09-21T10:03:15.532727+0000","last_active":"2024-09-21T10:03:43.846832+0000","last_peered":"2024-09-21T10:03:43.846832+0000","last_clean":"2024-09-21T10:03:43.846832+0000","last_became_active":"2024-09-21T10:01:56.596893+0000","last_became_peered":"2024-09-21T10:01:56.596893+0000","last_unstale":"2024-09-21T10:03:43.846832+0000","last_undegraded":"2024-09-21T10:03:43.846832+0000","last_fullsized":"2024-09-21T10:03:43.846832+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7180,"log_dups_size":0,"ondisk_log_size":7180,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-29T15:50:57.381992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75104256,"num_objects":1146,"num_object_clones":0,"num_object_copies":3438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5564,"num_write_kb":214720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2085,"num_bytes_recovered":136642560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":1251,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.476937+0000","last_change":"2024-09-21T10:00:16.913196+0000","last_active":"2024-09-21T10:03:34.476937+0000","last_peered":"2024-09-21T10:03:34.476937+0000","last_clean":"2024-09-21T10:03:34.476937+0000","last_became_active":"2024-09-21T09:59:43.370454+0000","last_became_peered":"2024-09-21T09:59:43.370454+0000","last_unstale":"2024-09-21T10:03:34.476937+0000","last_undegraded":"2024-09-21T10:03:34.476937+0000","last_fullsized":"2024-09-21T10:03:34.476937+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:16.913146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:16.913146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:23:33.788563+0000","scrub_duration":0.001342315,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1758,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.512648+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:03:39.512648+0000","last_peered":"2024-09-21T10:03:39.512648+0000","last_clean":"2024-09-21T10:03:39.512648+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:03:39.512648+0000","last_undegraded":"2024-09-21T10:03:39.512648+0000","last_fullsized":"2024-09-21T10:03:39.512648+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1713,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.514358+0000","last_change":"2024-09-21T10:01:40.414547+0000","last_active":"2024-09-21T10:03:39.514358+0000","last_peered":"2024-09-21T10:03:39.514358+0000","last_clean":"2024-09-21T10:03:39.514358+0000","last_became_active":"2024-09-21T10:01:40.414243+0000","last_became_peered":"2024-09-21T10:01:40.414243+0000","last_unstale":"2024-09-21T10:03:39.514358+0000","last_undegraded":"2024-09-21T10:03:39.514358+0000","last_fullsized":"2024-09-21T10:03:39.514358+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:55:26.044481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":493,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.520138+0000","last_change":"2024-09-21T10:01:40.421427+0000","last_active":"2024-09-21T10:03:39.520138+0000","last_peered":"2024-09-21T10:03:39.520138+0000","last_clean":"2024-09-21T10:03:39.520138+0000","last_became_active":"2024-09-21T10:01:40.421047+0000","last_became_peered":"2024-09-21T10:01:40.421047+0000","last_unstale":"2024-09-21T10:03:39.520138+0000","last_undegraded":"2024-09-21T10:03:39.520138+0000","last_fullsized":"2024-09-21T10:03:39.520138+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:48.640296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.7","version":"1181'7329","reported_seq":7386,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:45.488674+0000","last_change":"2024-09-21T10:01:27.359695+0000","last_active":"2024-09-21T10:03:45.488674+0000","last_peered":"2024-09-21T10:03:45.488674+0000","last_clean":"2024-09-21T10:03:45.488674+0000","last_became_active":"2024-09-21T10:01:27.358189+0000","last_became_peered":"2024-09-21T10:01:27.358189+0000","last_unstale":"2024-09-21T10:03:45.488674+0000","last_undegraded":"2024-09-21T10:03:45.488674+0000","last_fullsized":"2024-09-21T10:03:45.488674+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7329,"log_dups_size":0,"ondisk_log_size":7329,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73564160,"num_objects":1122,"num_object_clones":0,"num_object_copies":3366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5666,"num_write_kb":217248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":480,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.520567+0000","last_change":"2024-09-21T10:03:19.415715+0000","last_active":"2024-09-21T10:03:39.520567+0000","last_peered":"2024-09-21T10:03:39.520567+0000","last_clean":"2024-09-21T10:03:39.520567+0000","last_became_active":"2024-09-21T09:59:43.371384+0000","last_became_peered":"2024-09-21T09:59:43.371384+0000","last_unstale":"2024-09-21T10:03:39.520567+0000","last_undegraded":"2024-09-21T10:03:39.520567+0000","last_fullsized":"2024-09-21T10:03:39.520567+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-30T13:30:29.665440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":138,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479797+0000","last_change":"2024-09-21T10:01:40.426865+0000","last_active":"2024-09-21T10:03:34.479797+0000","last_peered":"2024-09-21T10:03:34.479797+0000","last_clean":"2024-09-21T10:03:34.479797+0000","last_became_active":"2024-09-21T10:01:40.413152+0000","last_became_peered":"2024-09-21T10:01:40.413152+0000","last_unstale":"2024-09-21T10:03:34.479797+0000","last_undegraded":"2024-09-21T10:03:34.479797+0000","last_fullsized":"2024-09-21T10:03:34.479797+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:59.034627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1143,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479116+0000","last_change":"2024-09-21T10:00:16.356367+0000","last_active":"2024-09-21T10:03:34.479116+0000","last_peered":"2024-09-21T10:03:34.479116+0000","last_clean":"2024-09-21T10:03:34.479116+0000","last_became_active":"2024-09-21T10:00:16.356079+0000","last_became_peered":"2024-09-21T10:00:16.356079+0000","last_unstale":"2024-09-21T10:03:34.479116+0000","last_undegraded":"2024-09-21T10:03:34.479116+0000","last_fullsized":"2024-09-21T10:03:34.479116+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:12.934640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":624,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.513072+0000","last_change":"2024-09-21T10:01:40.421262+0000","last_active":"2024-09-21T10:03:39.513072+0000","last_peered":"2024-09-21T10:03:39.513072+0000","last_clean":"2024-09-21T10:03:39.513072+0000","last_became_active":"2024-09-21T10:01:40.420959+0000","last_became_peered":"2024-09-21T10:01:40.420959+0000","last_unstale":"2024-09-21T10:03:39.513072+0000","last_undegraded":"2024-09-21T10:03:39.513072+0000","last_fullsized":"2024-09-21T10:03:39.513072+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:06.866421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1","version":"1181'7463","reported_seq":7560,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:45.899164+0000","last_change":"2024-09-21T10:03:39.448818+0000","last_active":"2024-09-21T10:03:45.899164+0000","last_peered":"2024-09-21T10:03:45.899164+0000","last_clean":"2024-09-21T10:03:45.899164+0000","last_became_active":"2024-09-21T10:01:27.378346+0000","last_became_peered":"2024-09-21T10:01:27.378346+0000","last_unstale":"2024-09-21T10:03:45.899164+0000","last_undegraded":"2024-09-21T10:03:45.899164+0000","last_fullsized":"2024-09-21T10:03:45.899164+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7463,"log_dups_size":0,"ondisk_log_size":7463,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75399168,"num_objects":1150,"num_object_clones":0,"num_object_copies":3450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5780,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":70,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.481824+0000","last_change":"2024-09-21T10:00:46.494180+0000","last_active":"2024-09-21T10:03:34.481824+0000","last_peered":"2024-09-21T10:03:34.481824+0000","last_clean":"2024-09-21T10:03:34.481824+0000","last_became_active":"2024-09-21T10:00:16.355386+0000","last_became_peered":"2024-09-21T10:00:16.355386+0000","last_unstale":"2024-09-21T10:03:34.481824+0000","last_undegraded":"2024-09-21T10:03:34.481824+0000","last_fullsized":"2024-09-21T10:03:34.481824+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:46.494129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:46.494129+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:11.193593+0000","scrub_duration":0.00183863,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":714,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.530692+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T10:03:39.530692+0000","last_peered":"2024-09-21T10:03:39.530692+0000","last_clean":"2024-09-21T10:03:39.530692+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T10:03:39.530692+0000","last_undegraded":"2024-09-21T10:03:39.530692+0000","last_fullsized":"2024-09-21T10:03:39.530692+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":522,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.519941+0000","last_change":"2024-09-21T10:01:40.418328+0000","last_active":"2024-09-21T10:03:39.519941+0000","last_peered":"2024-09-21T10:03:39.519941+0000","last_clean":"2024-09-21T10:03:39.519941+0000","last_became_active":"2024-09-21T10:01:40.418011+0000","last_became_peered":"2024-09-21T10:01:40.418011+0000","last_unstale":"2024-09-21T10:03:39.519941+0000","last_undegraded":"2024-09-21T10:03:39.519941+0000","last_fullsized":"2024-09-21T10:03:39.519941+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:58.716988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":484,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.478618+0000","last_change":"2024-09-21T10:01:40.421225+0000","last_active":"2024-09-21T10:03:34.478618+0000","last_peered":"2024-09-21T10:03:34.478618+0000","last_clean":"2024-09-21T10:03:34.478618+0000","last_became_active":"2024-09-21T10:01:40.420909+0000","last_became_peered":"2024-09-21T10:01:40.420909+0000","last_unstale":"2024-09-21T10:03:34.478618+0000","last_undegraded":"2024-09-21T10:03:34.478618+0000","last_fullsized":"2024-09-21T10:03:34.478618+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:03.811941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"1181'7230","reported_seq":11532,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:46.189571+0000","last_change":"2024-09-21T10:01:55.586239+0000","last_active":"2024-09-21T10:03:46.189571+0000","last_peered":"2024-09-21T10:03:46.189571+0000","last_clean":"2024-09-21T10:03:46.189571+0000","last_became_active":"2024-09-21T10:01:55.581482+0000","last_became_peered":"2024-09-21T10:01:55.581482+0000","last_unstale":"2024-09-21T10:03:46.189571+0000","last_undegraded":"2024-09-21T10:03:46.189571+0000","last_fullsized":"2024-09-21T10:03:46.189571+0000","mapping_epoch":1160,"log_start":"1145'1146","ondisk_log_start":"1145'1146","created":1142,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6084,"log_dups_size":0,"ondisk_log_size":6084,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69009408,"num_objects":1053,"num_object_clones":0,"num_object_copies":3159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5613,"num_write_kb":213312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2093,"num_bytes_recovered":137166848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":70,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479065+0000","last_change":"2024-09-21T10:00:20.637858+0000","last_active":"2024-09-21T10:03:34.479065+0000","last_peered":"2024-09-21T10:03:34.479065+0000","last_clean":"2024-09-21T10:03:34.479065+0000","last_became_active":"2024-09-21T10:00:16.353796+0000","last_became_peered":"2024-09-21T10:00:16.353796+0000","last_unstale":"2024-09-21T10:03:34.479065+0000","last_undegraded":"2024-09-21T10:03:34.479065+0000","last_fullsized":"2024-09-21T10:03:34.479065+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:20.637806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:20.637806+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:12:16.884495+0000","scrub_duration":0.0016753919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":550,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.512906+0000","last_change":"2024-09-21T10:01:40.414536+0000","last_active":"2024-09-21T10:03:39.512906+0000","last_peered":"2024-09-21T10:03:39.512906+0000","last_clean":"2024-09-21T10:03:39.512906+0000","last_became_active":"2024-09-21T10:01:40.413913+0000","last_became_peered":"2024-09-21T10:01:40.413913+0000","last_unstale":"2024-09-21T10:03:39.512906+0000","last_undegraded":"2024-09-21T10:03:39.512906+0000","last_fullsized":"2024-09-21T10:03:39.512906+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:27.994971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1116,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.477344+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T10:03:34.477344+0000","last_peered":"2024-09-21T10:03:34.477344+0000","last_clean":"2024-09-21T10:03:34.477344+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T10:03:34.477344+0000","last_undegraded":"2024-09-21T10:03:34.477344+0000","last_fullsized":"2024-09-21T10:03:34.477344+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1774,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.518877+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T10:03:39.518877+0000","last_peered":"2024-09-21T10:03:39.518877+0000","last_clean":"2024-09-21T10:03:39.518877+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T10:03:39.518877+0000","last_undegraded":"2024-09-21T10:03:39.518877+0000","last_fullsized":"2024-09-21T10:03:39.518877+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1181'7431","reported_seq":7531,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:46.354001+0000","last_change":"2024-09-21T10:01:27.384197+0000","last_active":"2024-09-21T10:03:46.354001+0000","last_peered":"2024-09-21T10:03:46.354001+0000","last_clean":"2024-09-21T10:03:46.354001+0000","last_became_active":"2024-09-21T10:01:27.382492+0000","last_became_peered":"2024-09-21T10:01:27.382492+0000","last_unstale":"2024-09-21T10:03:46.354001+0000","last_undegraded":"2024-09-21T10:03:46.354001+0000","last_fullsized":"2024-09-21T10:03:46.354001+0000","mapping_epoch":1142,"log_start":"1145'1445","ondisk_log_start":"1145'1445","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5986,"log_dups_size":0,"ondisk_log_size":5986,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72843264,"num_objects":1111,"num_object_clones":0,"num_object_copies":3333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5785,"num_write_kb":220704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":90,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479107+0000","last_change":"2024-09-21T10:00:16.356251+0000","last_active":"2024-09-21T10:03:34.479107+0000","last_peered":"2024-09-21T10:03:34.479107+0000","last_clean":"2024-09-21T10:03:34.479107+0000","last_became_active":"2024-09-21T10:00:16.355833+0000","last_became_peered":"2024-09-21T10:00:16.355833+0000","last_unstale":"2024-09-21T10:03:34.479107+0000","last_undegraded":"2024-09-21T10:03:34.479107+0000","last_fullsized":"2024-09-21T10:03:34.479107+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T09:59:47.767654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:59:47.767654+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:50.939198+0000","scrub_duration":0.006614371,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1709,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.523034+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T10:03:39.523034+0000","last_peered":"2024-09-21T10:03:39.523034+0000","last_clean":"2024-09-21T10:03:39.523034+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T10:03:39.523034+0000","last_undegraded":"2024-09-21T10:03:39.523034+0000","last_fullsized":"2024-09-21T10:03:39.523034+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1684,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.478732+0000","last_change":"2024-09-21T10:01:40.423209+0000","last_active":"2024-09-21T10:03:34.478732+0000","last_peered":"2024-09-21T10:03:34.478732+0000","last_clean":"2024-09-21T10:03:34.478732+0000","last_became_active":"2024-09-21T10:01:40.422890+0000","last_became_peered":"2024-09-21T10:01:40.422890+0000","last_unstale":"2024-09-21T10:03:34.478732+0000","last_undegraded":"2024-09-21T10:03:34.478732+0000","last_fullsized":"2024-09-21T10:03:34.478732+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:10.683126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1707,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.521451+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T10:03:39.521451+0000","last_peered":"2024-09-21T10:03:39.521451+0000","last_clean":"2024-09-21T10:03:39.521451+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T10:03:39.521451+0000","last_undegraded":"2024-09-21T10:03:39.521451+0000","last_fullsized":"2024-09-21T10:03:39.521451+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1181'7451","reported_seq":7558,"reported_epoch":1181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:03:46.320241+0000","last_change":"2024-09-21T10:01:41.614201+0000","last_active":"2024-09-21T10:03:46.320241+0000","last_peered":"2024-09-21T10:03:46.320241+0000","last_clean":"2024-09-21T10:01:39.401120+0000","last_became_active":"2024-09-21T10:01:41.609480+0000","last_became_peered":"2024-09-21T10:01:41.609480+0000","last_unstale":"2024-09-21T10:03:46.320241+0000","last_undegraded":"2024-09-21T10:01:41.546996+0000","last_fullsized":"2024-09-21T10:01:41.408102+0000","mapping_epoch":1156,"log_start":"1145'1420","ondisk_log_start":"1145'1420","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6031,"log_dups_size":0,"ondisk_log_size":6031,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71860224,"num_objects":1096,"num_object_clones":0,"num_object_copies":3288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2143,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5776,"num_write_kb":219936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"2,3","objects":-1047}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":139,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.522132+0000","last_change":"2024-09-21T10:00:22.052497+0000","last_active":"2024-09-21T10:03:39.522132+0000","last_peered":"2024-09-21T10:03:39.522132+0000","last_clean":"2024-09-21T10:03:39.522132+0000","last_became_active":"2024-09-21T09:59:43.368157+0000","last_became_peered":"2024-09-21T09:59:43.368157+0000","last_unstale":"2024-09-21T10:03:39.522132+0000","last_undegraded":"2024-09-21T10:03:39.522132+0000","last_fullsized":"2024-09-21T10:03:39.522132+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-28T12:21:36.382264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1798,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.521634+0000","last_change":"2024-09-21T10:01:40.433067+0000","last_active":"2024-09-21T10:03:39.521634+0000","last_peered":"2024-09-21T10:03:39.521634+0000","last_clean":"2024-09-21T10:03:39.521634+0000","last_became_active":"2024-09-21T10:01:40.420622+0000","last_became_peered":"2024-09-21T10:01:40.420622+0000","last_unstale":"2024-09-21T10:03:39.521634+0000","last_undegraded":"2024-09-21T10:03:39.521634+0000","last_fullsized":"2024-09-21T10:03:39.521634+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.383227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":60,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.481754+0000","last_change":"2024-09-21T10:00:16.355102+0000","last_active":"2024-09-21T10:03:34.481754+0000","last_peered":"2024-09-21T10:03:34.481754+0000","last_clean":"2024-09-21T10:03:34.481754+0000","last_became_active":"2024-09-21T10:00:16.354473+0000","last_became_peered":"2024-09-21T10:00:16.354473+0000","last_unstale":"2024-09-21T10:03:34.481754+0000","last_undegraded":"2024-09-21T10:03:34.481754+0000","last_fullsized":"2024-09-21T10:03:34.481754+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:49:35.449111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1125,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.478815+0000","last_change":"2024-09-21T10:00:16.355601+0000","last_active":"2024-09-21T10:03:34.478815+0000","last_peered":"2024-09-21T10:03:34.478815+0000","last_clean":"2024-09-21T10:03:34.478815+0000","last_became_active":"2024-09-21T10:00:16.355110+0000","last_became_peered":"2024-09-21T10:00:16.355110+0000","last_unstale":"2024-09-21T10:03:34.478815+0000","last_undegraded":"2024-09-21T10:03:34.478815+0000","last_fullsized":"2024-09-21T10:03:34.478815+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:10.590252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.10","version":"1181'7211","reported_seq":12551,"reported_epoch":1181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:03:46.326899+0000","last_change":"2024-09-21T10:01:43.611910+0000","last_active":"2024-09-21T10:03:46.326899+0000","last_peered":"2024-09-21T10:03:46.326899+0000","last_clean":"2024-09-21T10:01:32.765736+0000","last_became_active":"2024-09-21T10:01:43.608034+0000","last_became_peered":"2024-09-21T10:01:43.608034+0000","last_unstale":"2024-09-21T10:03:46.326899+0000","last_undegraded":"2024-09-21T10:01:41.524927+0000","last_fullsized":"2024-09-21T10:01:41.408209+0000","mapping_epoch":1156,"log_start":"1145'925","ondisk_log_start":"1145'925","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6286,"log_dups_size":0,"ondisk_log_size":6286,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77201408,"num_objects":1178,"num_object_clones":0,"num_object_copies":3534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":481,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5594,"num_write_kb":216704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1849,"num_bytes_recovered":121176064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2","5"],"object_location_counts":[{"shards":"2,3","objects":697}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1152,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479036+0000","last_change":"2024-09-21T10:01:40.414117+0000","last_active":"2024-09-21T10:03:34.479036+0000","last_peered":"2024-09-21T10:03:34.479036+0000","last_clean":"2024-09-21T10:03:34.479036+0000","last_became_active":"2024-09-21T10:01:40.413186+0000","last_became_peered":"2024-09-21T10:01:40.413186+0000","last_unstale":"2024-09-21T10:03:34.479036+0000","last_undegraded":"2024-09-21T10:03:34.479036+0000","last_fullsized":"2024-09-21T10:03:34.479036+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:17:05.368450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1168,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479246+0000","last_change":"2024-09-21T10:00:16.371426+0000","last_active":"2024-09-21T10:03:34.479246+0000","last_peered":"2024-09-21T10:03:34.479246+0000","last_clean":"2024-09-21T10:03:34.479246+0000","last_became_active":"2024-09-21T10:00:16.357486+0000","last_became_peered":"2024-09-21T10:00:16.357486+0000","last_unstale":"2024-09-21T10:03:34.479246+0000","last_undegraded":"2024-09-21T10:03:34.479246+0000","last_fullsized":"2024-09-21T10:03:34.479246+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:02.923682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.11","version":"1181'7473","reported_seq":9487,"reported_epoch":1181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:03:46.299863+0000","last_change":"2024-09-21T10:01:43.633724+0000","last_active":"2024-09-21T10:03:46.299863+0000","last_peered":"2024-09-21T10:03:46.299863+0000","last_clean":"2024-09-21T10:01:33.111456+0000","last_became_active":"2024-09-21T10:01:36.436541+0000","last_became_peered":"2024-09-21T10:01:36.436541+0000","last_unstale":"2024-09-21T10:03:46.299863+0000","last_undegraded":"2024-09-21T10:01:43.633368+0000","last_fullsized":"2024-09-21T10:01:36.387971+0000","mapping_epoch":1153,"log_start":"1145'1454","ondisk_log_start":"1145'1454","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6019,"log_dups_size":0,"ondisk_log_size":6019,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73564160,"num_objects":1123,"num_object_clones":0,"num_object_copies":3369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1021,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5791,"num_write_kb":221216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":960,"num_bytes_recovered":62914560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,6","objects":102}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1685,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479935+0000","last_change":"2024-09-21T10:01:40.413147+0000","last_active":"2024-09-21T10:03:34.479935+0000","last_peered":"2024-09-21T10:03:34.479935+0000","last_clean":"2024-09-21T10:03:34.479935+0000","last_became_active":"2024-09-21T10:01:40.412833+0000","last_became_peered":"2024-09-21T10:01:40.412833+0000","last_unstale":"2024-09-21T10:03:34.479935+0000","last_undegraded":"2024-09-21T10:03:34.479935+0000","last_fullsized":"2024-09-21T10:03:34.479935+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:33.262350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1107,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.477165+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T10:03:34.477165+0000","last_peered":"2024-09-21T10:03:34.477165+0000","last_clean":"2024-09-21T10:03:34.477165+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T10:03:34.477165+0000","last_undegraded":"2024-09-21T10:03:34.477165+0000","last_fullsized":"2024-09-21T10:03:34.477165+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.12","version":"1181'7393","reported_seq":11195,"reported_epoch":1181,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:03:46.344783+0000","last_change":"2024-09-21T10:03:22.634376+0000","last_active":"2024-09-21T10:03:46.344783+0000","last_peered":"2024-09-21T10:03:46.344783+0000","last_clean":"2024-09-21T10:01:39.381638+0000","last_became_active":"2024-09-21T10:01:41.609703+0000","last_became_peered":"2024-09-21T10:01:41.609703+0000","last_unstale":"2024-09-21T10:03:46.344783+0000","last_undegraded":"2024-09-21T10:01:41.442786+0000","last_fullsized":"2024-09-21T10:01:41.415807+0000","mapping_epoch":1156,"log_start":"1145'1231","ondisk_log_start":"1145'1231","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6162,"log_dups_size":0,"ondisk_log_size":6162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73433088,"num_objects":1121,"num_object_clones":0,"num_object_copies":3363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":273,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5719,"num_write_kb":218848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1849,"num_bytes_recovered":33423360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"2,3","objects":848}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":561,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.477481+0000","last_change":"2024-09-21T10:01:40.427109+0000","last_active":"2024-09-21T10:03:34.477481+0000","last_peered":"2024-09-21T10:03:34.477481+0000","last_clean":"2024-09-21T10:03:34.477481+0000","last_became_active":"2024-09-21T10:01:40.426821+0000","last_became_peered":"2024-09-21T10:01:40.426821+0000","last_unstale":"2024-09-21T10:03:34.477481+0000","last_undegraded":"2024-09-21T10:03:34.477481+0000","last_fullsized":"2024-09-21T10:03:34.477481+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:12.260797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1696,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.523032+0000","last_change":"2024-09-21T10:00:16.357151+0000","last_active":"2024-09-21T10:03:39.523032+0000","last_peered":"2024-09-21T10:03:39.523032+0000","last_clean":"2024-09-21T10:03:39.523032+0000","last_became_active":"2024-09-21T10:00:16.356864+0000","last_became_peered":"2024-09-21T10:00:16.356864+0000","last_unstale":"2024-09-21T10:03:39.523032+0000","last_undegraded":"2024-09-21T10:03:39.523032+0000","last_fullsized":"2024-09-21T10:03:39.523032+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:52.851887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"1181'7324","reported_seq":7424,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:46.326341+0000","last_change":"2024-09-21T10:01:27.391083+0000","last_active":"2024-09-21T10:03:46.326341+0000","last_peered":"2024-09-21T10:03:46.326341+0000","last_clean":"2024-09-21T10:03:46.326341+0000","last_became_active":"2024-09-21T10:01:27.387803+0000","last_became_peered":"2024-09-21T10:01:27.387803+0000","last_unstale":"2024-09-21T10:03:46.326341+0000","last_undegraded":"2024-09-21T10:03:46.326341+0000","last_fullsized":"2024-09-21T10:03:46.326341+0000","mapping_epoch":1148,"log_start":"1145'1125","ondisk_log_start":"1145'1125","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6199,"log_dups_size":0,"ondisk_log_size":6199,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79462400,"num_objects":1213,"num_object_clones":0,"num_object_copies":3639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5679,"num_write_kb":220512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1782,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.520729+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T10:03:39.520729+0000","last_peered":"2024-09-21T10:03:39.520729+0000","last_clean":"2024-09-21T10:03:39.520729+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T10:03:39.520729+0000","last_undegraded":"2024-09-21T10:03:39.520729+0000","last_fullsized":"2024-09-21T10:03:39.520729+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1720,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.521364+0000","last_change":"2024-09-21T10:01:40.426702+0000","last_active":"2024-09-21T10:03:39.521364+0000","last_peered":"2024-09-21T10:03:39.521364+0000","last_clean":"2024-09-21T10:03:39.521364+0000","last_became_active":"2024-09-21T10:01:40.426034+0000","last_became_peered":"2024-09-21T10:01:40.426034+0000","last_unstale":"2024-09-21T10:03:39.521364+0000","last_undegraded":"2024-09-21T10:03:39.521364+0000","last_fullsized":"2024-09-21T10:03:39.521364+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:20.619421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.481784+0000","last_change":"2024-09-21T10:01:40.414393+0000","last_active":"2024-09-21T10:03:34.481784+0000","last_peered":"2024-09-21T10:03:34.481784+0000","last_clean":"2024-09-21T10:03:34.481784+0000","last_became_active":"2024-09-21T10:01:40.414096+0000","last_became_peered":"2024-09-21T10:01:40.414096+0000","last_unstale":"2024-09-21T10:03:34.481784+0000","last_undegraded":"2024-09-21T10:03:34.481784+0000","last_fullsized":"2024-09-21T10:03:34.481784+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:32.540479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1724,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.521889+0000","last_change":"2024-09-21T10:01:40.434475+0000","last_active":"2024-09-21T10:03:39.521889+0000","last_peered":"2024-09-21T10:03:39.521889+0000","last_clean":"2024-09-21T10:03:39.521889+0000","last_became_active":"2024-09-21T10:01:40.432013+0000","last_became_peered":"2024-09-21T10:01:40.432013+0000","last_unstale":"2024-09-21T10:03:39.521889+0000","last_undegraded":"2024-09-21T10:03:39.521889+0000","last_fullsized":"2024-09-21T10:03:39.521889+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:44.373378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1786,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.521445+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T10:03:39.521445+0000","last_peered":"2024-09-21T10:03:39.521445+0000","last_clean":"2024-09-21T10:03:39.521445+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T10:03:39.521445+0000","last_undegraded":"2024-09-21T10:03:39.521445+0000","last_fullsized":"2024-09-21T10:03:39.521445+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":613,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.523232+0000","last_change":"2024-09-21T10:01:40.437120+0000","last_active":"2024-09-21T10:03:39.523232+0000","last_peered":"2024-09-21T10:03:39.523232+0000","last_clean":"2024-09-21T10:03:39.523232+0000","last_became_active":"2024-09-21T10:01:40.436275+0000","last_became_peered":"2024-09-21T10:01:40.436275+0000","last_unstale":"2024-09-21T10:03:39.523232+0000","last_undegraded":"2024-09-21T10:03:39.523232+0000","last_fullsized":"2024-09-21T10:03:39.523232+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:25.322111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1705,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.530420+0000","last_change":"2024-09-21T10:01:40.427770+0000","last_active":"2024-09-21T10:03:39.530420+0000","last_peered":"2024-09-21T10:03:39.530420+0000","last_clean":"2024-09-21T10:03:39.530420+0000","last_became_active":"2024-09-21T10:01:40.427458+0000","last_became_peered":"2024-09-21T10:01:40.427458+0000","last_unstale":"2024-09-21T10:03:39.530420+0000","last_undegraded":"2024-09-21T10:03:39.530420+0000","last_fullsized":"2024-09-21T10:03:39.530420+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:38.605007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1151,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.478765+0000","last_change":"2024-09-21T10:01:40.420505+0000","last_active":"2024-09-21T10:03:34.478765+0000","last_peered":"2024-09-21T10:03:34.478765+0000","last_clean":"2024-09-21T10:03:34.478765+0000","last_became_active":"2024-09-21T10:01:40.420263+0000","last_became_peered":"2024-09-21T10:01:40.420263+0000","last_unstale":"2024-09-21T10:03:34.478765+0000","last_undegraded":"2024-09-21T10:03:34.478765+0000","last_fullsized":"2024-09-21T10:03:34.478765+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:23.452408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1744,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.512984+0000","last_change":"2024-09-21T10:01:40.420219+0000","last_active":"2024-09-21T10:03:39.512984+0000","last_peered":"2024-09-21T10:03:39.512984+0000","last_clean":"2024-09-21T10:03:39.512984+0000","last_became_active":"2024-09-21T10:01:40.419839+0000","last_became_peered":"2024-09-21T10:01:40.419839+0000","last_unstale":"2024-09-21T10:03:39.512984+0000","last_undegraded":"2024-09-21T10:03:39.512984+0000","last_fullsized":"2024-09-21T10:03:39.512984+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:59.370554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1775,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.519281+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T10:03:39.519281+0000","last_peered":"2024-09-21T10:03:39.519281+0000","last_clean":"2024-09-21T10:03:39.519281+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T10:03:39.519281+0000","last_undegraded":"2024-09-21T10:03:39.519281+0000","last_fullsized":"2024-09-21T10:03:39.519281+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.18","version":"1181'7494","reported_seq":7594,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:46.909561+0000","last_change":"2024-09-21T10:01:27.370045+0000","last_active":"2024-09-21T10:03:46.909561+0000","last_peered":"2024-09-21T10:03:46.909561+0000","last_clean":"2024-09-21T10:03:46.909561+0000","last_became_active":"2024-09-21T10:01:27.368411+0000","last_became_peered":"2024-09-21T10:01:27.368411+0000","last_unstale":"2024-09-21T10:03:46.909561+0000","last_undegraded":"2024-09-21T10:03:46.909561+0000","last_fullsized":"2024-09-21T10:03:46.909561+0000","mapping_epoch":1148,"log_start":"0'0","ondisk_log_start":"0'0","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7494,"log_dups_size":0,"ondisk_log_size":7494,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74907648,"num_objects":1143,"num_object_clones":0,"num_object_copies":3429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5831,"num_write_kb":223168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1149,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479011+0000","last_change":"2024-09-21T10:01:40.432190+0000","last_active":"2024-09-21T10:03:34.479011+0000","last_peered":"2024-09-21T10:03:34.479011+0000","last_clean":"2024-09-21T10:03:34.479011+0000","last_became_active":"2024-09-21T10:01:40.431847+0000","last_became_peered":"2024-09-21T10:01:40.431847+0000","last_unstale":"2024-09-21T10:03:34.479011+0000","last_undegraded":"2024-09-21T10:03:34.479011+0000","last_fullsized":"2024-09-21T10:03:34.479011+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:32.573309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":117,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479091+0000","last_change":"2024-09-21T10:01:40.420069+0000","last_active":"2024-09-21T10:03:34.479091+0000","last_peered":"2024-09-21T10:03:34.479091+0000","last_clean":"2024-09-21T10:03:34.479091+0000","last_became_active":"2024-09-21T10:01:40.416524+0000","last_became_peered":"2024-09-21T10:01:40.416524+0000","last_unstale":"2024-09-21T10:03:34.479091+0000","last_undegraded":"2024-09-21T10:03:34.479091+0000","last_fullsized":"2024-09-21T10:03:34.479091+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:36.273679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"1181'7261","reported_seq":11489,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:43.853201+0000","last_change":"2024-09-21T10:03:04.307452+0000","last_active":"2024-09-21T10:03:43.853201+0000","last_peered":"2024-09-21T10:03:43.853201+0000","last_clean":"2024-09-21T10:03:43.853201+0000","last_became_active":"2024-09-21T10:03:04.296838+0000","last_became_peered":"2024-09-21T10:03:04.296838+0000","last_unstale":"2024-09-21T10:03:43.853201+0000","last_undegraded":"2024-09-21T10:03:43.853201+0000","last_fullsized":"2024-09-21T10:03:43.853201+0000","mapping_epoch":1175,"log_start":"0'0","ondisk_log_start":"0'0","created":1148,"last_epoch_clean":1176,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7261,"log_dups_size":0,"ondisk_log_size":7261,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84738048,"num_objects":1293,"num_object_clones":0,"num_object_copies":3879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5657,"num_write_kb":222400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2058,"num_bytes_recovered":115474432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1139,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.478729+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T10:03:34.478729+0000","last_peered":"2024-09-21T10:03:34.478729+0000","last_clean":"2024-09-21T10:03:34.478729+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T10:03:34.478729+0000","last_undegraded":"2024-09-21T10:03:34.478729+0000","last_fullsized":"2024-09-21T10:03:34.478729+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":131,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479154+0000","last_change":"2024-09-21T10:00:16.366351+0000","last_active":"2024-09-21T10:03:34.479154+0000","last_peered":"2024-09-21T10:03:34.479154+0000","last_clean":"2024-09-21T10:03:34.479154+0000","last_became_active":"2024-09-21T10:00:16.355385+0000","last_became_peered":"2024-09-21T10:00:16.355385+0000","last_unstale":"2024-09-21T10:03:34.479154+0000","last_undegraded":"2024-09-21T10:03:34.479154+0000","last_fullsized":"2024-09-21T10:03:34.479154+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:29.014677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1709,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.523410+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:03:39.523410+0000","last_peered":"2024-09-21T10:03:39.523410+0000","last_clean":"2024-09-21T10:03:39.523410+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:03:39.523410+0000","last_undegraded":"2024-09-21T10:03:39.523410+0000","last_fullsized":"2024-09-21T10:03:39.523410+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1709,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.522690+0000","last_change":"2024-09-21T10:00:16.361160+0000","last_active":"2024-09-21T10:03:39.522690+0000","last_peered":"2024-09-21T10:03:39.522690+0000","last_clean":"2024-09-21T10:03:39.522690+0000","last_became_active":"2024-09-21T10:00:16.360524+0000","last_became_peered":"2024-09-21T10:00:16.360524+0000","last_unstale":"2024-09-21T10:03:39.522690+0000","last_undegraded":"2024-09-21T10:03:39.522690+0000","last_fullsized":"2024-09-21T10:03:39.522690+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:52.747211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1152,"reported_epoch":1180,"state":"active+clean","last_fresh":"2024-09-21T10:03:34.479439+0000","last_change":"2024-09-21T10:01:40.432238+0000","last_active":"2024-09-21T10:03:34.479439+0000","last_peered":"2024-09-21T10:03:34.479439+0000","last_clean":"2024-09-21T10:03:34.479439+0000","last_became_active":"2024-09-21T10:01:40.431883+0000","last_became_peered":"2024-09-21T10:01:40.431883+0000","last_unstale":"2024-09-21T10:03:34.479439+0000","last_undegraded":"2024-09-21T10:03:34.479439+0000","last_fullsized":"2024-09-21T10:03:34.479439+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:30.528533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1790,"reported_epoch":1181,"state":"active+clean","last_fresh":"2024-09-21T10:03:39.520828+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T10:03:39.520828+0000","last_peered":"2024-09-21T10:03:39.520828+0000","last_clean":"2024-09-21T10:03:39.520828+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T10:03:39.520828+0000","last_undegraded":"2024-09-21T10:03:39.520828+0000","last_fullsized":"2024-09-21T10:03:39.520828+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":2412576792,"num_objects":36814,"num_object_clones":0,"num_object_copies":110442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6530,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36814,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":168214,"num_write_kb":6272001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20693,"num_bytes_recovered":1234173952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8405463040,"data_stored":8405450824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":175719,"ondisk_log_size":175719,"up":78,"acting":73,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":9628,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":477367,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":107,"num_read_kb":272,"num_write":55,"num_write_kb":592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":542359,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397855,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1480776,"kb_used_data":1175104,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92264376,"statfs":{"total":95995035648,"available":94478721024,"internally_reserved":0,"allocated":1203306496,"data_stored":1200748605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594083,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1212932,"kb_used_data":907260,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92532220,"statfs":{"total":95995035648,"available":94752993280,"internally_reserved":0,"allocated":929034240,"data_stored":926480334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790313,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":946108,"kb_used_data":636276,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92799044,"statfs":{"total":95995035648,"available":95026221056,"internally_reserved":0,"allocated":651546624,"data_stored":648997256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986540,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1110020,"kb_used_data":812668,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92635132,"statfs":{"total":95995035648,"available":94858375168,"internally_reserved":0,"allocated":832172032,"data_stored":829618159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999182,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1929592,"kb_used_data":1449648,"kb_used_omap":25,"kb_used_meta":479526,"kb_avail":91815560,"statfs":{"total":95995035648,"available":94019133440,"internally_reserved":0,"allocated":1484439552,"data_stored":1481893747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":491035591},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111125,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1610424,"kb_used_data":1151536,"kb_used_omap":31,"kb_used_meta":458848,"kb_avail":92134728,"statfs":{"total":95995035648,"available":94345961472,"internally_reserved":0,"allocated":1179172864,"data_stored":1176627557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":469860584},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2203318223068,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335010,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1595004,"kb_used_data":1244212,"kb_used_omap":31,"kb_used_meta":350560,"kb_avail":92150148,"statfs":{"total":95995035648,"available":94361751552,"internally_reserved":0,"allocated":1274073088,"data_stored":1271523172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":358973536},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1264390144,"data_stored":1264386072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":964624384,"data_stored":964624384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1154351104,"data_stored":1154351104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1471483904,"data_stored":1471479832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":814874624,"data_stored":814874624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":638189568,"data_stored":638189568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":905449472,"data_stored":905445400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1192099840,"data_stored":1192099840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:03:53.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:03:53.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:53.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:53.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:53.403 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:53.427 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:53.529 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:53.555 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:53.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:53.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:53.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:53.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:53.759 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:53 smithi017 ceph-mon[102913]: pgmap v1566: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 5808/96177 objects degraded (6.039%); 0 B/s, 173 objects/s recovering 2024-09-21T10:03:53.759 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:53 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5808/96177 objects degraded (6.039%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:53.759 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:53 smithi017 ceph-mon[105151]: pgmap v1566: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 5808/96177 objects degraded (6.039%); 0 B/s, 173 objects/s recovering 2024-09-21T10:03:53.759 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:53 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5808/96177 objects degraded (6.039%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:53.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:53.831 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:53.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:53.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:53.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:53.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:53 smithi134 ceph-mon[90085]: pgmap v1566: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 5808/96177 objects degraded (6.039%); 0 B/s, 173 objects/s recovering 2024-09-21T10:03:53.997 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:53 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5808/96177 objects degraded (6.039%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:54.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:54.111 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:54.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:54.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:54.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:03:54.254 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:54.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:54.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:54.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:54.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:54.520 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:54.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:54.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:54.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:54.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:54.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:54.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:54.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.843 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:54.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:54.944 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:54.973 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:54.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:55.075 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:55.104 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:55.143 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:03:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:03:54] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.51.0" 2024-09-21T10:03:55.206 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:55.234 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:55.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:55.336 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:55.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:55.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:55.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:55.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:55.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:55.618 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:55.655 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:55.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:55.757 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:55.786 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:55.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:55.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:55.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:55.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:55 smithi017 ceph-mon[102913]: pgmap v1567: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 964 op/s; 5808/92823 objects degraded (6.257%); 0 B/s, 149 objects/s recovering 2024-09-21T10:03:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:55 smithi017 ceph-mon[105151]: pgmap v1567: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 964 op/s; 5808/92823 objects degraded (6.257%); 0 B/s, 149 objects/s recovering 2024-09-21T10:03:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:55 smithi134 ceph-mon[90085]: pgmap v1567: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 964 op/s; 5808/92823 objects degraded (6.257%); 0 B/s, 149 objects/s recovering 2024-09-21T10:03:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:56.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:56.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:56.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:56.114 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:56.125 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:56.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:56.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.236 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:56.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:56.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:56.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:56.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:56.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:56.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:56.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:56.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:56.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:56.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:56.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:56.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:56.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:56.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:56.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:57.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:57.083 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:57.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:57.185 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:57.211 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:57.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:57.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:57.342 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:57.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:57.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:57.469 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:57.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:57.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:57.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:03:57.609 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:57.630 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:03:57.711 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:57.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:57.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:57.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:57.883 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:57.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:57 smithi017 ceph-mon[102913]: pgmap v1568: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 5583/87780 objects degraded (6.360%); 0 B/s, 118 objects/s recovering 2024-09-21T10:03:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2577052739' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:57 smithi017 ceph-mon[105151]: pgmap v1568: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 5583/87780 objects degraded (6.360%); 0 B/s, 118 objects/s recovering 2024-09-21T10:03:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2577052739' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:57.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:57 smithi134 ceph-mon[90085]: pgmap v1568: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 5583/87780 objects degraded (6.360%); 0 B/s, 118 objects/s recovering 2024-09-21T10:03:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2577052739' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:58.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:58.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:58.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:58.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:58.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:58.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:58.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:58.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:58.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:58.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:58.570 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:58.605 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:58.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:58.707 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:58.736 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:58.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:58.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:58.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:58 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:58 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2577052739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:03:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:58 smithi017 ceph-mon[102913]: osdmap e1184: 8 total, 8 up, 7 in 2024-09-21T10:03:58.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:58 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5583/87780 objects degraded (6.360%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:58.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:58 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:58.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:58 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2577052739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:03:58.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:58 smithi017 ceph-mon[105151]: osdmap e1184: 8 total, 8 up, 7 in 2024-09-21T10:03:58.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:58 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5583/87780 objects degraded (6.360%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:58.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:58 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:58 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2577052739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:03:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:58 smithi134 ceph-mon[90085]: osdmap e1184: 8 total, 8 up, 7 in 2024-09-21T10:03:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:58 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5583/87780 objects degraded (6.360%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:59.025 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:59.127 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:59.161 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:59.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:59.263 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:59.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:59.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:59.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:59.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:59.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:59.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:59.594 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:03:59.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:59.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:59.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:59.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:59.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:59.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:03:59.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:59.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:03:59 smithi017 ceph-mon[102913]: pgmap v1570: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 5258/81654 objects degraded (6.439%); 0 B/s, 158 objects/s recovering 2024-09-21T10:03:59.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:03:59 smithi017 ceph-mon[105151]: pgmap v1570: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 5258/81654 objects degraded (6.439%); 0 B/s, 158 objects/s recovering 2024-09-21T10:03:59.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:03:59 smithi134 ceph-mon[90085]: pgmap v1570: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.7 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 5258/81654 objects degraded (6.439%); 0 B/s, 158 objects/s recovering 2024-09-21T10:04:00.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:00.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:00.109 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:00.138 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:00.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:00.239 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.271 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:00.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:00.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:00.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.403 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:00.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:04:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:04:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:00.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:00.534 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:00.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:00.635 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:00.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:00.767 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:00.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:00.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:00.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:00.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:00.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:01.056 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:01.083 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:01.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:01.120 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:01.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:01.213 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:01.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:01.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:01.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:01.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:01.447 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:01.475 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:04:01.479 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:04:01.480 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:01.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:01.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:01.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:01.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:01.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:01.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:01.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:01.890 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:01.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:04:01.929 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:01 smithi017 ceph-mon[102913]: pgmap v1571: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 870 op/s; 5258/77814 objects degraded (6.757%); 0 B/s, 102 objects/s recovering 2024-09-21T10:04:01.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:01 smithi017 ceph-mon[105151]: pgmap v1571: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 870 op/s; 5258/77814 objects degraded (6.757%); 0 B/s, 102 objects/s recovering 2024-09-21T10:04:01.992 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:01 smithi134 ceph-mon[90085]: pgmap v1571: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 870 op/s; 5258/77814 objects degraded (6.757%); 0 B/s, 102 objects/s recovering 2024-09-21T10:04:02.020 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:02.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:02.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:02.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:02.260 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:02.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:02.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:02.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:02.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:02.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:02.535 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:02.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:02.677 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:02.707 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:02.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:02.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:02.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:02.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:02 smithi017 ceph-mon[102913]: from='client.35673 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:02.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:02 smithi017 ceph-mon[105151]: from='client.35673 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:02.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:02.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:02 smithi134 ceph-mon[90085]: from='client.35673 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:03.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:03.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:03.228 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:03.254 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:03.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:03.356 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:03.389 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:03.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:03.490 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:03.523 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:03.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:03.624 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:03.647 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:03.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:03.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:03.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:03.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:03 smithi017 ceph-mon[102913]: pgmap v1572: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.3 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 993 op/s; 4722/66375 objects degraded (7.114%); 0 B/s, 108 objects/s recovering 2024-09-21T10:04:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:03 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 4722/66375 objects degraded (7.114%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:03 smithi017 ceph-mon[105151]: pgmap v1572: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.3 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 993 op/s; 4722/66375 objects degraded (7.114%); 0 B/s, 108 objects/s recovering 2024-09-21T10:04:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:03 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 4722/66375 objects degraded (7.114%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:03.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:03 smithi134 ceph-mon[90085]: pgmap v1572: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.3 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 993 op/s; 4722/66375 objects degraded (7.114%); 0 B/s, 108 objects/s recovering 2024-09-21T10:04:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:03 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 4722/66375 objects degraded (7.114%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:04.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:04.060 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:60910 6.8 e1085/1078 6:115de437:::benchmark_data_smithi017_149547_object60909:head v 1085'3857, mlcod=1085'3855)", 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.397470+0000", 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 417.65768814900002, 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14857488399999999, 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:04:04.061 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397470+0000", 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397470+0000", 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397470+0000", 2024-09-21T10:04:04.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6100000000000001e-07 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397502+0000", 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1847999999999997e-05 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397510+0000", 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4270000000000008e-06 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397516+0000", 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7509999999999997e-06 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.398288+0000", 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00077210400000000002 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.398307+0000", 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8601e-05 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.546045+0000", 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.147737592 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:60913 6.f 6:fefdacc2:::benchmark_data_smithi017_149547_object60912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.397930+0000", 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 417.657228143, 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19911648100000001, 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.065 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60913 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397930+0000", 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397930+0000", 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397931+0000", 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.674e-06 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397958+0000", 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6211e-05 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397971+0000", 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3062e-05 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.397981+0000", 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0169999999999999e-05 2024-09-21T10:04:04.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.403821+0000", 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058398900000000004 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.403974+0000", 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015353400000000001 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:04.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.404113+0000", 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000139018 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.408776+0000", 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046623130000000004 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.437344+0000", 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028568758 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.596960+0000", 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15961568800000001 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.597004+0000", 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4168000000000001e-05 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.597046+0000", 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1995e-05 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.071 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:60915 6.0 e1085/1078)", 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.402516+0000", 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 417.65264133800002, 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.161702771, 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402516+0000", 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402516+0000", 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402517+0000", 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1999999999999998e-07 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402543+0000", 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5154e-05 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402558+0000", 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5007e-05 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.402566+0000", 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3739999999999998e-06 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.564201+0000", 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16163555499999999 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.564219+0000", 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7761000000000002e-05 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45173.0:60914 6.8 e1085/1078)", 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.409836+0000", 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 417.64532191000001, 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15982787300000001, 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.409836+0000", 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.409836+0000", 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.409836+0000", 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8299999999999998e-07 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.409872+0000", 2024-09-21T10:04:04.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6238000000000001e-05 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.409879+0000", 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5570000000000001e-06 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.409884+0000", 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9080000000000003e-06 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.569639+0000", 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.159754971 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.569664+0000", 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4916e-05 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:123239 6.f 6:f8834be7:::benchmark_data_smithi017_149547_object4160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:11.002234+0000", 2024-09-21T10:04:04.079 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 353.05292335000001, 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.149642569, 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123239 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.002234+0000", 2024-09-21T10:04:04.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.002234+0000", 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.002237+0000", 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3190000000000002e-06 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.002244+0000", 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0010000000000004e-06 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.002293+0000", 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9481000000000003e-05 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.002303+0000", 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9350000000000006e-06 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.002366+0000", 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2694000000000007e-05 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.002480+0000", 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113726 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.002596+0000", 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116879 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.008794+0000", 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061970569999999997 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.008964+0000", 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016999100000000001 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.151769+0000", 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.142805502 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.151830+0000", 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1414000000000005e-05 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.151877+0000", 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6570000000000003e-05 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:123251 6.13 6:ca3ca4a4:::benchmark_data_smithi017_149547_object4172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T10:04:04.085 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:11.011022+0000", 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 353.04413562000002, 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15699486900000001, 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123251 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.011022+0000", 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.011022+0000", 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.011024+0000", 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.561e-06 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.011029+0000", 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9309999999999999e-06 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.011041+0000", 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1969e-05 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.011048+0000", 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9720000000000003e-06 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.011110+0000", 2024-09-21T10:04:04.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2580000000000001e-05 2024-09-21T10:04:04.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.011260+0000", 2024-09-21T10:04:04.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014962999999999999 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3,6", 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.011406+0000", 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014579400000000001 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013371+0000", 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019658240000000001 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013458+0000", 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6246000000000001e-05 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.014683+0000", 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001225198 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.166810+0000", 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15212673099999999 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.166873+0000", 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3540999999999994e-05 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.168017+0000", 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011438920000000001 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:123252 6.f 6:fe23691d:::benchmark_data_smithi017_149547_object4173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-09-21T10:04:04.092 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:11.013254+0000", 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 353.04190341899999, 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15042488900000001, 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123252 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013254+0000", 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013254+0000", 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013256+0000", 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5379999999999999e-06 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013261+0000", 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9729999999999997e-06 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013271+0000", 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0531e-05 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013278+0000", 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2090000000000002e-06 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013487+0000", 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000209311 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013614+0000", 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012707199999999999 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.013723+0000", 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109324 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.016807+0000", 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003084051 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.016887+0000", 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9436000000000003e-05 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.163549+0000", 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14666262699999999 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.163630+0000", 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0887e-05 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:11.163679+0000", 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8930000000000001e-05 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.098 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:175241 6.13 6:cfd98e43:::benchmark_data_smithi017_149547_object56162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:08.290955+0000", 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.76420272799999, 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14707041500000001, 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 175241 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.290955+0000", 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.290955+0000", 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.290956+0000", 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2500000000000004e-07 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.290961+0000", 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6360000000000002e-06 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.290971+0000", 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0489000000000001e-05 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.290978+0000", 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1770000000000001e-06 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.291032+0000", 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3399000000000002e-05 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.291164+0000", 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000132173 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3,6", 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.291283+0000", 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119454 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.302302+0000", 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011019126000000001 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.302347+0000", 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5011e-05 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.302362+0000", 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4739000000000001e-05 2024-09-21T10:04:04.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.436503+0000", 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.134141232 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.436580+0000", 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6772000000000003e-05 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.438025+0000", 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001445382 2024-09-21T10:04:04.104 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:175244 6.f 6:f18c813b:::benchmark_data_smithi017_149547_object56165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:08.292997+0000", 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.76216039799999, 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.179269501, 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:04.105 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 175244 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.292997+0000", 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.292997+0000", 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.292999+0000", 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.376e-06 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.293004+0000", 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3460000000000001e-06 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.293015+0000", 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1285e-05 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.293023+0000", 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1539999999999996e-06 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.293086+0000", 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3894999999999997e-05 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.293187+0000", 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010081699999999999 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.293287+0000", 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010004 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.448363+0000", 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15507520399999999 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.454148+0000", 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005785994 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.471996+0000", 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017847470000000001 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.472211+0000", 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021458499999999999 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.472267+0000", 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6335000000000002e-05 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.110 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:175247 6.f 6:fa04c605:::benchmark_data_smithi017_149547_object56168:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:08.294839+0000", 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.76031849700001, 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18573033899999999, 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:04.111 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 175247 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.294839+0000", 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.294839+0000", 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.294841+0000", 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.578e-06 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.294847+0000", 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2620000000000004e-06 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.294860+0000", 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2872e-05 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.294868+0000", 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1170000000000004e-06 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.297750+0000", 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028817600000000001 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.297855+0000", 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105078 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.298263+0000", 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040756100000000002 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.448400+0000", 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.150137829 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.454154+0000", 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057535379999999999 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.480456+0000", 2024-09-21T10:04:04.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026301939999999999 2024-09-21T10:04:04.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.480521+0000", 2024-09-21T10:04:04.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5042000000000005e-05 2024-09-21T10:04:04.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.480570+0000", 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8761999999999998e-05 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:175252 6.f 6:f97a1314:::benchmark_data_smithi017_149547_object56173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:08.300972+0000", 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.75418548699997, 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17973899900000001, 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.117 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 175252 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.300972+0000", 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.300972+0000", 2024-09-21T10:04:04.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.300974+0000", 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5200000000000001e-06 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.300979+0000", 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6809999999999997e-06 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.300991+0000", 2024-09-21T10:04:04.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.258e-05 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.300999+0000", 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.729e-06 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.436242+0000", 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13524280399999999 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.436369+0000", 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127105 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.438109+0000", 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001739907 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.448435+0000", 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010326722 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.454160+0000", 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057250740000000001 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.480621+0000", 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026460556999999999 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.480668+0000", 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6600000000000001e-05 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.480711+0000", 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3720000000000002e-05 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:175255 6.f 6:fdaf1a85:::benchmark_data_smithi017_149547_object56176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1114)", 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:08.302716+0000", 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 295.75244177399998, 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17813868399999999, 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.123 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 175255 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.302716+0000", 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.302716+0000", 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.302717+0000", 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3689999999999999e-06 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.302722+0000", 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8439999999999997e-06 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.436147+0000", 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13342438700000001 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.436169+0000", 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2123000000000001e-05 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.442768+0000", 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065993019999999996 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.442889+0000", 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121315 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.443010+0000", 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120703 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.448472+0000", 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054615920000000004 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.454165+0000", 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056932950000000001 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.480763+0000", 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026598199999999999 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.480814+0000", 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0435000000000001e-05 2024-09-21T10:04:04.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:08.480855+0000", 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1118999999999997e-05 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.129 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53004 7.e 7:7bfbabea:::benchmark_data_smithi017_180590_object53003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1147)", 2024-09-21T10:04:04.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:04.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:04.131 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.340426+0000", 2024-09-21T10:04:04.131 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 157.71473169800001, 2024-09-21T10:04:04.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15705764899999999, 2024-09-21T10:04:04.131 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.131 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.131 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53004 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.340426+0000", 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.340426+0000", 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.340428+0000", 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9130000000000001e-06 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.340457+0000", 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8911000000000002e-05 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.340471+0000", 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4085e-05 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.340480+0000", 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2809999999999997e-06 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345202+0000", 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047222159999999996 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345361+0000", 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015882 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345667+0000", 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030591700000000001 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.358813+0000", 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013145644999999999 2024-09-21T10:04:04.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.358824+0000", 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1292e-05 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.497368+0000", 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13854371900000001 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.497434+0000", 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6459999999999997e-05 2024-09-21T10:04:04.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.497484+0000", 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.939e-05 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53009 7.e 7:75bb5ff9:::benchmark_data_smithi017_180590_object53008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1147)", 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.344969+0000", 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 157.71018847400001, 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.152643374, 2024-09-21T10:04:04.137 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53009 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.344969+0000", 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.344969+0000", 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.344970+0000", 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7700000000000004e-07 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.344991+0000", 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1162000000000002e-05 2024-09-21T10:04:04.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.344998+0000", 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1670000000000002e-06 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345003+0000", 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1209999999999998e-06 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.347807+0000", 2024-09-21T10:04:04.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002803538 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.347953+0000", 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014660699999999999 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348098+0000", 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144041 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.358860+0000", 2024-09-21T10:04:04.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010762594 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.365110+0000", 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0062499360000000002 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.497525+0000", 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13241493900000001 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.497571+0000", 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6357999999999999e-05 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.497613+0000", 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1334000000000001e-05 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53007 7.18 7:1c743eac:::benchmark_data_smithi017_180590_object53006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.345705+0000", 2024-09-21T10:04:04.143 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 157.70945279399999, 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0290580650000001, 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53007 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345705+0000", 2024-09-21T10:04:04.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345705+0000", 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345706+0000", 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.232e-06 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345731+0000", 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4842e-05 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345741+0000", 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0114e-05 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345748+0000", 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5699999999999998e-06 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.469569+0000", 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.12382101 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.469577+0000", 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8490000000000006e-06 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.370274+0000", 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.900697466 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.370408+0000", 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133802 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.370543+0000", 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013473900000000001 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374294+0000", 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003751739 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374333+0000", 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8186999999999999e-05 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374685+0000", 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035275299999999997 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374728+0000", 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3074999999999997e-05 2024-09-21T10:04:04.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374763+0000", 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4687000000000002e-05 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPush(7.b 1162/1156 [PushOp(7:dbf0f70c:::benchmark_data_smithi017_180590_object2556:head, version: 1145'154, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:dbf0f70c:::benchmark_data_smithi017_180590_object2556:head@1145'154, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:56.621539+0000", 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 127.433619202, 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15143771, 2024-09-21T10:04:04.150 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.621539+0000", 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.621539+0000", 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.621548+0000", 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2490000000000007e-06 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.621627+0000", 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8787000000000003e-05 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.767798+0000", 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14617165500000001 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.767865+0000", 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6820000000000001e-05 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771164+0000", 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032985979999999998 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771179+0000", 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5469e-05 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.772976+0000", 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017971319999999999 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35520.0:152147 7.10 e1176/1156 7:0e36332e:::benchmark_data_smithi017_180590_object42587:head v 1176'6272, mlcod=1146'2571)", 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:03:13.407693+0000", 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 50.647464429999999, 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34189436699999998, 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.154 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407693+0000", 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407693+0000", 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407694+0000", 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.048e-06 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407702+0000", 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.345e-06 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407712+0000", 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0546e-05 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407720+0000", 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.25e-06 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407775+0000", 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5878999999999998e-05 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407801+0000", 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5588e-05 2024-09-21T10:04:04.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411581+0000", 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037798039999999999 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749588+0000", 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33800690700000002 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35520.0:152175 7.5 e1176/1148 7:a0037212:::benchmark_data_smithi017_180590_object42615:head v 1176'6406, mlcod=1176'6404)", 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:03:13.410900+0000", 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 50.644257682999999, 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34129278699999999, 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410900+0000", 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410900+0000", 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410901+0000", 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6899999999999996e-07 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410908+0000", 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8179999999999998e-06 2024-09-21T10:04:04.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410917+0000", 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0380000000000005e-06 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410923+0000", 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0889999999999996e-06 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749002+0000", 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.338079081 2024-09-21T10:04:04.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749031+0000", 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8997e-05 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.752149+0000", 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031175489999999998 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.752193+0000", 2024-09-21T10:04:04.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4246000000000003e-05 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35520.0:152169 7.a e1176/1142 7:50705924:::benchmark_data_smithi017_180590_object42609:head v 1176'9589, mlcod=1176'9586)", 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:03:13.411045+0000", 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 50.644112481999997, 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34690485799999998, 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411045+0000", 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411045+0000", 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411046+0000", 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9000000000000005e-07 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411053+0000", 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8859999999999998e-06 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.748947+0000", 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33789343100000002 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.748966+0000", 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9604000000000001e-05 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749042+0000", 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6174000000000006e-05 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749065+0000", 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2821999999999999e-05 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.757905+0000", 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0088397830000000004 2024-09-21T10:04:04.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.757950+0000", 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5167999999999999e-05 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPush(7.b 1176/1156 [PushOp(7:d7bc6bf0:::benchmark_data_smithi017_180590_object56111:head, version: 1149'3496, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:d7bc6bf0:::benchmark_data_smithi017_180590_object56111:head@1149'3496, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:03:13.412303+0000", 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 50.642854532000001, 2024-09-21T10:04:04.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34021348600000001, 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.412303+0000", 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.412303+0000", 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:04.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.412304+0000", 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8299999999999996e-07 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.412341+0000", 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7271000000000003e-05 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.748967+0000", 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33662544999999999 2024-09-21T10:04:04.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749005+0000", 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8247999999999997e-05 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749112+0000", 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010668299999999999 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:04.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749126+0000", 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4139e-05 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.752517+0000", 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033910120000000001 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:04.171 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:04:04.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:04.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:04.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:04.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:04.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:04.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:04.585 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:04.687 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:04.716 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1571,"stamp":"2024-09-21T10:04:00.173415+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1698586236,"num_objects":25938,"num_object_clones":0,"num_object_copies":77814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5258,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25938,"num_whiteouts":0,"num_read":108,"num_read_kb":273,"num_write":179234,"num_write_kb":6272624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21992,"num_bytes_recovered":1234725977,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":176737,"ondisk_log_size":176737,"up":327,"acting":323,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":329,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7861552,"kb_used_data":5361972,"kb_used_omap":95,"kb_used_meta":2499104,"kb_avail":648354512,"statfs":{"total":671965249536,"available":663915020288,"internally_reserved":0,"allocated":5490659328,"data_stored":5472743356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97681,"internal_metadata":2559083119},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,6,8,4,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-548012032,"num_objects":-8362,"num_object_clones":0,"num_object_copies":-25086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-985,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8362,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":985,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.601124"},"pg_stats":[{"pgid":"7.17","version":"1183'7670","reported_seq":7728,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:55.479139+0000","last_change":"2024-09-21T10:01:27.381508+0000","last_active":"2024-09-21T10:03:55.479139+0000","last_peered":"2024-09-21T10:03:55.479139+0000","last_clean":"2024-09-21T10:03:55.479139+0000","last_became_active":"2024-09-21T10:01:27.375972+0000","last_became_peered":"2024-09-21T10:01:27.375972+0000","last_unstale":"2024-09-21T10:03:55.479139+0000","last_undegraded":"2024-09-21T10:03:55.479139+0000","last_fullsized":"2024-09-21T10:03:55.479139+0000","mapping_epoch":1148,"log_start":"0'0","ondisk_log_start":"0'0","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7670,"log_dups_size":0,"ondisk_log_size":7670,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56459264,"num_objects":862,"num_object_clones":0,"num_object_copies":2586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6008,"num_write_kb":219808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1710,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575717+0000","last_change":"2024-09-21T10:01:40.433720+0000","last_active":"2024-09-21T10:03:50.575717+0000","last_peered":"2024-09-21T10:03:50.575717+0000","last_clean":"2024-09-21T10:03:50.575717+0000","last_became_active":"2024-09-21T10:01:40.433360+0000","last_became_peered":"2024-09-21T10:01:40.433360+0000","last_unstale":"2024-09-21T10:03:50.575717+0000","last_undegraded":"2024-09-21T10:03:50.575717+0000","last_fullsized":"2024-09-21T10:03:50.575717+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:08.257176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1108,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.579739+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T10:03:49.579739+0000","last_peered":"2024-09-21T10:03:49.579739+0000","last_clean":"2024-09-21T10:03:49.579739+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T10:03:49.579739+0000","last_undegraded":"2024-09-21T10:03:49.579739+0000","last_fullsized":"2024-09-21T10:03:49.579739+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.16","version":"1184'7780","reported_seq":7917,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:58.910798+0000","last_change":"2024-09-21T10:03:10.420081+0000","last_active":"2024-09-21T10:03:58.910798+0000","last_peered":"2024-09-21T10:03:58.910798+0000","last_clean":"2024-09-21T10:03:58.910798+0000","last_became_active":"2024-09-21T10:01:27.383294+0000","last_became_peered":"2024-09-21T10:01:27.383294+0000","last_unstale":"2024-09-21T10:03:58.910798+0000","last_undegraded":"2024-09-21T10:03:58.910798+0000","last_fullsized":"2024-09-21T10:03:58.910798+0000","mapping_epoch":1148,"log_start":"1145'464","ondisk_log_start":"1145'464","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"1176'6333","last_scrub_stamp":"2024-09-21T10:03:10.419712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:03:10.419712+0000","objects_scrubbed":2183,"log_size":7316,"log_dups_size":0,"ondisk_log_size":7316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:18.821196+0000","scrub_duration":884,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47251456,"num_objects":721,"num_object_clones":0,"num_object_copies":2163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1462,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":616,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580036+0000","last_change":"2024-09-21T09:57:39.541887+0000","last_active":"2024-09-21T10:03:49.580036+0000","last_peered":"2024-09-21T10:03:49.580036+0000","last_clean":"2024-09-21T10:03:49.580036+0000","last_became_active":"2024-09-21T09:57:39.541598+0000","last_became_peered":"2024-09-21T09:57:39.541598+0000","last_unstale":"2024-09-21T10:03:49.580036+0000","last_undegraded":"2024-09-21T10:03:49.580036+0000","last_fullsized":"2024-09-21T10:03:49.580036+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:41.615525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1789,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.646838+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T10:03:57.646838+0000","last_peered":"2024-09-21T10:03:57.646838+0000","last_clean":"2024-09-21T10:03:57.646838+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T10:03:57.646838+0000","last_undegraded":"2024-09-21T10:03:57.646838+0000","last_fullsized":"2024-09-21T10:03:57.646838+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.15","version":"1183'7683","reported_seq":7840,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:55.980846+0000","last_change":"2024-09-21T10:01:59.676028+0000","last_active":"2024-09-21T10:03:55.980846+0000","last_peered":"2024-09-21T10:03:55.980846+0000","last_clean":"2024-09-21T10:03:55.980846+0000","last_became_active":"2024-09-21T10:01:27.368645+0000","last_became_peered":"2024-09-21T10:01:27.368645+0000","last_unstale":"2024-09-21T10:03:55.980846+0000","last_undegraded":"2024-09-21T10:03:55.980846+0000","last_fullsized":"2024-09-21T10:03:55.980846+0000","mapping_epoch":1148,"log_start":"1145'608","ondisk_log_start":"1145'608","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"1162'4060","last_scrub_stamp":"2024-09-21T10:01:59.675861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:01:59.675861+0000","objects_scrubbed":2420,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:02.268701+0000","scrub_duration":0.74397808499999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54788096,"num_objects":836,"num_object_clones":0,"num_object_copies":2508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3630,"num_write_kb":65472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1790,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.646732+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T10:03:57.646732+0000","last_peered":"2024-09-21T10:03:57.646732+0000","last_clean":"2024-09-21T10:03:57.646732+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T10:03:57.646732+0000","last_undegraded":"2024-09-21T10:03:57.646732+0000","last_fullsized":"2024-09-21T10:03:57.646732+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1109,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580250+0000","last_change":"2024-09-21T10:00:16.358888+0000","last_active":"2024-09-21T10:03:49.580250+0000","last_peered":"2024-09-21T10:03:49.580250+0000","last_clean":"2024-09-21T10:03:49.580250+0000","last_became_active":"2024-09-21T10:00:16.357219+0000","last_became_peered":"2024-09-21T10:00:16.357219+0000","last_unstale":"2024-09-21T10:03:49.580250+0000","last_undegraded":"2024-09-21T10:03:49.580250+0000","last_fullsized":"2024-09-21T10:03:49.580250+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:13.887798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.14","version":"1184'7610","reported_seq":11757,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:58.927269+0000","last_change":"2024-09-21T10:03:21.493141+0000","last_active":"2024-09-21T10:03:58.927269+0000","last_peered":"2024-09-21T10:03:58.927269+0000","last_clean":"2024-09-21T10:03:58.927269+0000","last_became_active":"2024-09-21T10:02:18.295754+0000","last_became_peered":"2024-09-21T10:02:18.295754+0000","last_unstale":"2024-09-21T10:03:58.927269+0000","last_undegraded":"2024-09-21T10:03:58.927269+0000","last_fullsized":"2024-09-21T10:03:58.927269+0000","mapping_epoch":1165,"log_start":"1145'163","ondisk_log_start":"1145'163","created":1148,"last_epoch_clean":1166,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7447,"log_dups_size":0,"ondisk_log_size":7447,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-29T05:19:32.789172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52822016,"num_objects":806,"num_object_clones":0,"num_object_copies":2418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5994,"num_write_kb":217600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2014,"num_bytes_recovered":131989504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1710,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575355+0000","last_change":"2024-09-21T10:01:40.431450+0000","last_active":"2024-09-21T10:03:50.575355+0000","last_peered":"2024-09-21T10:03:50.575355+0000","last_clean":"2024-09-21T10:03:50.575355+0000","last_became_active":"2024-09-21T10:01:40.431140+0000","last_became_peered":"2024-09-21T10:01:40.431140+0000","last_unstale":"2024-09-21T10:03:50.575355+0000","last_undegraded":"2024-09-21T10:03:50.575355+0000","last_fullsized":"2024-09-21T10:03:50.575355+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:37.604030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":72,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.576459+0000","last_change":"2024-09-21T10:00:16.377280+0000","last_active":"2024-09-21T10:03:49.576459+0000","last_peered":"2024-09-21T10:03:49.576459+0000","last_clean":"2024-09-21T10:03:49.576459+0000","last_became_active":"2024-09-21T10:00:16.353074+0000","last_became_peered":"2024-09-21T10:00:16.353074+0000","last_unstale":"2024-09-21T10:03:49.576459+0000","last_undegraded":"2024-09-21T10:03:49.576459+0000","last_fullsized":"2024-09-21T10:03:49.576459+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:35.382228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1183'11972","reported_seq":20386,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:55.411744+0000","last_change":"2024-09-21T10:03:24.437255+0000","last_active":"2024-09-21T10:03:55.411744+0000","last_peered":"2024-09-21T10:03:55.411744+0000","last_clean":"2024-09-21T10:03:55.411744+0000","last_became_active":"2024-09-21T10:03:24.413749+0000","last_became_peered":"2024-09-21T10:03:24.413749+0000","last_unstale":"2024-09-21T10:03:55.411744+0000","last_undegraded":"2024-09-21T10:03:55.411744+0000","last_fullsized":"2024-09-21T10:03:55.411744+0000","mapping_epoch":1178,"log_start":"1164'5500","ondisk_log_start":"1164'5500","created":1142,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6472,"log_dups_size":0,"ondisk_log_size":6472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:40.139577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113377280,"num_objects":1730,"num_object_clones":0,"num_object_copies":5190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11972,"num_write_kb":438464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4161,"num_bytes_recovered":272695296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1107,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580975+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T10:03:49.580975+0000","last_peered":"2024-09-21T10:03:49.580975+0000","last_clean":"2024-09-21T10:03:49.580975+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T10:03:49.580975+0000","last_undegraded":"2024-09-21T10:03:49.580975+0000","last_fullsized":"2024-09-21T10:03:49.580975+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":115,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.579956+0000","last_change":"2024-09-21T10:00:16.357541+0000","last_active":"2024-09-21T10:03:49.579956+0000","last_peered":"2024-09-21T10:03:49.579956+0000","last_clean":"2024-09-21T10:03:49.579956+0000","last_became_active":"2024-09-21T10:00:16.357209+0000","last_became_peered":"2024-09-21T10:00:16.357209+0000","last_unstale":"2024-09-21T10:03:49.579956+0000","last_undegraded":"2024-09-21T10:03:49.579956+0000","last_fullsized":"2024-09-21T10:03:49.579956+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:20.732470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"1183'12170","reported_seq":12268,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.362135+0000","last_change":"2024-09-21T10:00:46.477601+0000","last_active":"2024-09-21T10:03:56.362135+0000","last_peered":"2024-09-21T10:03:56.362135+0000","last_clean":"2024-09-21T10:03:56.362135+0000","last_became_active":"2024-09-21T10:00:46.477208+0000","last_became_peered":"2024-09-21T10:00:46.477208+0000","last_unstale":"2024-09-21T10:03:56.362135+0000","last_undegraded":"2024-09-21T10:03:56.362135+0000","last_fullsized":"2024-09-21T10:03:56.362135+0000","mapping_epoch":1142,"log_start":"1168'7000","ondisk_log_start":"1168'7000","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5170,"log_dups_size":0,"ondisk_log_size":5170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:55.418857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111017984,"num_objects":1694,"num_object_clones":0,"num_object_copies":5082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12170,"num_write_kb":443648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1122,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580725+0000","last_change":"2024-09-21T10:01:40.429966+0000","last_active":"2024-09-21T10:03:49.580725+0000","last_peered":"2024-09-21T10:03:49.580725+0000","last_clean":"2024-09-21T10:03:49.580725+0000","last_became_active":"2024-09-21T10:01:40.419541+0000","last_became_peered":"2024-09-21T10:01:40.419541+0000","last_unstale":"2024-09-21T10:03:49.580725+0000","last_undegraded":"2024-09-21T10:03:49.580725+0000","last_fullsized":"2024-09-21T10:03:49.580725+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:10.848843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1151,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578910+0000","last_change":"2024-09-21T10:01:40.423234+0000","last_active":"2024-09-21T10:03:49.578910+0000","last_peered":"2024-09-21T10:03:49.578910+0000","last_clean":"2024-09-21T10:03:49.578910+0000","last_became_active":"2024-09-21T10:01:40.422854+0000","last_became_peered":"2024-09-21T10:01:40.422854+0000","last_unstale":"2024-09-21T10:03:49.578910+0000","last_undegraded":"2024-09-21T10:03:49.578910+0000","last_fullsized":"2024-09-21T10:03:49.578910+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:51.607040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1184'7613","reported_seq":11812,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:58.926680+0000","last_change":"2024-09-21T10:02:40.410442+0000","last_active":"2024-09-21T10:03:58.926680+0000","last_peered":"2024-09-21T10:03:58.926680+0000","last_clean":"2024-09-21T10:03:58.926680+0000","last_became_active":"2024-09-21T10:02:40.405438+0000","last_became_peered":"2024-09-21T10:02:40.405438+0000","last_unstale":"2024-09-21T10:03:58.926680+0000","last_undegraded":"2024-09-21T10:03:58.926680+0000","last_fullsized":"2024-09-21T10:03:58.926680+0000","mapping_epoch":1170,"log_start":"1145'262","ondisk_log_start":"1145'262","created":1142,"last_epoch_clean":1171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7351,"log_dups_size":0,"ondisk_log_size":7351,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49479680,"num_objects":755,"num_object_clones":0,"num_object_copies":2265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6009,"num_write_kb":216448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2040,"num_bytes_recovered":133693440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":1118,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580228+0000","last_change":"2024-09-21T10:00:16.373327+0000","last_active":"2024-09-21T10:03:49.580228+0000","last_peered":"2024-09-21T10:03:49.580228+0000","last_clean":"2024-09-21T10:03:49.580228+0000","last_became_active":"2024-09-21T10:00:16.360853+0000","last_became_peered":"2024-09-21T10:00:16.360853+0000","last_unstale":"2024-09-21T10:03:49.580228+0000","last_undegraded":"2024-09-21T10:03:49.580228+0000","last_fullsized":"2024-09-21T10:03:49.580228+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:18.160233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.576754+0000","last_change":"2024-09-21T10:01:40.417926+0000","last_active":"2024-09-21T10:03:49.576754+0000","last_peered":"2024-09-21T10:03:49.576754+0000","last_clean":"2024-09-21T10:03:49.576754+0000","last_became_active":"2024-09-21T10:01:40.417540+0000","last_became_peered":"2024-09-21T10:01:40.417540+0000","last_unstale":"2024-09-21T10:03:49.576754+0000","last_undegraded":"2024-09-21T10:03:49.576754+0000","last_fullsized":"2024-09-21T10:03:49.576754+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:43.890839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1183'7666","reported_seq":7771,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.940796+0000","last_change":"2024-09-21T10:01:27.367802+0000","last_active":"2024-09-21T10:03:56.940796+0000","last_peered":"2024-09-21T10:03:56.940796+0000","last_clean":"2024-09-21T10:03:56.940796+0000","last_became_active":"2024-09-21T10:01:27.365655+0000","last_became_peered":"2024-09-21T10:01:27.365655+0000","last_unstale":"2024-09-21T10:03:56.940796+0000","last_undegraded":"2024-09-21T10:03:56.940796+0000","last_fullsized":"2024-09-21T10:03:56.940796+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7666,"log_dups_size":0,"ondisk_log_size":7666,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53542912,"num_objects":817,"num_object_clones":0,"num_object_copies":2451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6003,"num_write_kb":218240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":716,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575911+0000","last_change":"2024-09-21T10:00:16.360708+0000","last_active":"2024-09-21T10:03:50.575911+0000","last_peered":"2024-09-21T10:03:50.575911+0000","last_clean":"2024-09-21T10:03:50.575911+0000","last_became_active":"2024-09-21T10:00:16.360416+0000","last_became_peered":"2024-09-21T10:00:16.360416+0000","last_unstale":"2024-09-21T10:03:50.575911+0000","last_undegraded":"2024-09-21T10:03:50.575911+0000","last_fullsized":"2024-09-21T10:03:50.575911+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:59.090215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1798,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.646951+0000","last_change":"2024-09-21T10:01:40.425332+0000","last_active":"2024-09-21T10:03:57.646951+0000","last_peered":"2024-09-21T10:03:57.646951+0000","last_clean":"2024-09-21T10:03:57.646951+0000","last_became_active":"2024-09-21T10:01:40.425034+0000","last_became_peered":"2024-09-21T10:01:40.425034+0000","last_unstale":"2024-09-21T10:03:57.646951+0000","last_undegraded":"2024-09-21T10:03:57.646951+0000","last_fullsized":"2024-09-21T10:03:57.646951+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:57.701200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1184'12270","reported_seq":12370,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:58.925534+0000","last_change":"2024-09-21T10:00:46.479962+0000","last_active":"2024-09-21T10:03:58.925534+0000","last_peered":"2024-09-21T10:03:58.925534+0000","last_clean":"2024-09-21T10:03:58.925534+0000","last_became_active":"2024-09-21T10:00:46.479648+0000","last_became_peered":"2024-09-21T10:00:46.479648+0000","last_unstale":"2024-09-21T10:03:58.925534+0000","last_undegraded":"2024-09-21T10:03:58.925534+0000","last_fullsized":"2024-09-21T10:03:58.925534+0000","mapping_epoch":1142,"log_start":"1166'6300","ondisk_log_start":"1166'6300","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:22.382825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95289344,"num_objects":1454,"num_object_clones":0,"num_object_copies":4362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12270,"num_write_kb":439168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1253,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580782+0000","last_change":"2024-09-21T10:00:47.119852+0000","last_active":"2024-09-21T10:03:49.580782+0000","last_peered":"2024-09-21T10:03:49.580782+0000","last_clean":"2024-09-21T10:03:49.580782+0000","last_became_active":"2024-09-21T09:59:43.374051+0000","last_became_peered":"2024-09-21T09:59:43.374051+0000","last_unstale":"2024-09-21T10:03:49.580782+0000","last_undegraded":"2024-09-21T10:03:49.580782+0000","last_fullsized":"2024-09-21T10:03:49.580782+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:47.119793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:47.119793+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:23.087965+0000","scrub_duration":0.0013414659999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1713,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575444+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T10:03:50.575444+0000","last_peered":"2024-09-21T10:03:50.575444+0000","last_clean":"2024-09-21T10:03:50.575444+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T10:03:50.575444+0000","last_undegraded":"2024-09-21T10:03:50.575444+0000","last_fullsized":"2024-09-21T10:03:50.575444+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1144,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578816+0000","last_change":"2024-09-21T10:00:16.358954+0000","last_active":"2024-09-21T10:03:49.578816+0000","last_peered":"2024-09-21T10:03:49.578816+0000","last_clean":"2024-09-21T10:03:49.578816+0000","last_became_active":"2024-09-21T10:00:16.358416+0000","last_became_peered":"2024-09-21T10:00:16.358416+0000","last_unstale":"2024-09-21T10:03:49.578816+0000","last_undegraded":"2024-09-21T10:03:49.578816+0000","last_fullsized":"2024-09-21T10:03:49.578816+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:46.916076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1183'12220","reported_seq":12317,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.929165+0000","last_change":"2024-09-21T10:00:46.481001+0000","last_active":"2024-09-21T10:03:56.929165+0000","last_peered":"2024-09-21T10:03:56.929165+0000","last_clean":"2024-09-21T10:03:56.929165+0000","last_became_active":"2024-09-21T10:00:46.480689+0000","last_became_peered":"2024-09-21T10:00:46.480689+0000","last_unstale":"2024-09-21T10:03:56.929165+0000","last_undegraded":"2024-09-21T10:03:56.929165+0000","last_fullsized":"2024-09-21T10:03:56.929165+0000","mapping_epoch":1142,"log_start":"1162'4700","ondisk_log_start":"1162'4700","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7520,"log_dups_size":0,"ondisk_log_size":7520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:56.487842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101056512,"num_objects":1542,"num_object_clones":0,"num_object_copies":4626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12220,"num_write_kb":440384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":155,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.574233+0000","last_change":"2024-09-21T10:00:15.524812+0000","last_active":"2024-09-21T10:03:50.574233+0000","last_peered":"2024-09-21T10:03:50.574233+0000","last_clean":"2024-09-21T10:03:50.574233+0000","last_became_active":"2024-09-21T09:59:43.361751+0000","last_became_peered":"2024-09-21T09:59:43.361751+0000","last_unstale":"2024-09-21T10:03:50.574233+0000","last_undegraded":"2024-09-21T10:03:50.574233+0000","last_fullsized":"2024-09-21T10:03:50.574233+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.306777+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":72,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.576629+0000","last_change":"2024-09-21T10:01:40.415754+0000","last_active":"2024-09-21T10:03:49.576629+0000","last_peered":"2024-09-21T10:03:49.576629+0000","last_clean":"2024-09-21T10:03:49.576629+0000","last_became_active":"2024-09-21T10:01:40.415428+0000","last_became_peered":"2024-09-21T10:01:40.415428+0000","last_unstale":"2024-09-21T10:03:49.576629+0000","last_undegraded":"2024-09-21T10:03:49.576629+0000","last_fullsized":"2024-09-21T10:03:49.576629+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:53.323411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1151,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.579241+0000","last_change":"2024-09-21T10:01:40.414164+0000","last_active":"2024-09-21T10:03:49.579241+0000","last_peered":"2024-09-21T10:03:49.579241+0000","last_clean":"2024-09-21T10:03:49.579241+0000","last_became_active":"2024-09-21T10:01:40.413838+0000","last_became_peered":"2024-09-21T10:01:40.413838+0000","last_unstale":"2024-09-21T10:03:49.579241+0000","last_undegraded":"2024-09-21T10:03:49.579241+0000","last_fullsized":"2024-09-21T10:03:49.579241+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:13:24.380350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1183'12002","reported_seq":16256,"reported_epoch":1183,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:03:55.522424+0000","last_change":"2024-09-21T10:03:22.658284+0000","last_active":"2024-09-21T10:03:55.522424+0000","last_peered":"2024-09-21T10:03:55.522424+0000","last_clean":"2024-09-21T10:01:39.398694+0000","last_became_active":"2024-09-21T10:01:41.614592+0000","last_became_peered":"2024-09-21T10:01:41.614592+0000","last_unstale":"2024-09-21T10:03:55.522424+0000","last_undegraded":"2024-09-21T10:01:50.421725+0000","last_fullsized":"2024-09-21T10:01:41.406829+0000","mapping_epoch":1156,"log_start":"1164'5600","ondisk_log_start":"1164'5600","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6402,"log_dups_size":0,"ondisk_log_size":6402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:44:35.847340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110952472,"num_objects":1694,"num_object_clones":0,"num_object_copies":5082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2100,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1694,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12002,"num_write_kb":438209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2095,"num_bytes_recovered":88932352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,6","objects":-406}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1125,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578980+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T10:03:49.578980+0000","last_peered":"2024-09-21T10:03:49.578980+0000","last_clean":"2024-09-21T10:03:49.578980+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T10:03:49.578980+0000","last_undegraded":"2024-09-21T10:03:49.578980+0000","last_fullsized":"2024-09-21T10:03:49.578980+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1154,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578715+0000","last_change":"2024-09-21T10:01:40.424536+0000","last_active":"2024-09-21T10:03:49.578715+0000","last_peered":"2024-09-21T10:03:49.578715+0000","last_clean":"2024-09-21T10:03:49.578715+0000","last_became_active":"2024-09-21T10:01:40.424229+0000","last_became_peered":"2024-09-21T10:01:40.424229+0000","last_unstale":"2024-09-21T10:03:49.578715+0000","last_undegraded":"2024-09-21T10:03:49.578715+0000","last_fullsized":"2024-09-21T10:03:49.578715+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:10.771540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1183'11921","reported_seq":11970,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:55.504578+0000","last_change":"2024-09-21T10:00:46.477205+0000","last_active":"2024-09-21T10:03:55.504578+0000","last_peered":"2024-09-21T10:03:55.504578+0000","last_clean":"2024-09-21T10:03:55.504578+0000","last_became_active":"2024-09-21T10:00:46.476844+0000","last_became_peered":"2024-09-21T10:00:46.476844+0000","last_unstale":"2024-09-21T10:03:55.504578+0000","last_undegraded":"2024-09-21T10:03:55.504578+0000","last_fullsized":"2024-09-21T10:03:55.504578+0000","mapping_epoch":1142,"log_start":"1164'5500","ondisk_log_start":"1164'5500","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6421,"log_dups_size":0,"ondisk_log_size":6421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:29:19.716166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110559232,"num_objects":1687,"num_object_clones":0,"num_object_copies":5061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11921,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":144,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.573938+0000","last_change":"2024-09-21T10:00:16.351253+0000","last_active":"2024-09-21T10:03:50.573938+0000","last_peered":"2024-09-21T10:03:50.573938+0000","last_clean":"2024-09-21T10:03:50.573938+0000","last_became_active":"2024-09-21T10:00:16.351010+0000","last_became_peered":"2024-09-21T10:00:16.351010+0000","last_unstale":"2024-09-21T10:03:50.573938+0000","last_undegraded":"2024-09-21T10:03:50.573938+0000","last_fullsized":"2024-09-21T10:03:50.573938+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1711,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575188+0000","last_change":"2024-09-21T10:01:40.428561+0000","last_active":"2024-09-21T10:03:50.575188+0000","last_peered":"2024-09-21T10:03:50.575188+0000","last_clean":"2024-09-21T10:03:50.575188+0000","last_became_active":"2024-09-21T10:01:40.428250+0000","last_became_peered":"2024-09-21T10:01:40.428250+0000","last_unstale":"2024-09-21T10:03:50.575188+0000","last_undegraded":"2024-09-21T10:03:50.575188+0000","last_fullsized":"2024-09-21T10:03:50.575188+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:58.781370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":617,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575532+0000","last_change":"2024-09-21T10:01:40.423834+0000","last_active":"2024-09-21T10:03:50.575532+0000","last_peered":"2024-09-21T10:03:50.575532+0000","last_clean":"2024-09-21T10:03:50.575532+0000","last_became_active":"2024-09-21T10:01:40.423506+0000","last_became_peered":"2024-09-21T10:01:40.423506+0000","last_unstale":"2024-09-21T10:03:50.575532+0000","last_undegraded":"2024-09-21T10:03:50.575532+0000","last_fullsized":"2024-09-21T10:03:50.575532+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:11.358742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.6","version":"1184'7818","reported_seq":7954,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:58.815505+0000","last_change":"2024-09-21T10:03:12.185190+0000","last_active":"2024-09-21T10:03:58.815505+0000","last_peered":"2024-09-21T10:03:58.815505+0000","last_clean":"2024-09-21T10:03:58.815505+0000","last_became_active":"2024-09-21T10:01:27.352341+0000","last_became_peered":"2024-09-21T10:01:27.352341+0000","last_unstale":"2024-09-21T10:03:58.815505+0000","last_undegraded":"2024-09-21T10:03:58.815505+0000","last_fullsized":"2024-09-21T10:03:58.815505+0000","mapping_epoch":1142,"log_start":"1145'524","ondisk_log_start":"1145'524","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1176'6402","last_scrub_stamp":"2024-09-21T10:03:12.185040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:03:12.185040+0000","objects_scrubbed":2137,"log_size":7294,"log_dups_size":0,"ondisk_log_size":7294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:51.219397+0000","scrub_duration":683,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46727168,"num_objects":713,"num_object_clones":0,"num_object_copies":2139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1424,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1267,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580258+0000","last_change":"2024-09-21T10:00:49.116174+0000","last_active":"2024-09-21T10:03:49.580258+0000","last_peered":"2024-09-21T10:03:49.580258+0000","last_clean":"2024-09-21T10:03:49.580258+0000","last_became_active":"2024-09-21T10:00:16.360430+0000","last_became_peered":"2024-09-21T10:00:16.360430+0000","last_unstale":"2024-09-21T10:03:49.580258+0000","last_undegraded":"2024-09-21T10:03:49.580258+0000","last_fullsized":"2024-09-21T10:03:49.580258+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:49.116103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:49.116103+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:29.874911+0000","scrub_duration":0.0020386419999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":190,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.574152+0000","last_change":"2024-09-21T09:57:39.541461+0000","last_active":"2024-09-21T10:03:50.574152+0000","last_peered":"2024-09-21T10:03:50.574152+0000","last_clean":"2024-09-21T10:03:50.574152+0000","last_became_active":"2024-09-21T09:57:39.540927+0000","last_became_peered":"2024-09-21T09:57:39.540927+0000","last_unstale":"2024-09-21T10:03:50.574152+0000","last_undegraded":"2024-09-21T10:03:50.574152+0000","last_fullsized":"2024-09-21T10:03:50.574152+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:14.815783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1701,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.577270+0000","last_change":"2024-09-21T10:00:16.362131+0000","last_active":"2024-09-21T10:03:50.577270+0000","last_peered":"2024-09-21T10:03:50.577270+0000","last_clean":"2024-09-21T10:03:50.577270+0000","last_became_active":"2024-09-21T10:00:16.361722+0000","last_became_peered":"2024-09-21T10:00:16.361722+0000","last_unstale":"2024-09-21T10:03:50.577270+0000","last_undegraded":"2024-09-21T10:03:50.577270+0000","last_fullsized":"2024-09-21T10:03:50.577270+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:38.717426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":62,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.579068+0000","last_change":"2024-09-21T10:00:16.361540+0000","last_active":"2024-09-21T10:03:49.579068+0000","last_peered":"2024-09-21T10:03:49.579068+0000","last_clean":"2024-09-21T10:03:49.579068+0000","last_became_active":"2024-09-21T10:00:16.361086+0000","last_became_peered":"2024-09-21T10:00:16.361086+0000","last_unstale":"2024-09-21T10:03:49.579068+0000","last_undegraded":"2024-09-21T10:03:49.579068+0000","last_fullsized":"2024-09-21T10:03:49.579068+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:14.645896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1183'7585","reported_seq":7738,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.005089+0000","last_change":"2024-09-21T10:01:58.715049+0000","last_active":"2024-09-21T10:03:56.005089+0000","last_peered":"2024-09-21T10:03:56.005089+0000","last_clean":"2024-09-21T10:03:56.005089+0000","last_became_active":"2024-09-21T10:01:27.359228+0000","last_became_peered":"2024-09-21T10:01:27.359228+0000","last_unstale":"2024-09-21T10:03:56.005089+0000","last_undegraded":"2024-09-21T10:03:56.005089+0000","last_fullsized":"2024-09-21T10:03:56.005089+0000","mapping_epoch":1142,"log_start":"1145'230","ondisk_log_start":"1145'230","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1162'3998","last_scrub_stamp":"2024-09-21T10:01:58.714971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:01:58.714971+0000","objects_scrubbed":2286,"log_size":7355,"log_dups_size":0,"ondisk_log_size":7355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:21.752075+0000","scrub_duration":0.73731484599999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54263808,"num_objects":828,"num_object_clones":0,"num_object_copies":2484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3600,"num_write_kb":68544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":67,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.576675+0000","last_change":"2024-09-21T10:01:40.412152+0000","last_active":"2024-09-21T10:03:49.576675+0000","last_peered":"2024-09-21T10:03:49.576675+0000","last_clean":"2024-09-21T10:03:49.576675+0000","last_became_active":"2024-09-21T10:01:40.411508+0000","last_became_peered":"2024-09-21T10:01:40.411508+0000","last_unstale":"2024-09-21T10:03:49.576675+0000","last_undegraded":"2024-09-21T10:03:49.576675+0000","last_fullsized":"2024-09-21T10:03:49.576675+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:21.317838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:21.317838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1145,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580186+0000","last_change":"2024-09-21T10:01:40.428393+0000","last_active":"2024-09-21T10:03:49.580186+0000","last_peered":"2024-09-21T10:03:49.580186+0000","last_clean":"2024-09-21T10:03:49.580186+0000","last_became_active":"2024-09-21T10:01:40.428204+0000","last_became_peered":"2024-09-21T10:01:40.428204+0000","last_unstale":"2024-09-21T10:03:49.580186+0000","last_undegraded":"2024-09-21T10:03:49.580186+0000","last_fullsized":"2024-09-21T10:03:49.580186+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:23:02.136035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1790,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.649238+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T10:03:57.649238+0000","last_peered":"2024-09-21T10:03:57.649238+0000","last_clean":"2024-09-21T10:03:57.649238+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T10:03:57.649238+0000","last_undegraded":"2024-09-21T10:03:57.649238+0000","last_fullsized":"2024-09-21T10:03:57.649238+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1809,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.646478+0000","last_change":"2024-09-21T10:01:40.421887+0000","last_active":"2024-09-21T10:03:57.646478+0000","last_peered":"2024-09-21T10:03:57.646478+0000","last_clean":"2024-09-21T10:03:57.646478+0000","last_became_active":"2024-09-21T10:01:40.420730+0000","last_became_peered":"2024-09-21T10:01:40.420730+0000","last_unstale":"2024-09-21T10:03:57.646478+0000","last_undegraded":"2024-09-21T10:03:57.646478+0000","last_fullsized":"2024-09-21T10:03:57.646478+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:51.504051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"1184'7642","reported_seq":11931,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:58.938459+0000","last_change":"2024-09-21T10:03:15.532727+0000","last_active":"2024-09-21T10:03:58.938459+0000","last_peered":"2024-09-21T10:03:58.938459+0000","last_clean":"2024-09-21T10:03:58.938459+0000","last_became_active":"2024-09-21T10:01:56.596893+0000","last_became_peered":"2024-09-21T10:01:56.596893+0000","last_unstale":"2024-09-21T10:03:58.938459+0000","last_undegraded":"2024-09-21T10:03:58.938459+0000","last_fullsized":"2024-09-21T10:03:58.938459+0000","mapping_epoch":1161,"log_start":"1145'345","ondisk_log_start":"1145'345","created":1142,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7297,"log_dups_size":0,"ondisk_log_size":7297,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-29T15:50:57.381992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44826624,"num_objects":684,"num_object_clones":0,"num_object_copies":2052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6026,"num_write_kb":214720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2085,"num_bytes_recovered":136642560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":1253,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580421+0000","last_change":"2024-09-21T10:00:16.913196+0000","last_active":"2024-09-21T10:03:49.580421+0000","last_peered":"2024-09-21T10:03:49.580421+0000","last_clean":"2024-09-21T10:03:49.580421+0000","last_became_active":"2024-09-21T09:59:43.370454+0000","last_became_peered":"2024-09-21T09:59:43.370454+0000","last_unstale":"2024-09-21T10:03:49.580421+0000","last_undegraded":"2024-09-21T10:03:49.580421+0000","last_fullsized":"2024-09-21T10:03:49.580421+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:16.913146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:16.913146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:23:33.788563+0000","scrub_duration":0.001342315,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1762,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.573860+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:03:50.573860+0000","last_peered":"2024-09-21T10:03:50.573860+0000","last_clean":"2024-09-21T10:03:50.573860+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:03:50.573860+0000","last_undegraded":"2024-09-21T10:03:50.573860+0000","last_fullsized":"2024-09-21T10:03:50.573860+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1717,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.574333+0000","last_change":"2024-09-21T10:01:40.414547+0000","last_active":"2024-09-21T10:03:50.574333+0000","last_peered":"2024-09-21T10:03:50.574333+0000","last_clean":"2024-09-21T10:03:50.574333+0000","last_became_active":"2024-09-21T10:01:40.414243+0000","last_became_peered":"2024-09-21T10:01:40.414243+0000","last_unstale":"2024-09-21T10:03:50.574333+0000","last_undegraded":"2024-09-21T10:03:50.574333+0000","last_fullsized":"2024-09-21T10:03:50.574333+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:55:26.044481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":500,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.647669+0000","last_change":"2024-09-21T10:01:40.421427+0000","last_active":"2024-09-21T10:03:57.647669+0000","last_peered":"2024-09-21T10:03:57.647669+0000","last_clean":"2024-09-21T10:03:57.647669+0000","last_became_active":"2024-09-21T10:01:40.421047+0000","last_became_peered":"2024-09-21T10:01:40.421047+0000","last_unstale":"2024-09-21T10:03:57.647669+0000","last_undegraded":"2024-09-21T10:03:57.647669+0000","last_fullsized":"2024-09-21T10:03:57.647669+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:48.640296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.7","version":"1183'7618","reported_seq":7677,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:55.362114+0000","last_change":"2024-09-21T10:01:27.359695+0000","last_active":"2024-09-21T10:03:55.362114+0000","last_peered":"2024-09-21T10:03:55.362114+0000","last_clean":"2024-09-21T10:03:55.362114+0000","last_became_active":"2024-09-21T10:01:27.358189+0000","last_became_peered":"2024-09-21T10:01:27.358189+0000","last_unstale":"2024-09-21T10:03:55.362114+0000","last_undegraded":"2024-09-21T10:03:55.362114+0000","last_fullsized":"2024-09-21T10:03:55.362114+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7618,"log_dups_size":0,"ondisk_log_size":7618,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54624256,"num_objects":833,"num_object_clones":0,"num_object_copies":2499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5955,"num_write_kb":217248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":487,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.649153+0000","last_change":"2024-09-21T10:03:19.415715+0000","last_active":"2024-09-21T10:03:57.649153+0000","last_peered":"2024-09-21T10:03:57.649153+0000","last_clean":"2024-09-21T10:03:57.649153+0000","last_became_active":"2024-09-21T09:59:43.371384+0000","last_became_peered":"2024-09-21T09:59:43.371384+0000","last_unstale":"2024-09-21T10:03:57.649153+0000","last_undegraded":"2024-09-21T10:03:57.649153+0000","last_fullsized":"2024-09-21T10:03:57.649153+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-30T13:30:29.665440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":140,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.576329+0000","last_change":"2024-09-21T10:01:40.426865+0000","last_active":"2024-09-21T10:03:49.576329+0000","last_peered":"2024-09-21T10:03:49.576329+0000","last_clean":"2024-09-21T10:03:49.576329+0000","last_became_active":"2024-09-21T10:01:40.413152+0000","last_became_peered":"2024-09-21T10:01:40.413152+0000","last_unstale":"2024-09-21T10:03:49.576329+0000","last_undegraded":"2024-09-21T10:03:49.576329+0000","last_fullsized":"2024-09-21T10:03:49.576329+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:59.034627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1145,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578971+0000","last_change":"2024-09-21T10:00:16.356367+0000","last_active":"2024-09-21T10:03:49.578971+0000","last_peered":"2024-09-21T10:03:49.578971+0000","last_clean":"2024-09-21T10:03:49.578971+0000","last_became_active":"2024-09-21T10:00:16.356079+0000","last_became_peered":"2024-09-21T10:00:16.356079+0000","last_unstale":"2024-09-21T10:03:49.578971+0000","last_undegraded":"2024-09-21T10:03:49.578971+0000","last_fullsized":"2024-09-21T10:03:49.578971+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:12.934640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":628,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.574280+0000","last_change":"2024-09-21T10:01:40.421262+0000","last_active":"2024-09-21T10:03:50.574280+0000","last_peered":"2024-09-21T10:03:50.574280+0000","last_clean":"2024-09-21T10:03:50.574280+0000","last_became_active":"2024-09-21T10:01:40.420959+0000","last_became_peered":"2024-09-21T10:01:40.420959+0000","last_unstale":"2024-09-21T10:03:50.574280+0000","last_undegraded":"2024-09-21T10:03:50.574280+0000","last_fullsized":"2024-09-21T10:03:50.574280+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:06.866421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1","version":"1183'7779","reported_seq":7878,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.018490+0000","last_change":"2024-09-21T10:03:39.448818+0000","last_active":"2024-09-21T10:03:56.018490+0000","last_peered":"2024-09-21T10:03:56.018490+0000","last_clean":"2024-09-21T10:03:56.018490+0000","last_became_active":"2024-09-21T10:01:27.378346+0000","last_became_peered":"2024-09-21T10:01:27.378346+0000","last_unstale":"2024-09-21T10:03:56.018490+0000","last_undegraded":"2024-09-21T10:03:56.018490+0000","last_fullsized":"2024-09-21T10:03:56.018490+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7779,"log_dups_size":0,"ondisk_log_size":7779,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54689792,"num_objects":834,"num_object_clones":0,"num_object_copies":2502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6096,"num_write_kb":221792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":72,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.579193+0000","last_change":"2024-09-21T10:00:46.494180+0000","last_active":"2024-09-21T10:03:49.579193+0000","last_peered":"2024-09-21T10:03:49.579193+0000","last_clean":"2024-09-21T10:03:49.579193+0000","last_became_active":"2024-09-21T10:00:16.355386+0000","last_became_peered":"2024-09-21T10:00:16.355386+0000","last_unstale":"2024-09-21T10:03:49.579193+0000","last_undegraded":"2024-09-21T10:03:49.579193+0000","last_fullsized":"2024-09-21T10:03:49.579193+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:46.494129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:46.494129+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:11.193593+0000","scrub_duration":0.00183863,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":719,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.576843+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T10:03:50.576843+0000","last_peered":"2024-09-21T10:03:50.576843+0000","last_clean":"2024-09-21T10:03:50.576843+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T10:03:50.576843+0000","last_undegraded":"2024-09-21T10:03:50.576843+0000","last_fullsized":"2024-09-21T10:03:50.576843+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":529,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.647170+0000","last_change":"2024-09-21T10:01:40.418328+0000","last_active":"2024-09-21T10:03:57.647170+0000","last_peered":"2024-09-21T10:03:57.647170+0000","last_clean":"2024-09-21T10:03:57.647170+0000","last_became_active":"2024-09-21T10:01:40.418011+0000","last_became_peered":"2024-09-21T10:01:40.418011+0000","last_unstale":"2024-09-21T10:03:57.647170+0000","last_undegraded":"2024-09-21T10:03:57.647170+0000","last_fullsized":"2024-09-21T10:03:57.647170+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:58.716988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":486,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578215+0000","last_change":"2024-09-21T10:01:40.421225+0000","last_active":"2024-09-21T10:03:49.578215+0000","last_peered":"2024-09-21T10:03:49.578215+0000","last_clean":"2024-09-21T10:03:49.578215+0000","last_became_active":"2024-09-21T10:01:40.420909+0000","last_became_peered":"2024-09-21T10:01:40.420909+0000","last_unstale":"2024-09-21T10:03:49.578215+0000","last_undegraded":"2024-09-21T10:03:49.578215+0000","last_fullsized":"2024-09-21T10:03:49.578215+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:03.811941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"1183'7516","reported_seq":11822,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.387463+0000","last_change":"2024-09-21T10:01:55.586239+0000","last_active":"2024-09-21T10:03:56.387463+0000","last_peered":"2024-09-21T10:03:56.387463+0000","last_clean":"2024-09-21T10:03:56.387463+0000","last_became_active":"2024-09-21T10:01:55.581482+0000","last_became_peered":"2024-09-21T10:01:55.581482+0000","last_unstale":"2024-09-21T10:03:56.387463+0000","last_undegraded":"2024-09-21T10:03:56.387463+0000","last_fullsized":"2024-09-21T10:03:56.387463+0000","mapping_epoch":1160,"log_start":"1145'1709","ondisk_log_start":"1145'1709","created":1142,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5807,"log_dups_size":0,"ondisk_log_size":5807,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50266112,"num_objects":767,"num_object_clones":0,"num_object_copies":2301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5899,"num_write_kb":213312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2093,"num_bytes_recovered":137166848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":72,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.577111+0000","last_change":"2024-09-21T10:00:20.637858+0000","last_active":"2024-09-21T10:03:49.577111+0000","last_peered":"2024-09-21T10:03:49.577111+0000","last_clean":"2024-09-21T10:03:49.577111+0000","last_became_active":"2024-09-21T10:00:16.353796+0000","last_became_peered":"2024-09-21T10:00:16.353796+0000","last_unstale":"2024-09-21T10:03:49.577111+0000","last_undegraded":"2024-09-21T10:03:49.577111+0000","last_fullsized":"2024-09-21T10:03:49.577111+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:20.637806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:20.637806+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:12:16.884495+0000","scrub_duration":0.0016753919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":554,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.574095+0000","last_change":"2024-09-21T10:01:40.414536+0000","last_active":"2024-09-21T10:03:50.574095+0000","last_peered":"2024-09-21T10:03:50.574095+0000","last_clean":"2024-09-21T10:03:50.574095+0000","last_became_active":"2024-09-21T10:01:40.413913+0000","last_became_peered":"2024-09-21T10:01:40.413913+0000","last_unstale":"2024-09-21T10:03:50.574095+0000","last_undegraded":"2024-09-21T10:03:50.574095+0000","last_fullsized":"2024-09-21T10:03:50.574095+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:27.994971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1118,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580237+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T10:03:49.580237+0000","last_peered":"2024-09-21T10:03:49.580237+0000","last_clean":"2024-09-21T10:03:49.580237+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T10:03:49.580237+0000","last_undegraded":"2024-09-21T10:03:49.580237+0000","last_fullsized":"2024-09-21T10:03:49.580237+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1781,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.646994+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T10:03:57.646994+0000","last_peered":"2024-09-21T10:03:57.646994+0000","last_clean":"2024-09-21T10:03:57.646994+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T10:03:57.646994+0000","last_undegraded":"2024-09-21T10:03:57.646994+0000","last_fullsized":"2024-09-21T10:03:57.646994+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1183'7751","reported_seq":7856,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.398069+0000","last_change":"2024-09-21T10:01:27.384197+0000","last_active":"2024-09-21T10:03:56.398069+0000","last_peered":"2024-09-21T10:03:56.398069+0000","last_clean":"2024-09-21T10:03:56.398069+0000","last_became_active":"2024-09-21T10:01:27.382492+0000","last_became_peered":"2024-09-21T10:01:27.382492+0000","last_unstale":"2024-09-21T10:03:56.398069+0000","last_undegraded":"2024-09-21T10:03:56.398069+0000","last_fullsized":"2024-09-21T10:03:56.398069+0000","mapping_epoch":1142,"log_start":"1145'2081","ondisk_log_start":"1145'2081","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51871744,"num_objects":791,"num_object_clones":0,"num_object_copies":2373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6105,"num_write_kb":220704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":92,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578981+0000","last_change":"2024-09-21T10:00:16.356251+0000","last_active":"2024-09-21T10:03:49.578981+0000","last_peered":"2024-09-21T10:03:49.578981+0000","last_clean":"2024-09-21T10:03:49.578981+0000","last_became_active":"2024-09-21T10:00:16.355833+0000","last_became_peered":"2024-09-21T10:00:16.355833+0000","last_unstale":"2024-09-21T10:03:49.578981+0000","last_undegraded":"2024-09-21T10:03:49.578981+0000","last_fullsized":"2024-09-21T10:03:49.578981+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T09:59:47.767654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:59:47.767654+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:50.939198+0000","scrub_duration":0.006614371,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1713,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575577+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T10:03:50.575577+0000","last_peered":"2024-09-21T10:03:50.575577+0000","last_clean":"2024-09-21T10:03:50.575577+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T10:03:50.575577+0000","last_undegraded":"2024-09-21T10:03:50.575577+0000","last_fullsized":"2024-09-21T10:03:50.575577+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1686,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.576989+0000","last_change":"2024-09-21T10:01:40.423209+0000","last_active":"2024-09-21T10:03:49.576989+0000","last_peered":"2024-09-21T10:03:49.576989+0000","last_clean":"2024-09-21T10:03:49.576989+0000","last_became_active":"2024-09-21T10:01:40.422890+0000","last_became_peered":"2024-09-21T10:01:40.422890+0000","last_unstale":"2024-09-21T10:03:49.576989+0000","last_undegraded":"2024-09-21T10:03:49.576989+0000","last_fullsized":"2024-09-21T10:03:49.576989+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:10.683126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1711,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575006+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T10:03:50.575006+0000","last_peered":"2024-09-21T10:03:50.575006+0000","last_clean":"2024-09-21T10:03:50.575006+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T10:03:50.575006+0000","last_undegraded":"2024-09-21T10:03:50.575006+0000","last_fullsized":"2024-09-21T10:03:50.575006+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1183'7746","reported_seq":8831,"reported_epoch":1183,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:03:56.394837+0000","last_change":"2024-09-21T10:03:49.082009+0000","last_active":"2024-09-21T10:03:56.394837+0000","last_peered":"2024-09-21T10:03:56.394837+0000","last_clean":"2024-09-21T10:01:39.401120+0000","last_became_active":"2024-09-21T10:01:41.609480+0000","last_became_peered":"2024-09-21T10:01:41.609480+0000","last_unstale":"2024-09-21T10:03:56.394837+0000","last_undegraded":"2024-09-21T10:01:41.546996+0000","last_fullsized":"2024-09-21T10:01:41.408102+0000","mapping_epoch":1156,"log_start":"1145'1979","ondisk_log_start":"1145'1979","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5767,"log_dups_size":0,"ondisk_log_size":5767,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52527104,"num_objects":801,"num_object_clones":0,"num_object_copies":2403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1656,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6071,"num_write_kb":219936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":488,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"2,3","objects":-855}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":144,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575600+0000","last_change":"2024-09-21T10:00:22.052497+0000","last_active":"2024-09-21T10:03:50.575600+0000","last_peered":"2024-09-21T10:03:50.575600+0000","last_clean":"2024-09-21T10:03:50.575600+0000","last_became_active":"2024-09-21T09:59:43.368157+0000","last_became_peered":"2024-09-21T09:59:43.368157+0000","last_unstale":"2024-09-21T10:03:50.575600+0000","last_undegraded":"2024-09-21T10:03:50.575600+0000","last_fullsized":"2024-09-21T10:03:50.575600+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-28T12:21:36.382264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1804,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.646823+0000","last_change":"2024-09-21T10:01:40.433067+0000","last_active":"2024-09-21T10:03:57.646823+0000","last_peered":"2024-09-21T10:03:57.646823+0000","last_clean":"2024-09-21T10:03:57.646823+0000","last_became_active":"2024-09-21T10:01:40.420622+0000","last_became_peered":"2024-09-21T10:01:40.420622+0000","last_unstale":"2024-09-21T10:03:57.646823+0000","last_undegraded":"2024-09-21T10:03:57.646823+0000","last_fullsized":"2024-09-21T10:03:57.646823+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.383227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":62,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.577073+0000","last_change":"2024-09-21T10:00:16.355102+0000","last_active":"2024-09-21T10:03:49.577073+0000","last_peered":"2024-09-21T10:03:49.577073+0000","last_clean":"2024-09-21T10:03:49.577073+0000","last_became_active":"2024-09-21T10:00:16.354473+0000","last_became_peered":"2024-09-21T10:00:16.354473+0000","last_unstale":"2024-09-21T10:03:49.577073+0000","last_undegraded":"2024-09-21T10:03:49.577073+0000","last_fullsized":"2024-09-21T10:03:49.577073+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:49:35.449111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1127,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578780+0000","last_change":"2024-09-21T10:00:16.355601+0000","last_active":"2024-09-21T10:03:49.578780+0000","last_peered":"2024-09-21T10:03:49.578780+0000","last_clean":"2024-09-21T10:03:49.578780+0000","last_became_active":"2024-09-21T10:00:16.355110+0000","last_became_peered":"2024-09-21T10:00:16.355110+0000","last_unstale":"2024-09-21T10:03:49.578780+0000","last_undegraded":"2024-09-21T10:03:49.578780+0000","last_fullsized":"2024-09-21T10:03:49.578780+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:10.590252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.10","version":"1183'7519","reported_seq":12864,"reported_epoch":1183,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:03:56.367948+0000","last_change":"2024-09-21T10:01:43.611910+0000","last_active":"2024-09-21T10:03:56.367948+0000","last_peered":"2024-09-21T10:03:56.367948+0000","last_clean":"2024-09-21T10:01:32.765736+0000","last_became_active":"2024-09-21T10:01:43.608034+0000","last_became_peered":"2024-09-21T10:01:43.608034+0000","last_unstale":"2024-09-21T10:03:56.367948+0000","last_undegraded":"2024-09-21T10:01:41.524927+0000","last_fullsized":"2024-09-21T10:01:41.408209+0000","mapping_epoch":1156,"log_start":"1145'1569","ondisk_log_start":"1145'1569","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5950,"log_dups_size":0,"ondisk_log_size":5950,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57016320,"num_objects":870,"num_object_clones":0,"num_object_copies":2610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":481,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5902,"num_write_kb":216704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1849,"num_bytes_recovered":121176064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2],"avail_no_missing":["3","1","2","5"],"object_location_counts":[{"shards":"2,3","objects":389}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1154,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578821+0000","last_change":"2024-09-21T10:01:40.414117+0000","last_active":"2024-09-21T10:03:49.578821+0000","last_peered":"2024-09-21T10:03:49.578821+0000","last_clean":"2024-09-21T10:03:49.578821+0000","last_became_active":"2024-09-21T10:01:40.413186+0000","last_became_peered":"2024-09-21T10:01:40.413186+0000","last_unstale":"2024-09-21T10:03:49.578821+0000","last_undegraded":"2024-09-21T10:03:49.578821+0000","last_fullsized":"2024-09-21T10:03:49.578821+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:17:05.368450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1170,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578920+0000","last_change":"2024-09-21T10:00:16.371426+0000","last_active":"2024-09-21T10:03:49.578920+0000","last_peered":"2024-09-21T10:03:49.578920+0000","last_clean":"2024-09-21T10:03:49.578920+0000","last_became_active":"2024-09-21T10:00:16.357486+0000","last_became_peered":"2024-09-21T10:00:16.357486+0000","last_unstale":"2024-09-21T10:03:49.578920+0000","last_undegraded":"2024-09-21T10:03:49.578920+0000","last_fullsized":"2024-09-21T10:03:49.578920+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:02.923682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.11","version":"1183'7780","reported_seq":9799,"reported_epoch":1183,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:03:56.401290+0000","last_change":"2024-09-21T10:01:43.633724+0000","last_active":"2024-09-21T10:03:56.401290+0000","last_peered":"2024-09-21T10:03:56.401290+0000","last_clean":"2024-09-21T10:01:33.111456+0000","last_became_active":"2024-09-21T10:01:36.436541+0000","last_became_peered":"2024-09-21T10:01:36.436541+0000","last_unstale":"2024-09-21T10:03:56.401290+0000","last_undegraded":"2024-09-21T10:01:43.633368+0000","last_fullsized":"2024-09-21T10:01:36.387971+0000","mapping_epoch":1153,"log_start":"1145'2042","ondisk_log_start":"1145'2042","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5738,"log_dups_size":0,"ondisk_log_size":5738,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53444608,"num_objects":816,"num_object_clones":0,"num_object_copies":2448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1021,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6098,"num_write_kb":221216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":960,"num_bytes_recovered":62914560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[3,6],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,6","objects":-205}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1687,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.576546+0000","last_change":"2024-09-21T10:01:40.413147+0000","last_active":"2024-09-21T10:03:49.576546+0000","last_peered":"2024-09-21T10:03:49.576546+0000","last_clean":"2024-09-21T10:03:49.576546+0000","last_became_active":"2024-09-21T10:01:40.412833+0000","last_became_peered":"2024-09-21T10:01:40.412833+0000","last_unstale":"2024-09-21T10:03:49.576546+0000","last_undegraded":"2024-09-21T10:03:49.576546+0000","last_fullsized":"2024-09-21T10:03:49.576546+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:33.262350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1109,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580124+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T10:03:49.580124+0000","last_peered":"2024-09-21T10:03:49.580124+0000","last_clean":"2024-09-21T10:03:49.580124+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T10:03:49.580124+0000","last_undegraded":"2024-09-21T10:03:49.580124+0000","last_fullsized":"2024-09-21T10:03:49.580124+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.12","version":"1183'7669","reported_seq":12030,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.333962+0000","last_change":"2024-09-21T10:03:50.595866+0000","last_active":"2024-09-21T10:03:56.333962+0000","last_peered":"2024-09-21T10:03:56.333962+0000","last_clean":"2024-09-21T10:03:56.333962+0000","last_became_active":"2024-09-21T10:03:50.580944+0000","last_became_peered":"2024-09-21T10:03:50.580944+0000","last_unstale":"2024-09-21T10:03:56.333962+0000","last_undegraded":"2024-09-21T10:03:56.333962+0000","last_fullsized":"2024-09-21T10:03:56.333962+0000","mapping_epoch":1182,"log_start":"1145'1789","ondisk_log_start":"1145'1789","created":1148,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5880,"log_dups_size":0,"ondisk_log_size":5880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55345152,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5995,"num_write_kb":218848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":33423360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":563,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.580582+0000","last_change":"2024-09-21T10:01:40.427109+0000","last_active":"2024-09-21T10:03:49.580582+0000","last_peered":"2024-09-21T10:03:49.580582+0000","last_clean":"2024-09-21T10:03:49.580582+0000","last_became_active":"2024-09-21T10:01:40.426821+0000","last_became_peered":"2024-09-21T10:01:40.426821+0000","last_unstale":"2024-09-21T10:03:49.580582+0000","last_undegraded":"2024-09-21T10:03:49.580582+0000","last_fullsized":"2024-09-21T10:03:49.580582+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:12.260797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1700,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575423+0000","last_change":"2024-09-21T10:00:16.357151+0000","last_active":"2024-09-21T10:03:50.575423+0000","last_peered":"2024-09-21T10:03:50.575423+0000","last_clean":"2024-09-21T10:03:50.575423+0000","last_became_active":"2024-09-21T10:00:16.356864+0000","last_became_peered":"2024-09-21T10:00:16.356864+0000","last_unstale":"2024-09-21T10:03:50.575423+0000","last_undegraded":"2024-09-21T10:03:50.575423+0000","last_fullsized":"2024-09-21T10:03:50.575423+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:52.851887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"1183'7647","reported_seq":7752,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.317717+0000","last_change":"2024-09-21T10:01:27.391083+0000","last_active":"2024-09-21T10:03:56.317717+0000","last_peered":"2024-09-21T10:03:56.317717+0000","last_clean":"2024-09-21T10:03:56.317717+0000","last_became_active":"2024-09-21T10:01:27.387803+0000","last_became_peered":"2024-09-21T10:01:27.387803+0000","last_unstale":"2024-09-21T10:03:56.317717+0000","last_undegraded":"2024-09-21T10:03:56.317717+0000","last_fullsized":"2024-09-21T10:03:56.317717+0000","mapping_epoch":1148,"log_start":"1145'1794","ondisk_log_start":"1145'1794","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5853,"log_dups_size":0,"ondisk_log_size":5853,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58294272,"num_objects":890,"num_object_clones":0,"num_object_copies":2670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6002,"num_write_kb":220512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1788,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.647867+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T10:03:57.647867+0000","last_peered":"2024-09-21T10:03:57.647867+0000","last_clean":"2024-09-21T10:03:57.647867+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T10:03:57.647867+0000","last_undegraded":"2024-09-21T10:03:57.647867+0000","last_fullsized":"2024-09-21T10:03:57.647867+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1724,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.574924+0000","last_change":"2024-09-21T10:01:40.426702+0000","last_active":"2024-09-21T10:03:50.574924+0000","last_peered":"2024-09-21T10:03:50.574924+0000","last_clean":"2024-09-21T10:03:50.574924+0000","last_became_active":"2024-09-21T10:01:40.426034+0000","last_became_peered":"2024-09-21T10:01:40.426034+0000","last_unstale":"2024-09-21T10:03:50.574924+0000","last_undegraded":"2024-09-21T10:03:50.574924+0000","last_fullsized":"2024-09-21T10:03:50.574924+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:20.619421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":72,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.579109+0000","last_change":"2024-09-21T10:01:40.414393+0000","last_active":"2024-09-21T10:03:49.579109+0000","last_peered":"2024-09-21T10:03:49.579109+0000","last_clean":"2024-09-21T10:03:49.579109+0000","last_became_active":"2024-09-21T10:01:40.414096+0000","last_became_peered":"2024-09-21T10:01:40.414096+0000","last_unstale":"2024-09-21T10:03:49.579109+0000","last_undegraded":"2024-09-21T10:03:49.579109+0000","last_fullsized":"2024-09-21T10:03:49.579109+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:32.540479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1728,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575093+0000","last_change":"2024-09-21T10:01:40.434475+0000","last_active":"2024-09-21T10:03:50.575093+0000","last_peered":"2024-09-21T10:03:50.575093+0000","last_clean":"2024-09-21T10:03:50.575093+0000","last_became_active":"2024-09-21T10:01:40.432013+0000","last_became_peered":"2024-09-21T10:01:40.432013+0000","last_unstale":"2024-09-21T10:03:50.575093+0000","last_undegraded":"2024-09-21T10:03:50.575093+0000","last_fullsized":"2024-09-21T10:03:50.575093+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:44.373378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1792,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.646645+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T10:03:57.646645+0000","last_peered":"2024-09-21T10:03:57.646645+0000","last_clean":"2024-09-21T10:03:57.646645+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T10:03:57.646645+0000","last_undegraded":"2024-09-21T10:03:57.646645+0000","last_fullsized":"2024-09-21T10:03:57.646645+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":617,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575642+0000","last_change":"2024-09-21T10:01:40.437120+0000","last_active":"2024-09-21T10:03:50.575642+0000","last_peered":"2024-09-21T10:03:50.575642+0000","last_clean":"2024-09-21T10:03:50.575642+0000","last_became_active":"2024-09-21T10:01:40.436275+0000","last_became_peered":"2024-09-21T10:01:40.436275+0000","last_unstale":"2024-09-21T10:03:50.575642+0000","last_undegraded":"2024-09-21T10:03:50.575642+0000","last_fullsized":"2024-09-21T10:03:50.575642+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:25.322111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1709,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.576437+0000","last_change":"2024-09-21T10:01:40.427770+0000","last_active":"2024-09-21T10:03:50.576437+0000","last_peered":"2024-09-21T10:03:50.576437+0000","last_clean":"2024-09-21T10:03:50.576437+0000","last_became_active":"2024-09-21T10:01:40.427458+0000","last_became_peered":"2024-09-21T10:01:40.427458+0000","last_unstale":"2024-09-21T10:03:50.576437+0000","last_undegraded":"2024-09-21T10:03:50.576437+0000","last_fullsized":"2024-09-21T10:03:50.576437+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:38.605007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1153,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578338+0000","last_change":"2024-09-21T10:01:40.420505+0000","last_active":"2024-09-21T10:03:49.578338+0000","last_peered":"2024-09-21T10:03:49.578338+0000","last_clean":"2024-09-21T10:03:49.578338+0000","last_became_active":"2024-09-21T10:01:40.420263+0000","last_became_peered":"2024-09-21T10:01:40.420263+0000","last_unstale":"2024-09-21T10:03:49.578338+0000","last_undegraded":"2024-09-21T10:03:49.578338+0000","last_fullsized":"2024-09-21T10:03:49.578338+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:23.452408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1748,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.574192+0000","last_change":"2024-09-21T10:01:40.420219+0000","last_active":"2024-09-21T10:03:50.574192+0000","last_peered":"2024-09-21T10:03:50.574192+0000","last_clean":"2024-09-21T10:03:50.574192+0000","last_became_active":"2024-09-21T10:01:40.419839+0000","last_became_peered":"2024-09-21T10:01:40.419839+0000","last_unstale":"2024-09-21T10:03:50.574192+0000","last_undegraded":"2024-09-21T10:03:50.574192+0000","last_fullsized":"2024-09-21T10:03:50.574192+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:59.370554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1781,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.647072+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T10:03:57.647072+0000","last_peered":"2024-09-21T10:03:57.647072+0000","last_clean":"2024-09-21T10:03:57.647072+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T10:03:57.647072+0000","last_undegraded":"2024-09-21T10:03:57.647072+0000","last_fullsized":"2024-09-21T10:03:57.647072+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.18","version":"1183'7823","reported_seq":7928,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:56.928816+0000","last_change":"2024-09-21T10:01:27.370045+0000","last_active":"2024-09-21T10:03:56.928816+0000","last_peered":"2024-09-21T10:03:56.928816+0000","last_clean":"2024-09-21T10:03:56.928816+0000","last_became_active":"2024-09-21T10:01:27.368411+0000","last_became_peered":"2024-09-21T10:01:27.368411+0000","last_unstale":"2024-09-21T10:03:56.928816+0000","last_undegraded":"2024-09-21T10:03:56.928816+0000","last_fullsized":"2024-09-21T10:03:56.928816+0000","mapping_epoch":1148,"log_start":"0'0","ondisk_log_start":"0'0","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7823,"log_dups_size":0,"ondisk_log_size":7823,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53346304,"num_objects":814,"num_object_clones":0,"num_object_copies":2442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6160,"num_write_kb":223168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1151,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578755+0000","last_change":"2024-09-21T10:01:40.432190+0000","last_active":"2024-09-21T10:03:49.578755+0000","last_peered":"2024-09-21T10:03:49.578755+0000","last_clean":"2024-09-21T10:03:49.578755+0000","last_became_active":"2024-09-21T10:01:40.431847+0000","last_became_peered":"2024-09-21T10:01:40.431847+0000","last_unstale":"2024-09-21T10:03:49.578755+0000","last_undegraded":"2024-09-21T10:03:49.578755+0000","last_fullsized":"2024-09-21T10:03:49.578755+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:32.573309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":119,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578654+0000","last_change":"2024-09-21T10:01:40.420069+0000","last_active":"2024-09-21T10:03:49.578654+0000","last_peered":"2024-09-21T10:03:49.578654+0000","last_clean":"2024-09-21T10:03:49.578654+0000","last_became_active":"2024-09-21T10:01:40.416524+0000","last_became_peered":"2024-09-21T10:01:40.416524+0000","last_unstale":"2024-09-21T10:03:49.578654+0000","last_undegraded":"2024-09-21T10:03:49.578654+0000","last_fullsized":"2024-09-21T10:03:49.578654+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:36.273679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"1184'7726","reported_seq":11961,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:58.907956+0000","last_change":"2024-09-21T10:03:04.307452+0000","last_active":"2024-09-21T10:03:58.907956+0000","last_peered":"2024-09-21T10:03:58.907956+0000","last_clean":"2024-09-21T10:03:58.907956+0000","last_became_active":"2024-09-21T10:03:04.296838+0000","last_became_peered":"2024-09-21T10:03:04.296838+0000","last_unstale":"2024-09-21T10:03:58.907956+0000","last_undegraded":"2024-09-21T10:03:58.907956+0000","last_fullsized":"2024-09-21T10:03:58.907956+0000","mapping_epoch":1175,"log_start":"1145'458","ondisk_log_start":"1145'458","created":1148,"last_epoch_clean":1176,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7268,"log_dups_size":0,"ondisk_log_size":7268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54263808,"num_objects":828,"num_object_clones":0,"num_object_copies":2484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6122,"num_write_kb":222400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2058,"num_bytes_recovered":115474432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1141,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578284+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T10:03:49.578284+0000","last_peered":"2024-09-21T10:03:49.578284+0000","last_clean":"2024-09-21T10:03:49.578284+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T10:03:49.578284+0000","last_undegraded":"2024-09-21T10:03:49.578284+0000","last_fullsized":"2024-09-21T10:03:49.578284+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":133,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.579062+0000","last_change":"2024-09-21T10:00:16.366351+0000","last_active":"2024-09-21T10:03:49.579062+0000","last_peered":"2024-09-21T10:03:49.579062+0000","last_clean":"2024-09-21T10:03:49.579062+0000","last_became_active":"2024-09-21T10:00:16.355385+0000","last_became_peered":"2024-09-21T10:00:16.355385+0000","last_unstale":"2024-09-21T10:03:49.579062+0000","last_undegraded":"2024-09-21T10:03:49.579062+0000","last_fullsized":"2024-09-21T10:03:49.579062+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:29.014677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1713,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.575791+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:03:50.575791+0000","last_peered":"2024-09-21T10:03:50.575791+0000","last_clean":"2024-09-21T10:03:50.575791+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:03:50.575791+0000","last_undegraded":"2024-09-21T10:03:50.575791+0000","last_fullsized":"2024-09-21T10:03:50.575791+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1713,"reported_epoch":1183,"state":"active+clean","last_fresh":"2024-09-21T10:03:50.576546+0000","last_change":"2024-09-21T10:00:16.361160+0000","last_active":"2024-09-21T10:03:50.576546+0000","last_peered":"2024-09-21T10:03:50.576546+0000","last_clean":"2024-09-21T10:03:50.576546+0000","last_became_active":"2024-09-21T10:00:16.360524+0000","last_became_peered":"2024-09-21T10:00:16.360524+0000","last_unstale":"2024-09-21T10:03:50.576546+0000","last_undegraded":"2024-09-21T10:03:50.576546+0000","last_fullsized":"2024-09-21T10:03:50.576546+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:52.747211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1154,"reported_epoch":1182,"state":"active+clean","last_fresh":"2024-09-21T10:03:49.578155+0000","last_change":"2024-09-21T10:01:40.432238+0000","last_active":"2024-09-21T10:03:49.578155+0000","last_peered":"2024-09-21T10:03:49.578155+0000","last_clean":"2024-09-21T10:03:49.578155+0000","last_became_active":"2024-09-21T10:01:40.431883+0000","last_became_peered":"2024-09-21T10:01:40.431883+0000","last_unstale":"2024-09-21T10:03:49.578155+0000","last_undegraded":"2024-09-21T10:03:49.578155+0000","last_fullsized":"2024-09-21T10:03:49.578155+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:30.528533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1797,"reported_epoch":1184,"state":"active+clean","last_fresh":"2024-09-21T10:03:57.647962+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T10:03:57.647962+0000","last_peered":"2024-09-21T10:03:57.647962+0000","last_clean":"2024-09-21T10:03:57.647962+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T10:03:57.647962+0000","last_undegraded":"2024-09-21T10:03:57.647962+0000","last_fullsized":"2024-09-21T10:03:57.647962+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":1698103320,"num_objects":25912,"num_object_clones":0,"num_object_copies":77736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5258,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25912,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":179116,"num_write_kb":6272001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21965,"num_bytes_recovered":1234173952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6287339520,"data_stored":6287327304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":175579,"ondisk_log_size":175579,"up":78,"acting":74,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":9628,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":477367,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":107,"num_read_kb":272,"num_write":55,"num_write_kb":592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":542359,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397857,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1165728,"kb_used_data":868312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92579424,"statfs":{"total":95995035648,"available":94801330176,"internally_reserved":0,"allocated":889151488,"data_stored":886585612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594085,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":974556,"kb_used_data":673044,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92770596,"statfs":{"total":95995035648,"available":94997090304,"internally_reserved":0,"allocated":689197056,"data_stored":686635165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790315,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":805972,"kb_used_data":504460,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92939180,"statfs":{"total":95995035648,"available":95169720320,"internally_reserved":0,"allocated":516567040,"data_stored":514009687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986542,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":944860,"kb_used_data":635028,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92800292,"statfs":{"total":95995035648,"available":95027499008,"internally_reserved":0,"allocated":650268672,"data_stored":647706814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999184,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1548816,"kb_used_data":1060744,"kb_used_omap":25,"kb_used_meta":487846,"kb_avail":92196336,"statfs":{"total":95995035648,"available":94409048064,"internally_reserved":0,"allocated":1086201856,"data_stored":1083648066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":499555271},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,4,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111127,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1310992,"kb_used_data":852040,"kb_used_omap":31,"kb_used_meta":458848,"kb_avail":92434160,"statfs":{"total":95995035648,"available":94652579840,"internally_reserved":0,"allocated":872488960,"data_stored":869935668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":469860584},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,4,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2203318223070,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335013,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1110628,"kb_used_data":768344,"kb_used_omap":31,"kb_used_meta":342240,"kb_avail":92634524,"statfs":{"total":95995035648,"available":94857752576,"internally_reserved":0,"allocated":786784256,"data_stored":784222344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":350453856},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":777261056,"data_stored":777256984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":909574144,"data_stored":909574144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":847052800,"data_stored":847052800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1075580928,"data_stored":1075576856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":630784000,"data_stored":630784000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":504168448,"data_stored":504168448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":669192192,"data_stored":669188120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":873725952,"data_stored":873725952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:04:04.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:04:04.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.723 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:04.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:04.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:04.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:04.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:04.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:04:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:04:04] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.51.0" 2024-09-21T10:04:04.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.019 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:05.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:05.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:05.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:05.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:05.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:05.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:05.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:05.429 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.469 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:05.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:05.571 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:05.603 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:05.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:05.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:05.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:05.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:05.847 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:05.879 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:05.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:05.920 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:05 smithi017 ceph-mon[102913]: pgmap v1573: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.3 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 4722/62637 objects degraded (7.539%); 0 B/s, 108 objects/s recovering 2024-09-21T10:04:05.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:05 smithi017 ceph-mon[105151]: pgmap v1573: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.3 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 4722/62637 objects degraded (7.539%); 0 B/s, 108 objects/s recovering 2024-09-21T10:04:05.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:04:05.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:05 smithi134 ceph-mon[90085]: pgmap v1573: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.3 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 4722/62637 objects degraded (7.539%); 0 B/s, 108 objects/s recovering 2024-09-21T10:04:06.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:06.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:06.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:06.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:06.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:06.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:06.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:06.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:06.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:06.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:06.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:06.597 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.630 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:06.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:06.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:06.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:06.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:06.850 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:06.855 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:06.921 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:06.944 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:06.948 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:06.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:07.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:07.106 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:07.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:07.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:07.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:07.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:07.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:07.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:07.487 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.526 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:07.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:07.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:07.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:07.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:07.769 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:07.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.802 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:07.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:07.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:07.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:07.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:07 smithi017 ceph-mon[102913]: pgmap v1574: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.2 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 4502/57645 objects degraded (7.810%); 0 B/s, 108 objects/s recovering 2024-09-21T10:04:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:07 smithi017 ceph-mon[105151]: pgmap v1574: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.2 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 4502/57645 objects degraded (7.810%); 0 B/s, 108 objects/s recovering 2024-09-21T10:04:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:07 smithi134 ceph-mon[90085]: pgmap v1574: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.2 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 4502/57645 objects degraded (7.810%); 0 B/s, 108 objects/s recovering 2024-09-21T10:04:08.044 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:08.073 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:08.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:08.218 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:08.320 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:08.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.353 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:08.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:08.454 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:08.475 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:08.577 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:08.603 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:08.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:08.704 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:08.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:08.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:08.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:08.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:08.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:08 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 4502/57645 objects degraded (7.810%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:08.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:08 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 4502/57645 objects degraded (7.810%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:08.983 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:08 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 4502/57645 objects degraded (7.810%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:09.012 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:09.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:09.114 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:09.142 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:09.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:09.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:09.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:09.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:09.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:09.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:09.523 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:09.553 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:09.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:09.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:09.679 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:09.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:09.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:09.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:09.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:09.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:09 smithi017 ceph-mon[102913]: pgmap v1575: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.0 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 949 op/s; 4127/51606 objects degraded (7.997%); 0 B/s, 107 objects/s recovering 2024-09-21T10:04:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:09 smithi017 ceph-mon[105151]: pgmap v1575: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.0 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 949 op/s; 4127/51606 objects degraded (7.997%); 0 B/s, 107 objects/s recovering 2024-09-21T10:04:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:09.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:09 smithi134 ceph-mon[90085]: pgmap v1575: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.0 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 949 op/s; 4127/51606 objects degraded (7.997%); 0 B/s, 107 objects/s recovering 2024-09-21T10:04:10.062 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:10.095 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:10.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:10.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:10.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:10.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:10.337 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:10.382 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:10.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:04:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:04:10.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:10.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:10.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:10.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:10.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:10.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:10.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:10.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.803 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:10.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:10.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:10.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:10.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:11.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:11.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:11.173 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:11.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:11.306 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:11.334 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:11.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:11.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:11.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:11.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:11.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:11.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:11.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:11.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:11.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:11.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:11.914 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:11.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:11.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:11 smithi017 ceph-mon[102913]: pgmap v1576: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 996 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 4127/47847 objects degraded (8.625%); 0 B/s, 94 objects/s recovering 2024-09-21T10:04:11.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:11 smithi017 ceph-mon[105151]: pgmap v1576: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 996 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 4127/47847 objects degraded (8.625%); 0 B/s, 94 objects/s recovering 2024-09-21T10:04:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:11 smithi134 ceph-mon[90085]: pgmap v1576: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 996 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 4127/47847 objects degraded (8.625%); 0 B/s, 94 objects/s recovering 2024-09-21T10:04:12.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:12.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:12.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:12.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:12.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:12.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:12.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:12.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:12.424 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:12.453 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:12.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:12.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:12.595 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:12.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:12.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:12.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:12.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:12.838 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:12.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.868 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:12.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:12.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:13.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:13.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:13.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:13.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:13.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:13.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:13.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:13.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:13.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:13.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.426 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:13.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:13.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:13.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:13.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:13.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:13.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:13.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:13.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:13.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:13 smithi017 ceph-mon[102913]: pgmap v1577: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 760 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 3520/36528 objects degraded (9.636%); 0 B/s, 144 objects/s recovering 2024-09-21T10:04:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:13 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 3520/36528 objects degraded (9.636%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:13 smithi017 ceph-mon[105151]: pgmap v1577: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 760 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 3520/36528 objects degraded (9.636%); 0 B/s, 144 objects/s recovering 2024-09-21T10:04:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:13 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 3520/36528 objects degraded (9.636%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:13.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:13 smithi134 ceph-mon[90085]: pgmap v1577: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 760 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 3520/36528 objects degraded (9.636%); 0 B/s, 144 objects/s recovering 2024-09-21T10:04:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:13 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 3520/36528 objects degraded (9.636%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:14.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:14.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:14.111 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:14.141 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:14.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:14.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:14.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:14.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:14.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:14.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:14.529 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:14.556 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:14.658 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:14.688 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:14.762 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2017772222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:14.762 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:04:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:04:14] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.51.0" 2024-09-21T10:04:14.763 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2017772222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:14.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:14.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:14.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:14.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:14.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.980 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:14.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2017772222' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:15.081 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:15.106 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:15.208 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:15.241 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:15.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:15.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:15.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:15.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:15.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:15.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:15.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:15.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:15.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:15.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:15.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:15.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:15.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:15 smithi017 ceph-mon[105151]: pgmap v1578: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 676 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 3520/32523 objects degraded (10.823%); 0 B/s, 100 objects/s recovering 2024-09-21T10:04:15.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:15 smithi017 ceph-mon[102913]: pgmap v1578: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 676 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 3520/32523 objects degraded (10.823%); 0 B/s, 100 objects/s recovering 2024-09-21T10:04:15.857 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:04:15.857 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:04:15.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:04:15.859 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:04:15.859 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:04:15.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:04:15.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:04:15.920 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:15.950 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:15.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:15 smithi134 ceph-mon[90085]: pgmap v1578: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 676 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 3520/32523 objects degraded (10.823%); 0 B/s, 100 objects/s recovering 2024-09-21T10:04:16.052 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:16.082 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:16.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:16.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:16.212 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:16.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:16.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:16.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:16.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:16.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:16.611 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:16.642 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:16.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:16.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:16.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:16.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:16.935 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:16.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:17.037 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:17.062 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:17.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:17.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:17.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:17.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:17.270 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:17.270 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:17.304 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:17.330 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:17.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:17.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:17.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:17.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:17.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:17.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:17.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:17.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:17.753 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:04:17.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:17.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:17.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:17.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.893 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:17.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:17.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:17 smithi134 ceph-mon[90085]: pgmap v1579: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 567 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 982 op/s; 3271/27255 objects degraded (12.001%); 0 B/s, 120 objects/s recovering 2024-09-21T10:04:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1041583948' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:04:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:17 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:04:18.033 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:18.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:18.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:18.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:18.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:17 smithi017 ceph-mon[102913]: pgmap v1579: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 567 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 982 op/s; 3271/27255 objects degraded (12.001%); 0 B/s, 120 objects/s recovering 2024-09-21T10:04:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1041583948' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:04:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:17 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:04:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:17 smithi017 ceph-mon[105151]: pgmap v1579: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 567 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 982 op/s; 3271/27255 objects degraded (12.001%); 0 B/s, 120 objects/s recovering 2024-09-21T10:04:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1041583948' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:04:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:17 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:04:18.270 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:18.295 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:18.397 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:18.417 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:18.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:18.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:18.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:18.646 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:18.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:18.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:18.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:18.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:18.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:18.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:18.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:18 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:18 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:04:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:18 smithi017 ceph-mon[102913]: osdmap e1185: 8 total, 8 up, 7 in 2024-09-21T10:04:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:18 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 3271/27255 objects degraded (12.001%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:18 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:18 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:04:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:18 smithi017 ceph-mon[105151]: osdmap e1185: 8 total, 8 up, 7 in 2024-09-21T10:04:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:18 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 3271/27255 objects degraded (12.001%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:18 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:18 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:04:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:18 smithi134 ceph-mon[90085]: osdmap e1185: 8 total, 8 up, 7 in 2024-09-21T10:04:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:18 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 3271/27255 objects degraded (12.001%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:19.030 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:19.064 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:19.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:19.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:19.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:19.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:19.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:19.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:19.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:19.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:19.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:19.568 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:19.599 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:19.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:19.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:19.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:19.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:19.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:19.880 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:19.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:19.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.026 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:20.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:20.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:20.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:19 smithi017 ceph-mon[102913]: pgmap v1581: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 433 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 2796/20856 objects degraded (13.406%); 0 B/s, 133 objects/s recovering 2024-09-21T10:04:20.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:19 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:04:20.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:19 smithi017 ceph-mon[102913]: osdmap e1186: 8 total, 8 up, 7 in 2024-09-21T10:04:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:19 smithi017 ceph-mon[105151]: pgmap v1581: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 433 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 2796/20856 objects degraded (13.406%); 0 B/s, 133 objects/s recovering 2024-09-21T10:04:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:19 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:04:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:19 smithi017 ceph-mon[105151]: osdmap e1186: 8 total, 8 up, 7 in 2024-09-21T10:04:20.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:19 smithi134 ceph-mon[90085]: pgmap v1581: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 433 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 2796/20856 objects degraded (13.406%); 0 B/s, 133 objects/s recovering 2024-09-21T10:04:20.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:19 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:04:20.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:19 smithi134 ceph-mon[90085]: osdmap e1186: 8 total, 8 up, 7 in 2024-09-21T10:04:20.215 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:04:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:04:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:20.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:20.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:20.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:20.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:20.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:20.589 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:20.629 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:20.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.774 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:20.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:20.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.906 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:20.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:20.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:20 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:21.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:04:21.007 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:21.033 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:21.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:21.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:21.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:21.275 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:21.300 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:21.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:21.402 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.436 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:21.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:21.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:21.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:21.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:21.664 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.687 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:21.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:21.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:21.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.816 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:21.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:21.918 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:21.951 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:22.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:21 smithi134 ceph-mon[90085]: pgmap v1583: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 366 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 2796/17622 objects degraded (15.867%); 0 B/s, 90 objects/s recovering 2024-09-21T10:04:22.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:21 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:04:22.013 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:21 smithi134 ceph-mon[90085]: osdmap e1187: 8 total, 8 up, 7 in 2024-09-21T10:04:22.052 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:21 smithi017 ceph-mon[105151]: pgmap v1583: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 366 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 2796/17622 objects degraded (15.867%); 0 B/s, 90 objects/s recovering 2024-09-21T10:04:22.052 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:21 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:04:22.052 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:21 smithi017 ceph-mon[105151]: osdmap e1187: 8 total, 8 up, 7 in 2024-09-21T10:04:22.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:22.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:21 smithi017 ceph-mon[102913]: pgmap v1583: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 366 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 2796/17622 objects degraded (15.867%); 0 B/s, 90 objects/s recovering 2024-09-21T10:04:22.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:21 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:04:22.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:21 smithi017 ceph-mon[102913]: osdmap e1187: 8 total, 8 up, 7 in 2024-09-21T10:04:22.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:22.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:22.174 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:22.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:22.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:04:22.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:22.374 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:22.402 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:22.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:22.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:22.531 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:22.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:22.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:22.650 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:04:22.650 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 2796/17622 objects degraded (15.867%), 4 pgs degraded, 4 pgs undersized","count":8},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1257,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1188,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912006,"num_in_osds":7,"osd_in_since":1726912899,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":105},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":2}],"num_pgs":109,"num_pools":5,"num_objects":5874,"data_bytes":383671932,"bytes_used":4133031936,"bytes_avail":667832217600,"bytes_total":671965249536,"degraded_objects":2796,"degraded_total":17622,"degraded_ratio":0.15866530473272047,"recovering_objects_per_sec":90,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":724,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":787},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.coodep","status":"up:active","gid":14655}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":25,"modified":"2024-09-21T09:59:24.056708+0000","services":{}},"progress_events":{"8eee188d-5f7b-4865-8b50-9bf77cc8c66e":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 5s)","progress":0.96330273151397705,"add_to_ceph_s":true},"b01dc62e-dd1a-4a72-9ecf-5a6fd6d40ae0":{"message":"Upgrade to 19.3.0-5092-g62da54b4 (4m)\n [============................] (remaining: 6m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-09-21T10:04:22.659 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:22.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:22.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:22.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:22.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:22.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:22 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:04:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:22 smithi017 ceph-mon[102913]: osdmap e1188: 8 total, 8 up, 7 in 2024-09-21T10:04:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:22 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/251958821' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:04:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:22 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:04:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:22 smithi017 ceph-mon[105151]: osdmap e1188: 8 total, 8 up, 7 in 2024-09-21T10:04:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:22 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/251958821' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:04:23.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:23.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:22 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:04:23.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:22 smithi134 ceph-mon[90085]: osdmap e1188: 8 total, 8 up, 7 in 2024-09-21T10:04:23.070 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:22 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/251958821' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:04:23.079 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:23.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:23.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:23.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:23.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:23.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:23.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:23.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:23.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:23.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:23.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:23.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:23.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:23.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:23.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:23.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:23.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:23.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:04:23.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:23.877 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:23 smithi134 ceph-mon[90085]: pgmap v1586: 109 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 106 active+clean; 171 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 1673/8277 objects degraded (20.213%); 0 B/s, 252 objects/s recovering 2024-09-21T10:04:23.877 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:23 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:04:23.877 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:23 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 1673/8277 objects degraded (20.213%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:23.879 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:23.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:23.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:24.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:24.125 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:24.153 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:24.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:24.155 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 109557 objects 2024-09-21T10:04:24.155 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :114.228 2024-09-21T10:04:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:23 smithi017 ceph-mon[102913]: pgmap v1586: 109 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 106 active+clean; 171 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 1673/8277 objects degraded (20.213%); 0 B/s, 252 objects/s recovering 2024-09-21T10:04:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:23 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:04:24.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:23 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 1673/8277 objects degraded (20.213%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:24.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:23 smithi017 ceph-mon[105151]: pgmap v1586: 109 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 106 active+clean; 171 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 1673/8277 objects degraded (20.213%); 0 B/s, 252 objects/s recovering 2024-09-21T10:04:24.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:23 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:04:24.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:23 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 1673/8277 objects degraded (20.213%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:24.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:24.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:24.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:24.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:24.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:24.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:24.526 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:24.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:24.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:24.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:24.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:24.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:24.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:24.878 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:24.900 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:24.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:24 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:04:24.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:24 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:04:24.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:04:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:04:24] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.51.0" 2024-09-21T10:04:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:24 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:04:25.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:25.025 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:25.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:25.127 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:25.148 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:25.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:25.250 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:25.271 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:25.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:25.394 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:25.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:25.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:25.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.523 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:25.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:25.623 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:25.645 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:25.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:25.747 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:25.769 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:25.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:25.871 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:25.891 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:25.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:25.993 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.015 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:26.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:26.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:26.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:26.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:25 smithi017 ceph-mon[105151]: pgmap v1587: 109 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 106 active+clean; 110 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 1673/5331 objects degraded (31.382%); 0 B/s, 187 objects/s recovering 2024-09-21T10:04:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:26.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:26.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:25 smithi017 ceph-mon[102913]: pgmap v1587: 109 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 106 active+clean; 110 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 1673/5331 objects degraded (31.382%); 0 B/s, 187 objects/s recovering 2024-09-21T10:04:26.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:26.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:26.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:26.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:25 smithi134 ceph-mon[90085]: pgmap v1587: 109 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 106 active+clean; 110 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 1673/5331 objects degraded (31.382%); 0 B/s, 187 objects/s recovering 2024-09-21T10:04:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:26.265 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:26.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:26.367 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:26.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:26.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:26.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:26.616 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:26.638 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:26.740 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:26.761 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:26.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:26.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:26.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:26.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:26 smithi017 ceph-mon[102913]: osdmap e1189: 8 total, 8 up, 7 in 2024-09-21T10:04:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:26 smithi017 ceph-mon[102913]: pgmap v1589: 109 pgs: 1 creating+peering, 1 active+undersized+remapped, 1 peering, 106 active+clean; 49 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 0 B/s, 388 objects/s recovering 2024-09-21T10:04:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:26 smithi017 ceph-mon[102913]: 7.1 scrub starts 2024-09-21T10:04:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:26 smithi017 ceph-mon[102913]: 7.1 scrub ok 2024-09-21T10:04:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:26 smithi017 ceph-mon[105151]: osdmap e1189: 8 total, 8 up, 7 in 2024-09-21T10:04:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:26 smithi017 ceph-mon[105151]: pgmap v1589: 109 pgs: 1 creating+peering, 1 active+undersized+remapped, 1 peering, 106 active+clean; 49 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 0 B/s, 388 objects/s recovering 2024-09-21T10:04:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:26 smithi017 ceph-mon[105151]: 7.1 scrub starts 2024-09-21T10:04:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:26 smithi017 ceph-mon[105151]: 7.1 scrub ok 2024-09-21T10:04:26.987 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:27.008 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:27.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:27.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:27.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:27.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:26 smithi134 ceph-mon[90085]: osdmap e1189: 8 total, 8 up, 7 in 2024-09-21T10:04:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:26 smithi134 ceph-mon[90085]: pgmap v1589: 109 pgs: 1 creating+peering, 1 active+undersized+remapped, 1 peering, 106 active+clean; 49 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 0 B/s, 388 objects/s recovering 2024-09-21T10:04:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:26 smithi134 ceph-mon[90085]: 7.1 scrub starts 2024-09-21T10:04:27.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:26 smithi134 ceph-mon[90085]: 7.1 scrub ok 2024-09-21T10:04:27.254 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:27.293 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-09-21T10:04:27.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-09-21T10:04:27.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:27.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:27.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:27.481 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:27.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.504 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:27.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:27.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:27.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:27.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:27.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:27.751 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:27.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:27.852 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:27.873 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:27.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:27 smithi017 ceph-mon[102913]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:04:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:27 smithi017 ceph-mon[102913]: osdmap e1190: 8 total, 8 up, 7 in 2024-09-21T10:04:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:27 smithi017 ceph-mon[105151]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:04:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:27 smithi017 ceph-mon[105151]: osdmap e1190: 8 total, 8 up, 7 in 2024-09-21T10:04:27.975 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:27.998 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:27.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:28.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:28.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:28.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:28.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:28.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:27 smithi134 ceph-mon[90085]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:04:28.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:27 smithi134 ceph-mon[90085]: osdmap e1190: 8 total, 8 up, 7 in 2024-09-21T10:04:28.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:28.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:28.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:28.369 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:28.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:28.413 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:04:28.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:04:28.425 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:28.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:28.494 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:28.596 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:28.617 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:28.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:28.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:28.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:28.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:28.847 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:28.869 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:28.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:28.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:28.973 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:28.974 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:28.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:29.098 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:29.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:29.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:29.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:28 smithi017 ceph-mon[105151]: pgmap v1591: 109 pgs: 1 creating+peering, 1 active+undersized+remapped, 107 active+clean; 12 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 0 B/s, 188 objects/s recovering 2024-09-21T10:04:29.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:28 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-21T10:04:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:28 smithi017 ceph-mon[102913]: pgmap v1591: 109 pgs: 1 creating+peering, 1 active+undersized+remapped, 107 active+clean; 12 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 0 B/s, 188 objects/s recovering 2024-09-21T10:04:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:28 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-21T10:04:29.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:28 smithi134 ceph-mon[90085]: pgmap v1591: 109 pgs: 1 creating+peering, 1 active+undersized+remapped, 107 active+clean; 12 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 0 B/s, 188 objects/s recovering 2024-09-21T10:04:29.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:28 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-21T10:04:29.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:29.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:29.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:29.373 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:29.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:29.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:29.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:29.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:29.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:29.625 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:29.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:29.705 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:29.726 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:29.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:04:29.753 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:29.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:29.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:29.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:29 smithi017 ceph-mon[105151]: 7.9 scrub starts 2024-09-21T10:04:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:29 smithi017 ceph-mon[105151]: 7.9 scrub ok 2024-09-21T10:04:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:29 smithi017 ceph-mon[105151]: 7.3 scrub starts 2024-09-21T10:04:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:29 smithi017 ceph-mon[105151]: 7.3 scrub ok 2024-09-21T10:04:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:29 smithi017 ceph-mon[102913]: 7.9 scrub starts 2024-09-21T10:04:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:29 smithi017 ceph-mon[102913]: 7.9 scrub ok 2024-09-21T10:04:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:29 smithi017 ceph-mon[102913]: 7.3 scrub starts 2024-09-21T10:04:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:29 smithi017 ceph-mon[102913]: 7.3 scrub ok 2024-09-21T10:04:29.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:30.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:30.105 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.141 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:30.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:30.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:29 smithi134 ceph-mon[90085]: 7.9 scrub starts 2024-09-21T10:04:30.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:29 smithi134 ceph-mon[90085]: 7.9 scrub ok 2024-09-21T10:04:30.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:29 smithi134 ceph-mon[90085]: 7.3 scrub starts 2024-09-21T10:04:30.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:29 smithi134 ceph-mon[90085]: 7.3 scrub ok 2024-09-21T10:04:30.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:30.270 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:30.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:30.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:30.398 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:30.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:04:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:04:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:30.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:30.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:30.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:30.607 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:30.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:30.645 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:30.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:30.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:30.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:30.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:30.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:30.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:30.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:31.004 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:31.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[102913]: 7.19 scrub starts 2024-09-21T10:04:31.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[102913]: 7.19 scrub ok 2024-09-21T10:04:31.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[102913]: pgmap v1592: 109 pgs: 1 creating+peering, 1 active+undersized+remapped, 107 active+clean; 10 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 0 B/s, 151 objects/s recovering 2024-09-21T10:04:31.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[102913]: 7.0 scrub starts 2024-09-21T10:04:31.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[102913]: 7.0 scrub ok 2024-09-21T10:04:31.013 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[105151]: 7.19 scrub starts 2024-09-21T10:04:31.013 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[105151]: 7.19 scrub ok 2024-09-21T10:04:31.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[105151]: pgmap v1592: 109 pgs: 1 creating+peering, 1 active+undersized+remapped, 107 active+clean; 10 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 0 B/s, 151 objects/s recovering 2024-09-21T10:04:31.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[105151]: 7.0 scrub starts 2024-09-21T10:04:31.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:30 smithi017 ceph-mon[105151]: 7.0 scrub ok 2024-09-21T10:04:31.030 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:31.131 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:31.157 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:31.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:31.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:30 smithi134 ceph-mon[90085]: 7.19 scrub starts 2024-09-21T10:04:31.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:30 smithi134 ceph-mon[90085]: 7.19 scrub ok 2024-09-21T10:04:31.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:30 smithi134 ceph-mon[90085]: pgmap v1592: 109 pgs: 1 creating+peering, 1 active+undersized+remapped, 107 active+clean; 10 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 0 B/s, 151 objects/s recovering 2024-09-21T10:04:31.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:30 smithi134 ceph-mon[90085]: 7.0 scrub starts 2024-09-21T10:04:31.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:30 smithi134 ceph-mon[90085]: 7.0 scrub ok 2024-09-21T10:04:31.260 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:31.284 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:31.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:31.385 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:31.409 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:31.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:31.510 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:31.534 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:31.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:31.636 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:31.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:31.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:31.783 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:31.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:31.885 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:31.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.911 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:31.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:32.012 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:32.037 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:32.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:32.139 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:32.163 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:32.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:32.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:32.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:32.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:32 smithi017 ceph-mon[102913]: 7.7 scrub starts 2024-09-21T10:04:32.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:32 smithi017 ceph-mon[102913]: 7.7 scrub ok 2024-09-21T10:04:32.330 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:32 smithi017 ceph-mon[105151]: 7.7 scrub starts 2024-09-21T10:04:32.331 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:32 smithi017 ceph-mon[105151]: 7.7 scrub ok 2024-09-21T10:04:32.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:32.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:32 smithi134 ceph-mon[90085]: 7.7 scrub starts 2024-09-21T10:04:32.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:32 smithi134 ceph-mon[90085]: 7.7 scrub ok 2024-09-21T10:04:32.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:32.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:32.552 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:32.640 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:32.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.667 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:32.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:32.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:32.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:32.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:32.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:32.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:33.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:33.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:33.145 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:33.170 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:33 smithi017 ceph-mon[102913]: pgmap v1593: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 9.7 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:33 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:04:33.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:33 smithi017 ceph-mon[102913]: 7.12 scrub starts 2024-09-21T10:04:33.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:33 smithi017 ceph-mon[102913]: 7.12 scrub ok 2024-09-21T10:04:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:33 smithi017 ceph-mon[105151]: pgmap v1593: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 9.7 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:33 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:04:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:33 smithi017 ceph-mon[105151]: 7.12 scrub starts 2024-09-21T10:04:33.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:33 smithi017 ceph-mon[105151]: 7.12 scrub ok 2024-09-21T10:04:33.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:33.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:33.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:33.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:33.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:33.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:33 smithi134 ceph-mon[90085]: pgmap v1593: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 9.7 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:33 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:04:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:33 smithi134 ceph-mon[90085]: 7.12 scrub starts 2024-09-21T10:04:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:33 smithi134 ceph-mon[90085]: 7.12 scrub ok 2024-09-21T10:04:33.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:33.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:33.644 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:33.670 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:33.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:33.772 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:33.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:33.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:33.924 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:34.026 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:34.051 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:34.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:34.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:34.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:34.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:34.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[105151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:04:34.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-09-21T10:04:34.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:04:34.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[105151]: osdmap e1191: 8 total, 8 up, 7 in 2024-09-21T10:04:34.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[105151]: osdmap e1192: 8 total, 8 up, 7 in 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[105151]: 7.18 scrub starts 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[105151]: 7.18 scrub ok 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[105151]: 7.17 scrub starts 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[105151]: 7.17 scrub ok 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[102913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[102913]: osdmap e1191: 8 total, 8 up, 7 in 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[102913]: osdmap e1192: 8 total, 8 up, 7 in 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[102913]: 7.18 scrub starts 2024-09-21T10:04:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[102913]: 7.18 scrub ok 2024-09-21T10:04:34.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[102913]: 7.17 scrub starts 2024-09-21T10:04:34.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-mon[102913]: 7.17 scrub ok 2024-09-21T10:04:34.278 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:34.303 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:34.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:34.321 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:04:34.322 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:04:34.329 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:04:34.329 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:04:34.345 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:04:34.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:04:34.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:34.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:34.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:34.432 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:04:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:34 smithi134 ceph-mon[90085]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:04:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:34 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-09-21T10:04:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:04:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:34 smithi134 ceph-mon[90085]: osdmap e1191: 8 total, 8 up, 7 in 2024-09-21T10:04:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:34 smithi134 ceph-mon[90085]: osdmap e1192: 8 total, 8 up, 7 in 2024-09-21T10:04:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:34 smithi134 ceph-mon[90085]: 7.18 scrub starts 2024-09-21T10:04:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:34 smithi134 ceph-mon[90085]: 7.18 scrub ok 2024-09-21T10:04:34.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:34 smithi134 ceph-mon[90085]: 7.17 scrub starts 2024-09-21T10:04:34.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:34 smithi134 ceph-mon[90085]: 7.17 scrub ok 2024-09-21T10:04:34.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:34.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:34.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:34.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:34.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:34.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:34.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:34.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:34.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:34.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:34.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:34.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:04:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:04:34] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-09-21T10:04:35.028 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:35.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:35.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:35.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:35.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:35.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:35.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:35.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:35.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: pgmap v1596: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 9.2 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:35.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:04:35.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/236001291' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:35.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2001219644' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:35.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:35.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:04:35.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:04:35.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:04:35.397 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:35.397 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: osdmap e1193: 8 total, 8 up, 7 in 2024-09-21T10:04:35.397 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: 7.12 starting backfill to osd.0 from (0'0,0'0] MAX to 1188'8494 2024-09-21T10:04:35.397 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: 7.12 starting backfill to osd.6 from (0'0,0'0] MAX to 1188'8494 2024-09-21T10:04:35.397 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: 7.8 scrub starts 2024-09-21T10:04:35.397 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:35 smithi134 ceph-mon[90085]: 7.8 scrub ok 2024-09-21T10:04:35.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:35.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:35.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: pgmap v1596: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 9.2 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/236001291' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2001219644' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: osdmap e1193: 8 total, 8 up, 7 in 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: 7.12 starting backfill to osd.0 from (0'0,0'0] MAX to 1188'8494 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: 7.12 starting backfill to osd.6 from (0'0,0'0] MAX to 1188'8494 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: 7.8 scrub starts 2024-09-21T10:04:35.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[105151]: 7.8 scrub ok 2024-09-21T10:04:35.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: pgmap v1596: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 9.2 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:35.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:04:35.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/236001291' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:35.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2001219644' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:35.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:35.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:04:35.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:04:35.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:04:35.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:35.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: osdmap e1193: 8 total, 8 up, 7 in 2024-09-21T10:04:35.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: 7.12 starting backfill to osd.0 from (0'0,0'0] MAX to 1188'8494 2024-09-21T10:04:35.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: 7.12 starting backfill to osd.6 from (0'0,0'0] MAX to 1188'8494 2024-09-21T10:04:35.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: 7.8 scrub starts 2024-09-21T10:04:35.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:35 smithi017 ceph-mon[102913]: 7.8 scrub ok 2024-09-21T10:04:35.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:35.554 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:35.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:35.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:35.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:35.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:35.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:35.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:35.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:35.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:35.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:36.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:36.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:36.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:36.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:36.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:36.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:36.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:36.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:36.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:36.425 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:36.527 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:36.548 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:36.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:36.650 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:36.671 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:36.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[102913]: 7.11 scrub starts 2024-09-21T10:04:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[102913]: 7.11 scrub ok 2024-09-21T10:04:36.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[102913]: 7.4 scrub starts 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[102913]: 7.4 scrub ok 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[102913]: osdmap e1194: 8 total, 8 up, 7 in 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[102913]: 1.3 deep-scrub starts 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[102913]: 1.3 deep-scrub ok 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[105151]: 7.11 scrub starts 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[105151]: 7.11 scrub ok 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[105151]: 7.4 scrub starts 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[105151]: 7.4 scrub ok 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[105151]: osdmap e1194: 8 total, 8 up, 7 in 2024-09-21T10:04:36.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[105151]: 1.3 deep-scrub starts 2024-09-21T10:04:36.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:36 smithi017 ceph-mon[105151]: 1.3 deep-scrub ok 2024-09-21T10:04:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:36 smithi134 ceph-mon[90085]: 7.11 scrub starts 2024-09-21T10:04:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:36 smithi134 ceph-mon[90085]: 7.11 scrub ok 2024-09-21T10:04:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:36 smithi134 ceph-mon[90085]: 7.4 scrub starts 2024-09-21T10:04:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:36 smithi134 ceph-mon[90085]: 7.4 scrub ok 2024-09-21T10:04:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:36 smithi134 ceph-mon[90085]: osdmap e1194: 8 total, 8 up, 7 in 2024-09-21T10:04:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:36 smithi134 ceph-mon[90085]: 1.3 deep-scrub starts 2024-09-21T10:04:36.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:36 smithi134 ceph-mon[90085]: 1.3 deep-scrub ok 2024-09-21T10:04:36.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:36.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:36.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:36.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:36.919 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:36.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:37.021 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:37.042 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:37.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:37.144 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:37.165 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:37.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:37.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:37.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:37.392 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:37.413 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:37.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:37.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:37.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:37.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:37.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[102913]: pgmap v1599: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 8.9 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[102913]: 7.13 starting backfill to osd.0 from (0'0,0'0] MAX to 1188'8507 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[102913]: osdmap e1195: 8 total, 8 up, 7 in 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[102913]: 7.13 starting backfill to osd.7 from (0'0,0'0] MAX to 1188'8507 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[102913]: 7.13 scrub starts 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[102913]: 7.13 scrub ok 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[102913]: 7.12 scrub starts 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[102913]: 7.12 scrub ok 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[105151]: pgmap v1599: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 8.9 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[105151]: 7.13 starting backfill to osd.0 from (0'0,0'0] MAX to 1188'8507 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[105151]: osdmap e1195: 8 total, 8 up, 7 in 2024-09-21T10:04:37.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[105151]: 7.13 starting backfill to osd.7 from (0'0,0'0] MAX to 1188'8507 2024-09-21T10:04:37.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[105151]: 7.13 scrub starts 2024-09-21T10:04:37.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[105151]: 7.13 scrub ok 2024-09-21T10:04:37.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[105151]: 7.12 scrub starts 2024-09-21T10:04:37.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:37 smithi017 ceph-mon[105151]: 7.12 scrub ok 2024-09-21T10:04:37.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:37.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:37 smithi134 ceph-mon[90085]: pgmap v1599: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 8.9 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:37 smithi134 ceph-mon[90085]: 7.13 starting backfill to osd.0 from (0'0,0'0] MAX to 1188'8507 2024-09-21T10:04:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:37 smithi134 ceph-mon[90085]: osdmap e1195: 8 total, 8 up, 7 in 2024-09-21T10:04:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:37 smithi134 ceph-mon[90085]: 7.13 starting backfill to osd.7 from (0'0,0'0] MAX to 1188'8507 2024-09-21T10:04:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:37 smithi134 ceph-mon[90085]: 7.13 scrub starts 2024-09-21T10:04:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:37 smithi134 ceph-mon[90085]: 7.13 scrub ok 2024-09-21T10:04:37.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:37 smithi134 ceph-mon[90085]: 7.12 scrub starts 2024-09-21T10:04:37.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:37 smithi134 ceph-mon[90085]: 7.12 scrub ok 2024-09-21T10:04:37.766 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:37.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.789 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:37.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:37.890 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:37.911 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:38.013 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:38.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:04:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.035 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:38.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:38.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:38.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:38.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:38.259 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:38.281 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:38.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:38.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:38.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:38.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:38.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:38.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:38.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:38.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:38.652 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:38 smithi017 ceph-mon[102913]: 7.14 scrub starts 2024-09-21T10:04:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:38 smithi017 ceph-mon[102913]: 7.14 scrub ok 2024-09-21T10:04:38.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:38 smithi017 ceph-mon[102913]: osdmap e1196: 8 total, 8 up, 7 in 2024-09-21T10:04:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:38 smithi017 ceph-mon[105151]: 7.14 scrub starts 2024-09-21T10:04:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:38 smithi017 ceph-mon[105151]: 7.14 scrub ok 2024-09-21T10:04:38.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:38 smithi017 ceph-mon[105151]: osdmap e1196: 8 total, 8 up, 7 in 2024-09-21T10:04:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:38 smithi134 ceph-mon[90085]: 7.14 scrub starts 2024-09-21T10:04:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:38 smithi134 ceph-mon[90085]: 7.14 scrub ok 2024-09-21T10:04:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:38 smithi134 ceph-mon[90085]: osdmap e1196: 8 total, 8 up, 7 in 2024-09-21T10:04:38.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:38.776 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:38.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:38.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:38.900 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:38.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:39.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:39.026 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:39.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:39.127 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:39.153 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:39.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:39.255 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:39.279 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:39.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:04:39.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:39.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:39.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:39.505 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:39.530 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:39.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:39.632 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:39.656 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:39.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:39 smithi017 ceph-mon[105151]: pgmap v1602: 109 pgs: 1 active+clean+remapped, 1 peering, 107 active+clean; 6.6 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:39 smithi017 ceph-mon[105151]: 7.10 scrub starts 2024-09-21T10:04:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:39 smithi017 ceph-mon[105151]: 7.10 scrub ok 2024-09-21T10:04:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:39 smithi017 ceph-mon[105151]: osdmap e1197: 8 total, 8 up, 7 in 2024-09-21T10:04:39.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:39 smithi017 ceph-mon[102913]: pgmap v1602: 109 pgs: 1 active+clean+remapped, 1 peering, 107 active+clean; 6.6 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:39.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:39 smithi017 ceph-mon[102913]: 7.10 scrub starts 2024-09-21T10:04:39.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:39 smithi017 ceph-mon[102913]: 7.10 scrub ok 2024-09-21T10:04:39.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:39 smithi017 ceph-mon[102913]: osdmap e1197: 8 total, 8 up, 7 in 2024-09-21T10:04:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:39 smithi134 ceph-mon[90085]: pgmap v1602: 109 pgs: 1 active+clean+remapped, 1 peering, 107 active+clean; 6.6 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:39 smithi134 ceph-mon[90085]: 7.10 scrub starts 2024-09-21T10:04:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:39 smithi134 ceph-mon[90085]: 7.10 scrub ok 2024-09-21T10:04:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:39 smithi134 ceph-mon[90085]: osdmap e1197: 8 total, 8 up, 7 in 2024-09-21T10:04:39.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:39.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:39.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:39.884 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:39.909 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:39.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:40.011 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:40.040 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:40.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:40.141 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:40.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:40.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:40.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:40.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:40.395 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:40.426 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:40.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:04:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:04:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:40.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:40.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:40.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:40.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:40.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:40.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:40.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:04:40.801 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:04:40.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:40.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:40.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:40.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:41.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:41.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:41.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:41.182 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:41.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:41.284 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:41.308 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:41.410 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:41.434 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:41.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:41.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[102913]: pgmap v1604: 109 pgs: 1 peering, 108 active+clean; 6.1 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:41.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:41.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:41.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[102913]: 7.2 scrub starts 2024-09-21T10:04:41.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[102913]: 7.2 scrub ok 2024-09-21T10:04:41.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:41.535 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[102913]: from='client.45557 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:41.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[105151]: pgmap v1604: 109 pgs: 1 peering, 108 active+clean; 6.1 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:41.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:41.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:41.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[105151]: 7.2 scrub starts 2024-09-21T10:04:41.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[105151]: 7.2 scrub ok 2024-09-21T10:04:41.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:41.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:41 smithi017 ceph-mon[105151]: from='client.45557 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:41.536 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:41.561 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:41.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.562 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1604,"stamp":"2024-09-21T10:04:40.186794+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6446692,"num_objects":118,"num_object_clones":0,"num_object_copies":354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":118,"num_whiteouts":0,"num_read":108,"num_read_kb":273,"num_write":122992,"num_write_kb":3645968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14979,"num_bytes_recovered":516844633,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":166997,"ondisk_log_size":166997,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":332,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2850052,"kb_used_data":261648,"kb_used_omap":95,"kb_used_meta":2588192,"kb_avail":653366012,"statfs":{"total":671965249536,"available":669046796288,"internally_reserved":0,"allocated":267927552,"data_stored":249711962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97681,"internal_metadata":2650309231},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-3244032,"num_objects":-49,"num_object_clones":0,"num_object_copies":-147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-49,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-47885,"num_write_kb":-1532000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-6080,"num_bytes_recovered":-331546624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.761679"},"pg_stats":[{"pgid":"7.17","version":"1188'8537","reported_seq":8616,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501322+0000","last_change":"2024-09-21T10:04:33.984331+0000","last_active":"2024-09-21T10:04:34.501322+0000","last_peered":"2024-09-21T10:04:34.501322+0000","last_clean":"2024-09-21T10:04:34.501322+0000","last_became_active":"2024-09-21T10:01:27.375972+0000","last_became_peered":"2024-09-21T10:01:27.375972+0000","last_unstale":"2024-09-21T10:04:34.501322+0000","last_undegraded":"2024-09-21T10:04:34.501322+0000","last_fullsized":"2024-09-21T10:04:34.501322+0000","mapping_epoch":1148,"log_start":"1145'997","ondisk_log_start":"1145'997","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"1188'8537","last_scrub_stamp":"2024-09-21T10:04:33.984258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:33.984258+0000","objects_scrubbed":0,"log_size":7540,"log_dups_size":0,"ondisk_log_size":7540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:16.372925+0000","scrub_duration":0.0051918040000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1734,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449826+0000","last_change":"2024-09-21T10:01:40.433720+0000","last_active":"2024-09-21T10:04:36.449826+0000","last_peered":"2024-09-21T10:04:36.449826+0000","last_clean":"2024-09-21T10:04:36.449826+0000","last_became_active":"2024-09-21T10:01:40.433360+0000","last_became_peered":"2024-09-21T10:01:40.433360+0000","last_unstale":"2024-09-21T10:04:36.449826+0000","last_undegraded":"2024-09-21T10:04:36.449826+0000","last_fullsized":"2024-09-21T10:04:36.449826+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:08.257176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1119,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432220+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T10:04:34.432220+0000","last_peered":"2024-09-21T10:04:34.432220+0000","last_clean":"2024-09-21T10:04:34.432220+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T10:04:34.432220+0000","last_undegraded":"2024-09-21T10:04:34.432220+0000","last_fullsized":"2024-09-21T10:04:34.432220+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.16","version":"1188'8501","reported_seq":8666,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442649+0000","last_change":"2024-09-21T10:03:10.420081+0000","last_active":"2024-09-21T10:04:38.442649+0000","last_peered":"2024-09-21T10:04:38.442649+0000","last_clean":"2024-09-21T10:04:38.442649+0000","last_became_active":"2024-09-21T10:01:27.383294+0000","last_became_peered":"2024-09-21T10:01:27.383294+0000","last_unstale":"2024-09-21T10:04:38.442649+0000","last_undegraded":"2024-09-21T10:04:38.442649+0000","last_fullsized":"2024-09-21T10:04:38.442649+0000","mapping_epoch":1148,"log_start":"1145'1918","ondisk_log_start":"1145'1918","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"1176'6333","last_scrub_stamp":"2024-09-21T10:03:10.419712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:03:10.419712+0000","objects_scrubbed":2183,"log_size":6583,"log_dups_size":0,"ondisk_log_size":6583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:08.951806+0000","scrub_duration":884,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2183,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":627,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432187+0000","last_change":"2024-09-21T09:57:39.541887+0000","last_active":"2024-09-21T10:04:34.432187+0000","last_peered":"2024-09-21T10:04:34.432187+0000","last_clean":"2024-09-21T10:04:34.432187+0000","last_became_active":"2024-09-21T09:57:39.541598+0000","last_became_peered":"2024-09-21T09:57:39.541598+0000","last_unstale":"2024-09-21T10:04:34.432187+0000","last_undegraded":"2024-09-21T10:04:34.432187+0000","last_fullsized":"2024-09-21T10:04:34.432187+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:41.615525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1815,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.443087+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T10:04:38.443087+0000","last_peered":"2024-09-21T10:04:38.443087+0000","last_clean":"2024-09-21T10:04:38.443087+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T10:04:38.443087+0000","last_undegraded":"2024-09-21T10:04:38.443087+0000","last_fullsized":"2024-09-21T10:04:38.443087+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.15","version":"1188'8519","reported_seq":8687,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431967+0000","last_change":"2024-09-21T10:01:59.676028+0000","last_active":"2024-09-21T10:04:34.431967+0000","last_peered":"2024-09-21T10:04:34.431967+0000","last_clean":"2024-09-21T10:04:34.431967+0000","last_became_active":"2024-09-21T10:01:27.368645+0000","last_became_peered":"2024-09-21T10:01:27.368645+0000","last_unstale":"2024-09-21T10:04:34.431967+0000","last_undegraded":"2024-09-21T10:04:34.431967+0000","last_fullsized":"2024-09-21T10:04:34.431967+0000","mapping_epoch":1148,"log_start":"1145'2241","ondisk_log_start":"1145'2241","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"1162'4060","last_scrub_stamp":"2024-09-21T10:01:59.675861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:01:59.675861+0000","objects_scrubbed":2420,"log_size":6278,"log_dups_size":0,"ondisk_log_size":6278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:02.268701+0000","scrub_duration":0.74397808499999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4466,"num_write_kb":65472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1817,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442929+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T10:04:38.442929+0000","last_peered":"2024-09-21T10:04:38.442929+0000","last_clean":"2024-09-21T10:04:38.442929+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T10:04:38.442929+0000","last_undegraded":"2024-09-21T10:04:38.442929+0000","last_fullsized":"2024-09-21T10:04:38.442929+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1120,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432421+0000","last_change":"2024-09-21T10:00:16.358888+0000","last_active":"2024-09-21T10:04:34.432421+0000","last_peered":"2024-09-21T10:04:34.432421+0000","last_clean":"2024-09-21T10:04:34.432421+0000","last_became_active":"2024-09-21T10:00:16.357219+0000","last_became_peered":"2024-09-21T10:00:16.357219+0000","last_unstale":"2024-09-21T10:04:34.432421+0000","last_undegraded":"2024-09-21T10:04:34.432421+0000","last_fullsized":"2024-09-21T10:04:34.432421+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:13.887798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.14","version":"1188'8380","reported_seq":12563,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442560+0000","last_change":"2024-09-21T10:04:37.112623+0000","last_active":"2024-09-21T10:04:38.442560+0000","last_peered":"2024-09-21T10:04:38.442560+0000","last_clean":"2024-09-21T10:04:38.442560+0000","last_became_active":"2024-09-21T10:02:18.295754+0000","last_became_peered":"2024-09-21T10:02:18.295754+0000","last_unstale":"2024-09-21T10:04:38.442560+0000","last_undegraded":"2024-09-21T10:04:38.442560+0000","last_fullsized":"2024-09-21T10:04:38.442560+0000","mapping_epoch":1165,"log_start":"1145'1748","ondisk_log_start":"1145'1748","created":1148,"last_epoch_clean":1166,"parent":"0.0","parent_split_bits":5,"last_scrub":"1188'8380","last_scrub_stamp":"2024-09-21T10:04:37.112559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:37.112559+0000","objects_scrubbed":0,"log_size":6632,"log_dups_size":0,"ondisk_log_size":6632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:56:19.146271+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1734,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.450341+0000","last_change":"2024-09-21T10:01:40.431450+0000","last_active":"2024-09-21T10:04:36.450341+0000","last_peered":"2024-09-21T10:04:36.450341+0000","last_clean":"2024-09-21T10:04:36.450341+0000","last_became_active":"2024-09-21T10:01:40.431140+0000","last_became_peered":"2024-09-21T10:01:40.431140+0000","last_unstale":"2024-09-21T10:04:36.450341+0000","last_undegraded":"2024-09-21T10:04:36.450341+0000","last_fullsized":"2024-09-21T10:04:36.450341+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:37.604030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":83,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.500948+0000","last_change":"2024-09-21T10:00:16.377280+0000","last_active":"2024-09-21T10:04:34.500948+0000","last_peered":"2024-09-21T10:04:34.500948+0000","last_clean":"2024-09-21T10:04:34.500948+0000","last_became_active":"2024-09-21T10:00:16.353074+0000","last_became_peered":"2024-09-21T10:00:16.353074+0000","last_unstale":"2024-09-21T10:04:34.500948+0000","last_undegraded":"2024-09-21T10:04:34.500948+0000","last_fullsized":"2024-09-21T10:04:34.500948+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:35.382228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1188'13702","reported_seq":22127,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501004+0000","last_change":"2024-09-21T10:03:24.437255+0000","last_active":"2024-09-21T10:04:34.501004+0000","last_peered":"2024-09-21T10:04:34.501004+0000","last_clean":"2024-09-21T10:04:34.501004+0000","last_became_active":"2024-09-21T10:03:24.413749+0000","last_became_peered":"2024-09-21T10:03:24.413749+0000","last_unstale":"2024-09-21T10:04:34.501004+0000","last_undegraded":"2024-09-21T10:04:34.501004+0000","last_fullsized":"2024-09-21T10:04:34.501004+0000","mapping_epoch":1178,"log_start":"1169'7300","ondisk_log_start":"1169'7300","created":1142,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6402,"log_dups_size":0,"ondisk_log_size":6402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:40.139577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13702,"num_write_kb":438464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4161,"num_bytes_recovered":272695296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1118,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.433173+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T10:04:34.433173+0000","last_peered":"2024-09-21T10:04:34.433173+0000","last_clean":"2024-09-21T10:04:34.433173+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T10:04:34.433173+0000","last_undegraded":"2024-09-21T10:04:34.433173+0000","last_fullsized":"2024-09-21T10:04:34.433173+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":126,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431457+0000","last_change":"2024-09-21T10:00:16.357541+0000","last_active":"2024-09-21T10:04:34.431457+0000","last_peered":"2024-09-21T10:04:34.431457+0000","last_clean":"2024-09-21T10:04:34.431457+0000","last_became_active":"2024-09-21T10:00:16.357209+0000","last_became_peered":"2024-09-21T10:00:16.357209+0000","last_unstale":"2024-09-21T10:04:34.431457+0000","last_undegraded":"2024-09-21T10:04:34.431457+0000","last_fullsized":"2024-09-21T10:04:34.431457+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:20.732470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.a","version":"1188'13864","reported_seq":13988,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449833+0000","last_change":"2024-09-21T10:00:46.477601+0000","last_active":"2024-09-21T10:04:36.449833+0000","last_peered":"2024-09-21T10:04:36.449833+0000","last_clean":"2024-09-21T10:04:36.449833+0000","last_became_active":"2024-09-21T10:00:46.477208+0000","last_became_peered":"2024-09-21T10:00:46.477208+0000","last_unstale":"2024-09-21T10:04:36.449833+0000","last_undegraded":"2024-09-21T10:04:36.449833+0000","last_fullsized":"2024-09-21T10:04:36.449833+0000","mapping_epoch":1142,"log_start":"1172'8700","ondisk_log_start":"1172'8700","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5164,"log_dups_size":0,"ondisk_log_size":5164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:54.514879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13864,"num_write_kb":443648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1133,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.433012+0000","last_change":"2024-09-21T10:01:40.429966+0000","last_active":"2024-09-21T10:04:34.433012+0000","last_peered":"2024-09-21T10:04:34.433012+0000","last_clean":"2024-09-21T10:04:34.433012+0000","last_became_active":"2024-09-21T10:01:40.419541+0000","last_became_peered":"2024-09-21T10:01:40.419541+0000","last_unstale":"2024-09-21T10:04:34.433012+0000","last_undegraded":"2024-09-21T10:04:34.433012+0000","last_fullsized":"2024-09-21T10:04:34.433012+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:10.848843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1162,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432581+0000","last_change":"2024-09-21T10:01:40.423234+0000","last_active":"2024-09-21T10:04:34.432581+0000","last_peered":"2024-09-21T10:04:34.432581+0000","last_clean":"2024-09-21T10:04:34.432581+0000","last_became_active":"2024-09-21T10:01:40.422854+0000","last_became_peered":"2024-09-21T10:01:40.422854+0000","last_unstale":"2024-09-21T10:04:34.432581+0000","last_undegraded":"2024-09-21T10:04:34.432581+0000","last_fullsized":"2024-09-21T10:04:34.432581+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:51.607040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1188'8376","reported_seq":12611,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442060+0000","last_change":"2024-09-21T10:04:29.156901+0000","last_active":"2024-09-21T10:04:38.442060+0000","last_peered":"2024-09-21T10:04:38.442060+0000","last_clean":"2024-09-21T10:04:38.442060+0000","last_became_active":"2024-09-21T10:02:40.405438+0000","last_became_peered":"2024-09-21T10:02:40.405438+0000","last_unstale":"2024-09-21T10:04:38.442060+0000","last_undegraded":"2024-09-21T10:04:38.442060+0000","last_fullsized":"2024-09-21T10:04:38.442060+0000","mapping_epoch":1170,"log_start":"1145'1780","ondisk_log_start":"1145'1780","created":1142,"last_epoch_clean":1171,"parent":"0.0","parent_split_bits":0,"last_scrub":"1188'8376","last_scrub_stamp":"2024-09-21T10:04:29.156849+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:29.156849+0000","objects_scrubbed":0,"log_size":6596,"log_dups_size":0,"ondisk_log_size":6596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:19:03.431149+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":1129,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432341+0000","last_change":"2024-09-21T10:00:16.373327+0000","last_active":"2024-09-21T10:04:34.432341+0000","last_peered":"2024-09-21T10:04:34.432341+0000","last_clean":"2024-09-21T10:04:34.432341+0000","last_became_active":"2024-09-21T10:00:16.360853+0000","last_became_peered":"2024-09-21T10:00:16.360853+0000","last_unstale":"2024-09-21T10:04:34.432341+0000","last_undegraded":"2024-09-21T10:04:34.432341+0000","last_fullsized":"2024-09-21T10:04:34.432341+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:18.160233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.552190+0000","last_change":"2024-09-21T10:01:40.417926+0000","last_active":"2024-09-21T10:04:34.552190+0000","last_peered":"2024-09-21T10:04:34.552190+0000","last_clean":"2024-09-21T10:04:34.552190+0000","last_became_active":"2024-09-21T10:01:40.417540+0000","last_became_peered":"2024-09-21T10:01:40.417540+0000","last_unstale":"2024-09-21T10:04:34.552190+0000","last_undegraded":"2024-09-21T10:04:34.552190+0000","last_fullsized":"2024-09-21T10:04:34.552190+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:43.890839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1188'8491","reported_seq":8631,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.430962+0000","last_change":"2024-09-21T10:04:34.753334+0000","last_active":"2024-09-21T10:04:36.430962+0000","last_peered":"2024-09-21T10:04:36.430962+0000","last_clean":"2024-09-21T10:04:36.430962+0000","last_became_active":"2024-09-21T10:01:27.365655+0000","last_became_peered":"2024-09-21T10:01:27.365655+0000","last_unstale":"2024-09-21T10:04:36.430962+0000","last_undegraded":"2024-09-21T10:04:36.430962+0000","last_fullsized":"2024-09-21T10:04:36.430962+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1188'8491","last_scrub_stamp":"2024-09-21T10:04:34.753285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:34.753285+0000","objects_scrubbed":0,"log_size":8491,"log_dups_size":0,"ondisk_log_size":8491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:50.281444+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":741,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449832+0000","last_change":"2024-09-21T10:00:16.360708+0000","last_active":"2024-09-21T10:04:36.449832+0000","last_peered":"2024-09-21T10:04:36.449832+0000","last_clean":"2024-09-21T10:04:36.449832+0000","last_became_active":"2024-09-21T10:00:16.360416+0000","last_became_peered":"2024-09-21T10:00:16.360416+0000","last_unstale":"2024-09-21T10:04:36.449832+0000","last_undegraded":"2024-09-21T10:04:36.449832+0000","last_fullsized":"2024-09-21T10:04:36.449832+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:59.090215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1824,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442496+0000","last_change":"2024-09-21T10:01:40.425332+0000","last_active":"2024-09-21T10:04:38.442496+0000","last_peered":"2024-09-21T10:04:38.442496+0000","last_clean":"2024-09-21T10:04:38.442496+0000","last_became_active":"2024-09-21T10:01:40.425034+0000","last_became_peered":"2024-09-21T10:01:40.425034+0000","last_unstale":"2024-09-21T10:04:38.442496+0000","last_undegraded":"2024-09-21T10:04:38.442496+0000","last_fullsized":"2024-09-21T10:04:38.442496+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:57.701200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1188'13724","reported_seq":13852,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442579+0000","last_change":"2024-09-21T10:00:46.479962+0000","last_active":"2024-09-21T10:04:38.442579+0000","last_peered":"2024-09-21T10:04:38.442579+0000","last_clean":"2024-09-21T10:04:38.442579+0000","last_became_active":"2024-09-21T10:00:46.479648+0000","last_became_peered":"2024-09-21T10:00:46.479648+0000","last_unstale":"2024-09-21T10:04:38.442579+0000","last_undegraded":"2024-09-21T10:04:38.442579+0000","last_fullsized":"2024-09-21T10:04:38.442579+0000","mapping_epoch":1142,"log_start":"1171'7800","ondisk_log_start":"1171'7800","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5924,"log_dups_size":0,"ondisk_log_size":5924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:15.365700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13724,"num_write_kb":439168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1264,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431718+0000","last_change":"2024-09-21T10:00:47.119852+0000","last_active":"2024-09-21T10:04:34.431718+0000","last_peered":"2024-09-21T10:04:34.431718+0000","last_clean":"2024-09-21T10:04:34.431718+0000","last_became_active":"2024-09-21T09:59:43.374051+0000","last_became_peered":"2024-09-21T09:59:43.374051+0000","last_unstale":"2024-09-21T10:04:34.431718+0000","last_undegraded":"2024-09-21T10:04:34.431718+0000","last_fullsized":"2024-09-21T10:04:34.431718+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:47.119793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:47.119793+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:23.087965+0000","scrub_duration":0.0013414659999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1737,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449635+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T10:04:36.449635+0000","last_peered":"2024-09-21T10:04:36.449635+0000","last_clean":"2024-09-21T10:04:36.449635+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T10:04:36.449635+0000","last_undegraded":"2024-09-21T10:04:36.449635+0000","last_fullsized":"2024-09-21T10:04:36.449635+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1155,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431961+0000","last_change":"2024-09-21T10:00:16.358954+0000","last_active":"2024-09-21T10:04:34.431961+0000","last_peered":"2024-09-21T10:04:34.431961+0000","last_clean":"2024-09-21T10:04:34.431961+0000","last_became_active":"2024-09-21T10:00:16.358416+0000","last_became_peered":"2024-09-21T10:00:16.358416+0000","last_unstale":"2024-09-21T10:04:34.431961+0000","last_undegraded":"2024-09-21T10:04:34.431961+0000","last_fullsized":"2024-09-21T10:04:34.431961+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:46.916076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1188'13762","reported_seq":13886,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.431302+0000","last_change":"2024-09-21T10:00:46.481001+0000","last_active":"2024-09-21T10:04:36.431302+0000","last_peered":"2024-09-21T10:04:36.431302+0000","last_clean":"2024-09-21T10:04:36.431302+0000","last_became_active":"2024-09-21T10:00:46.480689+0000","last_became_peered":"2024-09-21T10:00:46.480689+0000","last_unstale":"2024-09-21T10:04:36.431302+0000","last_undegraded":"2024-09-21T10:04:36.431302+0000","last_fullsized":"2024-09-21T10:04:36.431302+0000","mapping_epoch":1142,"log_start":"1166'6200","ondisk_log_start":"1166'6200","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":7562,"log_dups_size":0,"ondisk_log_size":7562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:29.607989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13762,"num_write_kb":440384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":180,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.431158+0000","last_change":"2024-09-21T10:00:15.524812+0000","last_active":"2024-09-21T10:04:36.431158+0000","last_peered":"2024-09-21T10:04:36.431158+0000","last_clean":"2024-09-21T10:04:36.431158+0000","last_became_active":"2024-09-21T09:59:43.361751+0000","last_became_peered":"2024-09-21T09:59:43.361751+0000","last_unstale":"2024-09-21T10:04:36.431158+0000","last_undegraded":"2024-09-21T10:04:36.431158+0000","last_fullsized":"2024-09-21T10:04:36.431158+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.306777+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":83,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.552111+0000","last_change":"2024-09-21T10:01:40.415754+0000","last_active":"2024-09-21T10:04:34.552111+0000","last_peered":"2024-09-21T10:04:34.552111+0000","last_clean":"2024-09-21T10:04:34.552111+0000","last_became_active":"2024-09-21T10:01:40.415428+0000","last_became_peered":"2024-09-21T10:01:40.415428+0000","last_unstale":"2024-09-21T10:04:34.552111+0000","last_undegraded":"2024-09-21T10:04:34.552111+0000","last_fullsized":"2024-09-21T10:04:34.552111+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:53.323411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1162,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432675+0000","last_change":"2024-09-21T10:01:40.414164+0000","last_active":"2024-09-21T10:04:34.432675+0000","last_peered":"2024-09-21T10:04:34.432675+0000","last_clean":"2024-09-21T10:04:34.432675+0000","last_became_active":"2024-09-21T10:01:40.413838+0000","last_became_peered":"2024-09-21T10:01:40.413838+0000","last_unstale":"2024-09-21T10:04:34.432675+0000","last_undegraded":"2024-09-21T10:04:34.432675+0000","last_fullsized":"2024-09-21T10:04:34.432675+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:13:24.380350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1188'13696","reported_seq":22172,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501500+0000","last_change":"2024-09-21T10:04:26.828099+0000","last_active":"2024-09-21T10:04:34.501500+0000","last_peered":"2024-09-21T10:04:34.501500+0000","last_clean":"2024-09-21T10:04:34.501500+0000","last_became_active":"2024-09-21T10:04:26.794805+0000","last_became_peered":"2024-09-21T10:04:26.794805+0000","last_unstale":"2024-09-21T10:04:34.501500+0000","last_undegraded":"2024-09-21T10:04:34.501500+0000","last_fullsized":"2024-09-21T10:04:34.501500+0000","mapping_epoch":1189,"log_start":"1169'7300","ondisk_log_start":"1169'7300","created":1142,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6396,"log_dups_size":0,"ondisk_log_size":6396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:43:46.797537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13696,"num_write_kb":438209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4195,"num_bytes_recovered":88932352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1136,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432583+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T10:04:34.432583+0000","last_peered":"2024-09-21T10:04:34.432583+0000","last_clean":"2024-09-21T10:04:34.432583+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T10:04:34.432583+0000","last_undegraded":"2024-09-21T10:04:34.432583+0000","last_fullsized":"2024-09-21T10:04:34.432583+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1165,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432478+0000","last_change":"2024-09-21T10:01:40.424536+0000","last_active":"2024-09-21T10:04:34.432478+0000","last_peered":"2024-09-21T10:04:34.432478+0000","last_clean":"2024-09-21T10:04:34.432478+0000","last_became_active":"2024-09-21T10:01:40.424229+0000","last_became_peered":"2024-09-21T10:01:40.424229+0000","last_unstale":"2024-09-21T10:04:34.432478+0000","last_undegraded":"2024-09-21T10:04:34.432478+0000","last_fullsized":"2024-09-21T10:04:34.432478+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:10.771540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1188'13608","reported_seq":13668,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501350+0000","last_change":"2024-09-21T10:00:46.477205+0000","last_active":"2024-09-21T10:04:34.501350+0000","last_peered":"2024-09-21T10:04:34.501350+0000","last_clean":"2024-09-21T10:04:34.501350+0000","last_became_active":"2024-09-21T10:00:46.476844+0000","last_became_peered":"2024-09-21T10:00:46.476844+0000","last_unstale":"2024-09-21T10:04:34.501350+0000","last_undegraded":"2024-09-21T10:04:34.501350+0000","last_fullsized":"2024-09-21T10:04:34.501350+0000","mapping_epoch":1142,"log_start":"1169'7200","ondisk_log_start":"1169'7200","created":1142,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:29:19.716166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13608,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":170,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.431749+0000","last_change":"2024-09-21T10:04:20.777769+0000","last_active":"2024-09-21T10:04:36.431749+0000","last_peered":"2024-09-21T10:04:36.431749+0000","last_clean":"2024-09-21T10:04:36.431749+0000","last_became_active":"2024-09-21T10:00:16.351010+0000","last_became_peered":"2024-09-21T10:00:16.351010+0000","last_unstale":"2024-09-21T10:04:36.431749+0000","last_undegraded":"2024-09-21T10:04:36.431749+0000","last_fullsized":"2024-09-21T10:04:36.431749+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-29T15:53:23.454847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1736,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.450209+0000","last_change":"2024-09-21T10:01:40.428561+0000","last_active":"2024-09-21T10:04:36.450209+0000","last_peered":"2024-09-21T10:04:36.450209+0000","last_clean":"2024-09-21T10:04:36.450209+0000","last_became_active":"2024-09-21T10:01:40.428250+0000","last_became_peered":"2024-09-21T10:01:40.428250+0000","last_unstale":"2024-09-21T10:04:36.450209+0000","last_undegraded":"2024-09-21T10:04:36.450209+0000","last_fullsized":"2024-09-21T10:04:36.450209+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:58.781370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":641,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.450454+0000","last_change":"2024-09-21T10:01:40.423834+0000","last_active":"2024-09-21T10:04:36.450454+0000","last_peered":"2024-09-21T10:04:36.450454+0000","last_clean":"2024-09-21T10:04:36.450454+0000","last_became_active":"2024-09-21T10:01:40.423506+0000","last_became_peered":"2024-09-21T10:01:40.423506+0000","last_unstale":"2024-09-21T10:04:36.450454+0000","last_undegraded":"2024-09-21T10:04:36.450454+0000","last_fullsized":"2024-09-21T10:04:36.450454+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:11.358742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.6","version":"1188'8531","reported_seq":8695,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442562+0000","last_change":"2024-09-21T10:03:12.185190+0000","last_active":"2024-09-21T10:04:38.442562+0000","last_peered":"2024-09-21T10:04:38.442562+0000","last_clean":"2024-09-21T10:04:38.442562+0000","last_became_active":"2024-09-21T10:01:27.352341+0000","last_became_peered":"2024-09-21T10:01:27.352341+0000","last_unstale":"2024-09-21T10:04:38.442562+0000","last_undegraded":"2024-09-21T10:04:38.442562+0000","last_fullsized":"2024-09-21T10:04:38.442562+0000","mapping_epoch":1142,"log_start":"1145'1964","ondisk_log_start":"1145'1964","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1176'6402","last_scrub_stamp":"2024-09-21T10:03:12.185040+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:03:12.185040+0000","objects_scrubbed":2137,"log_size":6567,"log_dups_size":0,"ondisk_log_size":6567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:19.650457+0000","scrub_duration":683,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2137,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1278,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431668+0000","last_change":"2024-09-21T10:00:49.116174+0000","last_active":"2024-09-21T10:04:34.431668+0000","last_peered":"2024-09-21T10:04:34.431668+0000","last_clean":"2024-09-21T10:04:34.431668+0000","last_became_active":"2024-09-21T10:00:16.360430+0000","last_became_peered":"2024-09-21T10:00:16.360430+0000","last_unstale":"2024-09-21T10:04:34.431668+0000","last_undegraded":"2024-09-21T10:04:34.431668+0000","last_fullsized":"2024-09-21T10:04:34.431668+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:49.116103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:49.116103+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:29.874911+0000","scrub_duration":0.0020386419999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":214,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.431305+0000","last_change":"2024-09-21T09:57:39.541461+0000","last_active":"2024-09-21T10:04:36.431305+0000","last_peered":"2024-09-21T10:04:36.431305+0000","last_clean":"2024-09-21T10:04:36.431305+0000","last_became_active":"2024-09-21T09:57:39.540927+0000","last_became_peered":"2024-09-21T09:57:39.540927+0000","last_unstale":"2024-09-21T10:04:36.431305+0000","last_undegraded":"2024-09-21T10:04:36.431305+0000","last_fullsized":"2024-09-21T10:04:36.431305+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:14.815783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1725,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.450063+0000","last_change":"2024-09-21T10:00:16.362131+0000","last_active":"2024-09-21T10:04:36.450063+0000","last_peered":"2024-09-21T10:04:36.450063+0000","last_clean":"2024-09-21T10:04:36.450063+0000","last_became_active":"2024-09-21T10:00:16.361722+0000","last_became_peered":"2024-09-21T10:00:16.361722+0000","last_unstale":"2024-09-21T10:04:36.450063+0000","last_undegraded":"2024-09-21T10:04:36.450063+0000","last_fullsized":"2024-09-21T10:04:36.450063+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:38.717426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":73,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501393+0000","last_change":"2024-09-21T10:00:16.361540+0000","last_active":"2024-09-21T10:04:34.501393+0000","last_peered":"2024-09-21T10:04:34.501393+0000","last_clean":"2024-09-21T10:04:34.501393+0000","last_became_active":"2024-09-21T10:00:16.361086+0000","last_became_peered":"2024-09-21T10:00:16.361086+0000","last_unstale":"2024-09-21T10:04:34.501393+0000","last_undegraded":"2024-09-21T10:04:34.501393+0000","last_fullsized":"2024-09-21T10:04:34.501393+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:14.645896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1188'8413","reported_seq":8577,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432094+0000","last_change":"2024-09-21T10:01:58.715049+0000","last_active":"2024-09-21T10:04:34.432094+0000","last_peered":"2024-09-21T10:04:34.432094+0000","last_clean":"2024-09-21T10:04:34.432094+0000","last_became_active":"2024-09-21T10:01:27.359228+0000","last_became_peered":"2024-09-21T10:01:27.359228+0000","last_unstale":"2024-09-21T10:04:34.432094+0000","last_undegraded":"2024-09-21T10:04:34.432094+0000","last_fullsized":"2024-09-21T10:04:34.432094+0000","mapping_epoch":1142,"log_start":"1145'1843","ondisk_log_start":"1145'1843","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1162'3998","last_scrub_stamp":"2024-09-21T10:01:58.714971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:01:58.714971+0000","objects_scrubbed":2286,"log_size":6570,"log_dups_size":0,"ondisk_log_size":6570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:21.752075+0000","scrub_duration":0.73731484599999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4428,"num_write_kb":68544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":78,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501006+0000","last_change":"2024-09-21T10:01:40.412152+0000","last_active":"2024-09-21T10:04:34.501006+0000","last_peered":"2024-09-21T10:04:34.501006+0000","last_clean":"2024-09-21T10:04:34.501006+0000","last_became_active":"2024-09-21T10:01:40.411508+0000","last_became_peered":"2024-09-21T10:01:40.411508+0000","last_unstale":"2024-09-21T10:04:34.501006+0000","last_undegraded":"2024-09-21T10:04:34.501006+0000","last_fullsized":"2024-09-21T10:04:34.501006+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:21.317838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:21.317838+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1156,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431578+0000","last_change":"2024-09-21T10:01:40.428393+0000","last_active":"2024-09-21T10:04:34.431578+0000","last_peered":"2024-09-21T10:04:34.431578+0000","last_clean":"2024-09-21T10:04:34.431578+0000","last_became_active":"2024-09-21T10:01:40.428204+0000","last_became_peered":"2024-09-21T10:01:40.428204+0000","last_unstale":"2024-09-21T10:04:34.431578+0000","last_undegraded":"2024-09-21T10:04:34.431578+0000","last_fullsized":"2024-09-21T10:04:34.431578+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:23:02.136035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1816,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442196+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T10:04:38.442196+0000","last_peered":"2024-09-21T10:04:38.442196+0000","last_clean":"2024-09-21T10:04:38.442196+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T10:04:38.442196+0000","last_undegraded":"2024-09-21T10:04:38.442196+0000","last_fullsized":"2024-09-21T10:04:38.442196+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1836,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442763+0000","last_change":"2024-09-21T10:01:40.421887+0000","last_active":"2024-09-21T10:04:38.442763+0000","last_peered":"2024-09-21T10:04:38.442763+0000","last_clean":"2024-09-21T10:04:38.442763+0000","last_became_active":"2024-09-21T10:01:40.420730+0000","last_became_peered":"2024-09-21T10:01:40.420730+0000","last_unstale":"2024-09-21T10:04:38.442763+0000","last_undegraded":"2024-09-21T10:04:38.442763+0000","last_fullsized":"2024-09-21T10:04:38.442763+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:51.504051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"1188'8362","reported_seq":12687,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442440+0000","last_change":"2024-09-21T10:04:35.129926+0000","last_active":"2024-09-21T10:04:38.442440+0000","last_peered":"2024-09-21T10:04:38.442440+0000","last_clean":"2024-09-21T10:04:38.442440+0000","last_became_active":"2024-09-21T10:01:56.596893+0000","last_became_peered":"2024-09-21T10:01:56.596893+0000","last_unstale":"2024-09-21T10:04:38.442440+0000","last_undegraded":"2024-09-21T10:04:38.442440+0000","last_fullsized":"2024-09-21T10:04:38.442440+0000","mapping_epoch":1161,"log_start":"1145'1739","ondisk_log_start":"1145'1739","created":1142,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"1188'8362","last_scrub_stamp":"2024-09-21T10:04:35.129822+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:35.129822+0000","objects_scrubbed":0,"log_size":6623,"log_dups_size":0,"ondisk_log_size":6623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:19:15.099392+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":1264,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431964+0000","last_change":"2024-09-21T10:00:16.913196+0000","last_active":"2024-09-21T10:04:34.431964+0000","last_peered":"2024-09-21T10:04:34.431964+0000","last_clean":"2024-09-21T10:04:34.431964+0000","last_became_active":"2024-09-21T09:59:43.370454+0000","last_became_peered":"2024-09-21T09:59:43.370454+0000","last_unstale":"2024-09-21T10:04:34.431964+0000","last_undegraded":"2024-09-21T10:04:34.431964+0000","last_fullsized":"2024-09-21T10:04:34.431964+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:16.913146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:16.913146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:23:33.788563+0000","scrub_duration":0.001342315,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1786,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.431128+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:04:36.431128+0000","last_peered":"2024-09-21T10:04:36.431128+0000","last_clean":"2024-09-21T10:04:36.431128+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:04:36.431128+0000","last_undegraded":"2024-09-21T10:04:36.431128+0000","last_fullsized":"2024-09-21T10:04:36.431128+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1741,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.431092+0000","last_change":"2024-09-21T10:01:40.414547+0000","last_active":"2024-09-21T10:04:36.431092+0000","last_peered":"2024-09-21T10:04:36.431092+0000","last_clean":"2024-09-21T10:04:36.431092+0000","last_became_active":"2024-09-21T10:01:40.414243+0000","last_became_peered":"2024-09-21T10:01:40.414243+0000","last_unstale":"2024-09-21T10:04:36.431092+0000","last_undegraded":"2024-09-21T10:04:36.431092+0000","last_fullsized":"2024-09-21T10:04:36.431092+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:55:26.044481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":526,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442782+0000","last_change":"2024-09-21T10:01:40.421427+0000","last_active":"2024-09-21T10:04:38.442782+0000","last_peered":"2024-09-21T10:04:38.442782+0000","last_clean":"2024-09-21T10:04:38.442782+0000","last_became_active":"2024-09-21T10:01:40.421047+0000","last_became_peered":"2024-09-21T10:01:40.421047+0000","last_unstale":"2024-09-21T10:04:38.442782+0000","last_undegraded":"2024-09-21T10:04:38.442782+0000","last_fullsized":"2024-09-21T10:04:38.442782+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:48.640296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.7","version":"1188'8446","reported_seq":8526,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501399+0000","last_change":"2024-09-21T10:04:32.009730+0000","last_active":"2024-09-21T10:04:34.501399+0000","last_peered":"2024-09-21T10:04:34.501399+0000","last_clean":"2024-09-21T10:04:34.501399+0000","last_became_active":"2024-09-21T10:01:27.358189+0000","last_became_peered":"2024-09-21T10:01:27.358189+0000","last_unstale":"2024-09-21T10:04:34.501399+0000","last_undegraded":"2024-09-21T10:04:34.501399+0000","last_fullsized":"2024-09-21T10:04:34.501399+0000","mapping_epoch":1142,"log_start":"1145'788","ondisk_log_start":"1145'788","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1188'8446","last_scrub_stamp":"2024-09-21T10:04:32.009646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:32.009646+0000","objects_scrubbed":0,"log_size":7658,"log_dups_size":0,"ondisk_log_size":7658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:26.548851+0000","scrub_duration":0.0050810100000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":513,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442195+0000","last_change":"2024-09-21T10:03:19.415715+0000","last_active":"2024-09-21T10:04:38.442195+0000","last_peered":"2024-09-21T10:04:38.442195+0000","last_clean":"2024-09-21T10:04:38.442195+0000","last_became_active":"2024-09-21T09:59:43.371384+0000","last_became_peered":"2024-09-21T09:59:43.371384+0000","last_unstale":"2024-09-21T10:04:38.442195+0000","last_undegraded":"2024-09-21T10:04:38.442195+0000","last_fullsized":"2024-09-21T10:04:38.442195+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-30T13:30:29.665440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":151,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.551957+0000","last_change":"2024-09-21T10:01:40.426865+0000","last_active":"2024-09-21T10:04:34.551957+0000","last_peered":"2024-09-21T10:04:34.551957+0000","last_clean":"2024-09-21T10:04:34.551957+0000","last_became_active":"2024-09-21T10:01:40.413152+0000","last_became_peered":"2024-09-21T10:01:40.413152+0000","last_unstale":"2024-09-21T10:04:34.551957+0000","last_undegraded":"2024-09-21T10:04:34.551957+0000","last_fullsized":"2024-09-21T10:04:34.551957+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:59.034627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1156,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432853+0000","last_change":"2024-09-21T10:00:16.356367+0000","last_active":"2024-09-21T10:04:34.432853+0000","last_peered":"2024-09-21T10:04:34.432853+0000","last_clean":"2024-09-21T10:04:34.432853+0000","last_became_active":"2024-09-21T10:00:16.356079+0000","last_became_peered":"2024-09-21T10:00:16.356079+0000","last_unstale":"2024-09-21T10:04:34.432853+0000","last_undegraded":"2024-09-21T10:04:34.432853+0000","last_fullsized":"2024-09-21T10:04:34.432853+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:12.934640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":652,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.431529+0000","last_change":"2024-09-21T10:01:40.421262+0000","last_active":"2024-09-21T10:04:36.431529+0000","last_peered":"2024-09-21T10:04:36.431529+0000","last_clean":"2024-09-21T10:04:36.431529+0000","last_became_active":"2024-09-21T10:01:40.420959+0000","last_became_peered":"2024-09-21T10:01:40.420959+0000","last_unstale":"2024-09-21T10:04:36.431529+0000","last_undegraded":"2024-09-21T10:04:36.431529+0000","last_fullsized":"2024-09-21T10:04:36.431529+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:06.866421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1","version":"1188'8604","reported_seq":8739,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432154+0000","last_change":"2024-09-21T10:04:26.541588+0000","last_active":"2024-09-21T10:04:34.432154+0000","last_peered":"2024-09-21T10:04:34.432154+0000","last_clean":"2024-09-21T10:04:34.432154+0000","last_became_active":"2024-09-21T10:01:27.378346+0000","last_became_peered":"2024-09-21T10:01:27.378346+0000","last_unstale":"2024-09-21T10:04:34.432154+0000","last_undegraded":"2024-09-21T10:04:34.432154+0000","last_fullsized":"2024-09-21T10:04:34.432154+0000","mapping_epoch":1142,"log_start":"1145'1279","ondisk_log_start":"1145'1279","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1188'8604","last_scrub_stamp":"2024-09-21T10:04:26.541527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:26.541527+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:50:26.848592+0000","scrub_duration":0.0048044259999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":83,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501515+0000","last_change":"2024-09-21T10:00:46.494180+0000","last_active":"2024-09-21T10:04:34.501515+0000","last_peered":"2024-09-21T10:04:34.501515+0000","last_clean":"2024-09-21T10:04:34.501515+0000","last_became_active":"2024-09-21T10:00:16.355386+0000","last_became_peered":"2024-09-21T10:00:16.355386+0000","last_unstale":"2024-09-21T10:04:34.501515+0000","last_undegraded":"2024-09-21T10:04:34.501515+0000","last_fullsized":"2024-09-21T10:04:34.501515+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:46.494129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:46.494129+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:11.193593+0000","scrub_duration":0.00183863,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":743,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449940+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T10:04:36.449940+0000","last_peered":"2024-09-21T10:04:36.449940+0000","last_clean":"2024-09-21T10:04:36.449940+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T10:04:36.449940+0000","last_undegraded":"2024-09-21T10:04:36.449940+0000","last_fullsized":"2024-09-21T10:04:36.449940+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":555,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442678+0000","last_change":"2024-09-21T10:01:40.418328+0000","last_active":"2024-09-21T10:04:38.442678+0000","last_peered":"2024-09-21T10:04:38.442678+0000","last_clean":"2024-09-21T10:04:38.442678+0000","last_became_active":"2024-09-21T10:01:40.418011+0000","last_became_peered":"2024-09-21T10:01:40.418011+0000","last_unstale":"2024-09-21T10:04:38.442678+0000","last_undegraded":"2024-09-21T10:04:38.442678+0000","last_fullsized":"2024-09-21T10:04:38.442678+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:58.716988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":497,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431588+0000","last_change":"2024-09-21T10:01:40.421225+0000","last_active":"2024-09-21T10:04:34.431588+0000","last_peered":"2024-09-21T10:04:34.431588+0000","last_clean":"2024-09-21T10:04:34.431588+0000","last_became_active":"2024-09-21T10:01:40.420909+0000","last_became_peered":"2024-09-21T10:01:40.420909+0000","last_unstale":"2024-09-21T10:04:34.431588+0000","last_undegraded":"2024-09-21T10:04:34.431588+0000","last_fullsized":"2024-09-21T10:04:34.431588+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:03.811941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.0","version":"1188'8325","reported_seq":12665,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449452+0000","last_change":"2024-09-21T10:04:30.415047+0000","last_active":"2024-09-21T10:04:36.449452+0000","last_peered":"2024-09-21T10:04:36.449452+0000","last_clean":"2024-09-21T10:04:36.449452+0000","last_became_active":"2024-09-21T10:01:55.581482+0000","last_became_peered":"2024-09-21T10:01:55.581482+0000","last_unstale":"2024-09-21T10:04:36.449452+0000","last_undegraded":"2024-09-21T10:04:36.449452+0000","last_fullsized":"2024-09-21T10:04:36.449452+0000","mapping_epoch":1160,"log_start":"1147'3152","ondisk_log_start":"1147'3152","created":1142,"last_epoch_clean":1161,"parent":"0.0","parent_split_bits":0,"last_scrub":"1188'8325","last_scrub_stamp":"2024-09-21T10:04:30.414993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:30.414993+0000","objects_scrubbed":0,"log_size":5173,"log_dups_size":0,"ondisk_log_size":5173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:08.287747+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":83,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501383+0000","last_change":"2024-09-21T10:00:20.637858+0000","last_active":"2024-09-21T10:04:34.501383+0000","last_peered":"2024-09-21T10:04:34.501383+0000","last_clean":"2024-09-21T10:04:34.501383+0000","last_became_active":"2024-09-21T10:00:16.353796+0000","last_became_peered":"2024-09-21T10:00:16.353796+0000","last_unstale":"2024-09-21T10:04:34.501383+0000","last_undegraded":"2024-09-21T10:04:34.501383+0000","last_fullsized":"2024-09-21T10:04:34.501383+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:20.637806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:20.637806+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:12:16.884495+0000","scrub_duration":0.0016753919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":578,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.431235+0000","last_change":"2024-09-21T10:01:40.414536+0000","last_active":"2024-09-21T10:04:36.431235+0000","last_peered":"2024-09-21T10:04:36.431235+0000","last_clean":"2024-09-21T10:04:36.431235+0000","last_became_active":"2024-09-21T10:01:40.413913+0000","last_became_peered":"2024-09-21T10:01:40.413913+0000","last_unstale":"2024-09-21T10:04:36.431235+0000","last_undegraded":"2024-09-21T10:04:36.431235+0000","last_fullsized":"2024-09-21T10:04:36.431235+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:27.994971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1129,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432378+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T10:04:34.432378+0000","last_peered":"2024-09-21T10:04:34.432378+0000","last_clean":"2024-09-21T10:04:34.432378+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T10:04:34.432378+0000","last_undegraded":"2024-09-21T10:04:34.432378+0000","last_fullsized":"2024-09-21T10:04:34.432378+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1807,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442655+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T10:04:38.442655+0000","last_peered":"2024-09-21T10:04:38.442655+0000","last_clean":"2024-09-21T10:04:38.442655+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T10:04:38.442655+0000","last_undegraded":"2024-09-21T10:04:38.442655+0000","last_fullsized":"2024-09-21T10:04:38.442655+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1188'8572","reported_seq":8711,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449438+0000","last_change":"2024-09-21T10:04:29.374076+0000","last_active":"2024-09-21T10:04:36.449438+0000","last_peered":"2024-09-21T10:04:36.449438+0000","last_clean":"2024-09-21T10:04:36.449438+0000","last_became_active":"2024-09-21T10:01:27.382492+0000","last_became_peered":"2024-09-21T10:01:27.382492+0000","last_unstale":"2024-09-21T10:04:36.449438+0000","last_undegraded":"2024-09-21T10:04:36.449438+0000","last_fullsized":"2024-09-21T10:04:36.449438+0000","mapping_epoch":1142,"log_start":"1149'3459","ondisk_log_start":"1149'3459","created":1142,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":0,"last_scrub":"1188'8572","last_scrub_stamp":"2024-09-21T10:04:29.374020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:29.374020+0000","objects_scrubbed":0,"log_size":5113,"log_dups_size":0,"ondisk_log_size":5113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:29.367458+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":103,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432159+0000","last_change":"2024-09-21T10:00:16.356251+0000","last_active":"2024-09-21T10:04:34.432159+0000","last_peered":"2024-09-21T10:04:34.432159+0000","last_clean":"2024-09-21T10:04:34.432159+0000","last_became_active":"2024-09-21T10:00:16.355833+0000","last_became_peered":"2024-09-21T10:00:16.355833+0000","last_unstale":"2024-09-21T10:04:34.432159+0000","last_undegraded":"2024-09-21T10:04:34.432159+0000","last_fullsized":"2024-09-21T10:04:34.432159+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T09:59:47.767654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:59:47.767654+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:50.939198+0000","scrub_duration":0.006614371,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1738,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.450481+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T10:04:36.450481+0000","last_peered":"2024-09-21T10:04:36.450481+0000","last_clean":"2024-09-21T10:04:36.450481+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T10:04:36.450481+0000","last_undegraded":"2024-09-21T10:04:36.450481+0000","last_fullsized":"2024-09-21T10:04:36.450481+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1697,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501446+0000","last_change":"2024-09-21T10:01:40.423209+0000","last_active":"2024-09-21T10:04:34.501446+0000","last_peered":"2024-09-21T10:04:34.501446+0000","last_clean":"2024-09-21T10:04:34.501446+0000","last_became_active":"2024-09-21T10:01:40.422890+0000","last_became_peered":"2024-09-21T10:01:40.422890+0000","last_unstale":"2024-09-21T10:04:34.501446+0000","last_undegraded":"2024-09-21T10:04:34.501446+0000","last_fullsized":"2024-09-21T10:04:34.501446+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:10.683126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1735,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449461+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T10:04:36.449461+0000","last_peered":"2024-09-21T10:04:36.449461+0000","last_clean":"2024-09-21T10:04:36.449461+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T10:04:36.449461+0000","last_undegraded":"2024-09-21T10:04:36.449461+0000","last_fullsized":"2024-09-21T10:04:36.449461+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1188'8567","reported_seq":12999,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449959+0000","last_change":"2024-09-21T10:04:18.771690+0000","last_active":"2024-09-21T10:04:36.449959+0000","last_peered":"2024-09-21T10:04:36.449959+0000","last_clean":"2024-09-21T10:04:36.449959+0000","last_became_active":"2024-09-21T10:04:18.750524+0000","last_became_peered":"2024-09-21T10:04:18.750524+0000","last_unstale":"2024-09-21T10:04:36.449959+0000","last_undegraded":"2024-09-21T10:04:36.449959+0000","last_fullsized":"2024-09-21T10:04:36.449959+0000","mapping_epoch":1185,"log_start":"1149'3404","ondisk_log_start":"1149'3404","created":1142,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5163,"log_dups_size":0,"ondisk_log_size":5163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6892,"num_write_kb":219936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2144,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":168,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449575+0000","last_change":"2024-09-21T10:00:22.052497+0000","last_active":"2024-09-21T10:04:36.449575+0000","last_peered":"2024-09-21T10:04:36.449575+0000","last_clean":"2024-09-21T10:04:36.449575+0000","last_became_active":"2024-09-21T09:59:43.368157+0000","last_became_peered":"2024-09-21T09:59:43.368157+0000","last_unstale":"2024-09-21T10:04:36.449575+0000","last_undegraded":"2024-09-21T10:04:36.449575+0000","last_fullsized":"2024-09-21T10:04:36.449575+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-28T12:21:36.382264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1831,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.443036+0000","last_change":"2024-09-21T10:01:40.433067+0000","last_active":"2024-09-21T10:04:38.443036+0000","last_peered":"2024-09-21T10:04:38.443036+0000","last_clean":"2024-09-21T10:04:38.443036+0000","last_became_active":"2024-09-21T10:01:40.420622+0000","last_became_peered":"2024-09-21T10:01:40.420622+0000","last_unstale":"2024-09-21T10:04:38.443036+0000","last_undegraded":"2024-09-21T10:04:38.443036+0000","last_fullsized":"2024-09-21T10:04:38.443036+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.383227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":73,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501410+0000","last_change":"2024-09-21T10:00:16.355102+0000","last_active":"2024-09-21T10:04:34.501410+0000","last_peered":"2024-09-21T10:04:34.501410+0000","last_clean":"2024-09-21T10:04:34.501410+0000","last_became_active":"2024-09-21T10:00:16.354473+0000","last_became_peered":"2024-09-21T10:00:16.354473+0000","last_unstale":"2024-09-21T10:04:34.501410+0000","last_undegraded":"2024-09-21T10:04:34.501410+0000","last_fullsized":"2024-09-21T10:04:34.501410+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:49:35.449111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1138,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432375+0000","last_change":"2024-09-21T10:00:16.355601+0000","last_active":"2024-09-21T10:04:34.432375+0000","last_peered":"2024-09-21T10:04:34.432375+0000","last_clean":"2024-09-21T10:04:34.432375+0000","last_became_active":"2024-09-21T10:00:16.355110+0000","last_became_peered":"2024-09-21T10:00:16.355110+0000","last_unstale":"2024-09-21T10:04:34.432375+0000","last_undegraded":"2024-09-21T10:04:34.432375+0000","last_fullsized":"2024-09-21T10:04:34.432375+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:10.590252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.10","version":"1188'8347","reported_seq":14689,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449258+0000","last_change":"2024-09-21T10:04:21.789678+0000","last_active":"2024-09-21T10:04:36.449258+0000","last_peered":"2024-09-21T10:04:36.449258+0000","last_clean":"2024-09-21T10:04:36.449258+0000","last_became_active":"2024-09-21T10:04:21.766324+0000","last_became_peered":"2024-09-21T10:04:21.766324+0000","last_unstale":"2024-09-21T10:04:36.449258+0000","last_undegraded":"2024-09-21T10:04:36.449258+0000","last_fullsized":"2024-09-21T10:04:36.449258+0000","mapping_epoch":1187,"log_start":"1147'3179","ondisk_log_start":"1147'3179","created":1148,"last_epoch_clean":1188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:00:45.452921+0000","objects_scrubbed":0,"log_size":5168,"log_dups_size":0,"ondisk_log_size":5168,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2752512,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6730,"num_write_kb":216704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2330,"num_bytes_recovered":121176064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1165,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432687+0000","last_change":"2024-09-21T10:01:40.414117+0000","last_active":"2024-09-21T10:04:34.432687+0000","last_peered":"2024-09-21T10:04:34.432687+0000","last_clean":"2024-09-21T10:04:34.432687+0000","last_became_active":"2024-09-21T10:01:40.413186+0000","last_became_peered":"2024-09-21T10:01:40.413186+0000","last_unstale":"2024-09-21T10:04:34.432687+0000","last_undegraded":"2024-09-21T10:04:34.432687+0000","last_fullsized":"2024-09-21T10:04:34.432687+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:17:05.368450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1181,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432048+0000","last_change":"2024-09-21T10:00:16.371426+0000","last_active":"2024-09-21T10:04:34.432048+0000","last_peered":"2024-09-21T10:04:34.432048+0000","last_clean":"2024-09-21T10:04:34.432048+0000","last_became_active":"2024-09-21T10:00:16.357486+0000","last_became_peered":"2024-09-21T10:00:16.357486+0000","last_unstale":"2024-09-21T10:04:34.432048+0000","last_undegraded":"2024-09-21T10:04:34.432048+0000","last_fullsized":"2024-09-21T10:04:34.432048+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:02.923682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.11","version":"1188'8605","reported_seq":11780,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:35.080213+0000","last_change":"2024-09-21T10:04:35.080136+0000","last_active":"2024-09-21T10:04:35.080213+0000","last_peered":"2024-09-21T10:04:35.080213+0000","last_clean":"2024-09-21T10:04:35.080213+0000","last_became_active":"2024-09-21T10:04:26.793247+0000","last_became_peered":"2024-09-21T10:04:26.793247+0000","last_unstale":"2024-09-21T10:04:35.080213+0000","last_undegraded":"2024-09-21T10:04:35.080213+0000","last_fullsized":"2024-09-21T10:04:35.080213+0000","mapping_epoch":1189,"log_start":"1149'3437","ondisk_log_start":"1149'3437","created":1148,"last_epoch_clean":1190,"parent":"0.0","parent_split_bits":5,"last_scrub":"1188'8605","last_scrub_stamp":"2024-09-21T10:04:35.080036+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:35.080036+0000","objects_scrubbed":0,"log_size":5168,"log_dups_size":0,"ondisk_log_size":5168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:11.396181+0000","scrub_duration":0.0046519830000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1698,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.552056+0000","last_change":"2024-09-21T10:01:40.413147+0000","last_active":"2024-09-21T10:04:34.552056+0000","last_peered":"2024-09-21T10:04:34.552056+0000","last_clean":"2024-09-21T10:04:34.552056+0000","last_became_active":"2024-09-21T10:01:40.412833+0000","last_became_peered":"2024-09-21T10:01:40.412833+0000","last_unstale":"2024-09-21T10:04:34.552056+0000","last_undegraded":"2024-09-21T10:04:34.552056+0000","last_fullsized":"2024-09-21T10:04:34.552056+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:33.262350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1120,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432296+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T10:04:34.432296+0000","last_peered":"2024-09-21T10:04:34.432296+0000","last_clean":"2024-09-21T10:04:34.432296+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T10:04:34.432296+0000","last_undegraded":"2024-09-21T10:04:34.432296+0000","last_fullsized":"2024-09-21T10:04:34.432296+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.12","version":"1188'8494","reported_seq":12879,"reported_epoch":1194,"state":"peering","last_fresh":"2024-09-21T10:04:35.428367+0000","last_change":"2024-09-21T10:04:35.428609+0000","last_active":"2024-09-21T10:04:35.428332+0000","last_peered":"2024-09-21T10:04:34.574507+0000","last_clean":"2024-09-21T10:04:24.095099+0000","last_became_active":"2024-09-21T10:04:34.552877+0000","last_became_peered":"2024-09-21T10:04:34.552877+0000","last_unstale":"2024-09-21T10:04:35.428367+0000","last_undegraded":"2024-09-21T10:04:35.428367+0000","last_fullsized":"2024-09-21T10:04:35.428367+0000","mapping_epoch":1194,"log_start":"1149'3356","ondisk_log_start":"1149'3356","created":1148,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":5,"last_scrub":"1188'8494","last_scrub_stamp":"2024-09-21T10:04:32.393159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:32.393159+0000","objects_scrubbed":0,"log_size":5138,"log_dups_size":0,"ondisk_log_size":5138,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1277952,"num_objects":20,"num_object_clones":0,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6820,"num_write_kb":218848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":33423360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,3,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":574,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432938+0000","last_change":"2024-09-21T10:01:40.427109+0000","last_active":"2024-09-21T10:04:34.432938+0000","last_peered":"2024-09-21T10:04:34.432938+0000","last_clean":"2024-09-21T10:04:34.432938+0000","last_became_active":"2024-09-21T10:01:40.426821+0000","last_became_peered":"2024-09-21T10:01:40.426821+0000","last_unstale":"2024-09-21T10:04:34.432938+0000","last_undegraded":"2024-09-21T10:04:34.432938+0000","last_fullsized":"2024-09-21T10:04:34.432938+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:12.260797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1724,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449568+0000","last_change":"2024-09-21T10:00:16.357151+0000","last_active":"2024-09-21T10:04:36.449568+0000","last_peered":"2024-09-21T10:04:36.449568+0000","last_clean":"2024-09-21T10:04:36.449568+0000","last_became_active":"2024-09-21T10:00:16.356864+0000","last_became_peered":"2024-09-21T10:00:16.356864+0000","last_unstale":"2024-09-21T10:04:36.449568+0000","last_undegraded":"2024-09-21T10:04:36.449568+0000","last_fullsized":"2024-09-21T10:04:36.449568+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:52.851887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"1188'8507","reported_seq":8644,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.469907+0000","last_change":"2024-09-21T10:04:38.469593+0000","last_active":"2024-09-21T10:04:38.469907+0000","last_peered":"2024-09-21T10:04:38.469907+0000","last_clean":"2024-09-21T10:04:38.469907+0000","last_became_active":"2024-09-21T10:04:38.445285+0000","last_became_peered":"2024-09-21T10:04:38.445285+0000","last_unstale":"2024-09-21T10:04:38.469907+0000","last_undegraded":"2024-09-21T10:04:38.469907+0000","last_fullsized":"2024-09-21T10:04:38.469907+0000","mapping_epoch":1196,"log_start":"1149'3392","ondisk_log_start":"1149'3392","created":1148,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":5,"last_scrub":"1188'8507","last_scrub_stamp":"2024-09-21T10:04:36.773423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:36.773423+0000","objects_scrubbed":0,"log_size":5115,"log_dups_size":0,"ondisk_log_size":5115,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1933312,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6862,"num_write_kb":220512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1815,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442922+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T10:04:38.442922+0000","last_peered":"2024-09-21T10:04:38.442922+0000","last_clean":"2024-09-21T10:04:38.442922+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T10:04:38.442922+0000","last_undegraded":"2024-09-21T10:04:38.442922+0000","last_fullsized":"2024-09-21T10:04:38.442922+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1748,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449367+0000","last_change":"2024-09-21T10:01:40.426702+0000","last_active":"2024-09-21T10:04:36.449367+0000","last_peered":"2024-09-21T10:04:36.449367+0000","last_clean":"2024-09-21T10:04:36.449367+0000","last_became_active":"2024-09-21T10:01:40.426034+0000","last_became_peered":"2024-09-21T10:01:40.426034+0000","last_unstale":"2024-09-21T10:04:36.449367+0000","last_undegraded":"2024-09-21T10:04:36.449367+0000","last_fullsized":"2024-09-21T10:04:36.449367+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:20.619421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":83,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.501501+0000","last_change":"2024-09-21T10:01:40.414393+0000","last_active":"2024-09-21T10:04:34.501501+0000","last_peered":"2024-09-21T10:04:34.501501+0000","last_clean":"2024-09-21T10:04:34.501501+0000","last_became_active":"2024-09-21T10:01:40.414096+0000","last_became_peered":"2024-09-21T10:01:40.414096+0000","last_unstale":"2024-09-21T10:04:34.501501+0000","last_undegraded":"2024-09-21T10:04:34.501501+0000","last_fullsized":"2024-09-21T10:04:34.501501+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:32.540479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1752,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.450024+0000","last_change":"2024-09-21T10:01:40.434475+0000","last_active":"2024-09-21T10:04:36.450024+0000","last_peered":"2024-09-21T10:04:36.450024+0000","last_clean":"2024-09-21T10:04:36.450024+0000","last_became_active":"2024-09-21T10:01:40.432013+0000","last_became_peered":"2024-09-21T10:01:40.432013+0000","last_unstale":"2024-09-21T10:04:36.450024+0000","last_undegraded":"2024-09-21T10:04:36.450024+0000","last_fullsized":"2024-09-21T10:04:36.450024+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:44.373378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1818,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442912+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T10:04:38.442912+0000","last_peered":"2024-09-21T10:04:38.442912+0000","last_clean":"2024-09-21T10:04:38.442912+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T10:04:38.442912+0000","last_undegraded":"2024-09-21T10:04:38.442912+0000","last_fullsized":"2024-09-21T10:04:38.442912+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":642,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449729+0000","last_change":"2024-09-21T10:01:40.437120+0000","last_active":"2024-09-21T10:04:36.449729+0000","last_peered":"2024-09-21T10:04:36.449729+0000","last_clean":"2024-09-21T10:04:36.449729+0000","last_became_active":"2024-09-21T10:01:40.436275+0000","last_became_peered":"2024-09-21T10:01:40.436275+0000","last_unstale":"2024-09-21T10:04:36.449729+0000","last_undegraded":"2024-09-21T10:04:36.449729+0000","last_fullsized":"2024-09-21T10:04:36.449729+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:25.322111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1734,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449757+0000","last_change":"2024-09-21T10:01:40.427770+0000","last_active":"2024-09-21T10:04:36.449757+0000","last_peered":"2024-09-21T10:04:36.449757+0000","last_clean":"2024-09-21T10:04:36.449757+0000","last_became_active":"2024-09-21T10:01:40.427458+0000","last_became_peered":"2024-09-21T10:01:40.427458+0000","last_unstale":"2024-09-21T10:04:36.449757+0000","last_undegraded":"2024-09-21T10:04:36.449757+0000","last_fullsized":"2024-09-21T10:04:36.449757+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:38.605007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1164,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431808+0000","last_change":"2024-09-21T10:01:40.420505+0000","last_active":"2024-09-21T10:04:34.431808+0000","last_peered":"2024-09-21T10:04:34.431808+0000","last_clean":"2024-09-21T10:04:34.431808+0000","last_became_active":"2024-09-21T10:01:40.420263+0000","last_became_peered":"2024-09-21T10:01:40.420263+0000","last_unstale":"2024-09-21T10:04:34.431808+0000","last_undegraded":"2024-09-21T10:04:34.431808+0000","last_fullsized":"2024-09-21T10:04:34.431808+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:23.452408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1772,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.431369+0000","last_change":"2024-09-21T10:01:40.420219+0000","last_active":"2024-09-21T10:04:36.431369+0000","last_peered":"2024-09-21T10:04:36.431369+0000","last_clean":"2024-09-21T10:04:36.431369+0000","last_became_active":"2024-09-21T10:01:40.419839+0000","last_became_peered":"2024-09-21T10:01:40.419839+0000","last_unstale":"2024-09-21T10:04:36.431369+0000","last_undegraded":"2024-09-21T10:04:36.431369+0000","last_fullsized":"2024-09-21T10:04:36.431369+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:59.370554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1807,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442762+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T10:04:38.442762+0000","last_peered":"2024-09-21T10:04:38.442762+0000","last_clean":"2024-09-21T10:04:38.442762+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T10:04:38.442762+0000","last_undegraded":"2024-09-21T10:04:38.442762+0000","last_fullsized":"2024-09-21T10:04:38.442762+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.18","version":"1188'8629","reported_seq":8769,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.430947+0000","last_change":"2024-09-21T10:04:33.728844+0000","last_active":"2024-09-21T10:04:36.430947+0000","last_peered":"2024-09-21T10:04:36.430947+0000","last_clean":"2024-09-21T10:04:36.430947+0000","last_became_active":"2024-09-21T10:01:27.368411+0000","last_became_peered":"2024-09-21T10:01:27.368411+0000","last_unstale":"2024-09-21T10:04:36.430947+0000","last_undegraded":"2024-09-21T10:04:36.430947+0000","last_fullsized":"2024-09-21T10:04:36.430947+0000","mapping_epoch":1148,"log_start":"0'0","ondisk_log_start":"0'0","created":1148,"last_epoch_clean":1149,"parent":"0.0","parent_split_bits":5,"last_scrub":"1188'8629","last_scrub_stamp":"2024-09-21T10:04:33.728792+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:33.728792+0000","objects_scrubbed":0,"log_size":8629,"log_dups_size":0,"ondisk_log_size":8629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:14.578866+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1162,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432767+0000","last_change":"2024-09-21T10:01:40.432190+0000","last_active":"2024-09-21T10:04:34.432767+0000","last_peered":"2024-09-21T10:04:34.432767+0000","last_clean":"2024-09-21T10:04:34.432767+0000","last_became_active":"2024-09-21T10:01:40.431847+0000","last_became_peered":"2024-09-21T10:01:40.431847+0000","last_unstale":"2024-09-21T10:04:34.432767+0000","last_undegraded":"2024-09-21T10:04:34.432767+0000","last_fullsized":"2024-09-21T10:04:34.432767+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:32.573309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":130,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431969+0000","last_change":"2024-09-21T10:01:40.420069+0000","last_active":"2024-09-21T10:04:34.431969+0000","last_peered":"2024-09-21T10:04:34.431969+0000","last_clean":"2024-09-21T10:04:34.431969+0000","last_became_active":"2024-09-21T10:01:40.416524+0000","last_became_peered":"2024-09-21T10:01:40.416524+0000","last_unstale":"2024-09-21T10:04:34.431969+0000","last_undegraded":"2024-09-21T10:04:34.431969+0000","last_fullsized":"2024-09-21T10:04:34.431969+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:36.273679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"1188'8546","reported_seq":12817,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.442054+0000","last_change":"2024-09-21T10:04:30.133763+0000","last_active":"2024-09-21T10:04:38.442054+0000","last_peered":"2024-09-21T10:04:38.442054+0000","last_clean":"2024-09-21T10:04:38.442054+0000","last_became_active":"2024-09-21T10:03:04.296838+0000","last_became_peered":"2024-09-21T10:03:04.296838+0000","last_unstale":"2024-09-21T10:04:38.442054+0000","last_undegraded":"2024-09-21T10:04:38.442054+0000","last_fullsized":"2024-09-21T10:04:38.442054+0000","mapping_epoch":1175,"log_start":"1145'2093","ondisk_log_start":"1145'2093","created":1148,"last_epoch_clean":1176,"parent":"0.0","parent_split_bits":5,"last_scrub":"1188'8546","last_scrub_stamp":"2024-09-21T10:04:30.133699+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:00:45.452921+0000","last_clean_scrub_stamp":"2024-09-21T10:04:30.133699+0000","objects_scrubbed":0,"log_size":6453,"log_dups_size":0,"ondisk_log_size":6453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:53.047931+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1152,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431706+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T10:04:34.431706+0000","last_peered":"2024-09-21T10:04:34.431706+0000","last_clean":"2024-09-21T10:04:34.431706+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T10:04:34.431706+0000","last_undegraded":"2024-09-21T10:04:34.431706+0000","last_fullsized":"2024-09-21T10:04:34.431706+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":144,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.432714+0000","last_change":"2024-09-21T10:00:16.366351+0000","last_active":"2024-09-21T10:04:34.432714+0000","last_peered":"2024-09-21T10:04:34.432714+0000","last_clean":"2024-09-21T10:04:34.432714+0000","last_became_active":"2024-09-21T10:00:16.355385+0000","last_became_peered":"2024-09-21T10:00:16.355385+0000","last_unstale":"2024-09-21T10:04:34.432714+0000","last_undegraded":"2024-09-21T10:04:34.432714+0000","last_fullsized":"2024-09-21T10:04:34.432714+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:29.014677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1737,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.449892+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:04:36.449892+0000","last_peered":"2024-09-21T10:04:36.449892+0000","last_clean":"2024-09-21T10:04:36.449892+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:04:36.449892+0000","last_undegraded":"2024-09-21T10:04:36.449892+0000","last_fullsized":"2024-09-21T10:04:36.449892+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1737,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-09-21T10:04:36.450062+0000","last_change":"2024-09-21T10:00:16.361160+0000","last_active":"2024-09-21T10:04:36.450062+0000","last_peered":"2024-09-21T10:04:36.450062+0000","last_clean":"2024-09-21T10:04:36.450062+0000","last_became_active":"2024-09-21T10:00:16.360524+0000","last_became_peered":"2024-09-21T10:00:16.360524+0000","last_unstale":"2024-09-21T10:04:36.450062+0000","last_undegraded":"2024-09-21T10:04:36.450062+0000","last_fullsized":"2024-09-21T10:04:36.450062+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:52.747211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1165,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:04:34.431704+0000","last_change":"2024-09-21T10:01:40.432238+0000","last_active":"2024-09-21T10:04:34.431704+0000","last_peered":"2024-09-21T10:04:34.431704+0000","last_clean":"2024-09-21T10:04:34.431704+0000","last_became_active":"2024-09-21T10:01:40.431883+0000","last_became_peered":"2024-09-21T10:01:40.431883+0000","last_unstale":"2024-09-21T10:04:34.431704+0000","last_undegraded":"2024-09-21T10:04:34.431704+0000","last_fullsized":"2024-09-21T10:04:34.431704+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:30.528533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1823,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.443053+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T10:04:38.443053+0000","last_peered":"2024-09-21T10:04:38.443053+0000","last_clean":"2024-09-21T10:04:38.443053+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T10:04:38.443053+0000","last_undegraded":"2024-09-21T10:04:38.443053+0000","last_fullsized":"2024-09-21T10:04:38.443053+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":5963776,"num_objects":92,"num_object_clones":0,"num_object_copies":276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122874,"num_write_kb":3645345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14952,"num_bytes_recovered":516292608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":839319552,"data_stored":839319552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":165839,"ondisk_log_size":165839,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":9628,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":477367,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":107,"num_read_kb":272,"num_write":55,"num_write_kb":592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":542359,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397865,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318568,"kb_used_data":8732,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93426584,"statfs":{"total":95995035648,"available":95668822016,"internally_reserved":0,"allocated":8941568,"data_stored":6331747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594093,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318228,"kb_used_data":8392,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93426924,"statfs":{"total":95995035648,"available":95669170176,"internally_reserved":0,"allocated":8593408,"data_stored":5995668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790323,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":406172,"kb_used_data":100496,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93338980,"statfs":{"total":95995035648,"available":95579115520,"internally_reserved":0,"allocated":102907904,"data_stored":100306606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986550,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":423664,"kb_used_data":117976,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93321488,"statfs":{"total":95995035648,"available":95561203712,"internally_reserved":0,"allocated":120807424,"data_stored":118201621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999192,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":514912,"kb_used_data":8968,"kb_used_omap":25,"kb_used_meta":505894,"kb_avail":93230240,"statfs":{"total":95995035648,"available":95467765760,"internally_reserved":0,"allocated":9183232,"data_stored":6589569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":518036423},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111135,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":491432,"kb_used_data":8664,"kb_used_omap":31,"kb_used_meta":482720,"kb_avail":93253720,"statfs":{"total":95995035648,"available":95491809280,"internally_reserved":0,"allocated":8871936,"data_stored":6270621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":494305512},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":2203318223078,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335021,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":377076,"kb_used_data":8420,"kb_used_omap":31,"kb_used_meta":368608,"kb_avail":93368076,"statfs":{"total":95995035648,"available":95608909824,"internally_reserved":0,"allocated":8622080,"data_stored":6016130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":377454688},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":635371520,"data_stored":635371520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":112197632,"data_stored":112197632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":91750400,"data_stored":91750400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:04:41.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:41.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:41.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:41 smithi134 ceph-mon[90085]: pgmap v1604: 109 pgs: 1 peering, 108 active+clean; 6.1 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:41 smithi134 ceph-mon[90085]: 7.2 scrub starts 2024-09-21T10:04:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:41 smithi134 ceph-mon[90085]: 7.2 scrub ok 2024-09-21T10:04:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:41 smithi134 ceph-mon[90085]: from='client.45557 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:41.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:41.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:41.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:41.920 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:41.945 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:41.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:42.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:42.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:42.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:42.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:42.196 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:42.239 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:04:42.239 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:04:42.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:60919 6.6 6:60165c15:::benchmark_data_smithi017_149547_object60918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.425071+0000", 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 455.80407234500001, 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.015076965, 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60919 2024-09-21T10:04:42.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425071+0000", 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425071+0000", 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425073+0000", 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0320000000000002e-06 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425111+0000", 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7629000000000002e-05 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425134+0000", 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2532e-05 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425144+0000", 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0376e-05 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.573307+0000", 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.148163245 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.573317+0000", 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0359e-05 2024-09-21T10:04:42.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.424167+0000", 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.85084939599999998 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.424309+0000", 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014259799999999999 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:04:42.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.424460+0000", 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000150216 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.432200+0000", 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077398700000000003 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.435328+0000", 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031281989999999999 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440031+0000", 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047034499999999996 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440094+0000", 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2502000000000006e-05 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440148+0000", 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4561000000000002e-05 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.246 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:60922 6.6 6:63c23454:::benchmark_data_smithi017_149547_object60921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.425160+0000", 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 455.80398390800002, 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0152000210000001, 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60922 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.247 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425160+0000", 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425160+0000", 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425160+0000", 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6400000000000002e-07 2024-09-21T10:04:42.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425186+0000", 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6190000000000002e-05 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425194+0000", 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2300000000000002e-06 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.425199+0000", 2024-09-21T10:04:42.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1379999999999998e-06 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.573706+0000", 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14850719300000001 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.573718+0000", 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1591e-05 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.430125+0000", 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.85640711700000005 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.430297+0000", 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017236899999999999 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.430476+0000", 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017851100000000001 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.435369+0000", 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048929619999999998 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.435391+0000", 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2847e-05 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440270+0000", 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048781950000000001 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440321+0000", 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1585999999999997e-05 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.440360+0000", 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8528000000000001e-05 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90824 6.17 6:e9ce8460:::benchmark_data_smithi017_149547_object90823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.553125+0000", 2024-09-21T10:04:42.253 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.67601840200001, 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99456902999999997, 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90824 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553125+0000", 2024-09-21T10:04:42.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553125+0000", 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553127+0000", 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0099999999999998e-06 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553155+0000", 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7645e-05 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553167+0000", 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2021999999999999e-05 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553176+0000", 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4769999999999992e-06 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.554991+0000", 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001814782 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.554999+0000", 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.276e-06 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547582+0000", 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99258215699999996 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547694+0000", 2024-09-21T10:04:42.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112661 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90827 6.17 6:e8bdeb5b:::benchmark_data_smithi017_149547_object90826:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.553190+0000", 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.67595405499998, 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99469125599999997, 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.258 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.259 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:04:42.259 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.259 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.259 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.259 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90827 2024-09-21T10:04:42.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.259 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553190+0000", 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553190+0000", 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553190+0000", 2024-09-21T10:04:42.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0399999999999996e-07 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553211+0000", 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1285999999999999e-05 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553219+0000", 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1690000000000003e-06 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553224+0000", 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4940000000000003e-06 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555018+0000", 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017938120000000001 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555023+0000", 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5169999999999999e-06 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547798+0000", 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99277433599999998 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547881+0000", 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3138000000000003e-05 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90829 6.7 6:e6b999dd:::benchmark_data_smithi017_149547_object90828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.553231+0000", 2024-09-21T10:04:42.263 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.67591217199998, 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99863575900000001, 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90829 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553231+0000", 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553231+0000", 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553232+0000", 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5499999999999999e-07 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553253+0000", 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0780000000000001e-05 2024-09-21T10:04:42.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553259+0000", 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7429999999999996e-06 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553264+0000", 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9189999999999998e-06 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555044+0000", 2024-09-21T10:04:42.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001780139 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555052+0000", 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.593e-06 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551791+0000", 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99673869800000003 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551867+0000", 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6532000000000005e-05 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90837 6.17 6:e97fb59f:::benchmark_data_smithi017_149547_object90836:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.553271+0000", 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.67587246199997, 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99816365399999996, 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.268 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90837 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553271+0000", 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553271+0000", 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553272+0000", 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3300000000000003e-07 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553293+0000", 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0883e-05 2024-09-21T10:04:42.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553299+0000", 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6730000000000003e-06 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.553304+0000", 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0649999999999998e-06 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555072+0000", 2024-09-21T10:04:42.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017676949999999999 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.555079+0000", 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7669999999999998e-06 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.547928+0000", 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99284903599999996 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.548012+0000", 2024-09-21T10:04:42.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4330000000000001e-05 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551317+0000", 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033052939999999999 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551435+0000", 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117478 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90846 6.17 6:ea2175a0:::benchmark_data_smithi017_149547_object90845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.568722+0000", 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.660421703, 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98300621899999996, 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90846 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568722+0000", 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568722+0000", 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568724+0000", 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9489999999999999e-06 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568758+0000", 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4072000000000002e-05 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568772+0000", 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4515000000000001e-05 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.568781+0000", 2024-09-21T10:04:42.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.602e-06 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.570949+0000", 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002168279 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.570960+0000", 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0526999999999999e-05 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.548054+0000", 2024-09-21T10:04:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97709458800000004 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.548176+0000", 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001211 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.548342+0000", 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166222 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551477+0000", 2024-09-21T10:04:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031355599999999999 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551492+0000", 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4851999999999999e-05 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551538+0000", 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6167e-05 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551601+0000", 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2989000000000003e-05 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551728+0000", 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126797 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90859 6.7 6:e6380ed1:::benchmark_data_smithi017_149547_object90858:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.586953+0000", 2024-09-21T10:04:42.280 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.642190701, 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96900693999999998, 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90859 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.586953+0000", 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.586953+0000", 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.586953+0000", 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0100000000000005e-07 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.586998+0000", 2024-09-21T10:04:42.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4981000000000002e-05 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587006+0000", 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2790000000000001e-06 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587021+0000", 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4873e-05 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587910+0000", 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000889914 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587922+0000", 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1012999999999999e-05 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551891+0000", 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96396979800000004 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.551995+0000", 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103677 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.552141+0000", 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000145966 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.554708+0000", 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002566615 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555663+0000", 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00095549899999999995 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555876+0000", 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000212742 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555926+0000", 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0299000000000001e-05 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555960+0000", 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3782999999999998e-05 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90861 6.7 6:e1281cf9:::benchmark_data_smithi017_149547_object90860:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.587101+0000", 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.64204289000003, 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97326060400000003, 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.287 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90861 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587101+0000", 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587101+0000", 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587101+0000", 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2e-07 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587123+0000", 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.154e-05 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587133+0000", 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0246e-05 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587138+0000", 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8330000000000001e-06 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587947+0000", 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00080937800000000003 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.587954+0000", 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1130000000000003e-06 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.555986+0000", 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96803142600000003 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.556102+0000", 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011658 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:04:42.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.556236+0000", 2024-09-21T10:04:42.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133449 2024-09-21T10:04:42.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560018+0000", 2024-09-21T10:04:42.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037822099999999998 2024-09-21T10:04:42.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560063+0000", 2024-09-21T10:04:42.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5349000000000002e-05 2024-09-21T10:04:42.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560200+0000", 2024-09-21T10:04:42.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013658799999999999 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560313+0000", 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112865 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560361+0000", 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8606999999999997e-05 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.295 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:90865 6.7 6:e3d3d708:::benchmark_data_smithi017_149547_object90864:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:38.594875+0000", 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 423.634268163, 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96661452800000003, 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 90865 2024-09-21T10:04:42.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594875+0000", 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594875+0000", 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594876+0000", 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9000000000000005e-07 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594899+0000", 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2316000000000001e-05 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594914+0000", 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5498999999999998e-05 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.594922+0000", 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7829999999999999e-06 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.595060+0000", 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013757799999999999 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:38.595071+0000", 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1616e-05 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.558004+0000", 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96293270500000006 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.558199+0000", 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000194811 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.558344+0000", 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014554800000000001 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560148+0000", 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018039060000000001 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.560373+0000", 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000225028 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.561105+0000", 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00073150000000000005 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.561187+0000", 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1772000000000002e-05 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:39.561490+0000", 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030347600000000001 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:150250 6.11 6:88bfa6f6:::benchmark_data_smithi017_149547_object31171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-09-21T10:04:42.302 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:39.582468+0000", 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 362.64667558000002, 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.995005574, 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 150250 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582468+0000", 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582468+0000", 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582470+0000", 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5889999999999999e-06 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582475+0000", 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8679999999999998e-06 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582487+0000", 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2245e-05 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582495+0000", 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7470000000000005e-06 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582596+0000", 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101345 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582612+0000", 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5900999999999999e-05 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.572675+0000", 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99006340000000004 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.572815+0000", 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140223 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.573194+0000", 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037835199999999999 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577338+0000", 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041438459999999996 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577353+0000", 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5315000000000002e-05 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577387+0000", 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4332999999999998e-05 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577428+0000", 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0550999999999999e-05 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577474+0000", 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5859000000000003e-05 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:204393 6.13 6:cd3bd688:::benchmark_data_smithi017_149547_object85314:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:38.360732+0000", 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 303.86841190500002, 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.009112421, 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.309 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 204393 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360732+0000", 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360732+0000", 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360733+0000", 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.545e-06 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360738+0000", 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5120000000000002e-06 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360747+0000", 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7350000000000002e-06 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360753+0000", 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.827e-06 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.361569+0000", 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00081525800000000002 2024-09-21T10:04:42.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.361578+0000", 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1269999999999992e-06 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.369754+0000", 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.00817614 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.369844+0000", 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0277000000000003e-05 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:204403 6.13 6:c9e2c2d4:::benchmark_data_smithi017_149547_object85324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:38.362822+0000", 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 303.866321223, 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0171761130000001, 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.314 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 204403 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362822+0000", 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362822+0000", 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362823+0000", 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7599999999999996e-07 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362827+0000", 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2370000000000001e-06 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362835+0000", 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7960000000000001e-06 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362842+0000", 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4989999999999999e-06 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.365952+0000", 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031100170000000001 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.365987+0000", 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5614999999999997e-05 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.370197+0000", 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004209111 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.370300+0000", 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010301 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.370725+0000", 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042504000000000001 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379813+0000", 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0090883849999999992 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379869+0000", 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6437999999999997e-05 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379913+0000", 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3151000000000001e-05 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379961+0000", 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8149000000000003e-05 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379999+0000", 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7789e-05 2024-09-21T10:04:42.320 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:204414 6.13 6:cbe9328b:::benchmark_data_smithi017_149547_object85335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:38.371877+0000", 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 303.85726674300003, 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.01438549, 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.321 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:42.322 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.322 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 204414 2024-09-21T10:04:42.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.322 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371877+0000", 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371877+0000", 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371878+0000", 2024-09-21T10:04:42.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3099999999999996e-07 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371881+0000", 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6720000000000002e-06 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371890+0000", 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0599999999999997e-06 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371896+0000", 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6409999999999999e-06 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.375801+0000", 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039046380000000002 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.375810+0000", 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.448e-06 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.376959+0000", 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.001148868 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.377067+0000", 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010764000000000001 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.377390+0000", 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032288500000000002 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.380040+0000", 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002650857 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.386099+0000", 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006058211 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.386160+0000", 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1499000000000002e-05 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.386219+0000", 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8955000000000001e-05 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.386262+0000", 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3285000000000003e-05 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:235488 6.13 6:ce0cbf8e:::benchmark_data_smithi017_149547_object116409:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:15.357755+0000", 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 266.871389034, 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.033180937, 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 235488 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.329 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357755+0000", 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357755+0000", 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357755+0000", 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6800000000000001e-07 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357757+0000", 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.357e-06 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357765+0000", 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8050000000000007e-06 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357771+0000", 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5960000000000003e-06 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.365382+0000", 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076113500000000002 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.365391+0000", 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3559999999999993e-06 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.387335+0000", 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0219442320000001 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.387475+0000", 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014022899999999999 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.387625+0000", 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015034 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390759+0000", 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031338189999999999 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390825+0000", 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6186999999999995e-05 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390843+0000", 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7481000000000001e-05 2024-09-21T10:04:42.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390896+0000", 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3182e-05 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390936+0000", 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9535e-05 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.335 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:235503 6.13 6:cbfc1a84:::benchmark_data_smithi017_149547_object116424:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:15.377179+0000", 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 266.85196503200001, 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.019305262, 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 235503 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377179+0000", 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377179+0000", 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377180+0000", 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.623e-06 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377184+0000", 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0369999999999996e-06 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377194+0000", 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0172e-05 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377201+0000", 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5490000000000003e-06 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377430+0000", 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022947899999999999 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377439+0000", 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3080000000000008e-06 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390980+0000", 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.013541246 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.391103+0000", 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012309300000000001 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.391228+0000", 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124628 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396204+0000", 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049765 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396316+0000", 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111854 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396408+0000", 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2206000000000005e-05 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396444+0000", 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5416000000000001e-05 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396484+0000", 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0151000000000002e-05 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:235510 6.13 6:ce30ca24:::benchmark_data_smithi017_149547_object116431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:15.386849+0000", 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 266.84229487599998, 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.009715766, 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.342 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 235510 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386849+0000", 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386849+0000", 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386850+0000", 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4699999999999999e-06 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386857+0000", 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7499999999999997e-06 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386870+0000", 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2972999999999999e-05 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386879+0000", 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7069999999999998e-06 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386939+0000", 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0467999999999998e-05 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386949+0000", 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.611e-06 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.393738+0000", 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.006789538 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.393858+0000", 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119818 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.393980+0000", 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121723 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396342+0000", 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023616980000000002 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396395+0000", 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3122999999999999e-05 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396492+0000", 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6954999999999996e-05 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396532+0000", 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9941999999999998e-05 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396565+0000", 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2990000000000001e-05 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.348 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.349 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_2' removed 2024-09-21T10:04:42.349 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:235513 6.13 6:ccc4fd59:::benchmark_data_smithi017_149547_object116434:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-09-21T10:04:42.349 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:15.387276+0000", 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 266.841867568, 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.015868797, 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 235513 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387276+0000", 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387276+0000", 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387277+0000", 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0419999999999999e-06 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387282+0000", 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7600000000000002e-06 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387293+0000", 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1251e-05 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387301+0000", 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9729999999999997e-06 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387888+0000", 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00058712800000000002 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387897+0000", 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6920000000000008e-06 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396614+0000", 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0087173739999999 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396730+0000", 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011608 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396890+0000", 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015913200000000001 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.402889+0000", 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059999279999999999 2024-09-21T10:04:42.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.402939+0000", 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9267000000000001e-05 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.403024+0000", 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4907999999999994e-05 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.403091+0000", 2024-09-21T10:04:42.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7324999999999997e-05 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.403145+0000", 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3937000000000002e-05 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72192 7.0 7:034f8e73:::benchmark_data_smithi017_180590_object72191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1160)", 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:54.588801+0000", 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 167.64034269199999, 2024-09-21T10:04:42.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0011584870000001, 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72192 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588801+0000", 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588801+0000", 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588803+0000", 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6559999999999999e-06 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588827+0000", 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.446e-05 2024-09-21T10:04:42.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588839+0000", 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2384e-05 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588847+0000", 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4189999999999996e-06 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.591832+0000", 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002984675 2024-09-21T10:04:42.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.591840+0000", 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8179999999999993e-06 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.584201+0000", 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99236069699999996 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.584359+0000", 2024-09-21T10:04:42.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000157978 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.584534+0000", 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017445500000000001 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589781+0000", 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052477799999999996 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589837+0000", 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6205999999999997e-05 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589852+0000", 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4062e-05 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589911+0000", 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9385000000000001e-05 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589959+0000", 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8511999999999999e-05 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72222 7.0 7:00589716:::benchmark_data_smithi017_180590_object72221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1160)", 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:54.613264+0000", 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 167.61587941900001, 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98000705700000001, 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:04:42.363 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72222 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613264+0000", 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613264+0000", 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613266+0000", 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.68e-06 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613292+0000", 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5613e-05 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613303+0000", 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1161999999999999e-05 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613310+0000", 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5850000000000002e-06 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613375+0000", 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4787000000000002e-05 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613384+0000", 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4069999999999999e-06 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.586475+0000", 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97309033099999998 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.586636+0000", 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000161075 2024-09-21T10:04:42.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.586824+0000", 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018775399999999999 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.590000+0000", 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031762299999999999 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.590102+0000", 2024-09-21T10:04:42.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010170699999999999 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.593168+0000", 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030662179999999999 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.593225+0000", 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.681e-05 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:04:42.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.593271+0000", 2024-09-21T10:04:42.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6697999999999999e-05 2024-09-21T10:04:42.370 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.370 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.370 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.370 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:04:42.370 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:04:42.370 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:04:42.370 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:42.394 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:42.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:42 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/689180711' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:42.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:42 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:42.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:04:42.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:42 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/689180711' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:42.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:42 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:42.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:04:42.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:42.519 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:42.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:42.621 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:42.670 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:42.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:42 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/689180711' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:42 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:42.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:04:42.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:42.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:42.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:42.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:42.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:43.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:43.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:43.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:43.145 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:43.167 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:43.269 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:43.290 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:43.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:43.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:43.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:43.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:43.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:43.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:43.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:43 smithi017 ceph-mon[102913]: pgmap v1605: 109 pgs: 109 active+clean; 2.3 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:43.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:43 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:04:43.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:04:43.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:43 smithi017 ceph-mon[102913]: osdmap e1198: 8 total, 8 up, 7 in 2024-09-21T10:04:43.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:43 smithi017 ceph-mon[105151]: pgmap v1605: 109 pgs: 109 active+clean; 2.3 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:43.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:43 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:04:43.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:04:43.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:43 smithi017 ceph-mon[105151]: osdmap e1198: 8 total, 8 up, 7 in 2024-09-21T10:04:43.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:43.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:43.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:43 smithi134 ceph-mon[90085]: pgmap v1605: 109 pgs: 109 active+clean; 2.3 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:43 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:04:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:04:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:43 smithi134 ceph-mon[90085]: osdmap e1198: 8 total, 8 up, 7 in 2024-09-21T10:04:43.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:43.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.791 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:43.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:43.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:43.914 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:43.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:44.016 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:44.038 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:44.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:44.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:44.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:44.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:44.266 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:44.290 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:44.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:44.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:44.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:44.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:44.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:44.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:44.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:04:44.639 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:04:44.640 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:04:44.640 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-09-21T10:04:44.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_3 16 2024-09-21T10:04:44.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:44.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:44.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:44.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:44.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:44.891 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:44.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.918 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:44.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:44.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:04:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:04:44] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-09-21T10:04:45.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:45.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:45.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:45.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:45.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:45.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:45.270 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:45.274 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:45 smithi017 ceph-mon[102913]: pgmap v1607: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:45.295 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:45.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:45.382 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:45.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:45.420 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:45.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:45.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:45.547 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:45.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:45.649 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:45.675 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:45 smithi017 ceph-mon[105151]: pgmap v1607: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:45 smithi134 ceph-mon[90085]: pgmap v1607: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:45.776 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:45.801 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:45.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:45.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:45.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:45.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:45.997 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:04:46.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:46.065 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:46.166 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:46.187 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:46.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:46.261 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:46.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:46.310 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:46.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:46.411 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:46.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.433 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:46.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:46.448 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:04:46.448 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:04:46.465 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:04:46.466 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:04:46.482 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:04:46.482 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:04:46.498 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:04:46.498 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:04:46.514 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:04:46.515 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:04:46.530 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:04:46.530 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:04:46.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:46.546 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:04:46.546 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:04:46.557 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:46.562 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:04:46.562 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:04:46.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:46.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:46.710 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:46.710 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:46.783 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:46.806 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:46.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:46.908 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:46.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:46.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:47.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:47.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:47.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:47.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:47.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:47.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:47.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:47.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:47.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:47.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:47.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:47 smithi134 ceph-mon[90085]: pgmap v1608: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:47 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/222719716' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:47.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:47.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:47.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:47.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:47.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:47.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:47 smithi017 ceph-mon[105151]: pgmap v1608: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:47 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/222719716' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:47 smithi017 ceph-mon[102913]: pgmap v1608: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:47 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/222719716' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:47.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:47.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:47.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:47.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:47.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:48.025 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:48.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:48.142 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:48.142 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:48.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:48.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:48.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:48.237 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:48.238 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:48.274 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:48.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:48.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:48.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:48.420 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:48.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:04:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:04:48.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:04:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:04:48.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:48.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:48.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:48.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:48.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:48.670 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:04:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:04:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:48.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:48.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:48.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:48.824 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:04:48.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:48.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:48.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:49.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:49.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:49.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:49.067 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:49.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:49.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:49.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:49.269 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:49.291 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:49.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:49.392 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:49.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:04:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.414 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:49.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:49.445 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:49 smithi134 ceph-mon[90085]: pgmap v1609: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:49.515 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:49.537 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:49.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:49.639 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:49.660 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:49.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:49 smithi017 ceph-mon[102913]: pgmap v1609: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:49 smithi017 ceph-mon[105151]: pgmap v1609: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:49.761 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:49.782 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:49.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:49.884 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:49.905 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:50.007 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:50.028 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:50.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:50.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:50.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:50.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:50.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:50.381 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:50.402 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:50.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:50.435 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:04:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:50 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/463483354' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:50 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:04:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:04:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:50.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:50.526 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:50.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.656 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:50.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:50 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/463483354' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:50.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:50 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:50 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/463483354' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:50.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:50 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:50.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:50.783 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:50.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:50.885 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:50.889 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:50.889 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:50.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.910 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:50.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:51.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:51.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:51.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:51.136 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:51.160 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:51.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:51.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:51.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:51.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:51.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:51.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:51.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:51.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:51.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:51.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:51.637 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:51.659 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:51.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[105151]: pgmap v1610: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:04:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[105151]: osdmap e1199: 8 total, 8 up, 7 in 2024-09-21T10:04:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:51.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[102913]: pgmap v1610: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:51.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:51.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:04:51.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[102913]: osdmap e1199: 8 total, 8 up, 7 in 2024-09-21T10:04:51.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:51 smithi134 ceph-mon[90085]: pgmap v1610: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:51 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:51 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:04:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:51 smithi134 ceph-mon[90085]: osdmap e1199: 8 total, 8 up, 7 in 2024-09-21T10:04:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:51.761 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:51.783 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:51.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:04:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:51.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:51.907 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:51.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:51.927 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:52.009 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:52.031 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:52.071 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:52.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:52.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:52.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:52.259 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:52.280 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:52.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:52.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:52.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:52.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:52.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:52.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:52.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:52.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:52.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:52.776 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:52.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:52.878 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:52.899 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:52.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:53.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:53.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:53.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:53.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:53.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:53.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:53.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:53.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:53.279 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:04:53.280 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:04:53.377 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:53.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.398 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:53.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:53 smithi134 ceph-mon[90085]: pgmap v1612: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:53.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:53.521 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:53.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:53.581 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:53 smithi017 ceph-mon[102913]: pgmap v1612: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:53.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:53 smithi017 ceph-mon[105151]: pgmap v1612: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:53.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:53.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:53.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:53.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:53.761 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:53.761 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:53.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:53.872 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:53.894 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:53.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:04:53.995 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:53.999 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1612,"stamp":"2024-09-21T10:04:52.190023+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":429246,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":80,"num_read_kb":204,"num_write":104,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":498355,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1158,"ondisk_log_size":1158,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":251,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2830948,"kb_used_data":206484,"kb_used_omap":95,"kb_used_meta":2624288,"kb_avail":653385116,"statfs":{"total":671965249536,"available":669066358784,"internally_reserved":0,"allocated":211439616,"data_stored":193073098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97681,"internal_metadata":2687271535},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.920122"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1742,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.916034+0000","last_change":"2024-09-21T10:01:40.433720+0000","last_active":"2024-09-21T10:04:50.916034+0000","last_peered":"2024-09-21T10:04:50.916034+0000","last_clean":"2024-09-21T10:04:50.916034+0000","last_became_active":"2024-09-21T10:01:40.433360+0000","last_became_peered":"2024-09-21T10:01:40.433360+0000","last_unstale":"2024-09-21T10:04:50.916034+0000","last_undegraded":"2024-09-21T10:04:50.916034+0000","last_fullsized":"2024-09-21T10:04:50.916034+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:08.257176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1124,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.606921+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T10:04:42.606921+0000","last_peered":"2024-09-21T10:04:42.606921+0000","last_clean":"2024-09-21T10:04:42.606921+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T10:04:42.606921+0000","last_undegraded":"2024-09-21T10:04:42.606921+0000","last_fullsized":"2024-09-21T10:04:42.606921+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":632,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.607842+0000","last_change":"2024-09-21T09:57:39.541887+0000","last_active":"2024-09-21T10:04:42.607842+0000","last_peered":"2024-09-21T10:04:42.607842+0000","last_clean":"2024-09-21T10:04:42.607842+0000","last_became_active":"2024-09-21T09:57:39.541598+0000","last_became_peered":"2024-09-21T09:57:39.541598+0000","last_unstale":"2024-09-21T10:04:42.607842+0000","last_undegraded":"2024-09-21T10:04:42.607842+0000","last_fullsized":"2024-09-21T10:04:42.607842+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:41.615525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1817,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605213+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T10:04:42.605213+0000","last_peered":"2024-09-21T10:04:42.605213+0000","last_clean":"2024-09-21T10:04:42.605213+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T10:04:42.605213+0000","last_undegraded":"2024-09-21T10:04:42.605213+0000","last_fullsized":"2024-09-21T10:04:42.605213+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1819,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.610588+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T10:04:42.610588+0000","last_peered":"2024-09-21T10:04:42.610588+0000","last_clean":"2024-09-21T10:04:42.610588+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T10:04:42.610588+0000","last_undegraded":"2024-09-21T10:04:42.610588+0000","last_fullsized":"2024-09-21T10:04:42.610588+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1125,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.606355+0000","last_change":"2024-09-21T10:00:16.358888+0000","last_active":"2024-09-21T10:04:42.606355+0000","last_peered":"2024-09-21T10:04:42.606355+0000","last_clean":"2024-09-21T10:04:42.606355+0000","last_became_active":"2024-09-21T10:00:16.357219+0000","last_became_peered":"2024-09-21T10:00:16.357219+0000","last_unstale":"2024-09-21T10:04:42.606355+0000","last_undegraded":"2024-09-21T10:04:42.606355+0000","last_fullsized":"2024-09-21T10:04:42.606355+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:13.887798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1742,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.916410+0000","last_change":"2024-09-21T10:01:40.431450+0000","last_active":"2024-09-21T10:04:50.916410+0000","last_peered":"2024-09-21T10:04:50.916410+0000","last_clean":"2024-09-21T10:04:50.916410+0000","last_became_active":"2024-09-21T10:01:40.431140+0000","last_became_peered":"2024-09-21T10:01:40.431140+0000","last_unstale":"2024-09-21T10:04:50.916410+0000","last_undegraded":"2024-09-21T10:04:50.916410+0000","last_fullsized":"2024-09-21T10:04:50.916410+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:37.604030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":87,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440189+0000","last_change":"2024-09-21T10:00:16.377280+0000","last_active":"2024-09-21T10:04:38.440189+0000","last_peered":"2024-09-21T10:04:38.440189+0000","last_clean":"2024-09-21T10:04:38.440189+0000","last_became_active":"2024-09-21T10:00:16.353074+0000","last_became_peered":"2024-09-21T10:00:16.353074+0000","last_unstale":"2024-09-21T10:04:38.440189+0000","last_undegraded":"2024-09-21T10:04:38.440189+0000","last_fullsized":"2024-09-21T10:04:38.440189+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:35.382228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1123,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.607001+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T10:04:42.607001+0000","last_peered":"2024-09-21T10:04:42.607001+0000","last_clean":"2024-09-21T10:04:42.607001+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T10:04:42.607001+0000","last_undegraded":"2024-09-21T10:04:42.607001+0000","last_fullsized":"2024-09-21T10:04:42.607001+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":131,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.604426+0000","last_change":"2024-09-21T10:00:16.357541+0000","last_active":"2024-09-21T10:04:42.604426+0000","last_peered":"2024-09-21T10:04:42.604426+0000","last_clean":"2024-09-21T10:04:42.604426+0000","last_became_active":"2024-09-21T10:00:16.357209+0000","last_became_peered":"2024-09-21T10:00:16.357209+0000","last_unstale":"2024-09-21T10:04:42.604426+0000","last_undegraded":"2024-09-21T10:04:42.604426+0000","last_fullsized":"2024-09-21T10:04:42.604426+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:20.732470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1138,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.603748+0000","last_change":"2024-09-21T10:01:40.429966+0000","last_active":"2024-09-21T10:04:42.603748+0000","last_peered":"2024-09-21T10:04:42.603748+0000","last_clean":"2024-09-21T10:04:42.603748+0000","last_became_active":"2024-09-21T10:01:40.419541+0000","last_became_peered":"2024-09-21T10:01:40.419541+0000","last_unstale":"2024-09-21T10:04:42.603748+0000","last_undegraded":"2024-09-21T10:04:42.603748+0000","last_fullsized":"2024-09-21T10:04:42.603748+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:10.848843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1167,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605729+0000","last_change":"2024-09-21T10:01:40.423234+0000","last_active":"2024-09-21T10:04:42.605729+0000","last_peered":"2024-09-21T10:04:42.605729+0000","last_clean":"2024-09-21T10:04:42.605729+0000","last_became_active":"2024-09-21T10:01:40.422854+0000","last_became_peered":"2024-09-21T10:01:40.422854+0000","last_unstale":"2024-09-21T10:04:42.605729+0000","last_undegraded":"2024-09-21T10:04:42.605729+0000","last_fullsized":"2024-09-21T10:04:42.605729+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:51.607040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":1134,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.608045+0000","last_change":"2024-09-21T10:00:16.373327+0000","last_active":"2024-09-21T10:04:42.608045+0000","last_peered":"2024-09-21T10:04:42.608045+0000","last_clean":"2024-09-21T10:04:42.608045+0000","last_became_active":"2024-09-21T10:00:16.360853+0000","last_became_peered":"2024-09-21T10:00:16.360853+0000","last_unstale":"2024-09-21T10:04:42.608045+0000","last_undegraded":"2024-09-21T10:04:42.608045+0000","last_fullsized":"2024-09-21T10:04:42.608045+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:18.160233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":87,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440952+0000","last_change":"2024-09-21T10:01:40.417926+0000","last_active":"2024-09-21T10:04:38.440952+0000","last_peered":"2024-09-21T10:04:38.440952+0000","last_clean":"2024-09-21T10:04:38.440952+0000","last_became_active":"2024-09-21T10:01:40.417540+0000","last_became_peered":"2024-09-21T10:01:40.417540+0000","last_unstale":"2024-09-21T10:04:38.440952+0000","last_undegraded":"2024-09-21T10:04:38.440952+0000","last_fullsized":"2024-09-21T10:04:38.440952+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:43.890839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":749,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.916091+0000","last_change":"2024-09-21T10:00:16.360708+0000","last_active":"2024-09-21T10:04:50.916091+0000","last_peered":"2024-09-21T10:04:50.916091+0000","last_clean":"2024-09-21T10:04:50.916091+0000","last_became_active":"2024-09-21T10:00:16.360416+0000","last_became_peered":"2024-09-21T10:00:16.360416+0000","last_unstale":"2024-09-21T10:04:50.916091+0000","last_undegraded":"2024-09-21T10:04:50.916091+0000","last_fullsized":"2024-09-21T10:04:50.916091+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:59.090215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1826,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.606347+0000","last_change":"2024-09-21T10:01:40.425332+0000","last_active":"2024-09-21T10:04:42.606347+0000","last_peered":"2024-09-21T10:04:42.606347+0000","last_clean":"2024-09-21T10:04:42.606347+0000","last_became_active":"2024-09-21T10:01:40.425034+0000","last_became_peered":"2024-09-21T10:01:40.425034+0000","last_unstale":"2024-09-21T10:04:42.606347+0000","last_undegraded":"2024-09-21T10:04:42.606347+0000","last_fullsized":"2024-09-21T10:04:42.606347+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:57.701200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1269,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605910+0000","last_change":"2024-09-21T10:00:47.119852+0000","last_active":"2024-09-21T10:04:42.605910+0000","last_peered":"2024-09-21T10:04:42.605910+0000","last_clean":"2024-09-21T10:04:42.605910+0000","last_became_active":"2024-09-21T09:59:43.374051+0000","last_became_peered":"2024-09-21T09:59:43.374051+0000","last_unstale":"2024-09-21T10:04:42.605910+0000","last_undegraded":"2024-09-21T10:04:42.605910+0000","last_fullsized":"2024-09-21T10:04:42.605910+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:47.119793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:47.119793+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:23.087965+0000","scrub_duration":0.0013414659999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1746,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915392+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T10:04:50.915392+0000","last_peered":"2024-09-21T10:04:50.915392+0000","last_clean":"2024-09-21T10:04:50.915392+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T10:04:50.915392+0000","last_undegraded":"2024-09-21T10:04:50.915392+0000","last_fullsized":"2024-09-21T10:04:50.915392+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1160,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.603919+0000","last_change":"2024-09-21T10:00:16.358954+0000","last_active":"2024-09-21T10:04:42.603919+0000","last_peered":"2024-09-21T10:04:42.603919+0000","last_clean":"2024-09-21T10:04:42.603919+0000","last_became_active":"2024-09-21T10:00:16.358416+0000","last_became_peered":"2024-09-21T10:00:16.358416+0000","last_unstale":"2024-09-21T10:04:42.603919+0000","last_undegraded":"2024-09-21T10:04:42.603919+0000","last_fullsized":"2024-09-21T10:04:42.603919+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:46.916076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":188,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.914939+0000","last_change":"2024-09-21T10:00:15.524812+0000","last_active":"2024-09-21T10:04:50.914939+0000","last_peered":"2024-09-21T10:04:50.914939+0000","last_clean":"2024-09-21T10:04:50.914939+0000","last_became_active":"2024-09-21T09:59:43.361751+0000","last_became_peered":"2024-09-21T09:59:43.361751+0000","last_unstale":"2024-09-21T10:04:50.914939+0000","last_undegraded":"2024-09-21T10:04:50.914939+0000","last_fullsized":"2024-09-21T10:04:50.914939+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.306777+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":87,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440680+0000","last_change":"2024-09-21T10:01:40.415754+0000","last_active":"2024-09-21T10:04:38.440680+0000","last_peered":"2024-09-21T10:04:38.440680+0000","last_clean":"2024-09-21T10:04:38.440680+0000","last_became_active":"2024-09-21T10:01:40.415428+0000","last_became_peered":"2024-09-21T10:01:40.415428+0000","last_unstale":"2024-09-21T10:04:38.440680+0000","last_undegraded":"2024-09-21T10:04:38.440680+0000","last_fullsized":"2024-09-21T10:04:38.440680+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:53.323411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1167,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.602706+0000","last_change":"2024-09-21T10:01:40.414164+0000","last_active":"2024-09-21T10:04:42.602706+0000","last_peered":"2024-09-21T10:04:42.602706+0000","last_clean":"2024-09-21T10:04:42.602706+0000","last_became_active":"2024-09-21T10:01:40.413838+0000","last_became_peered":"2024-09-21T10:01:40.413838+0000","last_unstale":"2024-09-21T10:04:42.602706+0000","last_undegraded":"2024-09-21T10:04:42.602706+0000","last_fullsized":"2024-09-21T10:04:42.602706+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:13:24.380350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1141,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605786+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T10:04:42.605786+0000","last_peered":"2024-09-21T10:04:42.605786+0000","last_clean":"2024-09-21T10:04:42.605786+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T10:04:42.605786+0000","last_undegraded":"2024-09-21T10:04:42.605786+0000","last_fullsized":"2024-09-21T10:04:42.605786+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1170,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.602559+0000","last_change":"2024-09-21T10:01:40.424536+0000","last_active":"2024-09-21T10:04:42.602559+0000","last_peered":"2024-09-21T10:04:42.602559+0000","last_clean":"2024-09-21T10:04:42.602559+0000","last_became_active":"2024-09-21T10:01:40.424229+0000","last_became_peered":"2024-09-21T10:01:40.424229+0000","last_unstale":"2024-09-21T10:04:42.602559+0000","last_undegraded":"2024-09-21T10:04:42.602559+0000","last_fullsized":"2024-09-21T10:04:42.602559+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:10.771540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":178,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915068+0000","last_change":"2024-09-21T10:04:20.777769+0000","last_active":"2024-09-21T10:04:50.915068+0000","last_peered":"2024-09-21T10:04:50.915068+0000","last_clean":"2024-09-21T10:04:50.915068+0000","last_became_active":"2024-09-21T10:00:16.351010+0000","last_became_peered":"2024-09-21T10:00:16.351010+0000","last_unstale":"2024-09-21T10:04:50.915068+0000","last_undegraded":"2024-09-21T10:04:50.915068+0000","last_fullsized":"2024-09-21T10:04:50.915068+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-29T15:53:23.454847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1744,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915559+0000","last_change":"2024-09-21T10:01:40.428561+0000","last_active":"2024-09-21T10:04:50.915559+0000","last_peered":"2024-09-21T10:04:50.915559+0000","last_clean":"2024-09-21T10:04:50.915559+0000","last_became_active":"2024-09-21T10:01:40.428250+0000","last_became_peered":"2024-09-21T10:01:40.428250+0000","last_unstale":"2024-09-21T10:04:50.915559+0000","last_undegraded":"2024-09-21T10:04:50.915559+0000","last_fullsized":"2024-09-21T10:04:50.915559+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:58.781370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":649,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.916493+0000","last_change":"2024-09-21T10:01:40.423834+0000","last_active":"2024-09-21T10:04:50.916493+0000","last_peered":"2024-09-21T10:04:50.916493+0000","last_clean":"2024-09-21T10:04:50.916493+0000","last_became_active":"2024-09-21T10:01:40.423506+0000","last_became_peered":"2024-09-21T10:01:40.423506+0000","last_unstale":"2024-09-21T10:04:50.916493+0000","last_undegraded":"2024-09-21T10:04:50.916493+0000","last_fullsized":"2024-09-21T10:04:50.916493+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:11.358742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1283,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605792+0000","last_change":"2024-09-21T10:00:49.116174+0000","last_active":"2024-09-21T10:04:42.605792+0000","last_peered":"2024-09-21T10:04:42.605792+0000","last_clean":"2024-09-21T10:04:42.605792+0000","last_became_active":"2024-09-21T10:00:16.360430+0000","last_became_peered":"2024-09-21T10:00:16.360430+0000","last_unstale":"2024-09-21T10:04:42.605792+0000","last_undegraded":"2024-09-21T10:04:42.605792+0000","last_fullsized":"2024-09-21T10:04:42.605792+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:49.116103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:49.116103+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:29.874911+0000","scrub_duration":0.0020386419999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":222,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915992+0000","last_change":"2024-09-21T09:57:39.541461+0000","last_active":"2024-09-21T10:04:50.915992+0000","last_peered":"2024-09-21T10:04:50.915992+0000","last_clean":"2024-09-21T10:04:50.915992+0000","last_became_active":"2024-09-21T09:57:39.540927+0000","last_became_peered":"2024-09-21T09:57:39.540927+0000","last_unstale":"2024-09-21T10:04:50.915992+0000","last_undegraded":"2024-09-21T10:04:50.915992+0000","last_fullsized":"2024-09-21T10:04:50.915992+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:14.815783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1733,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915738+0000","last_change":"2024-09-21T10:00:16.362131+0000","last_active":"2024-09-21T10:04:50.915738+0000","last_peered":"2024-09-21T10:04:50.915738+0000","last_clean":"2024-09-21T10:04:50.915738+0000","last_became_active":"2024-09-21T10:00:16.361722+0000","last_became_peered":"2024-09-21T10:00:16.361722+0000","last_unstale":"2024-09-21T10:04:50.915738+0000","last_undegraded":"2024-09-21T10:04:50.915738+0000","last_fullsized":"2024-09-21T10:04:50.915738+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:38.717426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.441065+0000","last_change":"2024-09-21T10:00:16.361540+0000","last_active":"2024-09-21T10:04:38.441065+0000","last_peered":"2024-09-21T10:04:38.441065+0000","last_clean":"2024-09-21T10:04:38.441065+0000","last_became_active":"2024-09-21T10:00:16.361086+0000","last_became_peered":"2024-09-21T10:00:16.361086+0000","last_unstale":"2024-09-21T10:04:38.441065+0000","last_undegraded":"2024-09-21T10:04:38.441065+0000","last_fullsized":"2024-09-21T10:04:38.441065+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:14.645896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":92,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440256+0000","last_change":"2024-09-21T10:04:35.993794+0000","last_active":"2024-09-21T10:04:38.440256+0000","last_peered":"2024-09-21T10:04:38.440256+0000","last_clean":"2024-09-21T10:04:38.440256+0000","last_became_active":"2024-09-21T10:01:40.411508+0000","last_became_peered":"2024-09-21T10:01:40.411508+0000","last_unstale":"2024-09-21T10:04:38.440256+0000","last_undegraded":"2024-09-21T10:04:38.440256+0000","last_fullsized":"2024-09-21T10:04:38.440256+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:04:35.993739+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:58.250609+0000","scrub_duration":0.001400943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1161,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605707+0000","last_change":"2024-09-21T10:01:40.428393+0000","last_active":"2024-09-21T10:04:42.605707+0000","last_peered":"2024-09-21T10:04:42.605707+0000","last_clean":"2024-09-21T10:04:42.605707+0000","last_became_active":"2024-09-21T10:01:40.428204+0000","last_became_peered":"2024-09-21T10:01:40.428204+0000","last_unstale":"2024-09-21T10:04:42.605707+0000","last_undegraded":"2024-09-21T10:04:42.605707+0000","last_fullsized":"2024-09-21T10:04:42.605707+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:23:02.136035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1818,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605852+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T10:04:42.605852+0000","last_peered":"2024-09-21T10:04:42.605852+0000","last_clean":"2024-09-21T10:04:42.605852+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T10:04:42.605852+0000","last_undegraded":"2024-09-21T10:04:42.605852+0000","last_fullsized":"2024-09-21T10:04:42.605852+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1838,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.610344+0000","last_change":"2024-09-21T10:01:40.421887+0000","last_active":"2024-09-21T10:04:42.610344+0000","last_peered":"2024-09-21T10:04:42.610344+0000","last_clean":"2024-09-21T10:04:42.610344+0000","last_became_active":"2024-09-21T10:01:40.420730+0000","last_became_peered":"2024-09-21T10:01:40.420730+0000","last_unstale":"2024-09-21T10:04:42.610344+0000","last_undegraded":"2024-09-21T10:04:42.610344+0000","last_fullsized":"2024-09-21T10:04:42.610344+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:51.504051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":1269,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.604820+0000","last_change":"2024-09-21T10:00:16.913196+0000","last_active":"2024-09-21T10:04:42.604820+0000","last_peered":"2024-09-21T10:04:42.604820+0000","last_clean":"2024-09-21T10:04:42.604820+0000","last_became_active":"2024-09-21T09:59:43.370454+0000","last_became_peered":"2024-09-21T09:59:43.370454+0000","last_unstale":"2024-09-21T10:04:42.604820+0000","last_undegraded":"2024-09-21T10:04:42.604820+0000","last_fullsized":"2024-09-21T10:04:42.604820+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:16.913146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:16.913146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:23:33.788563+0000","scrub_duration":0.001342315,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1794,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915739+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:04:50.915739+0000","last_peered":"2024-09-21T10:04:50.915739+0000","last_clean":"2024-09-21T10:04:50.915739+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:04:50.915739+0000","last_undegraded":"2024-09-21T10:04:50.915739+0000","last_fullsized":"2024-09-21T10:04:50.915739+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1749,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915054+0000","last_change":"2024-09-21T10:01:40.414547+0000","last_active":"2024-09-21T10:04:50.915054+0000","last_peered":"2024-09-21T10:04:50.915054+0000","last_clean":"2024-09-21T10:04:50.915054+0000","last_became_active":"2024-09-21T10:01:40.414243+0000","last_became_peered":"2024-09-21T10:01:40.414243+0000","last_unstale":"2024-09-21T10:04:50.915054+0000","last_undegraded":"2024-09-21T10:04:50.915054+0000","last_fullsized":"2024-09-21T10:04:50.915054+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:55:26.044481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":528,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.608016+0000","last_change":"2024-09-21T10:01:40.421427+0000","last_active":"2024-09-21T10:04:42.608016+0000","last_peered":"2024-09-21T10:04:42.608016+0000","last_clean":"2024-09-21T10:04:42.608016+0000","last_became_active":"2024-09-21T10:01:40.421047+0000","last_became_peered":"2024-09-21T10:01:40.421047+0000","last_unstale":"2024-09-21T10:04:42.608016+0000","last_undegraded":"2024-09-21T10:04:42.608016+0000","last_fullsized":"2024-09-21T10:04:42.608016+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:48.640296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":515,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605766+0000","last_change":"2024-09-21T10:03:19.415715+0000","last_active":"2024-09-21T10:04:42.605766+0000","last_peered":"2024-09-21T10:04:42.605766+0000","last_clean":"2024-09-21T10:04:42.605766+0000","last_became_active":"2024-09-21T09:59:43.371384+0000","last_became_peered":"2024-09-21T09:59:43.371384+0000","last_unstale":"2024-09-21T10:04:42.605766+0000","last_undegraded":"2024-09-21T10:04:42.605766+0000","last_fullsized":"2024-09-21T10:04:42.605766+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-30T13:30:29.665440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":155,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440400+0000","last_change":"2024-09-21T10:01:40.426865+0000","last_active":"2024-09-21T10:04:38.440400+0000","last_peered":"2024-09-21T10:04:38.440400+0000","last_clean":"2024-09-21T10:04:38.440400+0000","last_became_active":"2024-09-21T10:01:40.413152+0000","last_became_peered":"2024-09-21T10:01:40.413152+0000","last_unstale":"2024-09-21T10:04:38.440400+0000","last_undegraded":"2024-09-21T10:04:38.440400+0000","last_fullsized":"2024-09-21T10:04:38.440400+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:59.034627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1161,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605080+0000","last_change":"2024-09-21T10:00:16.356367+0000","last_active":"2024-09-21T10:04:42.605080+0000","last_peered":"2024-09-21T10:04:42.605080+0000","last_clean":"2024-09-21T10:04:42.605080+0000","last_became_active":"2024-09-21T10:00:16.356079+0000","last_became_peered":"2024-09-21T10:00:16.356079+0000","last_unstale":"2024-09-21T10:04:42.605080+0000","last_undegraded":"2024-09-21T10:04:42.605080+0000","last_fullsized":"2024-09-21T10:04:42.605080+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:12.934640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":660,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915581+0000","last_change":"2024-09-21T10:01:40.421262+0000","last_active":"2024-09-21T10:04:50.915581+0000","last_peered":"2024-09-21T10:04:50.915581+0000","last_clean":"2024-09-21T10:04:50.915581+0000","last_became_active":"2024-09-21T10:01:40.420959+0000","last_became_peered":"2024-09-21T10:01:40.420959+0000","last_unstale":"2024-09-21T10:04:50.915581+0000","last_undegraded":"2024-09-21T10:04:50.915581+0000","last_fullsized":"2024-09-21T10:04:50.915581+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:06.866421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":87,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440855+0000","last_change":"2024-09-21T10:00:46.494180+0000","last_active":"2024-09-21T10:04:38.440855+0000","last_peered":"2024-09-21T10:04:38.440855+0000","last_clean":"2024-09-21T10:04:38.440855+0000","last_became_active":"2024-09-21T10:00:16.355386+0000","last_became_peered":"2024-09-21T10:00:16.355386+0000","last_unstale":"2024-09-21T10:04:38.440855+0000","last_undegraded":"2024-09-21T10:04:38.440855+0000","last_fullsized":"2024-09-21T10:04:38.440855+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:46.494129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:46.494129+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:11.193593+0000","scrub_duration":0.00183863,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":751,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.916270+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T10:04:50.916270+0000","last_peered":"2024-09-21T10:04:50.916270+0000","last_clean":"2024-09-21T10:04:50.916270+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T10:04:50.916270+0000","last_undegraded":"2024-09-21T10:04:50.916270+0000","last_fullsized":"2024-09-21T10:04:50.916270+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":557,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.610149+0000","last_change":"2024-09-21T10:01:40.418328+0000","last_active":"2024-09-21T10:04:42.610149+0000","last_peered":"2024-09-21T10:04:42.610149+0000","last_clean":"2024-09-21T10:04:42.610149+0000","last_became_active":"2024-09-21T10:01:40.418011+0000","last_became_peered":"2024-09-21T10:01:40.418011+0000","last_unstale":"2024-09-21T10:04:42.610149+0000","last_undegraded":"2024-09-21T10:04:42.610149+0000","last_fullsized":"2024-09-21T10:04:42.610149+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:58.716988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":502,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.602234+0000","last_change":"2024-09-21T10:01:40.421225+0000","last_active":"2024-09-21T10:04:42.602234+0000","last_peered":"2024-09-21T10:04:42.602234+0000","last_clean":"2024-09-21T10:04:42.602234+0000","last_became_active":"2024-09-21T10:01:40.420909+0000","last_became_peered":"2024-09-21T10:01:40.420909+0000","last_unstale":"2024-09-21T10:04:42.602234+0000","last_undegraded":"2024-09-21T10:04:42.602234+0000","last_fullsized":"2024-09-21T10:04:42.602234+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:03.811941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":87,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440981+0000","last_change":"2024-09-21T10:00:20.637858+0000","last_active":"2024-09-21T10:04:38.440981+0000","last_peered":"2024-09-21T10:04:38.440981+0000","last_clean":"2024-09-21T10:04:38.440981+0000","last_became_active":"2024-09-21T10:00:16.353796+0000","last_became_peered":"2024-09-21T10:00:16.353796+0000","last_unstale":"2024-09-21T10:04:38.440981+0000","last_undegraded":"2024-09-21T10:04:38.440981+0000","last_fullsized":"2024-09-21T10:04:38.440981+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:20.637806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:20.637806+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:12:16.884495+0000","scrub_duration":0.0016753919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":586,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915405+0000","last_change":"2024-09-21T10:01:40.414536+0000","last_active":"2024-09-21T10:04:50.915405+0000","last_peered":"2024-09-21T10:04:50.915405+0000","last_clean":"2024-09-21T10:04:50.915405+0000","last_became_active":"2024-09-21T10:01:40.413913+0000","last_became_peered":"2024-09-21T10:01:40.413913+0000","last_unstale":"2024-09-21T10:04:50.915405+0000","last_undegraded":"2024-09-21T10:04:50.915405+0000","last_fullsized":"2024-09-21T10:04:50.915405+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:27.994971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1134,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.608089+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T10:04:42.608089+0000","last_peered":"2024-09-21T10:04:42.608089+0000","last_clean":"2024-09-21T10:04:42.608089+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T10:04:42.608089+0000","last_undegraded":"2024-09-21T10:04:42.608089+0000","last_fullsized":"2024-09-21T10:04:42.608089+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1809,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.606893+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T10:04:42.606893+0000","last_peered":"2024-09-21T10:04:42.606893+0000","last_clean":"2024-09-21T10:04:42.606893+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T10:04:42.606893+0000","last_undegraded":"2024-09-21T10:04:42.606893+0000","last_fullsized":"2024-09-21T10:04:42.606893+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":108,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.602412+0000","last_change":"2024-09-21T10:00:16.356251+0000","last_active":"2024-09-21T10:04:42.602412+0000","last_peered":"2024-09-21T10:04:42.602412+0000","last_clean":"2024-09-21T10:04:42.602412+0000","last_became_active":"2024-09-21T10:00:16.355833+0000","last_became_peered":"2024-09-21T10:00:16.355833+0000","last_unstale":"2024-09-21T10:04:42.602412+0000","last_undegraded":"2024-09-21T10:04:42.602412+0000","last_fullsized":"2024-09-21T10:04:42.602412+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T09:59:47.767654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:59:47.767654+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:50.939198+0000","scrub_duration":0.006614371,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1746,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.916504+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T10:04:50.916504+0000","last_peered":"2024-09-21T10:04:50.916504+0000","last_clean":"2024-09-21T10:04:50.916504+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T10:04:50.916504+0000","last_undegraded":"2024-09-21T10:04:50.916504+0000","last_fullsized":"2024-09-21T10:04:50.916504+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1701,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440843+0000","last_change":"2024-09-21T10:01:40.423209+0000","last_active":"2024-09-21T10:04:38.440843+0000","last_peered":"2024-09-21T10:04:38.440843+0000","last_clean":"2024-09-21T10:04:38.440843+0000","last_became_active":"2024-09-21T10:01:40.422890+0000","last_became_peered":"2024-09-21T10:01:40.422890+0000","last_unstale":"2024-09-21T10:04:38.440843+0000","last_undegraded":"2024-09-21T10:04:38.440843+0000","last_fullsized":"2024-09-21T10:04:38.440843+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:10.683126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1744,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915374+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T10:04:50.915374+0000","last_peered":"2024-09-21T10:04:50.915374+0000","last_clean":"2024-09-21T10:04:50.915374+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T10:04:50.915374+0000","last_undegraded":"2024-09-21T10:04:50.915374+0000","last_fullsized":"2024-09-21T10:04:50.915374+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":176,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915948+0000","last_change":"2024-09-21T10:00:22.052497+0000","last_active":"2024-09-21T10:04:50.915948+0000","last_peered":"2024-09-21T10:04:50.915948+0000","last_clean":"2024-09-21T10:04:50.915948+0000","last_became_active":"2024-09-21T09:59:43.368157+0000","last_became_peered":"2024-09-21T09:59:43.368157+0000","last_unstale":"2024-09-21T10:04:50.915948+0000","last_undegraded":"2024-09-21T10:04:50.915948+0000","last_fullsized":"2024-09-21T10:04:50.915948+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-28T12:21:36.382264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1833,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.610685+0000","last_change":"2024-09-21T10:01:40.433067+0000","last_active":"2024-09-21T10:04:42.610685+0000","last_peered":"2024-09-21T10:04:42.610685+0000","last_clean":"2024-09-21T10:04:42.610685+0000","last_became_active":"2024-09-21T10:01:40.420622+0000","last_became_peered":"2024-09-21T10:01:40.420622+0000","last_unstale":"2024-09-21T10:04:42.610685+0000","last_undegraded":"2024-09-21T10:04:42.610685+0000","last_fullsized":"2024-09-21T10:04:42.610685+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.383227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":77,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440658+0000","last_change":"2024-09-21T10:00:16.355102+0000","last_active":"2024-09-21T10:04:38.440658+0000","last_peered":"2024-09-21T10:04:38.440658+0000","last_clean":"2024-09-21T10:04:38.440658+0000","last_became_active":"2024-09-21T10:00:16.354473+0000","last_became_peered":"2024-09-21T10:00:16.354473+0000","last_unstale":"2024-09-21T10:04:38.440658+0000","last_undegraded":"2024-09-21T10:04:38.440658+0000","last_fullsized":"2024-09-21T10:04:38.440658+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:49:35.449111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1143,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.605629+0000","last_change":"2024-09-21T10:00:16.355601+0000","last_active":"2024-09-21T10:04:42.605629+0000","last_peered":"2024-09-21T10:04:42.605629+0000","last_clean":"2024-09-21T10:04:42.605629+0000","last_became_active":"2024-09-21T10:00:16.355110+0000","last_became_peered":"2024-09-21T10:00:16.355110+0000","last_unstale":"2024-09-21T10:04:42.605629+0000","last_undegraded":"2024-09-21T10:04:42.605629+0000","last_fullsized":"2024-09-21T10:04:42.605629+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:10.590252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1170,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.604993+0000","last_change":"2024-09-21T10:01:40.414117+0000","last_active":"2024-09-21T10:04:42.604993+0000","last_peered":"2024-09-21T10:04:42.604993+0000","last_clean":"2024-09-21T10:04:42.604993+0000","last_became_active":"2024-09-21T10:01:40.413186+0000","last_became_peered":"2024-09-21T10:01:40.413186+0000","last_unstale":"2024-09-21T10:04:42.604993+0000","last_undegraded":"2024-09-21T10:04:42.604993+0000","last_fullsized":"2024-09-21T10:04:42.604993+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:17:05.368450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1186,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.603987+0000","last_change":"2024-09-21T10:00:16.371426+0000","last_active":"2024-09-21T10:04:42.603987+0000","last_peered":"2024-09-21T10:04:42.603987+0000","last_clean":"2024-09-21T10:04:42.603987+0000","last_became_active":"2024-09-21T10:00:16.357486+0000","last_became_peered":"2024-09-21T10:00:16.357486+0000","last_unstale":"2024-09-21T10:04:42.603987+0000","last_undegraded":"2024-09-21T10:04:42.603987+0000","last_fullsized":"2024-09-21T10:04:42.603987+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:02.923682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1702,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440570+0000","last_change":"2024-09-21T10:01:40.413147+0000","last_active":"2024-09-21T10:04:38.440570+0000","last_peered":"2024-09-21T10:04:38.440570+0000","last_clean":"2024-09-21T10:04:38.440570+0000","last_became_active":"2024-09-21T10:01:40.412833+0000","last_became_peered":"2024-09-21T10:01:40.412833+0000","last_unstale":"2024-09-21T10:04:38.440570+0000","last_undegraded":"2024-09-21T10:04:38.440570+0000","last_fullsized":"2024-09-21T10:04:38.440570+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:33.262350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1125,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.607942+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T10:04:42.607942+0000","last_peered":"2024-09-21T10:04:42.607942+0000","last_clean":"2024-09-21T10:04:42.607942+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T10:04:42.607942+0000","last_undegraded":"2024-09-21T10:04:42.607942+0000","last_fullsized":"2024-09-21T10:04:42.607942+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":579,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.603628+0000","last_change":"2024-09-21T10:01:40.427109+0000","last_active":"2024-09-21T10:04:42.603628+0000","last_peered":"2024-09-21T10:04:42.603628+0000","last_clean":"2024-09-21T10:04:42.603628+0000","last_became_active":"2024-09-21T10:01:40.426821+0000","last_became_peered":"2024-09-21T10:01:40.426821+0000","last_unstale":"2024-09-21T10:04:42.603628+0000","last_undegraded":"2024-09-21T10:04:42.603628+0000","last_fullsized":"2024-09-21T10:04:42.603628+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:12.260797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1732,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915568+0000","last_change":"2024-09-21T10:00:16.357151+0000","last_active":"2024-09-21T10:04:50.915568+0000","last_peered":"2024-09-21T10:04:50.915568+0000","last_clean":"2024-09-21T10:04:50.915568+0000","last_became_active":"2024-09-21T10:00:16.356864+0000","last_became_peered":"2024-09-21T10:00:16.356864+0000","last_unstale":"2024-09-21T10:04:50.915568+0000","last_undegraded":"2024-09-21T10:04:50.915568+0000","last_fullsized":"2024-09-21T10:04:50.915568+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:52.851887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1817,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.608204+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T10:04:42.608204+0000","last_peered":"2024-09-21T10:04:42.608204+0000","last_clean":"2024-09-21T10:04:42.608204+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T10:04:42.608204+0000","last_undegraded":"2024-09-21T10:04:42.608204+0000","last_fullsized":"2024-09-21T10:04:42.608204+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1756,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915286+0000","last_change":"2024-09-21T10:01:40.426702+0000","last_active":"2024-09-21T10:04:50.915286+0000","last_peered":"2024-09-21T10:04:50.915286+0000","last_clean":"2024-09-21T10:04:50.915286+0000","last_became_active":"2024-09-21T10:01:40.426034+0000","last_became_peered":"2024-09-21T10:01:40.426034+0000","last_unstale":"2024-09-21T10:04:50.915286+0000","last_undegraded":"2024-09-21T10:04:50.915286+0000","last_fullsized":"2024-09-21T10:04:50.915286+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:20.619421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:04:38.440732+0000","last_change":"2024-09-21T10:01:40.414393+0000","last_active":"2024-09-21T10:04:38.440732+0000","last_peered":"2024-09-21T10:04:38.440732+0000","last_clean":"2024-09-21T10:04:38.440732+0000","last_became_active":"2024-09-21T10:01:40.414096+0000","last_became_peered":"2024-09-21T10:01:40.414096+0000","last_unstale":"2024-09-21T10:04:38.440732+0000","last_undegraded":"2024-09-21T10:04:38.440732+0000","last_fullsized":"2024-09-21T10:04:38.440732+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:32.540479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1760,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915458+0000","last_change":"2024-09-21T10:01:40.434475+0000","last_active":"2024-09-21T10:04:50.915458+0000","last_peered":"2024-09-21T10:04:50.915458+0000","last_clean":"2024-09-21T10:04:50.915458+0000","last_became_active":"2024-09-21T10:01:40.432013+0000","last_became_peered":"2024-09-21T10:01:40.432013+0000","last_unstale":"2024-09-21T10:04:50.915458+0000","last_undegraded":"2024-09-21T10:04:50.915458+0000","last_fullsized":"2024-09-21T10:04:50.915458+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:44.373378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1820,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.610497+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T10:04:42.610497+0000","last_peered":"2024-09-21T10:04:42.610497+0000","last_clean":"2024-09-21T10:04:42.610497+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T10:04:42.610497+0000","last_undegraded":"2024-09-21T10:04:42.610497+0000","last_fullsized":"2024-09-21T10:04:42.610497+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":650,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915685+0000","last_change":"2024-09-21T10:01:40.437120+0000","last_active":"2024-09-21T10:04:50.915685+0000","last_peered":"2024-09-21T10:04:50.915685+0000","last_clean":"2024-09-21T10:04:50.915685+0000","last_became_active":"2024-09-21T10:01:40.436275+0000","last_became_peered":"2024-09-21T10:01:40.436275+0000","last_unstale":"2024-09-21T10:04:50.915685+0000","last_undegraded":"2024-09-21T10:04:50.915685+0000","last_fullsized":"2024-09-21T10:04:50.915685+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:25.322111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1742,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.916187+0000","last_change":"2024-09-21T10:01:40.427770+0000","last_active":"2024-09-21T10:04:50.916187+0000","last_peered":"2024-09-21T10:04:50.916187+0000","last_clean":"2024-09-21T10:04:50.916187+0000","last_became_active":"2024-09-21T10:01:40.427458+0000","last_became_peered":"2024-09-21T10:01:40.427458+0000","last_unstale":"2024-09-21T10:04:50.916187+0000","last_undegraded":"2024-09-21T10:04:50.916187+0000","last_fullsized":"2024-09-21T10:04:50.916187+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:38.605007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1169,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.602904+0000","last_change":"2024-09-21T10:01:40.420505+0000","last_active":"2024-09-21T10:04:42.602904+0000","last_peered":"2024-09-21T10:04:42.602904+0000","last_clean":"2024-09-21T10:04:42.602904+0000","last_became_active":"2024-09-21T10:01:40.420263+0000","last_became_peered":"2024-09-21T10:01:40.420263+0000","last_unstale":"2024-09-21T10:04:42.602904+0000","last_undegraded":"2024-09-21T10:04:42.602904+0000","last_fullsized":"2024-09-21T10:04:42.602904+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:23.452408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1780,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915497+0000","last_change":"2024-09-21T10:01:40.420219+0000","last_active":"2024-09-21T10:04:50.915497+0000","last_peered":"2024-09-21T10:04:50.915497+0000","last_clean":"2024-09-21T10:04:50.915497+0000","last_became_active":"2024-09-21T10:01:40.419839+0000","last_became_peered":"2024-09-21T10:01:40.419839+0000","last_unstale":"2024-09-21T10:04:50.915497+0000","last_undegraded":"2024-09-21T10:04:50.915497+0000","last_fullsized":"2024-09-21T10:04:50.915497+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:59.370554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1809,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.606971+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T10:04:42.606971+0000","last_peered":"2024-09-21T10:04:42.606971+0000","last_clean":"2024-09-21T10:04:42.606971+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T10:04:42.606971+0000","last_undegraded":"2024-09-21T10:04:42.606971+0000","last_fullsized":"2024-09-21T10:04:42.606971+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1167,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.604914+0000","last_change":"2024-09-21T10:01:40.432190+0000","last_active":"2024-09-21T10:04:42.604914+0000","last_peered":"2024-09-21T10:04:42.604914+0000","last_clean":"2024-09-21T10:04:42.604914+0000","last_became_active":"2024-09-21T10:01:40.431847+0000","last_became_peered":"2024-09-21T10:01:40.431847+0000","last_unstale":"2024-09-21T10:04:42.604914+0000","last_undegraded":"2024-09-21T10:04:42.604914+0000","last_fullsized":"2024-09-21T10:04:42.604914+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:32.573309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":135,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.603848+0000","last_change":"2024-09-21T10:01:40.420069+0000","last_active":"2024-09-21T10:04:42.603848+0000","last_peered":"2024-09-21T10:04:42.603848+0000","last_clean":"2024-09-21T10:04:42.603848+0000","last_became_active":"2024-09-21T10:01:40.416524+0000","last_became_peered":"2024-09-21T10:01:40.416524+0000","last_unstale":"2024-09-21T10:04:42.603848+0000","last_undegraded":"2024-09-21T10:04:42.603848+0000","last_fullsized":"2024-09-21T10:04:42.603848+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:36.273679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1157,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.602808+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T10:04:42.602808+0000","last_peered":"2024-09-21T10:04:42.602808+0000","last_clean":"2024-09-21T10:04:42.602808+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T10:04:42.602808+0000","last_undegraded":"2024-09-21T10:04:42.602808+0000","last_fullsized":"2024-09-21T10:04:42.602808+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":149,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.602507+0000","last_change":"2024-09-21T10:00:16.366351+0000","last_active":"2024-09-21T10:04:42.602507+0000","last_peered":"2024-09-21T10:04:42.602507+0000","last_clean":"2024-09-21T10:04:42.602507+0000","last_became_active":"2024-09-21T10:00:16.355385+0000","last_became_peered":"2024-09-21T10:00:16.355385+0000","last_unstale":"2024-09-21T10:04:42.602507+0000","last_undegraded":"2024-09-21T10:04:42.602507+0000","last_fullsized":"2024-09-21T10:04:42.602507+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:29.014677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1745,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.915826+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:04:50.915826+0000","last_peered":"2024-09-21T10:04:50.915826+0000","last_clean":"2024-09-21T10:04:50.915826+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:04:50.915826+0000","last_undegraded":"2024-09-21T10:04:50.915826+0000","last_fullsized":"2024-09-21T10:04:50.915826+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1746,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:04:50.916522+0000","last_change":"2024-09-21T10:00:16.361160+0000","last_active":"2024-09-21T10:04:50.916522+0000","last_peered":"2024-09-21T10:04:50.916522+0000","last_clean":"2024-09-21T10:04:50.916522+0000","last_became_active":"2024-09-21T10:00:16.360524+0000","last_became_peered":"2024-09-21T10:00:16.360524+0000","last_unstale":"2024-09-21T10:04:50.916522+0000","last_undegraded":"2024-09-21T10:04:50.916522+0000","last_fullsized":"2024-09-21T10:04:50.916522+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:52.747211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1170,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.602993+0000","last_change":"2024-09-21T10:01:40.432238+0000","last_active":"2024-09-21T10:04:42.602993+0000","last_peered":"2024-09-21T10:04:42.602993+0000","last_clean":"2024-09-21T10:04:42.602993+0000","last_became_active":"2024-09-21T10:01:40.431883+0000","last_became_peered":"2024-09-21T10:01:40.431883+0000","last_unstale":"2024-09-21T10:04:42.602993+0000","last_undegraded":"2024-09-21T10:04:42.602993+0000","last_fullsized":"2024-09-21T10:04:42.602993+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:30.528533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1825,"reported_epoch":1198,"state":"active+clean","last_fresh":"2024-09-21T10:04:42.608284+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T10:04:42.608284+0000","last_peered":"2024-09-21T10:04:42.608284+0000","last_clean":"2024-09-21T10:04:42.608284+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T10:04:42.608284+0000","last_undegraded":"2024-09-21T10:04:42.608284+0000","last_fullsized":"2024-09-21T10:04:42.608284+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":9628,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":423697,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":80,"num_read_kb":204,"num_write":41,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488689,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397868,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318632,"kb_used_data":8796,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93426520,"statfs":{"total":95995035648,"available":95668756480,"internally_reserved":0,"allocated":9007104,"data_stored":6372621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594096,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318292,"kb_used_data":8456,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93426860,"statfs":{"total":95995035648,"available":95669104640,"internally_reserved":0,"allocated":8658944,"data_stored":6037010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790326,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":377500,"kb_used_data":71824,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93367652,"statfs":{"total":95995035648,"available":95608475648,"internally_reserved":0,"allocated":73547776,"data_stored":70921816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986553,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":396884,"kb_used_data":91212,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93348268,"statfs":{"total":95995035648,"available":95588626432,"internally_reserved":0,"allocated":93401088,"data_stored":90775115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999195,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533012,"kb_used_data":9032,"kb_used_omap":25,"kb_used_meta":523942,"kb_avail":93212140,"statfs":{"total":95995035648,"available":95449231360,"internally_reserved":0,"allocated":9248768,"data_stored":6630443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":536517575},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111138,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":491480,"kb_used_data":8716,"kb_used_omap":31,"kb_used_meta":482720,"kb_avail":93253672,"statfs":{"total":95995035648,"available":95491760128,"internally_reserved":0,"allocated":8925184,"data_stored":6303285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":494305512},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":2203318223080,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335023,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":395148,"kb_used_data":8448,"kb_used_omap":31,"kb_used_meta":386656,"kb_avail":93350004,"statfs":{"total":95995035648,"available":95590404096,"internally_reserved":0,"allocated":8650752,"data_stored":6032808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":395935840},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:04:54.002 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:04:54.003 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:04:54.017 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:54.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:54.119 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:54.141 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:54.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:54.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:54.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:54.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:54.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:54.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:54.438 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_3' created 2024-09-21T10:04:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:54 smithi017 ceph-mon[102913]: from='client.45581 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:54.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:54 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2866463476' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:04:54.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:54 smithi017 ceph-mon[105151]: from='client.45581 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:54.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:54 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2866463476' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:04:54.490 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:54.498 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:04:54.511 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:54.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:54.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:54.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:54.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:54.739 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:54 smithi134 ceph-mon[90085]: from='client.45581 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:54 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2866463476' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:04:54.754 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.763 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:54.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:54.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:54.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:54.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:55.006 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:04:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:04:54] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-09-21T10:04:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.011 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:55.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:55.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:55.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:55.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:55.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:55.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:55.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-09-21T10:04:55.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:55.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:55.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:55.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:55.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:55.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:55.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[102913]: pgmap v1613: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:55.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2866463476' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-21T10:04:55.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[102913]: osdmap e1200: 8 total, 8 up, 7 in 2024-09-21T10:04:55.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:55.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:55.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[105151]: pgmap v1613: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:55.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2866463476' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-21T10:04:55.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[105151]: osdmap e1200: 8 total, 8 up, 7 in 2024-09-21T10:04:55.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:55.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:55.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:55.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:55.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:55 smithi134 ceph-mon[90085]: pgmap v1613: 83 pgs: 83 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:55 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2866463476' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-21T10:04:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:55 smithi134 ceph-mon[90085]: osdmap e1200: 8 total, 8 up, 7 in 2024-09-21T10:04:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:04:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:55.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:55.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:55.810 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:55.814 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:04:55.814 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:04:55.834 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:04:55.834 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:04:55.850 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:55.854 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:04:55.854 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:04:55.871 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:04:55.872 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:04:55.879 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:55.887 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:04:55.887 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:04:55.901 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:55.903 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:04:55.903 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:04:55.919 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:04:55.919 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:04:55.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:04:55.936 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:04:56.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:56.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:56.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:56.128 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:56.149 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:56.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:56.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:56.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:56.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:56.376 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:56.397 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:56.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:56.489 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:04:56.489 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:04:56.490 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 0 2024-09-21T10:04:56.490 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:04:56.499 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:56.520 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:56.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:56.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:56.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:56 smithi017 ceph-mon[102913]: osdmap e1201: 8 total, 8 up, 7 in 2024-09-21T10:04:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:56 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/516323519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:56 smithi017 ceph-mon[102913]: osdmap e1202: 8 total, 8 up, 7 in 2024-09-21T10:04:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:56 smithi017 ceph-mon[105151]: osdmap e1201: 8 total, 8 up, 7 in 2024-09-21T10:04:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:56 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/516323519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:56.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:56 smithi017 ceph-mon[105151]: osdmap e1202: 8 total, 8 up, 7 in 2024-09-21T10:04:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:56 smithi134 ceph-mon[90085]: osdmap e1201: 8 total, 8 up, 7 in 2024-09-21T10:04:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:56 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/516323519' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:56 smithi134 ceph-mon[90085]: osdmap e1202: 8 total, 8 up, 7 in 2024-09-21T10:04:56.748 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:56.768 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:56.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:56.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:56.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:56.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:57.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:57.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:57.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:57.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:57.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:57.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:57.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:57.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:04:57.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:57.390 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:57.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:57.492 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:57.513 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:57.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:57.572 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:57.615 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:57.636 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:57.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:57.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:57 smithi134 ceph-mon[90085]: pgmap v1616: 99 pgs: 2 creating+activating, 10 unknown, 87 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:57.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:57 smithi134 ceph-mon[90085]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:57 smithi017 ceph-mon[102913]: pgmap v1616: 99 pgs: 2 creating+activating, 10 unknown, 87 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:57 smithi017 ceph-mon[102913]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:57 smithi017 ceph-mon[105151]: pgmap v1616: 99 pgs: 2 creating+activating, 10 unknown, 87 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:57.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:57 smithi017 ceph-mon[105151]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:57.738 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:57.759 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:57.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:57.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:57.882 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:57.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:57.984 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:58.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:58.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:58.106 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:58.128 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:58.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:58.230 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:58.250 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:58.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:58.352 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:58.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:58.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:58.476 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:58.498 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:58.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:58.600 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:58.621 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:58.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:58.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:58.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:58.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:58.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:04:58.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:58.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:58.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:58.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:58.998 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:58.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:59.100 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:59.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:59.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:59.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:59.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:59.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:59.337 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:04:59.338 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:04:59.338 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:04:59.338 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:04:59.338 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:04:59.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:59.370 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:59.471 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:59.492 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:59.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:59.594 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:59.616 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:59.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:59.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:04:59 smithi017 ceph-mon[102913]: pgmap v1618: 99 pgs: 2 creating+activating, 4 unknown, 93 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:59.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:04:59 smithi017 ceph-mon[105151]: pgmap v1618: 99 pgs: 2 creating+activating, 4 unknown, 93 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:59.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:59.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:59.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:04:59 smithi134 ceph-mon[90085]: pgmap v1618: 99 pgs: 2 creating+activating, 4 unknown, 93 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:04:59.842 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:59.864 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:04:59.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:59.889 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:59.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:59.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:04:59.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:59.994 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:05:00.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:00.111 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:00.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:00.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:00.235 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:00.241 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:00.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:05:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:05:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:00.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:00.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:00.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:00.461 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-09-21T10:05:00.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:00.466 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:05:00.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:00.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:00.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:00.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:00.633 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:00 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2710831300' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:00.633 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:00 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2268155661' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:00.633 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:00 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:00 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2710831300' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:00 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2268155661' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:00.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:00 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:00 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2710831300' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:00 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2268155661' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:00 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:00.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:00.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:00.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:00.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:00.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:00.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:00.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:01.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:01.106 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:01.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:01.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:01.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:01.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:01.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:01.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:01.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:01.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:01.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:01.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:01.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:01.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:01.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:01.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[102913]: pgmap v1619: 99 pgs: 2 creating+activating, 97 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:05:01.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:01.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2710831300' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:01.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:01.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[102913]: osdmap e1203: 8 total, 8 up, 7 in 2024-09-21T10:05:01.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[105151]: pgmap v1619: 99 pgs: 2 creating+activating, 97 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:05:01.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:01.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2710831300' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:01.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:01.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:01 smithi017 ceph-mon[105151]: osdmap e1203: 8 total, 8 up, 7 in 2024-09-21T10:05:01.692 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:01 smithi134 ceph-mon[90085]: pgmap v1619: 99 pgs: 2 creating+activating, 97 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:05:01.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:01 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:01.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:01 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2710831300' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:01.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:01 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:01.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:01 smithi134 ceph-mon[90085]: osdmap e1203: 8 total, 8 up, 7 in 2024-09-21T10:05:01.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:01.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:01.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:01.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:01.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:01.902 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:05:01.902 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:57298 6.d 6:bc9613a0:::benchmark_data_smithi017_149547_object57297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1084)", 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:03.965427+0000", 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 477.93352208900001, 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.035238209999999999, 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.903 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 57298 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.965427+0000", 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.965427+0000", 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.904 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.965427+0000", 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.965476+0000", 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9367999999999999e-05 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.965491+0000", 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4935e-05 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.905 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.965503+0000", 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2024e-05 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.965607+0000", 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010391300000000001 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.965882+0000", 2024-09-21T10:05:01.906 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00027457399999999998 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.966097+0000", 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00021546 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.968329+0000", 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0022320439999999999 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.907 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.984151+0000", 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.015822358000000002 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:04.000597+0000", 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.016445775999999999 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:04.000629+0000", 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1853999999999997e-05 2024-09-21T10:05:01.908 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:04.000666+0000", 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6436999999999997e-05 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:57310 6.a 6:527f20f8:::benchmark_data_smithi017_149547_object57309:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1084)", 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:03.971011+0000", 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 477.92793851599998, 2024-09-21T10:05:01.909 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.033330250999999998, 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 57310 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971011+0000", 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.910 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971011+0000", 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971010+0000", 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971074+0000", 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4585999999999995e-05 2024-09-21T10:05:01.911 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971089+0000", 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4979e-05 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971095+0000", 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5840000000000002e-06 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.912 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971134+0000", 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8945000000000001e-05 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971268+0000", 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000134809 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971407+0000", 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000138689 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.913 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.974969+0000", 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0035616240000000002 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.974997+0000", 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7796e-05 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:04.004200+0000", 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029203798 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.914 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:04.004245+0000", 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5053999999999999e-05 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:04.004341+0000", 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5723000000000005e-05 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.915 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:57311 6.a 6:5bfc1600:::benchmark_data_smithi017_149547_object57310:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1084)", 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:03.971510+0000", 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 477.92743911600002, 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.032990740999999997, 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 57311 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.916 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971510+0000", 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971510+0000", 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971510+0000", 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.917 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971571+0000", 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1366999999999997e-05 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971580+0000", 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.166e-06 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.971585+0000", 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.622e-06 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.918 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.973341+0000", 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0017561 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.973473+0000", 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013221899999999999 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.973614+0000", 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014077600000000001 2024-09-21T10:05:01.919 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.975085+0000", 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001471298 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:03.975483+0000", 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00039817999999999999 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:04.004390+0000", 2024-09-21T10:05:01.920 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.028907097999999999 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:04.004442+0000", 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1382999999999999e-05 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:04.004501+0000", 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9268000000000002e-05 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.921 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60725 6.7 6:ebff3bba:::benchmark_data_smithi017_149547_object60724:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.191382+0000", 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.70756754799999, 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.085674934999999994, 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60725 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.922 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.191382+0000", 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.191382+0000", 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.191381+0000", 2024-09-21T10:05:01.923 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.191439+0000", 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7843000000000002e-05 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.191445+0000", 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6420000000000004e-06 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.191448+0000", 2024-09-21T10:05:01.924 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8169999999999999e-06 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.191485+0000", 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7582e-05 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.191556+0000", 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1248999999999995e-05 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.925 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.191630+0000", 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3827999999999999e-05 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.193201+0000", 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0015702940000000001 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.193536+0000", 2024-09-21T10:05:01.926 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00033548399999999999 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.276967+0000", 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.083430947000000005 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.277019+0000", 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2086999999999998e-05 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.927 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.277057+0000", 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7688999999999998e-05 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60911 6.7 6:e0eb4451:::benchmark_data_smithi017_149547_object60910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1085)", 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.396419+0000", 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.50253052900001, 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.102635751, 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.928 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60911 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.396419+0000", 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.396419+0000", 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.929 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.930 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.930 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.930 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.396417+0000", 2024-09-21T10:05:01.930 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:05:01.931 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.931 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.931 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.931 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.396461+0000", 2024-09-21T10:05:01.931 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4422000000000003e-05 2024-09-21T10:05:01.931 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.931 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.931 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.396475+0000", 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3915e-05 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.396481+0000", 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9200000000000001e-06 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.396535+0000", 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3570999999999999e-05 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.932 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.396697+0000", 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000162134 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.396854+0000", 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000156847 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.402480+0000", 2024-09-21T10:05:01.933 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.005626313 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.403936+0000", 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001456127 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.499042+0000", 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.095106020999999999 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.934 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.499055+0000", 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.239e-05 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60911 6.7 6:e0eb4451:::benchmark_data_smithi017_149547_object60910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422605+0000", 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.47634435700002, 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0199858690000001, 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.935 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60911 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422605+0000", 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422605+0000", 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.936 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422604+0000", 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422729+0000", 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012542200000000001 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422744+0000", 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4102e-05 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.937 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422749+0000", 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7629999999999997e-06 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.507377+0000", 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.084627732999999997 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.507390+0000", 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2775000000000001e-05 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.938 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.435674+0000", 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.928284585 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.436442+0000", 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00076718299999999995 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.442534+0000", 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0060925140000000003 2024-09-21T10:05:01.939 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.442591+0000", 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6827000000000003e-05 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:63753 6.d 6:b30c5842:::benchmark_data_smithi017_149547_object63752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1087)", 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:08.937024+0000", 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 472.96192537899998, 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.050335082000000003, 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.940 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 63753 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.937024+0000", 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.937024+0000", 2024-09-21T10:05:01.941 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.937022+0000", 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.937063+0000", 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1034e-05 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.942 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.937078+0000", 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5571999999999999e-05 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.937086+0000", 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3010000000000001e-06 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.937133+0000", 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7546999999999997e-05 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.943 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.937235+0000", 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010186199999999999 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.937339+0000", 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000103795 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.939789+0000", 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002450004 2024-09-21T10:05:01.944 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.939807+0000", 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8108e-05 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.987293+0000", 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.047486222000000002 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.987330+0000", 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6848999999999999e-05 2024-09-21T10:05:01.945 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.987359+0000", 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9135000000000001e-05 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:63766 6.d 6:b4a3461f:::benchmark_data_smithi017_149547_object63765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1087)", 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:08.944241+0000", 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 472.95470882400002, 2024-09-21T10:05:01.946 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.043196471, 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 63766 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.944241+0000", 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.947 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.944241+0000", 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.944239+0000", 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.944302+0000", 2024-09-21T10:05:01.948 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2330999999999997e-05 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.944318+0000", 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6203000000000001e-05 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.944324+0000", 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7989999999999999e-06 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.949 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.944371+0000", 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7036999999999997e-05 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.944490+0000", 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011982899999999999 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.944612+0000", 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000121793 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.950 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.948952+0000", 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0043398960000000002 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.948983+0000", 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0426e-05 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.987389+0000", 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.038406158000000003 2024-09-21T10:05:01.951 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.987418+0000", 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.904e-05 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.987437+0000", 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9249000000000002e-05 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.952 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:63770 6.d 6:b72a38b2:::benchmark_data_smithi017_149547_object63769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1087)", 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:08.946413+0000", 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 472.95253675999999, 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.043904435999999998, 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 63770 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.953 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.946413+0000", 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.946413+0000", 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.954 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.946411+0000", 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.946447+0000", 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5904e-05 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.946457+0000", 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.7229999999999999e-06 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.955 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.946462+0000", 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1240000000000004e-06 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.946791+0000", 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000329153 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.946919+0000", 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000127589 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.956 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.947037+0000", 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000118495 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.949089+0000", 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020515239999999999 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.949401+0000", 2024-09-21T10:05:01.957 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000312362 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.990249+0000", 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.040847730999999998 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.990280+0000", 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1201000000000002e-05 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.958 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:08.990317+0000", 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6903000000000002e-05 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:84101 6.d 6:b7eda218:::benchmark_data_smithi017_149547_object84100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1098)", 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:32.973057+0000", 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 448.92589201499999, 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029406463000000001, 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.959 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 84101 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.973057+0000", 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.960 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.973057+0000", 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.973056+0000", 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.973210+0000", 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000154069 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.961 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.973227+0000", 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7054e-05 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.973234+0000", 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6379999999999998e-06 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.978571+0000", 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0053373099999999996 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.962 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.978695+0000", 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012423800000000001 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.978793+0000", 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.7447000000000004e-05 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.981284+0000", 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0024911109999999998 2024-09-21T10:05:01.963 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:32.982840+0000", 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0015559980000000001 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:33.002376+0000", 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.019535726 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:33.002429+0000", 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2992999999999999e-05 2024-09-21T10:05:01.964 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:33.002464+0000", 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5126000000000002e-05 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:85403 6.7 6:e5bf5672:::benchmark_data_smithi017_149547_object85402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:34.018216+0000", 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 447.88073353999999, 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.037181838000000002, 2024-09-21T10:05:01.965 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 85403 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.018216+0000", 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.966 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.018216+0000", 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.018215+0000", 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.018245+0000", 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9563000000000001e-05 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.967 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.018252+0000", 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6960000000000005e-06 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.018266+0000", 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3737000000000001e-05 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.020119+0000", 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001852507 2024-09-21T10:05:01.968 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.020255+0000", 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000135828 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.020395+0000", 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014034 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.021691+0000", 2024-09-21T10:05:01.969 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001296448 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.025243+0000", 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0035516179999999999 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.055193+0000", 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029950349000000001 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.970 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.055265+0000", 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1999e-05 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.055398+0000", 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013240499999999999 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:85406 6.7 6:e718d2c2:::benchmark_data_smithi017_149547_object85405:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:34.022957+0000", 2024-09-21T10:05:01.971 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 447.875992532, 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.032582380000000001, 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 85406 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.022957+0000", 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.972 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.022957+0000", 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.022956+0000", 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.023011+0000", 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4632999999999997e-05 2024-09-21T10:05:01.973 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.023023+0000", 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2408e-05 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.023027+0000", 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1200000000000004e-06 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.974 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.023063+0000", 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5576e-05 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.023189+0000", 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000126441 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.023333+0000", 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014346300000000001 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.975 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.024738+0000", 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0014058950000000001 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.026420+0000", 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001681288 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.055433+0000", 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029013223000000001 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.976 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.055481+0000", 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8137999999999999e-05 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.055539+0000", 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8115999999999997e-05 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.977 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:86154 6.d 6:b4ee64f2:::benchmark_data_smithi017_149547_object86153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:34.641885+0000", 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 447.257064861, 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.055238071999999999, 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 86154 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.978 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.641885+0000", 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.641885+0000", 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.641883+0000", 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.979 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.641918+0000", 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4817000000000002e-05 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.641932+0000", 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4282e-05 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.641938+0000", 2024-09-21T10:05:01.980 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.755e-06 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642128+0000", 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00018990899999999999 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642239+0000", 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011102799999999999 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642367+0000", 2024-09-21T10:05:01.981 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012803599999999999 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.652085+0000", 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0097184290000000007 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.652134+0000", 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9110999999999998e-05 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697054+0000", 2024-09-21T10:05:01.982 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.044919942999999997 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697093+0000", 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8500000000000001e-05 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697123+0000", 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9736000000000001e-05 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.983 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:86155 6.d 6:bfee500d:::benchmark_data_smithi017_149547_object86154:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:34.641982+0000", 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 447.25696745200003, 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.055219237999999997, 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 86155 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.984 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.641982+0000", 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.641982+0000", 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.641981+0000", 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.985 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642014+0000", 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2839000000000002e-05 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642028+0000", 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3740999999999999e-05 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642034+0000", 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6049999999999997e-06 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.986 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.644848+0000", 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0028148159999999999 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.644984+0000", 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000135778 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:05:01.987 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.645116+0000", 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132194 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.652127+0000", 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0070109530000000003 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.652139+0000", 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1982e-05 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697153+0000", 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.045013575 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697182+0000", 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9068e-05 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697201+0000", 2024-09-21T10:05:01.989 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9233e-05 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:86156 6.d 6:b2881978:::benchmark_data_smithi017_149547_object86155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:34.642039+0000", 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 447.25691043900002, 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.063084451999999999, 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.990 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 86156 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642039+0000", 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642039+0000", 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.991 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642038+0000", 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642066+0000", 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7976000000000001e-05 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.992 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642074+0000", 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.187e-06 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.642077+0000", 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8820000000000003e-06 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.647825+0000", 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0057470799999999999 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.993 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.647954+0000", 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012959899999999999 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.648171+0000", 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00021695199999999999 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.652147+0000", 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0039761939999999997 2024-09-21T10:05:01.994 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.652323+0000", 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000175606 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.705016+0000", 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.052692769 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.705077+0000", 2024-09-21T10:05:01.995 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1455e-05 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.705123+0000", 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6281e-05 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:86166 6.d 6:bf31a881:::benchmark_data_smithi017_149547_object86165:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:34.648229+0000", 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 447.250720242, 2024-09-21T10:05:01.996 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.056999321999999998, 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 86166 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.648229+0000", 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.997 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.648229+0000", 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.648228+0000", 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:01.998 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.648323+0000", 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.4772999999999995e-05 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.648335+0000", 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2435999999999999e-05 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.648341+0000", 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.964e-06 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:01.999 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.650419+0000", 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020780260000000002 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.650505+0000", 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5952999999999995e-05 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.650591+0000", 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5932000000000003e-05 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.000 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.652232+0000", 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0016412390000000001 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.652464+0000", 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00023177500000000001 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.705159+0000", 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.052694509 2024-09-21T10:05:02.001 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.705206+0000", 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6989e-05 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.705228+0000", 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2898e-05 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.002 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:86167 6.7 6:e364f546:::benchmark_data_smithi017_149547_object86166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:34.697353+0000", 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 447.20159625500003, 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.046021959000000001, 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 86167 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:02.003 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697353+0000", 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697353+0000", 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697352+0000", 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:02.004 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697413+0000", 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0940999999999998e-05 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697423+0000", 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0200999999999999e-05 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697427+0000", 2024-09-21T10:05:02.005 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6770000000000001e-06 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697459+0000", 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2484999999999998e-05 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697568+0000", 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000108984 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:05:02.006 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.697668+0000", 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.9938000000000004e-05 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.699117+0000", 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0014493100000000001 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.699398+0000", 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00028087000000000001 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.007 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.743285+0000", 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.043886439999999999 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.743341+0000", 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6019e-05 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.743375+0000", 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4459999999999999e-05 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.008 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:86545 6.7 6:e03204b2:::benchmark_data_smithi017_149547_object86544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:34.989477+0000", 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 446.909472682, 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.049682348000000001, 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 86545 2024-09-21T10:05:02.009 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.989477+0000", 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.989477+0000", 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:02.010 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.989475+0000", 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.989519+0000", 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3443999999999999e-05 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.989532+0000", 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3678e-05 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.011 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.989539+0000", 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2899999999999999e-06 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.989678+0000", 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013934099999999999 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.989810+0000", 2024-09-21T10:05:02.012 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013252199999999999 2024-09-21T10:05:02.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:02.014 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.014 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.014 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:05:02.014 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.989964+0000", 2024-09-21T10:05:02.014 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015352000000000001 2024-09-21T10:05:02.014 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.014 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.014 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.014 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.993331+0000", 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0033671980000000001 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:34.993440+0000", 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010857300000000001 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.039097+0000", 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.045657376 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.015 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.039134+0000", 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6474999999999999e-05 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.039159+0000", 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.5497999999999999e-05 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.016 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:86870 6.a 6:5e028079:::benchmark_data_smithi017_149547_object86869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:35.275400+0000", 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 446.62354901999998, 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.030051377000000001, 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 86870 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.275400+0000", 2024-09-21T10:05:02.017 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.275400+0000", 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.275399+0000", 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:02.018 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.275438+0000", 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8946000000000003e-05 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.275451+0000", 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3032e-05 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.275457+0000", 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9449999999999998e-06 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.019 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.275510+0000", 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3009e-05 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.275626+0000", 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000116267 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.275771+0000", 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014522100000000001 2024-09-21T10:05:02.020 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.278269+0000", 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002498105 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.278518+0000", 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00024901300000000002 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.305370+0000", 2024-09-21T10:05:02.021 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.026851660999999999 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.305415+0000", 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4951999999999999e-05 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:35.305452+0000", 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6891000000000003e-05 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:02.022 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:87982 6.a 6:5e6e7c03:::benchmark_data_smithi017_149547_object87981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:36.206701+0000", 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 445.69224845399998, 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029445664999999999, 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:02.023 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 87982 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.206701+0000", 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.206701+0000", 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.024 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.206700+0000", 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.206738+0000", 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8114000000000001e-05 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.206746+0000", 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.2889999999999998e-06 2024-09-21T10:05:02.025 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.206751+0000", 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4689999999999999e-06 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.206966+0000", 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00021557999999999999 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:02.026 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.207098+0000", 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000131557 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.207189+0000", 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1069000000000005e-05 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.209931+0000", 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002742249 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.027 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.210681+0000", 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00075038999999999998 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.236057+0000", 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.025375676999999999 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.236109+0000", 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1496000000000003e-05 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:02.028 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:02.029 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:36.236147+0000", 2024-09-21T10:05:02.029 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8043000000000002e-05 2024-09-21T10:05:02.029 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.029 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:02.029 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.029 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:02.029 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:02.029 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:05:02.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:02.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:02.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:02.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:02.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:02.317 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:02.340 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:02.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:02.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:02.565 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:02.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:02.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:02.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:02 smithi017 ceph-mon[102913]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:02.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:02 smithi017 ceph-mon[105151]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:02.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:02.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:02.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:02.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:02.834 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:02.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:02.881 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:05:02.881 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:02.890 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:05:02.890 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:02.906 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:05:02.907 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:02.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:05:02.923 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:02.935 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:02.938 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:05:02.938 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:02.953 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:05:02.953 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:02.956 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:02.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:02.968 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:05:02.969 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:02.984 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:05:02.984 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:02.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:02 smithi134 ceph-mon[90085]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:03.058 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:03.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:03.100 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:05:03.182 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:03.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:03.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:03.306 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:03.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:03.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:03.386 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-09-21T10:05:03.387 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:05:03.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:05:03.388 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:03.390 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:05:03.390 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:05:03.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:03.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:03.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:03.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.577 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:03.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:03.643 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-21T10:05:03.643 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:05:03.643 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_209235 2024-09-21T10:05:03.644 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:05:03.644 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:05:03.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:03 smithi017 ceph-mon[102913]: pgmap v1621: 99 pgs: 99 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:05:03.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:03 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1796483314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:03.670 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:03 smithi017 ceph-mon[105151]: pgmap v1621: 99 pgs: 99 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:05:03.670 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:03 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1796483314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:03.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:03.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:03.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:03.830 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:03.833 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:03.837 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:03 smithi134 ceph-mon[90085]: pgmap v1621: 99 pgs: 99 active+clean; 419 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2024-09-21T10:05:03.837 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:03 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1796483314' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:03.867 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:03.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:03.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:04.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:04.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:04.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:05:04.105 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:04.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.138 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:04.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:04.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:04.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:04.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:04.384 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:04.414 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:04.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:04.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:04.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:04.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:04.644 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3771 3755 234.674 234.688 0.00270339 0.00424846 2024-09-21T10:05:04.661 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:04.697 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:04.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:04.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:04.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:04.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:04.976 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:05.078 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:05.113 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:05.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:05:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:05:04] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.51.0" 2024-09-21T10:05:05.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:05.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:05.348 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:05.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:05.369 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:05.369 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:05.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:05.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:05.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:05.539 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:05 smithi017 ceph-mon[102913]: pgmap v1622: 99 pgs: 99 active+clean; 23 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 2.9 MiB/s wr, 45 op/s 2024-09-21T10:05:05.540 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:05 smithi017 ceph-mon[105151]: pgmap v1622: 99 pgs: 99 active+clean; 23 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 2.9 MiB/s wr, 45 op/s 2024-09-21T10:05:05.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:05.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:05.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:05.644 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 7333 7317 228.64 222.625 0.00892536 0.00435706 2024-09-21T10:05:05.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:05.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:05.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:05:05.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:05.800 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:05 smithi134 ceph-mon[90085]: pgmap v1622: 99 pgs: 99 active+clean; 23 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 2.9 MiB/s wr, 45 op/s 2024-09-21T10:05:05.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:05.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:05.918 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:05.957 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:06.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:06.377 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:05:06.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:06.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:06.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:06.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:06.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:06.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:06.644 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 10282 10266 213.859 184.312 0.00338549 0.00466857 2024-09-21T10:05:06.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:06.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:06 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:06.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:06 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:06 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:06.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:06.799 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:06.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:06.868 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:05:06.870 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:05:06.922 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:06.959 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:06.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:07.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:07.098 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:07.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:07.200 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:07.236 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:07.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:07.337 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:07.377 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:07.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:07.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:07.619 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:07.644 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 13293 13277 207.437 188.188 0.00213389 0.0048112 2024-09-21T10:05:07.645 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:07.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:07.747 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:07.782 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:07.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:07.844 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:07 smithi017 ceph-mon[102913]: pgmap v1623: 99 pgs: 99 active+clean; 205 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 21 MiB/s wr, 335 op/s 2024-09-21T10:05:07.845 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:07 smithi017 ceph-mon[105151]: pgmap v1623: 99 pgs: 99 active+clean; 205 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 21 MiB/s wr, 335 op/s 2024-09-21T10:05:07.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:07.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:07 smithi134 ceph-mon[90085]: pgmap v1623: 99 pgs: 99 active+clean; 205 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 21 MiB/s wr, 335 op/s 2024-09-21T10:05:08.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:08.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:08.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:08.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:08.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:08.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:08.287 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:08.320 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:08.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:08.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:08.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:08.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:08.482 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1623,"stamp":"2024-09-21T10:05:06.194181+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":215059646,"num_objects":3301,"num_object_clones":0,"num_object_copies":9903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3301,"num_whiteouts":0,"num_read":80,"num_read_kb":204,"num_write":3379,"num_write_kb":210075,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":498355,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4433,"ondisk_log_size":4433,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3491652,"kb_used_data":896316,"kb_used_omap":95,"kb_used_meta":2595168,"kb_avail":652724412,"statfs":{"total":671965249536,"available":668389797888,"internally_reserved":0,"allocated":917827584,"data_stored":899343157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97681,"internal_metadata":2657452655},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":214630400,"num_objects":3275,"num_object_clones":0,"num_object_copies":9825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3275,"num_write_kb":209600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.755202"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1750,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.739814+0000","last_change":"2024-09-21T10:01:40.433720+0000","last_active":"2024-09-21T10:05:00.739814+0000","last_peered":"2024-09-21T10:05:00.739814+0000","last_clean":"2024-09-21T10:05:00.739814+0000","last_became_active":"2024-09-21T10:01:40.433360+0000","last_became_peered":"2024-09-21T10:01:40.433360+0000","last_unstale":"2024-09-21T10:05:00.739814+0000","last_undegraded":"2024-09-21T10:05:00.739814+0000","last_fullsized":"2024-09-21T10:05:00.739814+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:08.257176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1128,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356975+0000","last_change":"2024-09-21T09:46:13.254520+0000","last_active":"2024-09-21T10:04:57.356975+0000","last_peered":"2024-09-21T10:04:57.356975+0000","last_clean":"2024-09-21T10:04:57.356975+0000","last_became_active":"2024-09-21T09:46:13.254169+0000","last_became_peered":"2024-09-21T09:46:13.254169+0000","last_unstale":"2024-09-21T10:04:57.356975+0000","last_undegraded":"2024-09-21T10:04:57.356975+0000","last_fullsized":"2024-09-21T10:04:57.356975+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:18.748455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":636,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356986+0000","last_change":"2024-09-21T09:57:39.541887+0000","last_active":"2024-09-21T10:04:57.356986+0000","last_peered":"2024-09-21T10:04:57.356986+0000","last_clean":"2024-09-21T10:04:57.356986+0000","last_became_active":"2024-09-21T09:57:39.541598+0000","last_became_peered":"2024-09-21T09:57:39.541598+0000","last_unstale":"2024-09-21T10:04:57.356986+0000","last_undegraded":"2024-09-21T10:04:57.356986+0000","last_fullsized":"2024-09-21T10:04:57.356986+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:41.615525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1827,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.464689+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T10:05:00.464689+0000","last_peered":"2024-09-21T10:05:00.464689+0000","last_clean":"2024-09-21T10:05:00.464689+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T10:05:00.464689+0000","last_undegraded":"2024-09-21T10:05:00.464689+0000","last_fullsized":"2024-09-21T10:05:00.464689+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1829,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.462799+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T10:05:00.462799+0000","last_peered":"2024-09-21T10:05:00.462799+0000","last_clean":"2024-09-21T10:05:00.462799+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T10:05:00.462799+0000","last_undegraded":"2024-09-21T10:05:00.462799+0000","last_fullsized":"2024-09-21T10:05:00.462799+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1129,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357020+0000","last_change":"2024-09-21T10:00:16.358888+0000","last_active":"2024-09-21T10:04:57.357020+0000","last_peered":"2024-09-21T10:04:57.357020+0000","last_clean":"2024-09-21T10:04:57.357020+0000","last_became_active":"2024-09-21T10:00:16.357219+0000","last_became_peered":"2024-09-21T10:00:16.357219+0000","last_unstale":"2024-09-21T10:04:57.357020+0000","last_undegraded":"2024-09-21T10:04:57.357020+0000","last_fullsized":"2024-09-21T10:04:57.357020+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:13.887798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1750,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.739802+0000","last_change":"2024-09-21T10:01:40.431450+0000","last_active":"2024-09-21T10:05:00.739802+0000","last_peered":"2024-09-21T10:05:00.739802+0000","last_clean":"2024-09-21T10:05:00.739802+0000","last_became_active":"2024-09-21T10:01:40.431140+0000","last_became_peered":"2024-09-21T10:01:40.431140+0000","last_unstale":"2024-09-21T10:05:00.739802+0000","last_undegraded":"2024-09-21T10:05:00.739802+0000","last_fullsized":"2024-09-21T10:05:00.739802+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:37.604030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":92,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440114+0000","last_change":"2024-09-21T10:00:16.377280+0000","last_active":"2024-09-21T10:04:56.440114+0000","last_peered":"2024-09-21T10:04:56.440114+0000","last_clean":"2024-09-21T10:04:56.440114+0000","last_became_active":"2024-09-21T10:00:16.353074+0000","last_became_peered":"2024-09-21T10:00:16.353074+0000","last_unstale":"2024-09-21T10:04:56.440114+0000","last_undegraded":"2024-09-21T10:04:56.440114+0000","last_fullsized":"2024-09-21T10:04:56.440114+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:35.382228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"1203'516","reported_seq":527,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:06.031804+0000","last_change":"2024-09-21T10:04:55.933124+0000","last_active":"2024-09-21T10:05:06.031804+0000","last_peered":"2024-09-21T10:05:06.031804+0000","last_clean":"2024-09-21T10:05:06.031804+0000","last_became_active":"2024-09-21T10:04:55.932770+0000","last_became_peered":"2024-09-21T10:04:55.932770+0000","last_unstale":"2024-09-21T10:05:06.031804+0000","last_undegraded":"2024-09-21T10:05:06.031804+0000","last_fullsized":"2024-09-21T10:05:06.031804+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":516,"log_dups_size":0,"ondisk_log_size":516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:20:30.187438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33816576,"num_objects":516,"num_object_clones":0,"num_object_copies":1548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":516,"num_write_kb":33024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1127,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357806+0000","last_change":"2024-09-21T09:45:05.087740+0000","last_active":"2024-09-21T10:04:57.357806+0000","last_peered":"2024-09-21T10:04:57.357806+0000","last_clean":"2024-09-21T10:04:57.357806+0000","last_became_active":"2024-09-21T09:45:05.087401+0000","last_became_peered":"2024-09-21T09:45:05.087401+0000","last_unstale":"2024-09-21T10:04:57.357806+0000","last_undegraded":"2024-09-21T10:04:57.357806+0000","last_fullsized":"2024-09-21T10:04:57.357806+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:58.776424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":135,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357036+0000","last_change":"2024-09-21T10:00:16.357541+0000","last_active":"2024-09-21T10:04:57.357036+0000","last_peered":"2024-09-21T10:04:57.357036+0000","last_clean":"2024-09-21T10:04:57.357036+0000","last_became_active":"2024-09-21T10:00:16.357209+0000","last_became_peered":"2024-09-21T10:00:16.357209+0000","last_unstale":"2024-09-21T10:04:57.357036+0000","last_undegraded":"2024-09-21T10:04:57.357036+0000","last_fullsized":"2024-09-21T10:04:57.357036+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:20.732470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"1203'427","reported_seq":438,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:05.522987+0000","last_change":"2024-09-21T10:04:55.446088+0000","last_active":"2024-09-21T10:05:05.522987+0000","last_peered":"2024-09-21T10:05:05.522987+0000","last_clean":"2024-09-21T10:05:05.522987+0000","last_became_active":"2024-09-21T10:04:55.445807+0000","last_became_peered":"2024-09-21T10:04:55.445807+0000","last_unstale":"2024-09-21T10:05:05.522987+0000","last_undegraded":"2024-09-21T10:05:05.522987+0000","last_fullsized":"2024-09-21T10:05:05.522987+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":427,"log_dups_size":0,"ondisk_log_size":427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:20:38.601988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27983872,"num_objects":427,"num_object_clones":0,"num_object_copies":1281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":427,"num_write_kb":27328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1142,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357798+0000","last_change":"2024-09-21T10:01:40.429966+0000","last_active":"2024-09-21T10:04:57.357798+0000","last_peered":"2024-09-21T10:04:57.357798+0000","last_clean":"2024-09-21T10:04:57.357798+0000","last_became_active":"2024-09-21T10:01:40.419541+0000","last_became_peered":"2024-09-21T10:01:40.419541+0000","last_unstale":"2024-09-21T10:04:57.357798+0000","last_undegraded":"2024-09-21T10:04:57.357798+0000","last_fullsized":"2024-09-21T10:04:57.357798+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:10.848843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1171,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356886+0000","last_change":"2024-09-21T10:01:40.423234+0000","last_active":"2024-09-21T10:04:57.356886+0000","last_peered":"2024-09-21T10:04:57.356886+0000","last_clean":"2024-09-21T10:04:57.356886+0000","last_became_active":"2024-09-21T10:01:40.422854+0000","last_became_peered":"2024-09-21T10:01:40.422854+0000","last_unstale":"2024-09-21T10:04:57.356886+0000","last_undegraded":"2024-09-21T10:04:57.356886+0000","last_fullsized":"2024-09-21T10:04:57.356886+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:51.607040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"1203'79","reported_seq":96,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.014112+0000","last_change":"2024-09-21T10:04:55.450875+0000","last_active":"2024-09-21T10:05:04.014112+0000","last_peered":"2024-09-21T10:05:04.014112+0000","last_clean":"2024-09-21T10:05:04.014112+0000","last_became_active":"2024-09-21T10:04:55.450004+0000","last_became_peered":"2024-09-21T10:04:55.450004+0000","last_unstale":"2024-09-21T10:05:04.014112+0000","last_undegraded":"2024-09-21T10:05:04.014112+0000","last_fullsized":"2024-09-21T10:05:04.014112+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:16.183501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5177344,"num_objects":79,"num_object_clones":0,"num_object_copies":237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":79,"num_write_kb":5056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":1138,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357308+0000","last_change":"2024-09-21T10:00:16.373327+0000","last_active":"2024-09-21T10:04:57.357308+0000","last_peered":"2024-09-21T10:04:57.357308+0000","last_clean":"2024-09-21T10:04:57.357308+0000","last_became_active":"2024-09-21T10:00:16.360853+0000","last_became_peered":"2024-09-21T10:00:16.360853+0000","last_unstale":"2024-09-21T10:04:57.357308+0000","last_undegraded":"2024-09-21T10:04:57.357308+0000","last_fullsized":"2024-09-21T10:04:57.357308+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:18.160233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440509+0000","last_change":"2024-09-21T10:01:40.417926+0000","last_active":"2024-09-21T10:04:56.440509+0000","last_peered":"2024-09-21T10:04:56.440509+0000","last_clean":"2024-09-21T10:04:56.440509+0000","last_became_active":"2024-09-21T10:01:40.417540+0000","last_became_peered":"2024-09-21T10:01:40.417540+0000","last_unstale":"2024-09-21T10:04:56.440509+0000","last_undegraded":"2024-09-21T10:04:56.440509+0000","last_fullsized":"2024-09-21T10:04:56.440509+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:43.890839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":17,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740644+0000","last_change":"2024-09-21T10:04:55.445188+0000","last_active":"2024-09-21T10:05:00.740644+0000","last_peered":"2024-09-21T10:05:00.740644+0000","last_clean":"2024-09-21T10:05:00.740644+0000","last_became_active":"2024-09-21T10:04:55.444846+0000","last_became_peered":"2024-09-21T10:04:55.444846+0000","last_unstale":"2024-09-21T10:05:00.740644+0000","last_undegraded":"2024-09-21T10:05:00.740644+0000","last_fullsized":"2024-09-21T10:05:00.740644+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.773030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":757,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740512+0000","last_change":"2024-09-21T10:00:16.360708+0000","last_active":"2024-09-21T10:05:00.740512+0000","last_peered":"2024-09-21T10:05:00.740512+0000","last_clean":"2024-09-21T10:05:00.740512+0000","last_became_active":"2024-09-21T10:00:16.360416+0000","last_became_peered":"2024-09-21T10:00:16.360416+0000","last_unstale":"2024-09-21T10:05:00.740512+0000","last_undegraded":"2024-09-21T10:05:00.740512+0000","last_fullsized":"2024-09-21T10:05:00.740512+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:59.090215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1836,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.461505+0000","last_change":"2024-09-21T10:01:40.425332+0000","last_active":"2024-09-21T10:05:00.461505+0000","last_peered":"2024-09-21T10:05:00.461505+0000","last_clean":"2024-09-21T10:05:00.461505+0000","last_became_active":"2024-09-21T10:01:40.425034+0000","last_became_peered":"2024-09-21T10:01:40.425034+0000","last_unstale":"2024-09-21T10:05:00.461505+0000","last_undegraded":"2024-09-21T10:05:00.461505+0000","last_fullsized":"2024-09-21T10:05:00.461505+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:57.701200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":17,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740016+0000","last_change":"2024-09-21T10:04:55.934631+0000","last_active":"2024-09-21T10:05:00.740016+0000","last_peered":"2024-09-21T10:05:00.740016+0000","last_clean":"2024-09-21T10:05:00.740016+0000","last_became_active":"2024-09-21T10:04:55.934314+0000","last_became_peered":"2024-09-21T10:04:55.934314+0000","last_unstale":"2024-09-21T10:05:00.740016+0000","last_undegraded":"2024-09-21T10:05:00.740016+0000","last_fullsized":"2024-09-21T10:05:00.740016+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:48:44.869782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1273,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356817+0000","last_change":"2024-09-21T10:00:47.119852+0000","last_active":"2024-09-21T10:04:57.356817+0000","last_peered":"2024-09-21T10:04:57.356817+0000","last_clean":"2024-09-21T10:04:57.356817+0000","last_became_active":"2024-09-21T09:59:43.374051+0000","last_became_peered":"2024-09-21T09:59:43.374051+0000","last_unstale":"2024-09-21T10:04:57.356817+0000","last_undegraded":"2024-09-21T10:04:57.356817+0000","last_fullsized":"2024-09-21T10:04:57.356817+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:47.119793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:47.119793+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:23.087965+0000","scrub_duration":0.0013414659999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1754,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740201+0000","last_change":"2024-09-21T09:46:47.115578+0000","last_active":"2024-09-21T10:05:00.740201+0000","last_peered":"2024-09-21T10:05:00.740201+0000","last_clean":"2024-09-21T10:05:00.740201+0000","last_became_active":"2024-09-21T09:46:47.115021+0000","last_became_peered":"2024-09-21T09:46:47.115021+0000","last_unstale":"2024-09-21T10:05:00.740201+0000","last_undegraded":"2024-09-21T10:05:00.740201+0000","last_fullsized":"2024-09-21T10:05:00.740201+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.051712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1164,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357587+0000","last_change":"2024-09-21T10:00:16.358954+0000","last_active":"2024-09-21T10:04:57.357587+0000","last_peered":"2024-09-21T10:04:57.357587+0000","last_clean":"2024-09-21T10:04:57.357587+0000","last_became_active":"2024-09-21T10:00:16.358416+0000","last_became_peered":"2024-09-21T10:00:16.358416+0000","last_unstale":"2024-09-21T10:04:57.357587+0000","last_undegraded":"2024-09-21T10:04:57.357587+0000","last_fullsized":"2024-09-21T10:04:57.357587+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:46.916076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1203'92","reported_seq":109,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.014597+0000","last_change":"2024-09-21T10:04:55.933597+0000","last_active":"2024-09-21T10:05:04.014597+0000","last_peered":"2024-09-21T10:05:04.014597+0000","last_clean":"2024-09-21T10:05:04.014597+0000","last_became_active":"2024-09-21T10:04:55.933296+0000","last_became_peered":"2024-09-21T10:04:55.933296+0000","last_unstale":"2024-09-21T10:05:04.014597+0000","last_undegraded":"2024-09-21T10:05:04.014597+0000","last_fullsized":"2024-09-21T10:05:04.014597+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:33:25.512434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6029312,"num_objects":92,"num_object_clones":0,"num_object_copies":276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92,"num_write_kb":5888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":196,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.461598+0000","last_change":"2024-09-21T10:00:15.524812+0000","last_active":"2024-09-21T10:05:00.461598+0000","last_peered":"2024-09-21T10:05:00.461598+0000","last_clean":"2024-09-21T10:05:00.461598+0000","last_became_active":"2024-09-21T09:59:43.361751+0000","last_became_peered":"2024-09-21T09:59:43.361751+0000","last_unstale":"2024-09-21T10:05:00.461598+0000","last_undegraded":"2024-09-21T10:05:00.461598+0000","last_fullsized":"2024-09-21T10:05:00.461598+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:30.306777+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":92,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440378+0000","last_change":"2024-09-21T10:01:40.415754+0000","last_active":"2024-09-21T10:04:56.440378+0000","last_peered":"2024-09-21T10:04:56.440378+0000","last_clean":"2024-09-21T10:04:56.440378+0000","last_became_active":"2024-09-21T10:01:40.415428+0000","last_became_peered":"2024-09-21T10:01:40.415428+0000","last_unstale":"2024-09-21T10:04:56.440378+0000","last_undegraded":"2024-09-21T10:04:56.440378+0000","last_fullsized":"2024-09-21T10:04:56.440378+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:53.323411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1171,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356998+0000","last_change":"2024-09-21T10:01:40.414164+0000","last_active":"2024-09-21T10:04:57.356998+0000","last_peered":"2024-09-21T10:04:57.356998+0000","last_clean":"2024-09-21T10:04:57.356998+0000","last_became_active":"2024-09-21T10:01:40.413838+0000","last_became_peered":"2024-09-21T10:01:40.413838+0000","last_unstale":"2024-09-21T10:04:57.356998+0000","last_undegraded":"2024-09-21T10:04:57.356998+0000","last_fullsized":"2024-09-21T10:04:57.356998+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:13:24.380350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"1203'543","reported_seq":554,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:06.040558+0000","last_change":"2024-09-21T10:04:55.933906+0000","last_active":"2024-09-21T10:05:06.040558+0000","last_peered":"2024-09-21T10:05:06.040558+0000","last_clean":"2024-09-21T10:05:06.040558+0000","last_became_active":"2024-09-21T10:04:55.933614+0000","last_became_peered":"2024-09-21T10:04:55.933614+0000","last_unstale":"2024-09-21T10:05:06.040558+0000","last_undegraded":"2024-09-21T10:05:06.040558+0000","last_fullsized":"2024-09-21T10:05:06.040558+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":543,"log_dups_size":0,"ondisk_log_size":543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:08:13.495272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35586048,"num_objects":543,"num_object_clones":0,"num_object_copies":1629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":543,"num_write_kb":34752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1145,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356951+0000","last_change":"2024-09-21T09:46:13.261601+0000","last_active":"2024-09-21T10:04:57.356951+0000","last_peered":"2024-09-21T10:04:57.356951+0000","last_clean":"2024-09-21T10:04:57.356951+0000","last_became_active":"2024-09-21T09:46:13.260250+0000","last_became_peered":"2024-09-21T09:46:13.260250+0000","last_unstale":"2024-09-21T10:04:57.356951+0000","last_undegraded":"2024-09-21T10:04:57.356951+0000","last_fullsized":"2024-09-21T10:04:57.356951+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:06.593462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1174,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357432+0000","last_change":"2024-09-21T10:01:40.424536+0000","last_active":"2024-09-21T10:04:57.357432+0000","last_peered":"2024-09-21T10:04:57.357432+0000","last_clean":"2024-09-21T10:04:57.357432+0000","last_became_active":"2024-09-21T10:01:40.424229+0000","last_became_peered":"2024-09-21T10:01:40.424229+0000","last_unstale":"2024-09-21T10:04:57.357432+0000","last_undegraded":"2024-09-21T10:04:57.357432+0000","last_fullsized":"2024-09-21T10:04:57.357432+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:10.771540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1203'395","reported_seq":406,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:05.545717+0000","last_change":"2024-09-21T10:04:55.935712+0000","last_active":"2024-09-21T10:05:05.545717+0000","last_peered":"2024-09-21T10:05:05.545717+0000","last_clean":"2024-09-21T10:05:05.545717+0000","last_became_active":"2024-09-21T10:04:55.935221+0000","last_became_peered":"2024-09-21T10:04:55.935221+0000","last_unstale":"2024-09-21T10:05:05.545717+0000","last_undegraded":"2024-09-21T10:05:05.545717+0000","last_fullsized":"2024-09-21T10:05:05.545717+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":395,"log_dups_size":0,"ondisk_log_size":395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:20:50.440837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25886720,"num_objects":395,"num_object_clones":0,"num_object_copies":1185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":395,"num_write_kb":25280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":186,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.463216+0000","last_change":"2024-09-21T10:04:20.777769+0000","last_active":"2024-09-21T10:05:00.463216+0000","last_peered":"2024-09-21T10:05:00.463216+0000","last_clean":"2024-09-21T10:05:00.463216+0000","last_became_active":"2024-09-21T10:00:16.351010+0000","last_became_peered":"2024-09-21T10:00:16.351010+0000","last_unstale":"2024-09-21T10:05:00.463216+0000","last_undegraded":"2024-09-21T10:05:00.463216+0000","last_fullsized":"2024-09-21T10:05:00.463216+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-29T15:53:23.454847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1752,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740377+0000","last_change":"2024-09-21T10:01:40.428561+0000","last_active":"2024-09-21T10:05:00.740377+0000","last_peered":"2024-09-21T10:05:00.740377+0000","last_clean":"2024-09-21T10:05:00.740377+0000","last_became_active":"2024-09-21T10:01:40.428250+0000","last_became_peered":"2024-09-21T10:01:40.428250+0000","last_unstale":"2024-09-21T10:05:00.740377+0000","last_undegraded":"2024-09-21T10:05:00.740377+0000","last_fullsized":"2024-09-21T10:05:00.740377+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:58.781370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":657,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740811+0000","last_change":"2024-09-21T10:01:40.423834+0000","last_active":"2024-09-21T10:05:00.740811+0000","last_peered":"2024-09-21T10:05:00.740811+0000","last_clean":"2024-09-21T10:05:00.740811+0000","last_became_active":"2024-09-21T10:01:40.423506+0000","last_became_peered":"2024-09-21T10:01:40.423506+0000","last_unstale":"2024-09-21T10:05:00.740811+0000","last_undegraded":"2024-09-21T10:05:00.740811+0000","last_fullsized":"2024-09-21T10:05:00.740811+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:11.358742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.9","version":"1203'464","reported_seq":475,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:05.545818+0000","last_change":"2024-09-21T10:04:55.934931+0000","last_active":"2024-09-21T10:05:05.545818+0000","last_peered":"2024-09-21T10:05:05.545818+0000","last_clean":"2024-09-21T10:05:05.545818+0000","last_became_active":"2024-09-21T10:04:55.934596+0000","last_became_peered":"2024-09-21T10:04:55.934596+0000","last_unstale":"2024-09-21T10:05:05.545818+0000","last_undegraded":"2024-09-21T10:05:05.545818+0000","last_fullsized":"2024-09-21T10:05:05.545818+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":464,"log_dups_size":0,"ondisk_log_size":464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:39:56.366258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30408704,"num_objects":464,"num_object_clones":0,"num_object_copies":1392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":464,"num_write_kb":29696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1287,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356715+0000","last_change":"2024-09-21T10:00:49.116174+0000","last_active":"2024-09-21T10:04:57.356715+0000","last_peered":"2024-09-21T10:04:57.356715+0000","last_clean":"2024-09-21T10:04:57.356715+0000","last_became_active":"2024-09-21T10:00:16.360430+0000","last_became_peered":"2024-09-21T10:00:16.360430+0000","last_unstale":"2024-09-21T10:04:57.356715+0000","last_undegraded":"2024-09-21T10:04:57.356715+0000","last_fullsized":"2024-09-21T10:04:57.356715+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:49.116103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:49.116103+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:29.874911+0000","scrub_duration":0.0020386419999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":230,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.464174+0000","last_change":"2024-09-21T09:57:39.541461+0000","last_active":"2024-09-21T10:05:00.464174+0000","last_peered":"2024-09-21T10:05:00.464174+0000","last_clean":"2024-09-21T10:05:00.464174+0000","last_became_active":"2024-09-21T09:57:39.540927+0000","last_became_peered":"2024-09-21T09:57:39.540927+0000","last_unstale":"2024-09-21T10:05:00.464174+0000","last_undegraded":"2024-09-21T10:05:00.464174+0000","last_fullsized":"2024-09-21T10:05:00.464174+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:14.815783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1741,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740029+0000","last_change":"2024-09-21T10:00:16.362131+0000","last_active":"2024-09-21T10:05:00.740029+0000","last_peered":"2024-09-21T10:05:00.740029+0000","last_clean":"2024-09-21T10:05:00.740029+0000","last_became_active":"2024-09-21T10:00:16.361722+0000","last_became_peered":"2024-09-21T10:00:16.361722+0000","last_unstale":"2024-09-21T10:05:00.740029+0000","last_undegraded":"2024-09-21T10:05:00.740029+0000","last_fullsized":"2024-09-21T10:05:00.740029+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:38.717426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":82,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440846+0000","last_change":"2024-09-21T10:00:16.361540+0000","last_active":"2024-09-21T10:04:56.440846+0000","last_peered":"2024-09-21T10:04:56.440846+0000","last_clean":"2024-09-21T10:04:56.440846+0000","last_became_active":"2024-09-21T10:00:16.361086+0000","last_became_peered":"2024-09-21T10:00:16.361086+0000","last_unstale":"2024-09-21T10:04:56.440846+0000","last_undegraded":"2024-09-21T10:04:56.440846+0000","last_fullsized":"2024-09-21T10:04:56.440846+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:08:14.645896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":17,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.463066+0000","last_change":"2024-09-21T10:04:55.934274+0000","last_active":"2024-09-21T10:05:00.463066+0000","last_peered":"2024-09-21T10:05:00.463066+0000","last_clean":"2024-09-21T10:05:00.463066+0000","last_became_active":"2024-09-21T10:04:55.933895+0000","last_became_peered":"2024-09-21T10:04:55.933895+0000","last_unstale":"2024-09-21T10:05:00.463066+0000","last_undegraded":"2024-09-21T10:05:00.463066+0000","last_fullsized":"2024-09-21T10:05:00.463066+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:40.821830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":97,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440646+0000","last_change":"2024-09-21T10:04:35.993794+0000","last_active":"2024-09-21T10:04:56.440646+0000","last_peered":"2024-09-21T10:04:56.440646+0000","last_clean":"2024-09-21T10:04:56.440646+0000","last_became_active":"2024-09-21T10:01:40.411508+0000","last_became_peered":"2024-09-21T10:01:40.411508+0000","last_unstale":"2024-09-21T10:04:56.440646+0000","last_undegraded":"2024-09-21T10:04:56.440646+0000","last_fullsized":"2024-09-21T10:04:56.440646+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:04:35.993739+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:58.250609+0000","scrub_duration":0.001400943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1165,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356633+0000","last_change":"2024-09-21T10:01:40.428393+0000","last_active":"2024-09-21T10:04:57.356633+0000","last_peered":"2024-09-21T10:04:57.356633+0000","last_clean":"2024-09-21T10:04:57.356633+0000","last_became_active":"2024-09-21T10:01:40.428204+0000","last_became_peered":"2024-09-21T10:01:40.428204+0000","last_unstale":"2024-09-21T10:04:57.356633+0000","last_undegraded":"2024-09-21T10:04:57.356633+0000","last_fullsized":"2024-09-21T10:04:57.356633+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:23:02.136035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1828,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.465396+0000","last_change":"2024-09-21T09:46:47.121915+0000","last_active":"2024-09-21T10:05:00.465396+0000","last_peered":"2024-09-21T10:05:00.465396+0000","last_clean":"2024-09-21T10:05:00.465396+0000","last_became_active":"2024-09-21T09:46:47.121599+0000","last_became_peered":"2024-09-21T09:46:47.121599+0000","last_unstale":"2024-09-21T10:05:00.465396+0000","last_undegraded":"2024-09-21T10:05:00.465396+0000","last_fullsized":"2024-09-21T10:05:00.465396+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:50:20.892884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1848,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.462564+0000","last_change":"2024-09-21T10:01:40.421887+0000","last_active":"2024-09-21T10:05:00.462564+0000","last_peered":"2024-09-21T10:05:00.462564+0000","last_clean":"2024-09-21T10:05:00.462564+0000","last_became_active":"2024-09-21T10:01:40.420730+0000","last_became_peered":"2024-09-21T10:01:40.420730+0000","last_unstale":"2024-09-21T10:05:00.462564+0000","last_undegraded":"2024-09-21T10:05:00.462564+0000","last_fullsized":"2024-09-21T10:05:00.462564+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:51.504051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":17,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740172+0000","last_change":"2024-09-21T10:04:55.445654+0000","last_active":"2024-09-21T10:05:00.740172+0000","last_peered":"2024-09-21T10:05:00.740172+0000","last_clean":"2024-09-21T10:05:00.740172+0000","last_became_active":"2024-09-21T10:04:55.445345+0000","last_became_peered":"2024-09-21T10:04:55.445345+0000","last_unstale":"2024-09-21T10:05:00.740172+0000","last_undegraded":"2024-09-21T10:05:00.740172+0000","last_fullsized":"2024-09-21T10:05:00.740172+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:24.375029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":1273,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356616+0000","last_change":"2024-09-21T10:00:16.913196+0000","last_active":"2024-09-21T10:04:57.356616+0000","last_peered":"2024-09-21T10:04:57.356616+0000","last_clean":"2024-09-21T10:04:57.356616+0000","last_became_active":"2024-09-21T09:59:43.370454+0000","last_became_peered":"2024-09-21T09:59:43.370454+0000","last_unstale":"2024-09-21T10:04:57.356616+0000","last_undegraded":"2024-09-21T10:04:57.356616+0000","last_fullsized":"2024-09-21T10:04:57.356616+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:16.913146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:16.913146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:23:33.788563+0000","scrub_duration":0.001342315,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1803,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.462907+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:05:00.462907+0000","last_peered":"2024-09-21T10:05:00.462907+0000","last_clean":"2024-09-21T10:05:00.462907+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:05:00.462907+0000","last_undegraded":"2024-09-21T10:05:00.462907+0000","last_fullsized":"2024-09-21T10:05:00.462907+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1757,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.461723+0000","last_change":"2024-09-21T10:01:40.414547+0000","last_active":"2024-09-21T10:05:00.461723+0000","last_peered":"2024-09-21T10:05:00.461723+0000","last_clean":"2024-09-21T10:05:00.461723+0000","last_became_active":"2024-09-21T10:01:40.414243+0000","last_became_peered":"2024-09-21T10:01:40.414243+0000","last_unstale":"2024-09-21T10:05:00.461723+0000","last_undegraded":"2024-09-21T10:05:00.461723+0000","last_fullsized":"2024-09-21T10:05:00.461723+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:55:26.044481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":538,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.464823+0000","last_change":"2024-09-21T10:01:40.421427+0000","last_active":"2024-09-21T10:05:00.464823+0000","last_peered":"2024-09-21T10:05:00.464823+0000","last_clean":"2024-09-21T10:05:00.464823+0000","last_became_active":"2024-09-21T10:01:40.421047+0000","last_became_peered":"2024-09-21T10:01:40.421047+0000","last_unstale":"2024-09-21T10:05:00.464823+0000","last_undegraded":"2024-09-21T10:05:00.464823+0000","last_fullsized":"2024-09-21T10:05:00.464823+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:48.640296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":17,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.739968+0000","last_change":"2024-09-21T10:04:55.934259+0000","last_active":"2024-09-21T10:05:00.739968+0000","last_peered":"2024-09-21T10:05:00.739968+0000","last_clean":"2024-09-21T10:05:00.739968+0000","last_became_active":"2024-09-21T10:04:55.933861+0000","last_became_peered":"2024-09-21T10:04:55.933861+0000","last_unstale":"2024-09-21T10:05:00.739968+0000","last_undegraded":"2024-09-21T10:05:00.739968+0000","last_fullsized":"2024-09-21T10:05:00.739968+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:22:32.497986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":525,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.465405+0000","last_change":"2024-09-21T10:03:19.415715+0000","last_active":"2024-09-21T10:05:00.465405+0000","last_peered":"2024-09-21T10:05:00.465405+0000","last_clean":"2024-09-21T10:05:00.465405+0000","last_became_active":"2024-09-21T09:59:43.371384+0000","last_became_peered":"2024-09-21T09:59:43.371384+0000","last_unstale":"2024-09-21T10:05:00.465405+0000","last_undegraded":"2024-09-21T10:05:00.465405+0000","last_fullsized":"2024-09-21T10:05:00.465405+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-30T13:30:29.665440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":160,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440146+0000","last_change":"2024-09-21T10:01:40.426865+0000","last_active":"2024-09-21T10:04:56.440146+0000","last_peered":"2024-09-21T10:04:56.440146+0000","last_clean":"2024-09-21T10:04:56.440146+0000","last_became_active":"2024-09-21T10:01:40.413152+0000","last_became_peered":"2024-09-21T10:01:40.413152+0000","last_unstale":"2024-09-21T10:04:56.440146+0000","last_undegraded":"2024-09-21T10:04:56.440146+0000","last_fullsized":"2024-09-21T10:04:56.440146+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:59.034627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1165,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357476+0000","last_change":"2024-09-21T10:00:16.356367+0000","last_active":"2024-09-21T10:04:57.357476+0000","last_peered":"2024-09-21T10:04:57.357476+0000","last_clean":"2024-09-21T10:04:57.357476+0000","last_became_active":"2024-09-21T10:00:16.356079+0000","last_became_peered":"2024-09-21T10:00:16.356079+0000","last_unstale":"2024-09-21T10:04:57.357476+0000","last_undegraded":"2024-09-21T10:04:57.357476+0000","last_fullsized":"2024-09-21T10:04:57.357476+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:12.934640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":668,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.463839+0000","last_change":"2024-09-21T10:01:40.421262+0000","last_active":"2024-09-21T10:05:00.463839+0000","last_peered":"2024-09-21T10:05:00.463839+0000","last_clean":"2024-09-21T10:05:00.463839+0000","last_became_active":"2024-09-21T10:01:40.420959+0000","last_became_peered":"2024-09-21T10:01:40.420959+0000","last_unstale":"2024-09-21T10:05:00.463839+0000","last_undegraded":"2024-09-21T10:05:00.463839+0000","last_fullsized":"2024-09-21T10:05:00.463839+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:06.866421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"1203'101","reported_seq":118,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.013899+0000","last_change":"2024-09-21T10:04:55.449598+0000","last_active":"2024-09-21T10:05:04.013899+0000","last_peered":"2024-09-21T10:05:04.013899+0000","last_clean":"2024-09-21T10:05:04.013899+0000","last_became_active":"2024-09-21T10:04:55.449025+0000","last_became_peered":"2024-09-21T10:04:55.449025+0000","last_unstale":"2024-09-21T10:05:04.013899+0000","last_undegraded":"2024-09-21T10:05:04.013899+0000","last_fullsized":"2024-09-21T10:05:04.013899+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":101,"log_dups_size":0,"ondisk_log_size":101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:57:57.650827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6619136,"num_objects":101,"num_object_clones":0,"num_object_copies":303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":6464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":92,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440781+0000","last_change":"2024-09-21T10:00:46.494180+0000","last_active":"2024-09-21T10:04:56.440781+0000","last_peered":"2024-09-21T10:04:56.440781+0000","last_clean":"2024-09-21T10:04:56.440781+0000","last_became_active":"2024-09-21T10:00:16.355386+0000","last_became_peered":"2024-09-21T10:00:16.355386+0000","last_unstale":"2024-09-21T10:04:56.440781+0000","last_undegraded":"2024-09-21T10:04:56.440781+0000","last_fullsized":"2024-09-21T10:04:56.440781+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:46.494129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:46.494129+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:11.193593+0000","scrub_duration":0.00183863,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":759,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740666+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T10:05:00.740666+0000","last_peered":"2024-09-21T10:05:00.740666+0000","last_clean":"2024-09-21T10:05:00.740666+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T10:05:00.740666+0000","last_undegraded":"2024-09-21T10:05:00.740666+0000","last_fullsized":"2024-09-21T10:05:00.740666+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":567,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.462206+0000","last_change":"2024-09-21T10:01:40.418328+0000","last_active":"2024-09-21T10:05:00.462206+0000","last_peered":"2024-09-21T10:05:00.462206+0000","last_clean":"2024-09-21T10:05:00.462206+0000","last_became_active":"2024-09-21T10:01:40.418011+0000","last_became_peered":"2024-09-21T10:01:40.418011+0000","last_unstale":"2024-09-21T10:05:00.462206+0000","last_undegraded":"2024-09-21T10:05:00.462206+0000","last_fullsized":"2024-09-21T10:05:00.462206+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:58.716988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":506,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357101+0000","last_change":"2024-09-21T10:01:40.421225+0000","last_active":"2024-09-21T10:04:57.357101+0000","last_peered":"2024-09-21T10:04:57.357101+0000","last_clean":"2024-09-21T10:04:57.357101+0000","last_became_active":"2024-09-21T10:01:40.420909+0000","last_became_peered":"2024-09-21T10:01:40.420909+0000","last_unstale":"2024-09-21T10:04:57.357101+0000","last_undegraded":"2024-09-21T10:04:57.357101+0000","last_fullsized":"2024-09-21T10:04:57.357101+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:03.811941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":17,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740765+0000","last_change":"2024-09-21T10:04:55.445651+0000","last_active":"2024-09-21T10:05:00.740765+0000","last_peered":"2024-09-21T10:05:00.740765+0000","last_clean":"2024-09-21T10:05:00.740765+0000","last_became_active":"2024-09-21T10:04:55.445264+0000","last_became_peered":"2024-09-21T10:04:55.445264+0000","last_unstale":"2024-09-21T10:05:00.740765+0000","last_undegraded":"2024-09-21T10:05:00.740765+0000","last_fullsized":"2024-09-21T10:05:00.740765+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:40.201370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":92,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440624+0000","last_change":"2024-09-21T10:00:20.637858+0000","last_active":"2024-09-21T10:04:56.440624+0000","last_peered":"2024-09-21T10:04:56.440624+0000","last_clean":"2024-09-21T10:04:56.440624+0000","last_became_active":"2024-09-21T10:00:16.353796+0000","last_became_peered":"2024-09-21T10:00:16.353796+0000","last_unstale":"2024-09-21T10:04:56.440624+0000","last_undegraded":"2024-09-21T10:04:56.440624+0000","last_fullsized":"2024-09-21T10:04:56.440624+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:20.637806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:20.637806+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:12:16.884495+0000","scrub_duration":0.0016753919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":594,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.463664+0000","last_change":"2024-09-21T10:01:40.414536+0000","last_active":"2024-09-21T10:05:00.463664+0000","last_peered":"2024-09-21T10:05:00.463664+0000","last_clean":"2024-09-21T10:05:00.463664+0000","last_became_active":"2024-09-21T10:01:40.413913+0000","last_became_peered":"2024-09-21T10:01:40.413913+0000","last_unstale":"2024-09-21T10:05:00.463664+0000","last_undegraded":"2024-09-21T10:05:00.463664+0000","last_fullsized":"2024-09-21T10:05:00.463664+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:27.994971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1138,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357539+0000","last_change":"2024-09-21T09:45:05.087820+0000","last_active":"2024-09-21T10:04:57.357539+0000","last_peered":"2024-09-21T10:04:57.357539+0000","last_clean":"2024-09-21T10:04:57.357539+0000","last_became_active":"2024-09-21T09:45:05.087515+0000","last_became_peered":"2024-09-21T09:45:05.087515+0000","last_unstale":"2024-09-21T10:04:57.357539+0000","last_undegraded":"2024-09-21T10:04:57.357539+0000","last_fullsized":"2024-09-21T10:04:57.357539+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:09.799807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1819,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.461590+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T10:05:00.461590+0000","last_peered":"2024-09-21T10:05:00.461590+0000","last_clean":"2024-09-21T10:05:00.461590+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T10:05:00.461590+0000","last_undegraded":"2024-09-21T10:05:00.461590+0000","last_fullsized":"2024-09-21T10:05:00.461590+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"1203'94","reported_seq":111,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.016264+0000","last_change":"2024-09-21T10:04:55.448819+0000","last_active":"2024-09-21T10:05:04.016264+0000","last_peered":"2024-09-21T10:05:04.016264+0000","last_clean":"2024-09-21T10:05:04.016264+0000","last_became_active":"2024-09-21T10:04:55.448338+0000","last_became_peered":"2024-09-21T10:04:55.448338+0000","last_unstale":"2024-09-21T10:05:04.016264+0000","last_undegraded":"2024-09-21T10:05:04.016264+0000","last_fullsized":"2024-09-21T10:05:04.016264+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":94,"log_dups_size":0,"ondisk_log_size":94,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:58.758191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6160384,"num_objects":94,"num_object_clones":0,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":94,"num_write_kb":6016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":112,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356734+0000","last_change":"2024-09-21T10:00:16.356251+0000","last_active":"2024-09-21T10:04:57.356734+0000","last_peered":"2024-09-21T10:04:57.356734+0000","last_clean":"2024-09-21T10:04:57.356734+0000","last_became_active":"2024-09-21T10:00:16.355833+0000","last_became_peered":"2024-09-21T10:00:16.355833+0000","last_unstale":"2024-09-21T10:04:57.356734+0000","last_undegraded":"2024-09-21T10:04:57.356734+0000","last_fullsized":"2024-09-21T10:04:57.356734+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T09:59:47.767654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:59:47.767654+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:50.939198+0000","scrub_duration":0.006614371,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1754,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740849+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T10:05:00.740849+0000","last_peered":"2024-09-21T10:05:00.740849+0000","last_clean":"2024-09-21T10:05:00.740849+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T10:05:00.740849+0000","last_undegraded":"2024-09-21T10:05:00.740849+0000","last_fullsized":"2024-09-21T10:05:00.740849+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1706,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440546+0000","last_change":"2024-09-21T10:01:40.423209+0000","last_active":"2024-09-21T10:04:56.440546+0000","last_peered":"2024-09-21T10:04:56.440546+0000","last_clean":"2024-09-21T10:04:56.440546+0000","last_became_active":"2024-09-21T10:01:40.422890+0000","last_became_peered":"2024-09-21T10:01:40.422890+0000","last_unstale":"2024-09-21T10:04:56.440546+0000","last_undegraded":"2024-09-21T10:04:56.440546+0000","last_fullsized":"2024-09-21T10:04:56.440546+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:10.683126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1752,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740315+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T10:05:00.740315+0000","last_peered":"2024-09-21T10:05:00.740315+0000","last_clean":"2024-09-21T10:05:00.740315+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T10:05:00.740315+0000","last_undegraded":"2024-09-21T10:05:00.740315+0000","last_fullsized":"2024-09-21T10:05:00.740315+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1203'564","reported_seq":575,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:06.086958+0000","last_change":"2024-09-21T10:04:55.932700+0000","last_active":"2024-09-21T10:05:06.086958+0000","last_peered":"2024-09-21T10:05:06.086958+0000","last_clean":"2024-09-21T10:05:06.086958+0000","last_became_active":"2024-09-21T10:04:55.932416+0000","last_became_peered":"2024-09-21T10:04:55.932416+0000","last_unstale":"2024-09-21T10:05:06.086958+0000","last_undegraded":"2024-09-21T10:05:06.086958+0000","last_fullsized":"2024-09-21T10:05:06.086958+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":564,"log_dups_size":0,"ondisk_log_size":564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.824627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36962304,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":564,"num_write_kb":36096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":184,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740273+0000","last_change":"2024-09-21T10:00:22.052497+0000","last_active":"2024-09-21T10:05:00.740273+0000","last_peered":"2024-09-21T10:05:00.740273+0000","last_clean":"2024-09-21T10:05:00.740273+0000","last_became_active":"2024-09-21T09:59:43.368157+0000","last_became_peered":"2024-09-21T09:59:43.368157+0000","last_unstale":"2024-09-21T10:05:00.740273+0000","last_undegraded":"2024-09-21T10:05:00.740273+0000","last_fullsized":"2024-09-21T10:05:00.740273+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:32:54.834683+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic deep scrub scheduled @ 2024-09-28T12:21:36.382264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1843,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.462884+0000","last_change":"2024-09-21T10:01:40.433067+0000","last_active":"2024-09-21T10:05:00.462884+0000","last_peered":"2024-09-21T10:05:00.462884+0000","last_clean":"2024-09-21T10:05:00.462884+0000","last_became_active":"2024-09-21T10:01:40.420622+0000","last_became_peered":"2024-09-21T10:01:40.420622+0000","last_unstale":"2024-09-21T10:05:00.462884+0000","last_undegraded":"2024-09-21T10:05:00.462884+0000","last_fullsized":"2024-09-21T10:05:00.462884+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:28.383227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":82,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440712+0000","last_change":"2024-09-21T10:00:16.355102+0000","last_active":"2024-09-21T10:04:56.440712+0000","last_peered":"2024-09-21T10:04:56.440712+0000","last_clean":"2024-09-21T10:04:56.440712+0000","last_became_active":"2024-09-21T10:00:16.354473+0000","last_became_peered":"2024-09-21T10:00:16.354473+0000","last_unstale":"2024-09-21T10:04:56.440712+0000","last_undegraded":"2024-09-21T10:04:56.440712+0000","last_fullsized":"2024-09-21T10:04:56.440712+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:49:35.449111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1147,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356759+0000","last_change":"2024-09-21T10:00:16.355601+0000","last_active":"2024-09-21T10:04:57.356759+0000","last_peered":"2024-09-21T10:04:57.356759+0000","last_clean":"2024-09-21T10:04:57.356759+0000","last_became_active":"2024-09-21T10:00:16.355110+0000","last_became_peered":"2024-09-21T10:00:16.355110+0000","last_unstale":"2024-09-21T10:04:57.356759+0000","last_undegraded":"2024-09-21T10:04:57.356759+0000","last_fullsized":"2024-09-21T10:04:57.356759+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:10.590252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1174,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357406+0000","last_change":"2024-09-21T10:01:40.414117+0000","last_active":"2024-09-21T10:04:57.357406+0000","last_peered":"2024-09-21T10:04:57.357406+0000","last_clean":"2024-09-21T10:04:57.357406+0000","last_became_active":"2024-09-21T10:01:40.413186+0000","last_became_peered":"2024-09-21T10:01:40.413186+0000","last_unstale":"2024-09-21T10:04:57.357406+0000","last_undegraded":"2024-09-21T10:04:57.357406+0000","last_fullsized":"2024-09-21T10:04:57.357406+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:17:05.368450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1190,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357558+0000","last_change":"2024-09-21T10:00:16.371426+0000","last_active":"2024-09-21T10:04:57.357558+0000","last_peered":"2024-09-21T10:04:57.357558+0000","last_clean":"2024-09-21T10:04:57.357558+0000","last_became_active":"2024-09-21T10:00:16.357486+0000","last_became_peered":"2024-09-21T10:00:16.357486+0000","last_unstale":"2024-09-21T10:04:57.357558+0000","last_undegraded":"2024-09-21T10:04:57.357558+0000","last_fullsized":"2024-09-21T10:04:57.357558+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:02.923682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1707,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440298+0000","last_change":"2024-09-21T10:01:40.413147+0000","last_active":"2024-09-21T10:04:56.440298+0000","last_peered":"2024-09-21T10:04:56.440298+0000","last_clean":"2024-09-21T10:04:56.440298+0000","last_became_active":"2024-09-21T10:01:40.412833+0000","last_became_peered":"2024-09-21T10:01:40.412833+0000","last_unstale":"2024-09-21T10:04:56.440298+0000","last_undegraded":"2024-09-21T10:04:56.440298+0000","last_fullsized":"2024-09-21T10:04:56.440298+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:33.262350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1129,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357105+0000","last_change":"2024-09-21T09:46:47.113452+0000","last_active":"2024-09-21T10:04:57.357105+0000","last_peered":"2024-09-21T10:04:57.357105+0000","last_clean":"2024-09-21T10:04:57.357105+0000","last_became_active":"2024-09-21T09:46:47.113187+0000","last_became_peered":"2024-09-21T09:46:47.113187+0000","last_unstale":"2024-09-21T10:04:57.357105+0000","last_undegraded":"2024-09-21T10:04:57.357105+0000","last_fullsized":"2024-09-21T10:04:57.357105+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:43.383721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":583,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357732+0000","last_change":"2024-09-21T10:01:40.427109+0000","last_active":"2024-09-21T10:04:57.357732+0000","last_peered":"2024-09-21T10:04:57.357732+0000","last_clean":"2024-09-21T10:04:57.357732+0000","last_became_active":"2024-09-21T10:01:40.426821+0000","last_became_peered":"2024-09-21T10:01:40.426821+0000","last_unstale":"2024-09-21T10:04:57.357732+0000","last_undegraded":"2024-09-21T10:04:57.357732+0000","last_fullsized":"2024-09-21T10:04:57.357732+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:12.260797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1740,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.739904+0000","last_change":"2024-09-21T10:00:16.357151+0000","last_active":"2024-09-21T10:05:00.739904+0000","last_peered":"2024-09-21T10:05:00.739904+0000","last_clean":"2024-09-21T10:05:00.739904+0000","last_became_active":"2024-09-21T10:00:16.356864+0000","last_became_peered":"2024-09-21T10:00:16.356864+0000","last_unstale":"2024-09-21T10:05:00.739904+0000","last_undegraded":"2024-09-21T10:05:00.739904+0000","last_fullsized":"2024-09-21T10:05:00.739904+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:52.851887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1827,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.465109+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T10:05:00.465109+0000","last_peered":"2024-09-21T10:05:00.465109+0000","last_clean":"2024-09-21T10:05:00.465109+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T10:05:00.465109+0000","last_undegraded":"2024-09-21T10:05:00.465109+0000","last_fullsized":"2024-09-21T10:05:00.465109+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1764,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740256+0000","last_change":"2024-09-21T10:01:40.426702+0000","last_active":"2024-09-21T10:05:00.740256+0000","last_peered":"2024-09-21T10:05:00.740256+0000","last_clean":"2024-09-21T10:05:00.740256+0000","last_became_active":"2024-09-21T10:01:40.426034+0000","last_became_peered":"2024-09-21T10:01:40.426034+0000","last_unstale":"2024-09-21T10:05:00.740256+0000","last_undegraded":"2024-09-21T10:05:00.740256+0000","last_fullsized":"2024-09-21T10:05:00.740256+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:20.619421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":92,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:56.440764+0000","last_change":"2024-09-21T10:01:40.414393+0000","last_active":"2024-09-21T10:04:56.440764+0000","last_peered":"2024-09-21T10:04:56.440764+0000","last_clean":"2024-09-21T10:04:56.440764+0000","last_became_active":"2024-09-21T10:01:40.414096+0000","last_became_peered":"2024-09-21T10:01:40.414096+0000","last_unstale":"2024-09-21T10:04:56.440764+0000","last_undegraded":"2024-09-21T10:04:56.440764+0000","last_fullsized":"2024-09-21T10:04:56.440764+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:32.540479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1768,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.739786+0000","last_change":"2024-09-21T10:01:40.434475+0000","last_active":"2024-09-21T10:05:00.739786+0000","last_peered":"2024-09-21T10:05:00.739786+0000","last_clean":"2024-09-21T10:05:00.739786+0000","last_became_active":"2024-09-21T10:01:40.432013+0000","last_became_peered":"2024-09-21T10:01:40.432013+0000","last_unstale":"2024-09-21T10:05:00.739786+0000","last_undegraded":"2024-09-21T10:05:00.739786+0000","last_fullsized":"2024-09-21T10:05:00.739786+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:44.373378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1831,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.462750+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T10:05:00.462750+0000","last_peered":"2024-09-21T10:05:00.462750+0000","last_clean":"2024-09-21T10:05:00.462750+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T10:05:00.462750+0000","last_undegraded":"2024-09-21T10:05:00.462750+0000","last_fullsized":"2024-09-21T10:05:00.462750+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":658,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740212+0000","last_change":"2024-09-21T10:01:40.437120+0000","last_active":"2024-09-21T10:05:00.740212+0000","last_peered":"2024-09-21T10:05:00.740212+0000","last_clean":"2024-09-21T10:05:00.740212+0000","last_became_active":"2024-09-21T10:01:40.436275+0000","last_became_peered":"2024-09-21T10:01:40.436275+0000","last_unstale":"2024-09-21T10:05:00.740212+0000","last_undegraded":"2024-09-21T10:05:00.740212+0000","last_fullsized":"2024-09-21T10:05:00.740212+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:25.322111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1750,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740415+0000","last_change":"2024-09-21T10:01:40.427770+0000","last_active":"2024-09-21T10:05:00.740415+0000","last_peered":"2024-09-21T10:05:00.740415+0000","last_clean":"2024-09-21T10:05:00.740415+0000","last_became_active":"2024-09-21T10:01:40.427458+0000","last_became_peered":"2024-09-21T10:01:40.427458+0000","last_unstale":"2024-09-21T10:05:00.740415+0000","last_undegraded":"2024-09-21T10:05:00.740415+0000","last_fullsized":"2024-09-21T10:05:00.740415+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:30:38.605007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1173,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357235+0000","last_change":"2024-09-21T10:01:40.420505+0000","last_active":"2024-09-21T10:04:57.357235+0000","last_peered":"2024-09-21T10:04:57.357235+0000","last_clean":"2024-09-21T10:04:57.357235+0000","last_became_active":"2024-09-21T10:01:40.420263+0000","last_became_peered":"2024-09-21T10:01:40.420263+0000","last_unstale":"2024-09-21T10:04:57.357235+0000","last_undegraded":"2024-09-21T10:04:57.357235+0000","last_fullsized":"2024-09-21T10:04:57.357235+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:23.452408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1788,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.463756+0000","last_change":"2024-09-21T10:01:40.420219+0000","last_active":"2024-09-21T10:05:00.463756+0000","last_peered":"2024-09-21T10:05:00.463756+0000","last_clean":"2024-09-21T10:05:00.463756+0000","last_became_active":"2024-09-21T10:01:40.419839+0000","last_became_peered":"2024-09-21T10:01:40.419839+0000","last_unstale":"2024-09-21T10:05:00.463756+0000","last_undegraded":"2024-09-21T10:05:00.463756+0000","last_fullsized":"2024-09-21T10:05:00.463756+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:59.370554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1820,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.461736+0000","last_change":"2024-09-21T09:45:05.094710+0000","last_active":"2024-09-21T10:05:00.461736+0000","last_peered":"2024-09-21T10:05:00.461736+0000","last_clean":"2024-09-21T10:05:00.461736+0000","last_became_active":"2024-09-21T09:45:05.094021+0000","last_became_peered":"2024-09-21T09:45:05.094021+0000","last_unstale":"2024-09-21T10:05:00.461736+0000","last_undegraded":"2024-09-21T10:05:00.461736+0000","last_fullsized":"2024-09-21T10:05:00.461736+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:37.081412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1171,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357383+0000","last_change":"2024-09-21T10:01:40.432190+0000","last_active":"2024-09-21T10:04:57.357383+0000","last_peered":"2024-09-21T10:04:57.357383+0000","last_clean":"2024-09-21T10:04:57.357383+0000","last_became_active":"2024-09-21T10:01:40.431847+0000","last_became_peered":"2024-09-21T10:01:40.431847+0000","last_unstale":"2024-09-21T10:04:57.357383+0000","last_undegraded":"2024-09-21T10:04:57.357383+0000","last_fullsized":"2024-09-21T10:04:57.357383+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:32.573309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":139,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357412+0000","last_change":"2024-09-21T10:01:40.420069+0000","last_active":"2024-09-21T10:04:57.357412+0000","last_peered":"2024-09-21T10:04:57.357412+0000","last_clean":"2024-09-21T10:04:57.357412+0000","last_became_active":"2024-09-21T10:01:40.416524+0000","last_became_peered":"2024-09-21T10:01:40.416524+0000","last_unstale":"2024-09-21T10:04:57.357412+0000","last_undegraded":"2024-09-21T10:04:57.357412+0000","last_fullsized":"2024-09-21T10:04:57.357412+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:36.273679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1161,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.357177+0000","last_change":"2024-09-21T09:46:47.114223+0000","last_active":"2024-09-21T10:04:57.357177+0000","last_peered":"2024-09-21T10:04:57.357177+0000","last_clean":"2024-09-21T10:04:57.357177+0000","last_became_active":"2024-09-21T09:46:47.113912+0000","last_became_peered":"2024-09-21T09:46:47.113912+0000","last_unstale":"2024-09-21T10:04:57.357177+0000","last_undegraded":"2024-09-21T10:04:57.357177+0000","last_fullsized":"2024-09-21T10:04:57.357177+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:49.234077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":153,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356818+0000","last_change":"2024-09-21T10:00:16.366351+0000","last_active":"2024-09-21T10:04:57.356818+0000","last_peered":"2024-09-21T10:04:57.356818+0000","last_clean":"2024-09-21T10:04:57.356818+0000","last_became_active":"2024-09-21T10:00:16.355385+0000","last_became_peered":"2024-09-21T10:00:16.355385+0000","last_unstale":"2024-09-21T10:04:57.356818+0000","last_undegraded":"2024-09-21T10:04:57.356818+0000","last_fullsized":"2024-09-21T10:04:57.356818+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:29.014677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1753,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740209+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:05:00.740209+0000","last_peered":"2024-09-21T10:05:00.740209+0000","last_clean":"2024-09-21T10:05:00.740209+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:05:00.740209+0000","last_undegraded":"2024-09-21T10:05:00.740209+0000","last_fullsized":"2024-09-21T10:05:00.740209+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1754,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.740701+0000","last_change":"2024-09-21T10:00:16.361160+0000","last_active":"2024-09-21T10:05:00.740701+0000","last_peered":"2024-09-21T10:05:00.740701+0000","last_clean":"2024-09-21T10:05:00.740701+0000","last_became_active":"2024-09-21T10:00:16.360524+0000","last_became_peered":"2024-09-21T10:00:16.360524+0000","last_unstale":"2024-09-21T10:05:00.740701+0000","last_undegraded":"2024-09-21T10:05:00.740701+0000","last_fullsized":"2024-09-21T10:05:00.740701+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:52.747211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1174,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:04:57.356309+0000","last_change":"2024-09-21T10:01:40.432238+0000","last_active":"2024-09-21T10:04:57.356309+0000","last_peered":"2024-09-21T10:04:57.356309+0000","last_clean":"2024-09-21T10:04:57.356309+0000","last_became_active":"2024-09-21T10:01:40.431883+0000","last_became_peered":"2024-09-21T10:01:40.431883+0000","last_unstale":"2024-09-21T10:04:57.356309+0000","last_undegraded":"2024-09-21T10:04:57.356309+0000","last_fullsized":"2024-09-21T10:04:57.356309+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:30.528533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1835,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:05:00.465169+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T10:05:00.465169+0000","last_peered":"2024-09-21T10:05:00.465169+0000","last_clean":"2024-09-21T10:05:00.465169+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T10:05:00.465169+0000","last_undegraded":"2024-09-21T10:05:00.465169+0000","last_fullsized":"2024-09-21T10:05:00.465169+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":214630400,"num_objects":3275,"num_object_clones":0,"num_object_copies":9825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3275,"num_write_kb":209600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":909246464,"data_stored":909246464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3275,"ondisk_log_size":3275,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":9628,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":423697,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":80,"num_read_kb":204,"num_write":41,"num_write_kb":444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488689,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397871,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":472852,"kb_used_data":171340,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93272300,"statfs":{"total":95995035648,"available":95510835200,"internally_reserved":0,"allocated":175452160,"data_stored":172802217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594099,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":402636,"kb_used_data":105284,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93342516,"statfs":{"total":95995035648,"available":95582736384,"internally_reserved":0,"allocated":107810816,"data_stored":105168930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790329,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":634120,"kb_used_data":328448,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93111032,"statfs":{"total":95995035648,"available":95345696768,"internally_reserved":0,"allocated":336330752,"data_stored":333689332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986556,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":536912,"kb_used_data":239560,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93208240,"statfs":{"total":95995035648,"available":95445237760,"internally_reserved":0,"allocated":245309440,"data_stored":242663515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999197,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533056,"kb_used_data":9068,"kb_used_omap":25,"kb_used_meta":523942,"kb_avail":93212096,"statfs":{"total":95995035648,"available":95449186304,"internally_reserved":0,"allocated":9285632,"data_stored":6655634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":536517575},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111140,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":491528,"kb_used_data":8764,"kb_used_omap":31,"kb_used_meta":482720,"kb_avail":93253624,"statfs":{"total":95995035648,"available":95491710976,"internally_reserved":0,"allocated":8974336,"data_stored":6336977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":494305512},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2203318223083,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335026,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":420548,"kb_used_data":33852,"kb_used_omap":31,"kb_used_meta":386656,"kb_avail":93324604,"statfs":{"total":95995035648,"available":95564394496,"internally_reserved":0,"allocated":34664448,"data_stored":32026552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":395935840},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":35717120,"data_stored":35717120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":241827840,"data_stored":241827840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":309985280,"data_stored":309985280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":142082048,"data_stored":142082048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":179634176,"data_stored":179634176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:05:08.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 11 2024-09-21T10:05:08.491 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:08.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:08.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:08.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:08.645 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 15821 15805 197.512 158 0.00361866 0.00505667 2024-09-21T10:05:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:08 smithi017 ceph-mon[102913]: from='client.35766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:08.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:08 smithi017 ceph-mon[105151]: from='client.35766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:08.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:08.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:08.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:08.857 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:08.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.892 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:08.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:08.908 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:08 smithi134 ceph-mon[90085]: from='client.35766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:08.992 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:09.025 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:09.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:09.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:09.259 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:09.297 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:09.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:09.400 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:09.437 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:09.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:09.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:09.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:09.645 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 18419 18403 191.654 162.375 0.00566295 0.00521144 2024-09-21T10:05:09.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:09.710 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:09.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:09.811 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:09.846 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:09 smithi134 ceph-mon[90085]: pgmap v1624: 99 pgs: 99 active+clean; 441 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 44 MiB/s wr, 704 op/s 2024-09-21T10:05:09.847 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:09.892 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:09.892 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:09.930 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:09 smithi017 ceph-mon[102913]: pgmap v1624: 99 pgs: 99 active+clean; 441 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 44 MiB/s wr, 704 op/s 2024-09-21T10:05:09.930 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:09 smithi017 ceph-mon[105151]: pgmap v1624: 99 pgs: 99 active+clean; 441 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 44 MiB/s wr, 704 op/s 2024-09-21T10:05:09.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:09.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:09.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:10.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:10.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:10.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:10.224 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:10.246 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:10.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:10.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:10.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:10.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:10.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:05:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:05:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:10.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:10.584 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:05:10.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:10.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:05:10.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 21058 21042 187.827 164.938 0.00534674 0.00531771 2024-09-21T10:05:10.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:10.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:10.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:10.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:10.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:10.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:10.942 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:10.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:10 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1166912649' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:10 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:10 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1166912649' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:10 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:10 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1166912649' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:10 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:11.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:11.052 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:11.081 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:11.183 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:11.219 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:11.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:11.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:11.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:11.461 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:11.494 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:11.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:11.596 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:11.626 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:11.645 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 23742 23726 185.316 167.75 0.00340112 0.00539085 2024-09-21T10:05:11.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:11.743 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:11 smithi017 ceph-mon[105151]: pgmap v1625: 99 pgs: 99 active+clean; 680 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:05:11.743 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:11 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:11.744 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:11 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:05:11.744 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:11 smithi017 ceph-mon[105151]: osdmap e1204: 8 total, 8 up, 7 in 2024-09-21T10:05:11.744 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:11 smithi017 ceph-mon[102913]: pgmap v1625: 99 pgs: 99 active+clean; 680 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:05:11.744 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:11 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:11.744 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:11 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:05:11.744 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:11 smithi017 ceph-mon[102913]: osdmap e1204: 8 total, 8 up, 7 in 2024-09-21T10:05:11.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:11.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:11.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:11.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:11.991 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:11 smithi134 ceph-mon[90085]: pgmap v1625: 99 pgs: 99 active+clean; 680 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:05:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:11 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:11 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:05:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:11 smithi134 ceph-mon[90085]: osdmap e1204: 8 total, 8 up, 7 in 2024-09-21T10:05:12.025 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:12.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:12.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:12.153 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:12.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:12.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:12.288 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:12.321 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:12.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:12.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:12.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:12.564 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:12.594 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:12.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 26051 26035 180.759 144.312 0.00851 0.0055265 2024-09-21T10:05:12.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:12.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:12.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:12.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:12.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:12.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:13.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:13.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:13.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:13.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:13.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:13.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:13.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:13.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:13.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:13.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:13.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:05:13.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:13.566 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:13.566 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:13.566 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:13.566 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:13.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:13.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:13.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:05:13.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 28245 28229 176.395 137.125 0.00232978 0.00566131 2024-09-21T10:05:13.678 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:13.700 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:13.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:13.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:13.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:13 smithi017 ceph-mon[102913]: pgmap v1627: 99 pgs: 99 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-21T10:05:13.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:13 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/829015653' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:13.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:13 smithi017 ceph-mon[105151]: pgmap v1627: 99 pgs: 99 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-21T10:05:13.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:13 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/829015653' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:13.952 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:13.980 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:13.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:13 smithi134 ceph-mon[90085]: pgmap v1627: 99 pgs: 99 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-21T10:05:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:13 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/829015653' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:14.082 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:14.117 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:14.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:14.218 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:14.252 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:14.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:14.354 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:14.393 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:14.442 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:05:14.442 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:05:14.442 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:05:14.442 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90845 6.a 6:5de4ee99:::benchmark_data_smithi017_149547_object90844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.571935+0000", 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 455.86697136200002, 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.085221939, 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90845 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.443 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.571935+0000", 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.571935+0000", 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.571934+0000", 2024-09-21T10:05:14.444 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572128+0000", 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019428100000000001 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572139+0000", 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1455e-05 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.445 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572144+0000", 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8890000000000001e-06 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572202+0000", 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8139e-05 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.572213+0000", 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0562e-05 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.446 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.640648+0000", 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.068435024 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642661+0000", 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002012715 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642814+0000", 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015295599999999999 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.447 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651043+0000", 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0082295990000000006 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651085+0000", 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1270000000000003e-05 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657094+0000", 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.006009258 2024-09-21T10:05:14.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657129+0000", 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5426999999999998e-05 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657157+0000", 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8014999999999999e-05 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.449 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90864 6.a 6:5a2b4d81:::benchmark_data_smithi017_149547_object90863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.594822+0000", 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 455.84408510399999, 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0624268729999999, 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90864 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.450 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594822+0000", 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594822+0000", 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594820+0000", 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594917+0000", 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6347000000000004e-05 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594928+0000", 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1263999999999999e-05 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594934+0000", 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9699999999999996e-06 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594987+0000", 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3377000000000001e-05 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.594997+0000", 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0078000000000001e-05 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.645551+0000", 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.050553455 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.645683+0000", 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132762 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.645832+0000", 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000148992 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651118+0000", 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0052851829999999997 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651160+0000", 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2794999999999999e-05 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657187+0000", 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0060262620000000001 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657217+0000", 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0439000000000001e-05 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.456 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.456 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.657248+0000", 2024-09-21T10:05:14.456 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1368999999999997e-05 2024-09-21T10:05:14.456 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90868 6.a 6:506b43e1:::benchmark_data_smithi017_149547_object90867:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.596312+0000", 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 455.84259494899999, 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.066231537, 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90868 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596312+0000", 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596312+0000", 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.458 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596311+0000", 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596386+0000", 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4889999999999996e-05 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596397+0000", 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1117999999999999e-05 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.459 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596402+0000", 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0420000000000002e-06 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596441+0000", 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8637000000000003e-05 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.596447+0000", 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6919999999999996e-06 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.648577+0000", 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0521300920000001 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.648714+0000", 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013687000000000001 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.648850+0000", 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013567900000000001 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653519+0000", 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0046685540000000001 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653542+0000", 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3515000000000001e-05 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.662408+0000", 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0088662189999999998 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.662484+0000", 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5351000000000004e-05 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.662543+0000", 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9657000000000002e-05 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.463 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90889 6.a 6:58dc9b35:::benchmark_data_smithi017_149547_object90888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:38.649533+0000", 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 455.789373635, 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0149460320000001, 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90889 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649533+0000", 2024-09-21T10:05:14.464 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649533+0000", 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649532+0000", 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649557+0000", 2024-09-21T10:05:14.465 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.5724e-05 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649570+0000", 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3212e-05 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649576+0000", 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.993e-06 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.466 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649633+0000", 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6147999999999998e-05 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:38.649640+0000", 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2370000000000003e-06 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651180+0000", 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.00154027 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.467 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651306+0000", 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012564299999999999 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.651449+0000", 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000143218 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653646+0000", 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021974989999999999 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.468 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.653979+0000", 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00033292499999999997 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.664396+0000", 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010416262000000001 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.664443+0000", 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7518999999999997e-05 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.469 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.664479+0000", 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5933000000000003e-05 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52999 7.17 7:e8d33f0f:::benchmark_data_smithi017_180590_object52998:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345686+0000", 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 228.093220702, 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0320069839999999, 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:14.470 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52999 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345686+0000", 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345686+0000", 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.471 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345684+0000", 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346356+0000", 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000671431 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346375+0000", 2024-09-21T10:05:14.472 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9690999999999998e-05 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346383+0000", 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0290000000000005e-06 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.411662+0000", 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.065278180000000005 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.473 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.411668+0000", 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7700000000000004e-06 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.377605+0000", 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96593649999999998 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.377693+0000", 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8140999999999995e-05 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.474 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:67241 7.b 7:dbb4712a:::benchmark_data_smithi017_180590_object67240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:43.673332+0000", 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 210.76557426100001, 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8185781040000002, 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 67241 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.475 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.673332+0000", 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.673332+0000", 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.673331+0000", 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.476 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.673433+0000", 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000101406 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.673448+0000", 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4980999999999999e-05 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.673454+0000", 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5440000000000004e-06 2024-09-21T10:05:14.477 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.448278+0000", 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.774824068 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.448366+0000", 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7745999999999996e-05 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.448471+0000", 2024-09-21T10:05:14.478 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010480900000000001 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.452076+0000", 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0036055810000000001 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.454550+0000", 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002473818 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.479 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.488511+0000", 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.033961077999999999 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.488848+0000", 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00033646199999999999 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.491911+0000", 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003062717 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.480 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:67256 7.b 7:db73d6cd:::benchmark_data_smithi017_180590_object67255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:43.692733+0000", 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 210.74617401699999, 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8062693989999996, 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 67256 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.481 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.692733+0000", 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.692733+0000", 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.692732+0000", 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.692820+0000", 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8444999999999998e-05 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.692865+0000", 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4576000000000001e-05 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.692880+0000", 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5483000000000001e-05 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.452112+0000", 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7592320079999997 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.452282+0000", 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016993100000000001 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.452489+0000", 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00020652499999999999 2024-09-21T10:05:14.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.456666+0000", 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0041769490000000001 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.460962+0000", 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0042967760000000004 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.495554+0000", 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.034591757000000001 2024-09-21T10:05:14.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.495611+0000", 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6916000000000002e-05 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.499002+0000", 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0033909819999999999 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.486 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:67312 7.b 7:d54ac32e:::benchmark_data_smithi017_180590_object67311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:43.753966+0000", 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 210.684940391, 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.748020372, 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 67312 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.487 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.753966+0000", 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.753966+0000", 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.753966+0000", 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.754003+0000", 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7122000000000001e-05 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.754012+0000", 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.6309999999999992e-06 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.754016+0000", 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0659999999999997e-06 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.454587+0000", 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7005715119999998 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.454681+0000", 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.3885e-05 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.454788+0000", 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000106586 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.490 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.461005+0000", 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0062175069999999997 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.461014+0000", 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.9570000000000008e-06 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.499056+0000", 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.038041823000000002 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.491 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.499117+0000", 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0961000000000001e-05 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.501987+0000", 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0028698370000000001 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:67315 7.b 7:d9948fee:::benchmark_data_smithi017_180590_object67314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:43.754910+0000", 2024-09-21T10:05:14.492 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 210.68399680799999, 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7505742829999997, 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 67315 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.754910+0000", 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.493 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.754910+0000", 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.754909+0000", 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.754983+0000", 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4182000000000002e-05 2024-09-21T10:05:14.494 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.495 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.495 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.495 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.755004+0000", 2024-09-21T10:05:14.495 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0798999999999999e-05 2024-09-21T10:05:14.495 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:14.496 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.496 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.496 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.755010+0000", 2024-09-21T10:05:14.496 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8860000000000001e-06 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.464719+0000", 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7097097229999996 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.464867+0000", 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014789 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.464989+0000", 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012155900000000001 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.472595+0000", 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0076063989999999998 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.472614+0000", 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8762000000000001e-05 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.498 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.502063+0000", 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029448875999999999 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.502126+0000", 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3146999999999997e-05 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.505484+0000", 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0033581230000000002 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.499 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:67325 7.b 7:de01242c:::benchmark_data_smithi017_180590_object67324:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:43.765375+0000", 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 210.67353137800001, 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7426950909999999, 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 67325 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.500 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.765375+0000", 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.765375+0000", 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.765375+0000", 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:14.501 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.765436+0000", 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1314000000000003e-05 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.765445+0000", 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.0040000000000005e-06 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.765451+0000", 2024-09-21T10:05:14.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.19e-06 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.470063+0000", 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.704612236 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.470192+0000", 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000128675 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.470376+0000", 2024-09-21T10:05:14.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00018384799999999999 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.477806+0000", 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0074304660000000002 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.480319+0000", 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002512692 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.504 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.505574+0000", 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.025254711999999999 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.505613+0000", 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8954e-05 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.508070+0000", 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0024578220000000001 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.505 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:67368 7.b 7:dd3be812:::benchmark_data_smithi017_180590_object67367:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:43.826587+0000", 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 210.612319649, 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.68937252, 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 67368 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.506 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.826587+0000", 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.826587+0000", 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.826582+0000", 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999948 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.826720+0000", 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013876399999999999 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.826767+0000", 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6291000000000001e-05 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.826772+0000", 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6189999999999999e-06 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.475244+0000", 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6484712989999997 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.475379+0000", 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013489200000000001 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.475501+0000", 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012218299999999999 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.483060+0000", 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0075591290000000004 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.483189+0000", 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000128987 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.511106+0000", 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.027917654 2024-09-21T10:05:14.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.512719+0000", 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0016122910000000001 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.515960+0000", 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032408099999999998 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:67390 7.b 7:d1b23022:::benchmark_data_smithi017_180590_object67389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:43.859262+0000", 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 210.579644602, 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.664807165, 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 67390 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.859262+0000", 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.859262+0000", 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.859260+0000", 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.859300+0000", 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9693000000000001e-05 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.859314+0000", 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4242e-05 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.859321+0000", 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3790000000000003e-06 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.477847+0000", 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6185263440000002 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.477998+0000", 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015065200000000001 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.478120+0000", 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012257300000000001 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.483172+0000", 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0050522659999999997 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.488413+0000", 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0052408480000000002 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.521027+0000", 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.032613709999999997 2024-09-21T10:05:14.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.521072+0000", 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5487e-05 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.524069+0000", 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029968450000000002 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.517 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:67400 7.b 7:d0dcb314:::benchmark_data_smithi017_180590_object67399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:43.901537+0000", 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 210.53736974399999, 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6258323700000004, 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 67400 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.518 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.901537+0000", 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.901537+0000", 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.901536+0000", 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.901591+0000", 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5566000000000003e-05 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.901601+0000", 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.7899999999999994e-06 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:43.901606+0000", 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5249999999999999e-06 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.480366+0000", 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5787599569999999 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.480553+0000", 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00018729 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.480723+0000", 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000169686 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.488474+0000", 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.007751366 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.491928+0000", 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0034539660000000002 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.524134+0000", 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.032205989999999997 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.522 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.524239+0000", 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000105289 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:50.527369+0000", 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0031299969999999998 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:162006 7.b 7:daa534de:::benchmark_data_smithi017_180590_object52446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:03:23.446986+0000", 2024-09-21T10:05:14.523 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 110.991920808, 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.99576651299999996, 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 162006 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.446986+0000", 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.446986+0000", 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.446985+0000", 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.446991+0000", 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8490000000000002e-06 2024-09-21T10:05:14.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.447005+0000", 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4426e-05 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.447012+0000", 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0500000000000003e-06 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.447067+0000", 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4815000000000003e-05 2024-09-21T10:05:14.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.447076+0000", 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.4150000000000005e-06 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.434324+0000", 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.98724713399999997 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.434403+0000", 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9330000000000001e-05 2024-09-21T10:05:14.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.434480+0000", 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.7013000000000003e-05 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442531+0000", 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0080509320000000002 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442565+0000", 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4199999999999998e-05 2024-09-21T10:05:14.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442667+0000", 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010215100000000001 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442715+0000", 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7784000000000002e-05 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442752+0000", 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7400000000000001e-05 2024-09-21T10:05:14.529 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:162024 7.b 7:d73a0b86:::benchmark_data_smithi017_180590_object52464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:03:23.466906+0000", 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 110.972001034, 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97600271500000002, 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.530 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 162024 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.466906+0000", 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.466906+0000", 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.466904+0000", 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.466911+0000", 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2030000000000003e-06 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.466928+0000", 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6728999999999999e-05 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.466934+0000", 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.037e-06 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.466987+0000", 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3198999999999997e-05 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.466997+0000", 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0254e-05 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.437434+0000", 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97043635299999997 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.437530+0000", 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5911999999999999e-05 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.437599+0000", 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9815999999999997e-05 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442817+0000", 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.005217172 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442824+0000", 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8180000000000004e-06 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442853+0000", 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.898e-05 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442888+0000", 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4999000000000002e-05 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442908+0000", 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.001e-05 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:162061 7.b 7:d566aab0:::benchmark_data_smithi017_180590_object52501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:03:23.495529+0000", 2024-09-21T10:05:14.536 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 110.943378161, 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.94749787699999999, 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 162061 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.495529+0000", 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.495529+0000", 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.495527+0000", 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.495534+0000", 2024-09-21T10:05:14.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5950000000000004e-06 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.495547+0000", 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3129000000000001e-05 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.495552+0000", 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4399999999999996e-06 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.495605+0000", 2024-09-21T10:05:14.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2939999999999998e-05 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.495614+0000", 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1260000000000004e-06 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.440116+0000", 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.94450165500000005 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.440203+0000", 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7167000000000001e-05 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.440279+0000", 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6327000000000003e-05 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442951+0000", 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0026714500000000001 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442957+0000", 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6340000000000003e-06 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.442988+0000", 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0660000000000001e-05 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.443010+0000", 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1503999999999999e-05 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.443026+0000", 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6852000000000001e-05 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:162062 7.b 7:de599839:::benchmark_data_smithi017_180590_object52502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:03:23.500361+0000", 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 110.93854616199999, 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95131778499999997, 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.543 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 162062 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.500361+0000", 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.500361+0000", 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.500359+0000", 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.500367+0000", 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8310000000000001e-06 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.500382+0000", 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5710999999999999e-05 2024-09-21T10:05:14.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.500390+0000", 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0679999999999999e-06 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.500444+0000", 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4412e-05 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.500455+0000", 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0798999999999999e-05 2024-09-21T10:05:14.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.443056+0000", 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.94260120999999997 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.443166+0000", 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010960500000000001 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.443458+0000", 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00029271400000000002 2024-09-21T10:05:14.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451560+0000", 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0081018500000000007 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451592+0000", 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2084e-05 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451600+0000", 2024-09-21T10:05:14.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5209999999999997e-06 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451648+0000", 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8591000000000003e-05 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451678+0000", 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0040999999999999e-05 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:162068 7.b 7:d4a40899:::benchmark_data_smithi017_180590_object52508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:03:23.502766+0000", 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 110.93614057800001, 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95846755500000003, 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 162068 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.502766+0000", 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.502766+0000", 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.502765+0000", 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.502773+0000", 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1839999999999999e-06 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.502790+0000", 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6481e-05 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.502796+0000", 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7729999999999996e-06 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.502841+0000", 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5537999999999997e-05 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.502853+0000", 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1606999999999999e-05 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.446614+0000", 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.94376169700000001 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.446723+0000", 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010909 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.446834+0000", 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000110117 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451693+0000", 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0048596469999999999 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451726+0000", 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2348999999999998e-05 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461147+0000", 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0094213200000000004 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461195+0000", 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8384000000000003e-05 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461234+0000", 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8411999999999997e-05 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:162096 7.b 7:da497ded:::benchmark_data_smithi017_180590_object52536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:03:23.532686+0000", 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 110.90622028, 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.92870396, 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.556 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 162096 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.532686+0000", 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.532686+0000", 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.532685+0000", 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.532691+0000", 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2599999999999996e-06 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.532700+0000", 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.2269999999999999e-06 2024-09-21T10:05:14.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.532704+0000", 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0840000000000002e-06 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.532746+0000", 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2172000000000002e-05 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.532756+0000", 2024-09-21T10:05:14.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0273e-05 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.449192+0000", 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.91643590500000005 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.449291+0000", 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8881000000000004e-05 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.449360+0000", 2024-09-21T10:05:14.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8615999999999995e-05 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451732+0000", 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0023716689999999999 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.454515+0000", 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0027830730000000001 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461309+0000", 2024-09-21T10:05:14.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0067941329999999999 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461359+0000", 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0720000000000002e-05 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461390+0000", 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0923999999999997e-05 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:162099 7.b 7:dd213a01:::benchmark_data_smithi017_180590_object52539:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:03:23.537737+0000", 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 110.901169595, 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.92379348800000005, 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 162099 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.537737+0000", 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.537737+0000", 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.537736+0000", 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.564 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.537749+0000", 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2847000000000001e-05 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.537761+0000", 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1810999999999999e-05 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.537765+0000", 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7299999999999996e-06 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.537808+0000", 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3117000000000001e-05 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:23.537815+0000", 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7649999999999996e-06 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451754+0000", 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.91393867900000003 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451828+0000", 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3714999999999995e-05 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.451902+0000", 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4746999999999994e-05 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461104+0000", 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0092011939999999993 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461279+0000", 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017572599999999999 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461470+0000", 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019065 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461492+0000", 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1634999999999999e-05 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:14.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:14.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:03:24.461531+0000", 2024-09-21T10:05:14.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9091999999999998e-05 2024-09-21T10:05:14.569 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.569 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.569 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.569 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:14.569 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:14.569 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:05:14.602 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:14.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:14.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 30226 30210 171.614 123.812 0.00333297 0.00582095 2024-09-21T10:05:14.703 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.740 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:14.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:14.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:14.861 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:14.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:14.963 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:15.000 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:15.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:15.032 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:05:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:05:14] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.51.0" 2024-09-21T10:05:15.101 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:15.123 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:15.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:15.260 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:15.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:15.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.382 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:15.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:15.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:15.507 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:05:15.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:15.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:15.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:15.644 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:15.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:15.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 31702 31686 165 92.25 0.00301268 0.00600934 2024-09-21T10:05:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:15 smithi134 ceph-mon[90085]: pgmap v1628: 99 pgs: 99 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-21T10:05:15.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:15.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:15.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:15.830 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:15.884 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:15.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:15.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:15 smithi017 ceph-mon[102913]: pgmap v1628: 99 pgs: 99 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-21T10:05:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:15 smithi017 ceph-mon[105151]: pgmap v1628: 99 pgs: 99 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-21T10:05:16.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:16.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:16.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:16.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:16.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:16.288 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:16.323 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:16.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:16.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:16.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:16.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:16.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:16.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:16.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 33649 33633 161.668 121.688 0.004749 0.00617373 2024-09-21T10:05:16.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:16.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:16.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:16.848 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:16.883 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:16.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:16.985 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:17.020 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:17.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:17.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:17.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:17.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:17.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:17.397 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:17.418 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:17.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:17.520 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:17.568 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:17.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:17.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 35563 35547 158.664 119.625 0.00643665 0.0062978 2024-09-21T10:05:17.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:17.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:17 smithi134 ceph-mon[90085]: pgmap v1629: 99 pgs: 99 active+clean; 1.7 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-09-21T10:05:17.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:17.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.848 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:17.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:17.949 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:17 smithi017 ceph-mon[102913]: pgmap v1629: 99 pgs: 99 active+clean; 1.7 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-09-21T10:05:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:17 smithi017 ceph-mon[105151]: pgmap v1629: 99 pgs: 99 active+clean; 1.7 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-09-21T10:05:17.982 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:18.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:18.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:36310 8.5 e1204/1200 8:a0a63ff6:::benchmark_data_smithi017_209235_object36309:head v 1204'2250, mlcod=1204'2247)", 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:05:18.146377+0000", 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.00038118100000000002, 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00040165699999999998, 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:18.150 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:05:18.146377+0000", 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:05:18.146377+0000", 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:05:18.146376+0000", 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:18.151 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:05:18.146397+0000", 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1580999999999999e-05 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:05:18.146412+0000", 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.469e-05 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:05:18.146419+0000", 2024-09-21T10:05:18.152 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.657e-06 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:05:18.146462+0000", 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3053000000000003e-05 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:05:18.146478+0000", 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6494000000000002e-05 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:18.153 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:18.154 INFO:teuthology.orchestra.run.smithi134.stdout: ], 2024-09-21T10:05:18.154 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 1 2024-09-21T10:05:18.154 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:05:18.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:18.258 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:18.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:18.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:18.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:18.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:18.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:18.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:18.636 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:18.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 37163 37147 154.753 100 0.00504987 0.00643297 2024-09-21T10:05:18.657 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:18.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:18.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:18.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:18.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:18.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:18.927 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:05:18.949 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:18.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:19.051 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:19.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.091 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:19.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:19.191 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:19.212 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:19.287 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:19.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:19.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:19.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:19.453 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:19.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.491 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:19.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:19.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:19.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:19.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:19.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 38559 38543 150.533 87.25 0.14046 0.00658561 2024-09-21T10:05:19.702 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:19 smithi134 ceph-mon[90085]: pgmap v1630: 99 pgs: 99 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 154 MiB/s wr, 2.47k op/s 2024-09-21T10:05:19.717 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.759 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:19.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:19.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:19.892 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:19.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:19 smithi017 ceph-mon[102913]: pgmap v1630: 99 pgs: 99 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 154 MiB/s wr, 2.47k op/s 2024-09-21T10:05:19.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:19 smithi017 ceph-mon[105151]: pgmap v1630: 99 pgs: 99 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 154 MiB/s wr, 2.47k op/s 2024-09-21T10:05:19.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:20.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:20.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:20.133 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:20.169 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:20.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:20.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:20.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:20.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:20.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:20.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:20.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:05:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:05:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:20.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:20.585 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:20.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:20.646 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 40494 40478 148.792 120.938 0.00964902 0.00671524 2024-09-21T10:05:20.687 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:20.707 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:20.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:20.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:20.849 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:20.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:20.952 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:20.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.988 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:20.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:21.089 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:21.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:05:21.127 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:21.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:21.220 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:05:21.220 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:05:21.220 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:05:21.220 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:05:21.220 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:05:21.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:21.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:21.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:21.355 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:21.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:21.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:21.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:21.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:21.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:21.626 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:21.647 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 42162 42146 146.317 104.25 0.00422506 0.00681874 2024-09-21T10:05:21.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.648 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:21.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:21.749 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:21.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.786 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:21.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:21.888 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:21.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:21.938 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:21 smithi134 ceph-mon[90085]: pgmap v1631: 99 pgs: 99 active+clean; 2.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-21T10:05:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:21 smithi017 ceph-mon[102913]: pgmap v1631: 99 pgs: 99 active+clean; 2.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-21T10:05:21.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:21 smithi017 ceph-mon[105151]: pgmap v1631: 99 pgs: 99 active+clean; 2.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-21T10:05:21.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:05:22.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:22.063 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:22.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:22.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:22.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:22.298 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:22.350 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:22.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:22.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:22.466 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:22.490 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:22.560 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:22.592 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:22.618 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:22.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:22.647 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 43897 43881 144.323 108.438 0.00479788 0.00692421 2024-09-21T10:05:22.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:22.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:22.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:22.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:22.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:22.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:22.966 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:23.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:23.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:23.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:23.146 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:23.178 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:23.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:23.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:23.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:23.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:23.421 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:23.455 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:23.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:23.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:23.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:23.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:23.647 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:05:23.646016+0000 min lat: 0.00121399 max lat: 0.158156 avg lat: 0.00699004 2024-09-21T10:05:23.647 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:05:23.647 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 45770 45754 142.959 117.062 0.00723303 0.00699004 2024-09-21T10:05:23.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:23.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:23 smithi017 ceph-mon[102913]: pgmap v1632: 99 pgs: 99 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-09-21T10:05:23.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:23 smithi017 ceph-mon[105151]: pgmap v1632: 99 pgs: 99 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-09-21T10:05:23.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:23.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:23.837 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:23.860 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:23.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:23 smithi134 ceph-mon[90085]: pgmap v1632: 99 pgs: 99 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-09-21T10:05:24.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:24.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:24.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:24.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:24.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:24.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:24.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:24.382 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:24.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:24.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:24.560 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:24.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:24.603 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 11 2024-09-21T10:05:24.648 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 47521 47505 141.356 109.438 0.0105231 0.00706912 2024-09-21T10:05:24.662 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:24.703 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:24.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:24.730 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/659690359' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:05:24.730 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:24 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:05:24.730 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/659690359' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:05:24.730 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:24 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:05:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/659690359' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:05:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:24 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-21T10:05:24.805 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:24.850 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:24.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:24.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:24.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:25.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:25.124 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:05:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:05:24] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.51.0" 2024-09-21T10:05:25.128 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:25.230 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:25.271 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:25.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:25.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:25.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:25.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60910 6.8 6:115de437:::benchmark_data_smithi017_149547_object60909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422268+0000", 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 499.10103820900002, 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0318208280000001, 2024-09-21T10:05:25.528 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60910 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422268+0000", 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422268+0000", 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422265+0000", 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422345+0000", 2024-09-21T10:05:25.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9598000000000006e-05 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422368+0000", 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3481000000000001e-05 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422376+0000", 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5209999999999997e-06 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522605+0000", 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.1002286 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522613+0000", 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7090000000000008e-06 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440150+0000", 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.91753706199999996 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440785+0000", 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00063470900000000001 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.454016+0000", 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013230936 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.454089+0000", 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2690999999999999e-05 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60912 6.3 6:c077dba9:::benchmark_data_smithi017_149547_object60911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422388+0000", 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 499.10091768000001, 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0068617500000001, 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60912 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.534 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422388+0000", 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422388+0000", 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422388+0000", 2024-09-21T10:05:25.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422462+0000", 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3874000000000005e-05 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422472+0000", 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.91e-06 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422476+0000", 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3610000000000003e-06 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.528785+0000", 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10630861 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.528795+0000", 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0499000000000001e-05 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.421255+0000", 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.89245958400000003 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.422521+0000", 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0012658389999999999 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.429183+0000", 2024-09-21T10:05:25.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0066627240000000001 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.429250+0000", 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6773000000000006e-05 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60914 6.18 6:1ac7c059:::benchmark_data_smithi017_149547_object60913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422482+0000", 2024-09-21T10:05:25.539 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 499.10082355999998, 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.044582076, 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60914 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422482+0000", 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422482+0000", 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422482+0000", 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422558+0000", 2024-09-21T10:05:25.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5767999999999997e-05 2024-09-21T10:05:25.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.542 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422567+0000", 2024-09-21T10:05:25.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6039999999999999e-06 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422571+0000", 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9609999999999999e-06 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522630+0000", 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10005905700000001 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522635+0000", 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6129999999999997e-06 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.456028+0000", 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.93339256800000003 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.456812+0000", 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00078471200000000002 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467017+0000", 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.01020501 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467065+0000", 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7205e-05 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.545 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60918 6.18 6:1b86a79a:::benchmark_data_smithi017_149547_object60917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422576+0000", 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 499.10073020200002, 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0446003159999999, 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60918 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.546 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422576+0000", 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422576+0000", 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422575+0000", 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422664+0000", 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8214999999999995e-05 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422673+0000", 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5880000000000004e-06 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422677+0000", 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9360000000000001e-06 2024-09-21T10:05:25.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522651+0000", 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.099973353000000001 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.522655+0000", 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3050000000000003e-06 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.456857+0000", 2024-09-21T10:05:25.549 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.93420210199999998 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.456995+0000", 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013829599999999999 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.457146+0000", 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015081799999999999 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.458255+0000", 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001108574 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.466295+0000", 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0080404480000000004 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467091+0000", 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00079628900000000005 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467119+0000", 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7991999999999999e-05 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.467176+0000", 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6798000000000001e-05 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.552 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:60923 6.15 6:a9be3957:::benchmark_data_smithi017_149547_object60922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:06.422681+0000", 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 499.10062457499998, 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0276440060000001, 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 60923 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422681+0000", 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422681+0000", 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422681+0000", 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422755+0000", 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3478000000000004e-05 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422764+0000", 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.7510000000000003e-06 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.422768+0000", 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8070000000000001e-06 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537758+0000", 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.114989657 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:06.537768+0000", 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8940000000000005e-06 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440664+0000", 2024-09-21T10:05:25.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.90289677300000004 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440778+0000", 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011325299999999999 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.440930+0000", 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015233299999999999 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.441879+0000", 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00094917099999999996 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.442913+0000", 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0010342750000000001 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.450243+0000", 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0073293229999999996 2024-09-21T10:05:25.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.450291+0000", 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7846999999999998e-05 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:07.450325+0000", 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4833999999999999e-05 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90914 6.12 6:4f1f32ba:::benchmark_data_smithi017_149547_object90913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:39.572975+0000", 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 465.950331457, 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0214511260000001, 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90914 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.572975+0000", 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.572975+0000", 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.572967+0000", 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999928 2024-09-21T10:05:25.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573027+0000", 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9933999999999999e-05 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573040+0000", 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2594e-05 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573045+0000", 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5669999999999996e-06 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573086+0000", 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1261999999999999e-05 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.573092+0000", 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0159999999999997e-06 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.547378+0000", 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97428627099999998 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.547494+0000", 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000116019 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.547596+0000", 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000102067 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.552155+0000", 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0045592899999999997 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.561091+0000", 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0089351870000000007 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.594334+0000", 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.033243237000000002 2024-09-21T10:05:25.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.594385+0000", 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0929e-05 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.594426+0000", 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0877999999999997e-05 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90932 6.12 6:4ca4244b:::benchmark_data_smithi017_149547_object90931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:39.642171+0000", 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 465.88113550999998, 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95359630500000003, 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90932 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642171+0000", 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642171+0000", 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642169+0000", 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:25.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642203+0000", 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3807999999999999e-05 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642214+0000", 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0739e-05 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642219+0000", 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0710000000000002e-06 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642299+0000", 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0338000000000002e-05 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.642308+0000", 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.6410000000000008e-06 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.570 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.548566+0000", 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.90625812500000003 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.548644+0000", 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8574999999999999e-05 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.548730+0000", 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5718999999999997e-05 2024-09-21T10:05:25.571 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.552167+0000", 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0034371950000000001 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.561123+0000", 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0089554820000000007 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595687+0000", 2024-09-21T10:05:25.572 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.034564267000000003 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595733+0000", 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6477999999999998e-05 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595767+0000", 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3416e-05 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:90934 6.12 6:4a514f78:::benchmark_data_smithi017_149547_object90933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:57:39.643035+0000", 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 465.88027080000001, 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95286870599999995, 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 90934 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643035+0000", 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643035+0000", 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643034+0000", 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:05:25.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643085+0000", 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0504000000000003e-05 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643098+0000", 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3852999999999999e-05 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643104+0000", 2024-09-21T10:05:25.576 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5509999999999999e-06 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643148+0000", 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4313000000000001e-05 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:39.643157+0000", 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3610000000000001e-06 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.549525+0000", 2024-09-21T10:05:25.577 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.90636815299999995 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.549638+0000", 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011333 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.549788+0000", 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000149856 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.553948+0000", 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0041600489999999999 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.561172+0000", 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0072234380000000004 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595814+0000", 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.034642138000000003 2024-09-21T10:05:25.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595862+0000", 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7945999999999998e-05 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:57:40.595904+0000", 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2296999999999998e-05 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:215116 6.3 6:c3180f7c:::benchmark_data_smithi017_149547_object96037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1123)", 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:52.411445+0000", 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 333.11186057600003, 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.056761965, 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215116 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.581 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411445+0000", 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411445+0000", 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411444+0000", 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.583 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.583 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411450+0000", 2024-09-21T10:05:25.583 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.959e-06 2024-09-21T10:05:25.583 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.583 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.583 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.583 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411459+0000", 2024-09-21T10:05:25.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8279999999999992e-06 2024-09-21T10:05:25.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.584 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.411464+0000", 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8319999999999996e-06 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.414029+0000", 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0025651929999999999 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.414038+0000", 2024-09-21T10:05:25.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.0250000000000008e-06 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.434973+0000", 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0209354340000001 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.435056+0000", 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.2681000000000003e-05 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:25.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.435461+0000", 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00040495199999999998 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.443579+0000", 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0081180230000000003 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468046+0000", 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.024467073999999998 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.587 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468107+0000", 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0485000000000001e-05 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468170+0000", 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3707999999999994e-05 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468207+0000", 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6993999999999999e-05 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.588 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:215170 6.3 6:c71c1ba7:::benchmark_data_smithi017_149547_object96091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1124)", 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:52.469683+0000", 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 333.05362255300003, 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.998898701, 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215170 2024-09-21T10:05:25.589 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469683+0000", 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469683+0000", 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469682+0000", 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469687+0000", 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7490000000000004e-06 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469704+0000", 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6920000000000001e-05 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469712+0000", 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5580000000000003e-06 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469753+0000", 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0760999999999998e-05 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.469761+0000", 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0930000000000003e-06 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.437472+0000", 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96771112500000001 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.437540+0000", 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8157999999999994e-05 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.437787+0000", 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00024686800000000002 2024-09-21T10:05:25.593 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.447907+0000", 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.01012044 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468254+0000", 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.020347054999999999 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468535+0000", 2024-09-21T10:05:25.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00028029800000000001 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468562+0000", 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7610000000000002e-05 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468582+0000", 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9896000000000001e-05 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45173.0:215213 6.3 6:c300d469:::benchmark_data_smithi017_149547_object96134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1124)", 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T09:59:52.526237+0000", 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 332.99706929600001, 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.94241559200000002, 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45173", 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215213 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526237+0000", 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526237+0000", 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526235+0000", 2024-09-21T10:05:25.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526241+0000", 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1870000000000002e-06 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526252+0000", 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0171e-05 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526257+0000", 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2390000000000002e-06 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526298+0000", 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0932e-05 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:52.526306+0000", 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1620000000000008e-06 2024-09-21T10:05:25.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.439549+0000", 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.91324313599999996 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.439617+0000", 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8078999999999997e-05 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.439893+0000", 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00027582399999999998 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.447915+0000", 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0080218749999999995 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468412+0000", 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.020497517999999999 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.601 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468609+0000", 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019681599999999999 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468633+0000", 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3538000000000001e-05 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T09:59:53.468652+0000", 2024-09-21T10:05:25.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9638000000000001e-05 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52978 7.5 7:a7209f46:::benchmark_data_smithi017_180590_object52977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345746+0000", 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 239.17756048699999, 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.014767234, 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:25.603 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52978 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345746+0000", 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345746+0000", 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.604 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345743+0000", 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345789+0000", 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5448000000000003e-05 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345813+0000", 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4360999999999999e-05 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.605 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345822+0000", 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.636e-06 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407517+0000", 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061694909999999999 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407525+0000", 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1599999999999998e-06 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.606 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360423+0000", 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95289824000000001 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360513+0000", 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.9736999999999998e-05 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.607 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52980 7.15 7:aaa003b0:::benchmark_data_smithi017_180590_object52979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345830+0000", 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 239.17747611499999, 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.024545134, 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52980 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.608 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345830+0000", 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345830+0000", 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345829+0000", 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.609 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345883+0000", 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3947999999999999e-05 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345891+0000", 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3499999999999999e-06 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345895+0000", 2024-09-21T10:05:25.610 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8569999999999997e-06 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407919+0000", 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.062024397000000002 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407925+0000", 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6980000000000003e-06 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370322+0000", 2024-09-21T10:05:25.611 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96239729100000004 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370375+0000", 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3143000000000003e-05 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52981 7.15 7:adf0cbd4:::benchmark_data_smithi017_180590_object52980:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345901+0000", 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 239.17740531000001, 2024-09-21T10:05:25.612 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.024740942, 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52981 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345901+0000", 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.613 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345901+0000", 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345900+0000", 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345941+0000", 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1102000000000001e-05 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.614 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345950+0000", 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8599999999999999e-06 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345955+0000", 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.707e-06 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407943+0000", 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061987918000000003 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407947+0000", 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6040000000000003e-06 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370604+0000", 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.962656399 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370642+0000", 2024-09-21T10:05:25.616 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7769999999999999e-05 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52985 7.5 7:a76eb8c9:::benchmark_data_smithi017_180590_object52984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.345962+0000", 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 239.17734418000001, 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.014771651, 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:25.617 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52985 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345962+0000", 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345962+0000", 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.618 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345961+0000", 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.345999+0000", 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7415e-05 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346008+0000", 2024-09-21T10:05:25.619 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.0149999999999992e-06 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346012+0000", 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1320000000000004e-06 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407966+0000", 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061954189 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407972+0000", 2024-09-21T10:05:25.620 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9390000000000002e-06 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360688+0000", 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95271553399999998 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360733+0000", 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5875999999999999e-05 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.621 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52987 7.5 7:a671d9ea:::benchmark_data_smithi017_180590_object52986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346016+0000", 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 239.17728968200001, 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0147849289999999, 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52987 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.622 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346016+0000", 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346016+0000", 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346016+0000", 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.623 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346053+0000", 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6952000000000001e-05 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346061+0000", 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5599999999999994e-06 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346065+0000", 2024-09-21T10:05:25.624 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7270000000000001e-06 2024-09-21T10:05:25.625 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.625 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.625 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.625 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407990+0000", 2024-09-21T10:05:25.625 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061924664999999997 2024-09-21T10:05:25.625 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.625 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:25.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.407996+0000", 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7790000000000001e-06 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360758+0000", 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95276244799999998 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360801+0000", 2024-09-21T10:05:25.627 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3266000000000002e-05 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52995 7.15 7:ab424fbb:::benchmark_data_smithi017_180590_object52994:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346069+0000", 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 239.177236959, 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.024628104, 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.628 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52995 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346069+0000", 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346069+0000", 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.629 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346069+0000", 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346103+0000", 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4833999999999999e-05 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.630 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346111+0000", 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6020000000000002e-06 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346115+0000", 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4589999999999998e-06 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408012+0000", 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061897677999999998 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.631 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408017+0000", 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5140000000000003e-06 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370663+0000", 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96264632299999997 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370697+0000", 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4124000000000001e-05 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.632 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:52998 7.15 7:aba51093:::benchmark_data_smithi017_180590_object52997:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346118+0000", 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 239.17718769699999, 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0246339739999999, 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:25.633 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 52998 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346118+0000", 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346118+0000", 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.634 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346118+0000", 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346152+0000", 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4393000000000001e-05 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346162+0000", 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8209999999999997e-06 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.635 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346166+0000", 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3799999999999998e-06 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408044+0000", 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061878264000000002 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408048+0000", 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3789999999999999e-06 2024-09-21T10:05:25.636 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370720+0000", 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96267132899999996 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370752+0000", 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2796999999999998e-05 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.637 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:53001 7.15 7:adde1279:::benchmark_data_smithi017_180590_object53000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346179+0000", 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 239.17712751400001, 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0246183339999999, 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 53001 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.638 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346179+0000", 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346179+0000", 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346178+0000", 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.639 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346243+0000", 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5024000000000006e-05 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346251+0000", 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4460000000000004e-06 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346269+0000", 2024-09-21T10:05:25.640 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8881000000000001e-05 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408064+0000", 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061794295999999999 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408068+0000", 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5619999999999997e-06 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.641 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370765+0000", 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96269701200000002 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.370797+0000", 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1537e-05 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.35520.0:53005 7.5 7:a05ee3e9:::benchmark_data_smithi017_180590_object53004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:25.642 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:01:26.346275+0000", 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 239.177031019, 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.022219387, 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.35520", 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 53005 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346275+0000", 2024-09-21T10:05:25.643 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346275+0000", 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346273+0000", 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346305+0000", 2024-09-21T10:05:25.644 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1532999999999998e-05 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346312+0000", 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2969999999999998e-06 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.346315+0000", 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4029999999999999e-06 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.645 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408087+0000", 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.061771271000000003 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:26.408092+0000", 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7649999999999999e-06 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360817+0000", 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95272487800000005 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.646 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.360914+0000", 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6275999999999998e-05 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.361012+0000", 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8227999999999995e-05 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.361817+0000", 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000805582 2024-09-21T10:05:25.647 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.365156+0000", 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003338597 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.368399+0000", 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032432709999999998 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.368457+0000", 2024-09-21T10:05:25.648 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7213999999999998e-05 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:01:27.368494+0000", 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7870000000000002e-05 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:05:25.649 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:05:25.650 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 49303 49287 139.993 111.375 0.00829868 0.00713859 2024-09-21T10:05:25.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:25.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:25.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:25.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:25.898 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:25.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[102913]: pgmap v1633: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-21T10:05:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T10:05:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[102913]: osdmap e1205: 8 total, 8 up, 7 in 2024-09-21T10:05:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[102913]: 1.a scrub starts 2024-09-21T10:05:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[102913]: 1.a scrub ok 2024-09-21T10:05:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[105151]: pgmap v1633: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-21T10:05:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T10:05:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[105151]: osdmap e1205: 8 total, 8 up, 7 in 2024-09-21T10:05:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[105151]: 1.a scrub starts 2024-09-21T10:05:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[105151]: 1.a scrub ok 2024-09-21T10:05:25.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:25 smithi134 ceph-mon[90085]: pgmap v1633: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-21T10:05:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:25 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-21T10:05:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:25 smithi134 ceph-mon[90085]: osdmap e1205: 8 total, 8 up, 7 in 2024-09-21T10:05:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:25 smithi134 ceph-mon[90085]: 1.a scrub starts 2024-09-21T10:05:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:25 smithi134 ceph-mon[90085]: 1.a scrub ok 2024-09-21T10:05:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:26.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:26.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:26.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:26.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:26.178 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:26.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:26.280 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:26.316 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:26.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:26.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:26.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:26.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:26.557 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:26.588 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:26.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:26.648 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 51087 51071 138.754 111.5 0.0149634 0.00720177 2024-09-21T10:05:26.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:26.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:26.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:26.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:05:26.832 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:26.859 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:26.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:26 smithi017 ceph-mon[102913]: 1.1 scrub starts 2024-09-21T10:05:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:26 smithi017 ceph-mon[102913]: 1.1 scrub ok 2024-09-21T10:05:26.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:26 smithi017 ceph-mon[102913]: 1.4 scrub starts 2024-09-21T10:05:26.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:26 smithi017 ceph-mon[102913]: 1.4 scrub ok 2024-09-21T10:05:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:26 smithi017 ceph-mon[105151]: 1.1 scrub starts 2024-09-21T10:05:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:26 smithi017 ceph-mon[105151]: 1.1 scrub ok 2024-09-21T10:05:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:26 smithi017 ceph-mon[105151]: 1.4 scrub starts 2024-09-21T10:05:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:26 smithi017 ceph-mon[105151]: 1.4 scrub ok 2024-09-21T10:05:26.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:26 smithi134 ceph-mon[90085]: 1.1 scrub starts 2024-09-21T10:05:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:26 smithi134 ceph-mon[90085]: 1.1 scrub ok 2024-09-21T10:05:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:26 smithi134 ceph-mon[90085]: 1.4 scrub starts 2024-09-21T10:05:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:26 smithi134 ceph-mon[90085]: 1.4 scrub ok 2024-09-21T10:05:27.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:27.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:27.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:27.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:27.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:27.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:27.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:27.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:27.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:27.544 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:27.571 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:27.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:27.627 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:05:27.648 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 52787 52771 137.397 106.25 0.0194753 0.00727278 2024-09-21T10:05:27.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:27.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:27.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:27.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:27.856 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:27.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:27 smithi017 ceph-mon[102913]: pgmap v1635: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-21T10:05:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:27 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3015281072' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:27 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:27 smithi017 ceph-mon[105151]: pgmap v1635: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-21T10:05:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:27 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3015281072' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:27 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:27.957 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:27 smithi134 ceph-mon[90085]: pgmap v1635: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-21T10:05:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:27 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3015281072' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:27 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:27.996 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:27.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:28.098 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:28.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:28.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:28.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:28.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:28.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:28.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:28.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:28.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:28.417 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:28.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:28.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:28.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:28.649 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 54563 54547 136.34 111 0.00528848 0.00732995 2024-09-21T10:05:28.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:28.694 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:28.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:28.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:28.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:28.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:28.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:28 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:05:28.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:28 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:05:28.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:28 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:28.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:28 smithi017 ceph-mon[102913]: osdmap e1206: 8 total, 8 up, 7 in 2024-09-21T10:05:28.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:28 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:05:28.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:28 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:05:28.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:28 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:28.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:28 smithi017 ceph-mon[105151]: osdmap e1206: 8 total, 8 up, 7 in 2024-09-21T10:05:28.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:28.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:28.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:28 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:05:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:28 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:05:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:28 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:28 smithi134 ceph-mon[90085]: osdmap e1206: 8 total, 8 up, 7 in 2024-09-21T10:05:29.077 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:29.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:29.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:29.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:29.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.246 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:29.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:29.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:29.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:29.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:29.490 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:29.520 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:29.622 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:29.649 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 56311 56295 135.298 109.25 0.0154599 0.00738569 2024-09-21T10:05:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.653 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:29.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:29.754 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:29.785 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:29.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:29.790 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:05:29.790 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:29.820 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:05:29.821 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:29.858 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:05:29.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:29.858 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:29 smithi017 ceph-mon[102913]: pgmap v1637: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:05:29.859 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:29 smithi017 ceph-mon[105151]: pgmap v1637: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:05:29.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:29.892 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:05:29.892 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:29.919 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:05:29.920 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:29.920 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:29.936 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:05:29.936 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:29.952 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:05:29.952 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:29.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:05:29.983 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:29 smithi134 ceph-mon[90085]: pgmap v1637: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:05:30.022 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:30.055 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:30.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:30.157 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:30.191 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:30.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:30.292 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:30.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:30.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:30.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:05:30.430 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:30.469 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:30.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:05:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:05:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:30.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:30.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:30.649 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 57894 57878 133.951 98.9375 0.00611168 0.0074597 2024-09-21T10:05:30.718 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:30.761 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:30.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:30.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:30.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:30 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3640117301' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:30 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3640117301' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:30 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3640117301' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:31.010 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:31.047 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:31.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:31.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:31.192 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:31.294 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:31.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:31.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:31.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:31.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:31.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:31.566 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:31.601 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:31.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:31.649 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 59561 59545 132.888 104.188 0.00707468 0.00752004 2024-09-21T10:05:31.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:31.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:31.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:31 smithi017 ceph-mon[102913]: pgmap v1638: 99 pgs: 99 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-21T10:05:31.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:31 smithi017 ceph-mon[105151]: pgmap v1638: 99 pgs: 99 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-21T10:05:31.846 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:31.880 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:31.981 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:31 smithi134 ceph-mon[90085]: pgmap v1638: 99 pgs: 99 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-21T10:05:31.995 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:32.011 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:32.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:32.091 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 6] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:05:32.092 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:05:32.092 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:05:32.093 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T10:05:32.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.4 2024-09-21T10:05:32.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:32.120 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:32.121 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:32.213 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:32.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:32.314 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:32.355 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:32.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:32.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:32.494 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:32 smithi134 systemd[1]: Stopping Ceph osd.4 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:05:32.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:32.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:32.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:32.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:32.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:32.649 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 60973 60957 131.349 88.25 0.017479 0.00760786 2024-09-21T10:05:32.743 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:32.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.766 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:32.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:32.830 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:32 smithi017 ceph-mon[102913]: osd.4 marked itself down and dead 2024-09-21T10:05:32.830 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:32 smithi017 ceph-mon[105151]: osd.4 marked itself down and dead 2024-09-21T10:05:32.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:32.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:32.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:32.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:32 smithi134 ceph-mon[90085]: osd.4 marked itself down and dead 2024-09-21T10:05:32.994 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[38931]: 2024-09-21T10:05:32.637+0000 7feec3121640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:05:32.994 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[38931]: 2024-09-21T10:05:32.637+0000 7feec3121640 -1 osd.4 1206 *** Got signal Terminated *** 2024-09-21T10:05:32.994 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[38931]: 2024-09-21T10:05:32.637+0000 7feec3121640 -1 osd.4 1206 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:05:33.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:33.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:33.114 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:33.138 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:33.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:33.264 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:33.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:33.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:33.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:33.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:33.491 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:33.514 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:33.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:33.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:05:33.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:05:33.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:33.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:33.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:33.649 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 60994 60978 127.015 1.3125 0.00293689 0.00761026 2024-09-21T10:05:33.659 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:33 smithi134 podman[147458]: 2024-09-21 10:05:33.249281475 +0000 UTC m=+0.819939956 container died 7ece30084c4106dfcefc517024b0beb57217a0039dcb59819185558da68d3e70 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:05:33.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:33.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:33.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:33.883 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:33.911 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:33 smithi017 ceph-mon[105151]: pgmap v1639: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-21T10:05:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:33 smithi017 ceph-mon[105151]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:05:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:33 smithi017 ceph-mon[105151]: osdmap e1207: 8 total, 7 up, 7 in 2024-09-21T10:05:33.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:33 smithi017 ceph-mon[102913]: pgmap v1639: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-21T10:05:33.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:33 smithi017 ceph-mon[102913]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:05:33.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:33 smithi017 ceph-mon[102913]: osdmap e1207: 8 total, 7 up, 7 in 2024-09-21T10:05:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:33 smithi134 ceph-mon[90085]: pgmap v1639: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-21T10:05:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:33 smithi134 ceph-mon[90085]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:05:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:33 smithi134 ceph-mon[90085]: osdmap e1207: 8 total, 7 up, 7 in 2024-09-21T10:05:34.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:34.044 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:34.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:34.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:34.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:34.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:34.302 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:34.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:34.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:34.421 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:34 smithi134 podman[147458]: 2024-09-21 10:05:34.117971587 +0000 UTC m=+1.688630064 container cleanup 7ece30084c4106dfcefc517024b0beb57217a0039dcb59819185558da68d3e70 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:05:34.422 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:34 smithi134 bash[147458]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4 2024-09-21T10:05:34.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:34.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:34.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:34.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:34.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:34.649 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 62459 62443 125.871 91.5625 0.006211 0.00793912 2024-09-21T10:05:34.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:34.744 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:34 smithi134 podman[147510]: 2024-09-21 10:05:34.419712 +0000 UTC m=+1.168696183 container remove 7ece30084c4106dfcefc517024b0beb57217a0039dcb59819185558da68d3e70 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:05:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:34 smithi134 ceph-mon[90085]: osdmap e1208: 8 total, 7 up, 7 in 2024-09-21T10:05:34.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:34.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:34.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:34.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:34.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:34 smithi017 ceph-mon[102913]: osdmap e1208: 8 total, 7 up, 7 in 2024-09-21T10:05:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:34 smithi017 ceph-mon[105151]: osdmap e1208: 8 total, 7 up, 7 in 2024-09-21T10:05:34.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:05:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:05:34] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-09-21T10:05:35.020 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:35.056 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:35.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:35.158 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:35.176 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:34 smithi134 podman[147650]: 2024-09-21 10:05:34.773172476 +0000 UTC m=+0.026226519 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T10:05:35.176 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:34 smithi134 podman[147650]: 2024-09-21 10:05:34.904216196 +0000 UTC m=+0.157270233 container create 3cb0769d211e075cfca56e796e98d0bb285ff5d1a7b3ca722c3a566f24eac1ae (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:05:35.188 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:35.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:35.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:35.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:35.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:35.429 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:35.435 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:35 smithi134 podman[147650]: 2024-09-21 10:05:35.174401861 +0000 UTC m=+0.427455902 container init 3cb0769d211e075cfca56e796e98d0bb285ff5d1a7b3ca722c3a566f24eac1ae (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T10:05:35.435 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:35 smithi134 podman[147650]: 2024-09-21 10:05:35.182533007 +0000 UTC m=+0.435587040 container start 3cb0769d211e075cfca56e796e98d0bb285ff5d1a7b3ca722c3a566f24eac1ae (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T10:05:35.435 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:35 smithi134 podman[147650]: 2024-09-21 10:05:35.300632727 +0000 UTC m=+0.553686770 container attach 3cb0769d211e075cfca56e796e98d0bb285ff5d1a7b3ca722c3a566f24eac1ae (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-21T10:05:35.464 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:35.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:35.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:35.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:35.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:35.649 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 63828 63812 124.611 85.5625 0.00894316 0.00801984 2024-09-21T10:05:35.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:35.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:35 smithi134 ceph-mon[90085]: pgmap v1642: 99 pgs: 5 active+undersized, 16 stale+active+clean, 3 active+undersized+degraded, 75 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 7698/179277 objects degraded (4.294%) 2024-09-21T10:05:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:35 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 7698/179277 objects degraded (4.294%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2756822547' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:35.744 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:05:35 smithi134 podman[147650]: 2024-09-21 10:05:35.433421121 +0000 UTC m=+0.686475155 container died 3cb0769d211e075cfca56e796e98d0bb285ff5d1a7b3ca722c3a566f24eac1ae (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:05:35.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:35.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:35.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:35 smithi017 ceph-mon[102913]: pgmap v1642: 99 pgs: 5 active+undersized, 16 stale+active+clean, 3 active+undersized+degraded, 75 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 7698/179277 objects degraded (4.294%) 2024-09-21T10:05:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:35 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 7698/179277 objects degraded (4.294%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2756822547' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:35 smithi017 ceph-mon[105151]: pgmap v1642: 99 pgs: 5 active+undersized, 16 stale+active+clean, 3 active+undersized+degraded, 75 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 7698/179277 objects degraded (4.294%) 2024-09-21T10:05:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:35 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 7698/179277 objects degraded (4.294%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2756822547' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:35.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:36.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.012 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:36.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:36.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:36.147 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:36.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:36.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:36.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:36.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:36.277 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.4.service' 2024-09-21T10:05:36.334 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:05:36.334 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T10:05:36.373 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:36.408 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:36.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:36.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:36.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:36.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:36.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:36.649 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 65294 65278 123.611 91.625 0.0042762 0.00808437 2024-09-21T10:05:36.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:36.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:36.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:36.807 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:36.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:36.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:36.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:36.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:36 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:36 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2756822547' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:05:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:36 smithi017 ceph-mon[102913]: osdmap e1209: 8 total, 7 up, 7 in 2024-09-21T10:05:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:36 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:36 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2756822547' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:05:36.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:36 smithi017 ceph-mon[105151]: osdmap e1209: 8 total, 7 up, 7 in 2024-09-21T10:05:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:36 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:36 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2756822547' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:05:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:36 smithi134 ceph-mon[90085]: osdmap e1209: 8 total, 7 up, 7 in 2024-09-21T10:05:37.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:37.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:37.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:37.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:37.214 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:37.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:37.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:37.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:37.453 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:37.480 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:37.582 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:37.613 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:37.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:37.650 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 66638 66622 122.446 84 0.0111178 0.00816161 2024-09-21T10:05:37.715 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:37.736 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:37.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:37.837 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:37.866 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:37.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:37 smithi017 ceph-mon[102913]: pgmap v1644: 99 pgs: 16 active+undersized, 9 stale+active+clean, 11 active+undersized+degraded, 63 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 11600/183717 objects degraded (6.314%) 2024-09-21T10:05:37.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:37 smithi017 ceph-mon[105151]: pgmap v1644: 99 pgs: 16 active+undersized, 9 stale+active+clean, 11 active+undersized+degraded, 63 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 11600/183717 objects degraded (6.314%) 2024-09-21T10:05:37.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:37.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:37.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:37 smithi134 ceph-mon[90085]: pgmap v1644: 99 pgs: 16 active+undersized, 9 stale+active+clean, 11 active+undersized+degraded, 63 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 11600/183717 objects degraded (6.314%) 2024-09-21T10:05:38.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:38.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:38.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:38.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:38.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:38.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:38.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.403 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:38.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:38.464 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:05:38.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:38.536 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:38.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:38.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:38.650 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 67923 67907 121.242 80.3125 0.00973671 0.00824309 2024-09-21T10:05:38.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:38.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:38.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:38.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:38.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:38.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:38.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.937 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:38.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:39.038 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:39.066 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:39.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:39.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:39.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:39.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:39.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:39.433 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:39.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.463 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:39.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:39.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:39.599 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:39.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:39.650 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 69081 69065 119.885 72.375 0.00730116 0.00833624 2024-09-21T10:05:39.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:39.729 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:39.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:39.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:39.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:39 smithi017 ceph-mon[102913]: pgmap v1645: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 11 KiB/s rd, 61 MiB/s wr, 975 op/s; 27877/191403 objects degraded (14.565%); 7.2 KiB/s, 0 objects/s recovering 2024-09-21T10:05:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:39 smithi017 ceph-mon[105151]: pgmap v1645: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 11 KiB/s rd, 61 MiB/s wr, 975 op/s; 27877/191403 objects degraded (14.565%); 7.2 KiB/s, 0 objects/s recovering 2024-09-21T10:05:39.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:39.983 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:39.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:39 smithi134 ceph-mon[90085]: pgmap v1645: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 11 KiB/s rd, 61 MiB/s wr, 975 op/s; 27877/191403 objects degraded (14.565%); 7.2 KiB/s, 0 objects/s recovering 2024-09-21T10:05:40.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:40.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:40.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:40.252 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:40.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.395 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:40.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:05:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:05:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:40.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:40.524 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:40.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:40.626 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:40.636 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:40.650 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 70393 70377 118.86 82 0.00893566 0.0084081 2024-09-21T10:05:40.651 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:40.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:40.753 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:40.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:40.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:40.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:40.914 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:40.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:41.016 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:41.042 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:41.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:41.144 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:41.174 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:41.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:41.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:41.306 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:41.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:41.334 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 6] out_osds: [1] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2024-09-21T10:05:41.334 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:05:41.335 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-21T10:05:41.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:05:41.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:41.435 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:41.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:41 smithi017 ceph-mon[102913]: pgmap v1646: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.0 KiB/s rd, 66 MiB/s wr, 1.07k op/s; 28766/198018 objects degraded (14.527%); 5.7 KiB/s, 0 objects/s recovering 2024-09-21T10:05:41.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:41.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:41 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 28766/198018 objects degraded (14.527%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:41 smithi017 ceph-mon[105151]: pgmap v1646: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.0 KiB/s rd, 66 MiB/s wr, 1.07k op/s; 28766/198018 objects degraded (14.527%); 5.7 KiB/s, 0 objects/s recovering 2024-09-21T10:05:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:41.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:41 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 28766/198018 objects degraded (14.527%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:41.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:41.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:41.650 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 71768 71752 117.993 85.9375 0.00351206 0.00846825 2024-09-21T10:05:41.676 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:41.697 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:41.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:41 smithi134 ceph-mon[90085]: pgmap v1646: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.0 KiB/s rd, 66 MiB/s wr, 1.07k op/s; 28766/198018 objects degraded (14.527%); 5.7 KiB/s, 0 objects/s recovering 2024-09-21T10:05:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:41 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 28766/198018 objects degraded (14.527%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:41.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:41.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:41.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:41.942 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:41.973 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:42.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:42.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:42.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:42.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:42.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:42.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:42.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:42.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:42.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:42.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:42.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:42.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:42.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 73154 73138 117.182 86.625 0.017456 0.00852881 2024-09-21T10:05:42.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:42.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:42.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:42.811 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:42.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:42.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:42.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:42.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:43.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:43.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:43.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:43.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:43.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:43.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:43.341 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:43.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:43.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:43.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:43.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:43.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:43.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:43.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:43 smithi017 ceph-mon[102913]: pgmap v1647: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 7.9 KiB/s rd, 90 MiB/s wr, 1.45k op/s; 30735/211119 objects degraded (14.558%); 5.0 KiB/s, 0 objects/s recovering 2024-09-21T10:05:43.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:43 smithi017 ceph-mon[105151]: pgmap v1647: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 7.9 KiB/s rd, 90 MiB/s wr, 1.45k op/s; 30735/211119 objects degraded (14.558%); 5.0 KiB/s, 0 objects/s recovering 2024-09-21T10:05:43.652 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:05:43.651618+0000 min lat: 0.00121399 max lat: 1.06457 avg lat: 0.00857582 2024-09-21T10:05:43.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:05:43.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 74617 74601 116.539 91.4375 0.0189053 0.00857582 2024-09-21T10:05:43.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:43.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:43 smithi134 ceph-mon[90085]: pgmap v1647: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 7.9 KiB/s rd, 90 MiB/s wr, 1.45k op/s; 30735/211119 objects degraded (14.558%); 5.0 KiB/s, 0 objects/s recovering 2024-09-21T10:05:43.764 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:43.790 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:43.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:43.891 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:43.917 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:43.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:44.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:44.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:44.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:44.162 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:44.193 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:44.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:44.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:44.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:44.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:44.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:44.460 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:44.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:44.592 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:44.618 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:44.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 75915 75899 115.675 81.125 0.0135364 0.00863992 2024-09-21T10:05:44.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:44.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:44.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:44.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:05:44.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:44.903 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:05:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:05:44] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.51.0" 2024-09-21T10:05:44.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:45.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:45.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:45.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:45.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:45.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:45.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:45.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:45.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:45.476 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:45.505 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:45.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:45.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:45 smithi017 ceph-mon[102913]: pgmap v1648: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 6.8 KiB/s rd, 80 MiB/s wr, 1.29k op/s; 31623/217794 objects degraded (14.520%); 4.3 KiB/s, 0 objects/s recovering 2024-09-21T10:05:45.543 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:45 smithi017 ceph-mon[105151]: pgmap v1648: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 6.8 KiB/s rd, 80 MiB/s wr, 1.29k op/s; 31623/217794 objects degraded (14.520%); 4.3 KiB/s, 0 objects/s recovering 2024-09-21T10:05:45.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:45.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:45.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:45.652 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 77119 77103 114.712 75.25 0.00305663 0.00871265 2024-09-21T10:05:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:45 smithi134 ceph-mon[90085]: pgmap v1648: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 6.8 KiB/s rd, 80 MiB/s wr, 1.29k op/s; 31623/217794 objects degraded (14.520%); 4.3 KiB/s, 0 objects/s recovering 2024-09-21T10:05:45.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:45.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:45.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:45.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:45.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:46.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:46.081 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:46.183 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:46.219 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:46.321 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:46.345 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:05:46.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:46.347 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:46.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:46.386 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:05:46.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:46.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:46 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 32033/222693 objects degraded (14.384%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:46.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/622421968' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:46.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:46 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 32033/222693 objects degraded (14.384%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:46.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/622421968' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:46.429 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:05:46.429 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:46.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:46.470 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:05:46.470 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:46.470 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:05:46.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:46 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 32033/222693 objects degraded (14.384%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/622421968' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:46.503 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:05:46.504 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:46.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:05:46.535 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:46.572 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:05:46.573 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:46.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:46.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:46.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:46.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 78536 78520 114.104 88.5625 0.017134 0.00875903 2024-09-21T10:05:46.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:46.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:46.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:46.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:46.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:46.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:47.010 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:47.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:47.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:47.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:47.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:47.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:47.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:47.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:47.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:47.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:47 smithi017 ceph-mon[102913]: pgmap v1649: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 6.4 KiB/s rd, 79 MiB/s wr, 1.27k op/s; 32033/222693 objects degraded (14.384%); 4.1 KiB/s, 0 objects/s recovering 2024-09-21T10:05:47.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:47 smithi017 ceph-mon[105151]: pgmap v1649: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 6.4 KiB/s rd, 79 MiB/s wr, 1.27k op/s; 32033/222693 objects degraded (14.384%); 4.1 KiB/s, 0 objects/s recovering 2024-09-21T10:05:47.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:47.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:47.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:47.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:47.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:47.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:47.638 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:05:47.638 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:05:47.638 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:05:47.638 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:05:47.638 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:05:47.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 79949 79933 113.518 88.3125 0.00923043 0.00880335 2024-09-21T10:05:47.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:47 smithi134 ceph-mon[90085]: pgmap v1649: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 6.4 KiB/s rd, 79 MiB/s wr, 1.27k op/s; 32033/222693 objects degraded (14.384%); 4.1 KiB/s, 0 objects/s recovering 2024-09-21T10:05:47.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:47.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:47.872 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:47.900 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:47.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:48.002 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:48.034 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:48.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:48.135 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:48.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:48.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:48.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:48.292 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:48.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:48.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:48.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.433 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:48.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:48.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.576 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:48.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:48.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 81283 81267 112.848 83.375 0.0101632 0.00885548 2024-09-21T10:05:48.677 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.713 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:48.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:48.743 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:05:48.745 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:05:48.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.854 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:48.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:48.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:48.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.998 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:48.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:49.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:49.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:49.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:49.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:49.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:49.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:49.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:49.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:49.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:49.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:49.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:49.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:49 smithi017 ceph-mon[102913]: pgmap v1650: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 5.7 KiB/s rd, 83 MiB/s wr, 1.33k op/s; 33717/231609 objects degraded (14.558%); 4.4 KiB/s, 0 objects/s recovering 2024-09-21T10:05:49.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:05:49.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:49 smithi017 ceph-mon[102913]: from='client.45659 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:49.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:49 smithi017 ceph-mon[105151]: pgmap v1650: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 5.7 KiB/s rd, 83 MiB/s wr, 1.33k op/s; 33717/231609 objects degraded (14.558%); 4.4 KiB/s, 0 objects/s recovering 2024-09-21T10:05:49.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:05:49.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:49 smithi017 ceph-mon[105151]: from='client.45659 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:49.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 82570 82554 112.143 80.4375 0.0108929 0.00891233 2024-09-21T10:05:49.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:05:49.691 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:05:49.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:49 smithi134 ceph-mon[90085]: pgmap v1650: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 5.7 KiB/s rd, 83 MiB/s wr, 1.33k op/s; 33717/231609 objects degraded (14.558%); 4.4 KiB/s, 0 objects/s recovering 2024-09-21T10:05:49.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:05:49.694 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:49 smithi134 ceph-mon[90085]: from='client.45659 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:49.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:49.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:49.817 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:49.845 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:49.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:49.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:49.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:50.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:50.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:50.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:50.261 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:50.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:50.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:05:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:05:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:50.501 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:50.539 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:50.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:50.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:50.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 83704 83688 111.265 70.875 0.00219939 0.00897316 2024-09-21T10:05:50.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:50.765 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:50.794 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:50.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:50.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:51.035 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:51.056 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:51.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:51.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:51.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:51.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:51.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:51.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:51.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:51.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:51.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:51.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:51.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:51.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:51.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:51.653 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 84655 84639 110.185 59.4375 0.00361004 0.00906732 2024-09-21T10:05:51.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:51.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:05:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:51 smithi017 ceph-mon[102913]: pgmap v1651: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 34558/237846 objects degraded (14.530%) 2024-09-21T10:05:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:51 smithi017 ceph-mon[105151]: pgmap v1651: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 34558/237846 objects degraded (14.530%) 2024-09-21T10:05:51.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:51.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:51 smithi134 ceph-mon[90085]: pgmap v1651: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 34558/237846 objects degraded (14.530%) 2024-09-21T10:05:51.812 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:51.845 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:51.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:51.946 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:51.977 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:52.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:52.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:52.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:52.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:52.223 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:52.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:05:52.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:52.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:52.370 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:52.399 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:52.501 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:52.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:52 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 36525/250800 objects degraded (14.563%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:52.528 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:52 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 36525/250800 objects degraded (14.563%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:52.543 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:52.645 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:52.654 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 85866 85850 109.48 75.6875 0.00685169 0.00912934 2024-09-21T10:05:52.680 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:52.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:52 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 36525/250800 objects degraded (14.563%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:52.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:52.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:52.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:52.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:52.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:52.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:53.002 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1650,"stamp":"2024-09-21T10:05:48.214283+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5058521459,"num_objects":77203,"num_object_clones":0,"num_object_copies":231609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33717,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77203,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":77254,"num_write_kb":4939507,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":53848,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78335,"ondisk_log_size":78335,"up":252,"acting":252,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17426388,"kb_used_data":14349724,"kb_used_omap":95,"kb_used_meta":3076512,"kb_avail":638789676,"statfs":{"total":671965249536,"available":654120628224,"internally_reserved":0,"allocated":14694117376,"data_stored":14675477057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97681,"internal_metadata":3150348911},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1046270374,"num_objects":15964,"num_object_clones":0,"num_object_copies":47892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22117,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15964,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":15978,"num_write_kb":1021844,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005500"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1763,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.680439+0000","last_change":"2024-09-21T10:01:40.433720+0000","last_active":"2024-09-21T10:05:35.680439+0000","last_peered":"2024-09-21T10:05:35.680439+0000","last_clean":"2024-09-21T10:05:35.680439+0000","last_became_active":"2024-09-21T10:01:40.433360+0000","last_became_peered":"2024-09-21T10:01:40.433360+0000","last_unstale":"2024-09-21T10:05:35.680439+0000","last_undegraded":"2024-09-21T10:05:35.680439+0000","last_fullsized":"2024-09-21T10:05:35.680439+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:08.257176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":1141,"reported_epoch":1208,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:33.666133+0000","last_change":"2024-09-21T10:05:33.665837+0000","last_active":"2024-09-21T10:05:33.666133+0000","last_peered":"2024-09-21T10:05:33.666133+0000","last_clean":"2024-09-21T10:05:27.632417+0000","last_became_active":"2024-09-21T10:05:33.665837+0000","last_became_peered":"2024-09-21T10:05:33.665837+0000","last_unstale":"2024-09-21T10:05:33.666133+0000","last_undegraded":"2024-09-21T10:05:33.652521+0000","last_fullsized":"2024-09-21T10:05:33.652402+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:31.134487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":642,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.653024+0000","last_change":"2024-09-21T09:57:39.541887+0000","last_active":"2024-09-21T10:05:33.653024+0000","last_peered":"2024-09-21T10:05:33.653024+0000","last_clean":"2024-09-21T10:05:33.653024+0000","last_became_active":"2024-09-21T09:57:39.541598+0000","last_became_peered":"2024-09-21T09:57:39.541598+0000","last_unstale":"2024-09-21T10:05:33.653024+0000","last_undegraded":"2024-09-21T10:05:33.653024+0000","last_fullsized":"2024-09-21T10:05:33.653024+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:41.615525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1839,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.677038+0000","last_change":"2024-09-21T09:46:47.115404+0000","last_active":"2024-09-21T10:05:35.677038+0000","last_peered":"2024-09-21T10:05:35.677038+0000","last_clean":"2024-09-21T10:05:35.677038+0000","last_became_active":"2024-09-21T09:46:47.115097+0000","last_became_peered":"2024-09-21T09:46:47.115097+0000","last_unstale":"2024-09-21T10:05:35.677038+0000","last_undegraded":"2024-09-21T10:05:35.677038+0000","last_fullsized":"2024-09-21T10:05:35.677038+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:29.299036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1841,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.684895+0000","last_change":"2024-09-21T09:46:47.116796+0000","last_active":"2024-09-21T10:05:35.684895+0000","last_peered":"2024-09-21T10:05:35.684895+0000","last_clean":"2024-09-21T10:05:35.684895+0000","last_became_active":"2024-09-21T09:46:47.116092+0000","last_became_peered":"2024-09-21T09:46:47.116092+0000","last_unstale":"2024-09-21T10:05:35.684895+0000","last_undegraded":"2024-09-21T10:05:35.684895+0000","last_fullsized":"2024-09-21T10:05:35.684895+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.762803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1142,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.658376+0000","last_change":"2024-09-21T10:05:33.658107+0000","last_active":"2024-09-21T10:05:33.658376+0000","last_peered":"2024-09-21T10:05:33.658376+0000","last_clean":"2024-09-21T10:05:27.632012+0000","last_became_active":"2024-09-21T10:05:33.658107+0000","last_became_peered":"2024-09-21T10:05:33.658107+0000","last_unstale":"2024-09-21T10:05:33.658376+0000","last_undegraded":"2024-09-21T10:05:33.658376+0000","last_fullsized":"2024-09-21T10:05:33.652338+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:52:34.246843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1762,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.680079+0000","last_change":"2024-09-21T10:01:40.431450+0000","last_active":"2024-09-21T10:05:35.680079+0000","last_peered":"2024-09-21T10:05:35.680079+0000","last_clean":"2024-09-21T10:05:35.680079+0000","last_became_active":"2024-09-21T10:01:40.431140+0000","last_became_peered":"2024-09-21T10:01:40.431140+0000","last_unstale":"2024-09-21T10:05:35.680079+0000","last_undegraded":"2024-09-21T10:05:35.680079+0000","last_fullsized":"2024-09-21T10:05:35.680079+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:37.604030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":98,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.655284+0000","last_change":"2024-09-21T10:00:16.377280+0000","last_active":"2024-09-21T10:05:33.655284+0000","last_peered":"2024-09-21T10:05:33.655284+0000","last_clean":"2024-09-21T10:05:33.655284+0000","last_became_active":"2024-09-21T10:00:16.353074+0000","last_became_peered":"2024-09-21T10:00:16.353074+0000","last_unstale":"2024-09-21T10:05:33.655284+0000","last_undegraded":"2024-09-21T10:05:33.655284+0000","last_fullsized":"2024-09-21T10:05:33.655284+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:35.382228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"1209'4985","reported_seq":5009,"reported_epoch":1209,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:47.093278+0000","last_change":"2024-09-21T10:05:33.689923+0000","last_active":"2024-09-21T10:05:47.093278+0000","last_peered":"2024-09-21T10:05:47.093278+0000","last_clean":"2024-09-21T10:05:32.600239+0000","last_became_active":"2024-09-21T10:05:33.689923+0000","last_became_peered":"2024-09-21T10:05:33.689923+0000","last_unstale":"2024-09-21T10:05:47.093278+0000","last_undegraded":"2024-09-21T10:05:33.660589+0000","last_fullsized":"2024-09-21T10:05:33.660463+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4985,"log_dups_size":0,"ondisk_log_size":4985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:17.498541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326696960,"num_objects":4985,"num_object_clones":0,"num_object_copies":14955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4985,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4985,"num_write_kb":319040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":4985}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":1140,"reported_epoch":1208,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:33.660651+0000","last_change":"2024-09-21T10:05:33.660382+0000","last_active":"2024-09-21T10:05:33.660651+0000","last_peered":"2024-09-21T10:05:33.660651+0000","last_clean":"2024-09-21T10:05:27.633184+0000","last_became_active":"2024-09-21T10:05:33.660382+0000","last_became_peered":"2024-09-21T10:05:33.660382+0000","last_unstale":"2024-09-21T10:05:33.660651+0000","last_undegraded":"2024-09-21T10:05:33.653645+0000","last_fullsized":"2024-09-21T10:05:33.653515+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:03.968124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":141,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.652143+0000","last_change":"2024-09-21T10:00:16.357541+0000","last_active":"2024-09-21T10:05:33.652143+0000","last_peered":"2024-09-21T10:05:33.652143+0000","last_clean":"2024-09-21T10:05:33.652143+0000","last_became_active":"2024-09-21T10:00:16.357209+0000","last_became_peered":"2024-09-21T10:00:16.357209+0000","last_unstale":"2024-09-21T10:05:33.652143+0000","last_undegraded":"2024-09-21T10:05:33.652143+0000","last_fullsized":"2024-09-21T10:05:33.652143+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:59:20.732470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"1209'4671","reported_seq":4694,"reported_epoch":1209,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:45.544859+0000","last_change":"2024-09-21T10:05:33.658757+0000","last_active":"2024-09-21T10:05:45.544859+0000","last_peered":"2024-09-21T10:05:45.544859+0000","last_clean":"2024-09-21T10:05:32.643126+0000","last_became_active":"2024-09-21T10:05:33.658757+0000","last_became_peered":"2024-09-21T10:05:33.658757+0000","last_unstale":"2024-09-21T10:05:45.544859+0000","last_undegraded":"2024-09-21T10:05:33.654898+0000","last_fullsized":"2024-09-21T10:05:33.654703+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4671,"log_dups_size":0,"ondisk_log_size":4671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:17.421277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306118656,"num_objects":4671,"num_object_clones":0,"num_object_copies":14013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4671,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4671,"num_write_kb":298944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"6,7","objects":4671}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":1155,"reported_epoch":1208,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:33.661269+0000","last_change":"2024-09-21T10:05:33.660952+0000","last_active":"2024-09-21T10:05:33.661269+0000","last_peered":"2024-09-21T10:05:33.661269+0000","last_clean":"2024-09-21T10:05:27.633016+0000","last_became_active":"2024-09-21T10:05:33.660952+0000","last_became_peered":"2024-09-21T10:05:33.660952+0000","last_unstale":"2024-09-21T10:05:33.661269+0000","last_undegraded":"2024-09-21T10:05:33.653745+0000","last_fullsized":"2024-09-21T10:05:33.653618+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:45:24.986482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1127,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.678658+0000","last_change":"2024-09-21T10:05:33.670376+0000","last_active":"2024-09-21T10:05:35.678658+0000","last_peered":"2024-09-21T10:05:35.678658+0000","last_clean":"2024-09-21T10:01:36.402509+0000","last_became_active":"2024-09-21T10:05:33.670376+0000","last_became_peered":"2024-09-21T10:05:33.670376+0000","last_unstale":"2024-09-21T10:05:35.678658+0000","last_undegraded":"2024-09-21T10:05:35.678658+0000","last_fullsized":"2024-09-21T10:05:33.655002+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:50.021479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.6","version":"1209'4749","reported_seq":4778,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:44.049297+0000","last_change":"2024-09-21T10:04:55.450875+0000","last_active":"2024-09-21T10:05:44.049297+0000","last_peered":"2024-09-21T10:05:44.049297+0000","last_clean":"2024-09-21T10:05:44.049297+0000","last_became_active":"2024-09-21T10:04:55.450004+0000","last_became_peered":"2024-09-21T10:04:55.450004+0000","last_unstale":"2024-09-21T10:05:44.049297+0000","last_undegraded":"2024-09-21T10:05:44.049297+0000","last_fullsized":"2024-09-21T10:05:44.049297+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4749,"log_dups_size":0,"ondisk_log_size":4749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:16.183501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311230464,"num_objects":4749,"num_object_clones":0,"num_object_copies":14247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4749,"num_write_kb":303936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":1144,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.653584+0000","last_change":"2024-09-21T10:00:16.373327+0000","last_active":"2024-09-21T10:05:33.653584+0000","last_peered":"2024-09-21T10:05:33.653584+0000","last_clean":"2024-09-21T10:05:33.653584+0000","last_became_active":"2024-09-21T10:00:16.360853+0000","last_became_peered":"2024-09-21T10:00:16.360853+0000","last_unstale":"2024-09-21T10:05:33.653584+0000","last_undegraded":"2024-09-21T10:05:33.653584+0000","last_fullsized":"2024-09-21T10:05:33.653584+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:18.160233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":98,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.655509+0000","last_change":"2024-09-21T10:01:40.417926+0000","last_active":"2024-09-21T10:05:33.655509+0000","last_peered":"2024-09-21T10:05:33.655509+0000","last_clean":"2024-09-21T10:05:33.655509+0000","last_became_active":"2024-09-21T10:01:40.417540+0000","last_became_peered":"2024-09-21T10:01:40.417540+0000","last_unstale":"2024-09-21T10:05:33.655509+0000","last_undegraded":"2024-09-21T10:05:33.655509+0000","last_fullsized":"2024-09-21T10:05:33.655509+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:43.890839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1209'4911","reported_seq":4940,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:46.524614+0000","last_change":"2024-09-21T10:04:55.445188+0000","last_active":"2024-09-21T10:05:46.524614+0000","last_peered":"2024-09-21T10:05:46.524614+0000","last_clean":"2024-09-21T10:05:46.524614+0000","last_became_active":"2024-09-21T10:04:55.444846+0000","last_became_peered":"2024-09-21T10:04:55.444846+0000","last_unstale":"2024-09-21T10:05:46.524614+0000","last_undegraded":"2024-09-21T10:05:46.524614+0000","last_fullsized":"2024-09-21T10:05:46.524614+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4911,"log_dups_size":0,"ondisk_log_size":4911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:09.773030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321847296,"num_objects":4911,"num_object_clones":0,"num_object_copies":14733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4911,"num_write_kb":314304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":769,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.680652+0000","last_change":"2024-09-21T10:00:16.360708+0000","last_active":"2024-09-21T10:05:35.680652+0000","last_peered":"2024-09-21T10:05:35.680652+0000","last_clean":"2024-09-21T10:05:35.680652+0000","last_became_active":"2024-09-21T10:00:16.360416+0000","last_became_peered":"2024-09-21T10:00:16.360416+0000","last_unstale":"2024-09-21T10:05:35.680652+0000","last_undegraded":"2024-09-21T10:05:35.680652+0000","last_fullsized":"2024-09-21T10:05:35.680652+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:59.090215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1853,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.672809+0000","last_change":"2024-09-21T10:05:33.675759+0000","last_active":"2024-09-21T10:05:35.672809+0000","last_peered":"2024-09-21T10:05:35.672809+0000","last_clean":"2024-09-21T10:05:27.628966+0000","last_became_active":"2024-09-21T10:05:33.675759+0000","last_became_peered":"2024-09-21T10:05:33.675759+0000","last_unstale":"2024-09-21T10:05:35.672809+0000","last_undegraded":"2024-09-21T10:05:35.672809+0000","last_fullsized":"2024-09-21T10:05:33.657525+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:05.073600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"1209'4833","reported_seq":4864,"reported_epoch":1209,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:46.545137+0000","last_change":"2024-09-21T10:05:33.690535+0000","last_active":"2024-09-21T10:05:46.545137+0000","last_peered":"2024-09-21T10:05:46.545137+0000","last_clean":"2024-09-21T10:05:32.639360+0000","last_became_active":"2024-09-21T10:05:33.690535+0000","last_became_peered":"2024-09-21T10:05:33.690535+0000","last_unstale":"2024-09-21T10:05:46.545137+0000","last_undegraded":"2024-09-21T10:05:33.655207+0000","last_fullsized":"2024-09-21T10:05:33.655091+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4833,"log_dups_size":0,"ondisk_log_size":4833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:07.748972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316735488,"num_objects":4833,"num_object_clones":0,"num_object_copies":14499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4833,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4833,"num_write_kb":309312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":4833}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1279,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.652190+0000","last_change":"2024-09-21T10:00:47.119852+0000","last_active":"2024-09-21T10:05:33.652190+0000","last_peered":"2024-09-21T10:05:33.652190+0000","last_clean":"2024-09-21T10:05:33.652190+0000","last_became_active":"2024-09-21T09:59:43.374051+0000","last_became_peered":"2024-09-21T09:59:43.374051+0000","last_unstale":"2024-09-21T10:05:33.652190+0000","last_undegraded":"2024-09-21T10:05:33.652190+0000","last_fullsized":"2024-09-21T10:05:33.652190+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:47.119793+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:47.119793+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:21:23.087965+0000","scrub_duration":0.0013414659999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1770,"reported_epoch":1209,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:35.681587+0000","last_change":"2024-09-21T10:05:33.673313+0000","last_active":"2024-09-21T10:05:35.681587+0000","last_peered":"2024-09-21T10:05:35.681587+0000","last_clean":"2024-09-21T10:05:27.641438+0000","last_became_active":"2024-09-21T10:05:33.673313+0000","last_became_peered":"2024-09-21T10:05:33.673313+0000","last_unstale":"2024-09-21T10:05:35.681587+0000","last_undegraded":"2024-09-21T10:05:33.664773+0000","last_fullsized":"2024-09-21T10:05:33.664662+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:13:35.974047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1094,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.681311+0000","last_change":"2024-09-21T10:05:33.671864+0000","last_active":"2024-09-21T10:05:35.681311+0000","last_peered":"2024-09-21T10:05:35.681311+0000","last_clean":"2024-09-21T10:00:01.324528+0000","last_became_active":"2024-09-21T10:05:33.671864+0000","last_became_peered":"2024-09-21T10:05:33.671864+0000","last_unstale":"2024-09-21T10:05:35.681311+0000","last_undegraded":"2024-09-21T10:05:35.681311+0000","last_fullsized":"2024-09-21T10:05:33.658719+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:56.623195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"1209'4790","reported_seq":4819,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:43.994208+0000","last_change":"2024-09-21T10:04:55.933597+0000","last_active":"2024-09-21T10:05:43.994208+0000","last_peered":"2024-09-21T10:05:43.994208+0000","last_clean":"2024-09-21T10:05:43.994208+0000","last_became_active":"2024-09-21T10:04:55.933296+0000","last_became_peered":"2024-09-21T10:04:55.933296+0000","last_unstale":"2024-09-21T10:05:43.994208+0000","last_undegraded":"2024-09-21T10:05:43.994208+0000","last_fullsized":"2024-09-21T10:05:43.994208+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4790,"log_dups_size":0,"ondisk_log_size":4790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:33:25.512434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313917440,"num_objects":4790,"num_object_clones":0,"num_object_copies":14370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4790,"num_write_kb":306560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":209,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.675603+0000","last_change":"2024-09-21T10:00:15.524812+0000","last_active":"2024-09-21T10:05:35.675603+0000","last_peered":"2024-09-21T10:05:35.675603+0000","last_clean":"2024-09-21T10:05:35.675603+0000","last_became_active":"2024-09-21T09:59:43.361751+0000","last_became_peered":"2024-09-21T09:59:43.361751+0000","last_unstale":"2024-09-21T10:05:35.675603+0000","last_undegraded":"2024-09-21T10:05:35.675603+0000","last_fullsized":"2024-09-21T10:05:35.675603+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:20:34.289873+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":105,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.669392+0000","last_change":"2024-09-21T10:05:33.669087+0000","last_active":"2024-09-21T10:05:33.669392+0000","last_peered":"2024-09-21T10:05:33.669392+0000","last_clean":"2024-09-21T10:05:27.625316+0000","last_became_active":"2024-09-21T10:05:33.669087+0000","last_became_peered":"2024-09-21T10:05:33.669087+0000","last_unstale":"2024-09-21T10:05:33.669392+0000","last_undegraded":"2024-09-21T10:05:33.669392+0000","last_fullsized":"2024-09-21T10:05:33.654994+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:37:56.362284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1127,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.684024+0000","last_change":"2024-09-21T10:05:33.669690+0000","last_active":"2024-09-21T10:05:35.684024+0000","last_peered":"2024-09-21T10:05:35.684024+0000","last_clean":"2024-09-21T10:01:36.400674+0000","last_became_active":"2024-09-21T10:05:33.669690+0000","last_became_peered":"2024-09-21T10:05:33.669690+0000","last_unstale":"2024-09-21T10:05:35.684024+0000","last_undegraded":"2024-09-21T10:05:35.684024+0000","last_fullsized":"2024-09-21T10:05:33.662704+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:55.425167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"1209'4689","reported_seq":4713,"reported_epoch":1209,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:44.035814+0000","last_change":"2024-09-21T10:05:33.691575+0000","last_active":"2024-09-21T10:05:44.035814+0000","last_peered":"2024-09-21T10:05:44.035814+0000","last_clean":"2024-09-21T10:05:32.637461+0000","last_became_active":"2024-09-21T10:05:33.691575+0000","last_became_peered":"2024-09-21T10:05:33.691575+0000","last_unstale":"2024-09-21T10:05:44.035814+0000","last_undegraded":"2024-09-21T10:05:33.657505+0000","last_fullsized":"2024-09-21T10:05:33.657324+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4689,"log_dups_size":0,"ondisk_log_size":4689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:12:32.157646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307298304,"num_objects":4689,"num_object_clones":0,"num_object_copies":14067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4689,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4689,"num_write_kb":300096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4689}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":503,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.683505+0000","last_change":"2024-09-21T10:05:33.667867+0000","last_active":"2024-09-21T10:05:35.683505+0000","last_peered":"2024-09-21T10:05:35.683505+0000","last_clean":"2024-09-21T09:45:59.078640+0000","last_became_active":"2024-09-21T10:05:33.667867+0000","last_became_peered":"2024-09-21T10:05:33.667867+0000","last_unstale":"2024-09-21T10:05:35.683505+0000","last_undegraded":"2024-09-21T10:05:35.683505+0000","last_fullsized":"2024-09-21T10:05:33.661835+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:01.897155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1129,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.669520+0000","last_change":"2024-09-21T10:05:33.669258+0000","last_active":"2024-09-21T10:05:33.669520+0000","last_peered":"2024-09-21T10:05:33.669520+0000","last_clean":"2024-09-21T10:01:36.400926+0000","last_became_active":"2024-09-21T10:05:33.669258+0000","last_became_peered":"2024-09-21T10:05:33.669258+0000","last_unstale":"2024-09-21T10:05:33.669520+0000","last_undegraded":"2024-09-21T10:05:33.669520+0000","last_fullsized":"2024-09-21T10:05:33.656417+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:03:47.942149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.3","version":"1209'4729","reported_seq":4746,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:45.540908+0000","last_change":"2024-09-21T10:04:55.935712+0000","last_active":"2024-09-21T10:05:45.540908+0000","last_peered":"2024-09-21T10:05:45.540908+0000","last_clean":"2024-09-21T10:05:45.540908+0000","last_became_active":"2024-09-21T10:04:55.935221+0000","last_became_peered":"2024-09-21T10:04:55.935221+0000","last_unstale":"2024-09-21T10:05:45.540908+0000","last_undegraded":"2024-09-21T10:05:45.540908+0000","last_fullsized":"2024-09-21T10:05:45.540908+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4729,"log_dups_size":0,"ondisk_log_size":4729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:20:50.440837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309919744,"num_objects":4729,"num_object_clones":0,"num_object_copies":14187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4729,"num_write_kb":302656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":207,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.684248+0000","last_change":"2024-09-21T10:05:24.687658+0000","last_active":"2024-09-21T10:05:35.684248+0000","last_peered":"2024-09-21T10:05:35.684248+0000","last_clean":"2024-09-21T10:05:35.684248+0000","last_became_active":"2024-09-21T10:00:16.351010+0000","last_became_peered":"2024-09-21T10:00:16.351010+0000","last_unstale":"2024-09-21T10:05:35.684248+0000","last_undegraded":"2024-09-21T10:05:35.684248+0000","last_fullsized":"2024-09-21T10:05:35.684248+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:24.686892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:24.686892+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:31.755123+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1768,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.681022+0000","last_change":"2024-09-21T10:05:33.661843+0000","last_active":"2024-09-21T10:05:35.681022+0000","last_peered":"2024-09-21T10:05:35.681022+0000","last_clean":"2024-09-21T10:05:27.641408+0000","last_became_active":"2024-09-21T10:05:33.661843+0000","last_became_peered":"2024-09-21T10:05:33.661843+0000","last_unstale":"2024-09-21T10:05:35.681022+0000","last_undegraded":"2024-09-21T10:05:35.681022+0000","last_fullsized":"2024-09-21T10:05:33.654545+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:01.028265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":673,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.681278+0000","last_change":"2024-09-21T10:05:33.665430+0000","last_active":"2024-09-21T10:05:35.681278+0000","last_peered":"2024-09-21T10:05:35.681278+0000","last_clean":"2024-09-21T10:05:27.641804+0000","last_became_active":"2024-09-21T10:05:33.665430+0000","last_became_peered":"2024-09-21T10:05:33.665430+0000","last_unstale":"2024-09-21T10:05:35.681278+0000","last_undegraded":"2024-09-21T10:05:35.681278+0000","last_fullsized":"2024-09-21T10:05:33.659869+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:55:08.740267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.9","version":"1209'4884","reported_seq":4901,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:45.531062+0000","last_change":"2024-09-21T10:04:55.934931+0000","last_active":"2024-09-21T10:05:45.531062+0000","last_peered":"2024-09-21T10:05:45.531062+0000","last_clean":"2024-09-21T10:05:45.531062+0000","last_became_active":"2024-09-21T10:04:55.934596+0000","last_became_peered":"2024-09-21T10:04:55.934596+0000","last_unstale":"2024-09-21T10:05:45.531062+0000","last_undegraded":"2024-09-21T10:05:45.531062+0000","last_fullsized":"2024-09-21T10:05:45.531062+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4884,"log_dups_size":0,"ondisk_log_size":4884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:39:56.366258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320077824,"num_objects":4884,"num_object_clones":0,"num_object_copies":14652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4884,"num_write_kb":312576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":1293,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.652613+0000","last_change":"2024-09-21T10:00:49.116174+0000","last_active":"2024-09-21T10:05:33.652613+0000","last_peered":"2024-09-21T10:05:33.652613+0000","last_clean":"2024-09-21T10:05:33.652613+0000","last_became_active":"2024-09-21T10:00:16.360430+0000","last_became_peered":"2024-09-21T10:00:16.360430+0000","last_unstale":"2024-09-21T10:05:33.652613+0000","last_undegraded":"2024-09-21T10:05:33.652613+0000","last_fullsized":"2024-09-21T10:05:33.652613+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:49.116103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:49.116103+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:29.874911+0000","scrub_duration":0.0020386419999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":242,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.685004+0000","last_change":"2024-09-21T09:57:39.541461+0000","last_active":"2024-09-21T10:05:35.685004+0000","last_peered":"2024-09-21T10:05:35.685004+0000","last_clean":"2024-09-21T10:05:35.685004+0000","last_became_active":"2024-09-21T09:57:39.540927+0000","last_became_peered":"2024-09-21T09:57:39.540927+0000","last_unstale":"2024-09-21T10:05:35.685004+0000","last_undegraded":"2024-09-21T10:05:35.685004+0000","last_fullsized":"2024-09-21T10:05:35.685004+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:14.815783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1754,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.679276+0000","last_change":"2024-09-21T10:00:16.362131+0000","last_active":"2024-09-21T10:05:35.679276+0000","last_peered":"2024-09-21T10:05:35.679276+0000","last_clean":"2024-09-21T10:05:35.679276+0000","last_became_active":"2024-09-21T10:00:16.361722+0000","last_became_peered":"2024-09-21T10:00:16.361722+0000","last_unstale":"2024-09-21T10:05:35.679276+0000","last_undegraded":"2024-09-21T10:05:35.679276+0000","last_fullsized":"2024-09-21T10:05:35.679276+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:38.717426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":95,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.659002+0000","last_change":"2024-09-21T10:05:33.658773+0000","last_active":"2024-09-21T10:05:33.659002+0000","last_peered":"2024-09-21T10:05:33.659002+0000","last_clean":"2024-09-21T10:05:27.625911+0000","last_became_active":"2024-09-21T10:05:33.658773+0000","last_became_peered":"2024-09-21T10:05:33.658773+0000","last_unstale":"2024-09-21T10:05:33.659002+0000","last_undegraded":"2024-09-21T10:05:33.659002+0000","last_fullsized":"2024-09-21T10:05:33.654427+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:37.261328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1209'5080","reported_seq":5108,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:47.075292+0000","last_change":"2024-09-21T10:04:55.934274+0000","last_active":"2024-09-21T10:05:47.075292+0000","last_peered":"2024-09-21T10:05:47.075292+0000","last_clean":"2024-09-21T10:05:47.075292+0000","last_became_active":"2024-09-21T10:04:55.933895+0000","last_became_peered":"2024-09-21T10:04:55.933895+0000","last_unstale":"2024-09-21T10:05:47.075292+0000","last_undegraded":"2024-09-21T10:05:47.075292+0000","last_fullsized":"2024-09-21T10:05:47.075292+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":5080,"log_dups_size":0,"ondisk_log_size":5080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:40.821830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332922880,"num_objects":5080,"num_object_clones":0,"num_object_copies":15240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5080,"num_write_kb":325120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":110,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.690189+0000","last_change":"2024-09-21T10:05:33.689917+0000","last_active":"2024-09-21T10:05:33.690189+0000","last_peered":"2024-09-21T10:05:33.690189+0000","last_clean":"2024-09-21T10:05:27.625486+0000","last_became_active":"2024-09-21T10:05:33.689917+0000","last_became_peered":"2024-09-21T10:05:33.689917+0000","last_unstale":"2024-09-21T10:05:33.690189+0000","last_undegraded":"2024-09-21T10:05:33.690189+0000","last_fullsized":"2024-09-21T10:05:33.655343+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:04:35.993739+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:43.017674+0000","scrub_duration":0.001400943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1171,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.652538+0000","last_change":"2024-09-21T10:01:40.428393+0000","last_active":"2024-09-21T10:05:33.652538+0000","last_peered":"2024-09-21T10:05:33.652538+0000","last_clean":"2024-09-21T10:05:33.652538+0000","last_became_active":"2024-09-21T10:01:40.428204+0000","last_became_peered":"2024-09-21T10:01:40.428204+0000","last_unstale":"2024-09-21T10:05:33.652538+0000","last_undegraded":"2024-09-21T10:05:33.652538+0000","last_fullsized":"2024-09-21T10:05:33.652538+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:23:02.136035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1844,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.673810+0000","last_change":"2024-09-21T10:05:33.673510+0000","last_active":"2024-09-21T10:05:35.673810+0000","last_peered":"2024-09-21T10:05:35.673810+0000","last_clean":"2024-09-21T10:05:27.633062+0000","last_became_active":"2024-09-21T10:05:33.673510+0000","last_became_peered":"2024-09-21T10:05:33.673510+0000","last_unstale":"2024-09-21T10:05:35.673810+0000","last_undegraded":"2024-09-21T10:05:35.673810+0000","last_fullsized":"2024-09-21T10:05:33.655278+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:21:48.812935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1860,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.682960+0000","last_change":"2024-09-21T10:01:40.421887+0000","last_active":"2024-09-21T10:05:35.682960+0000","last_peered":"2024-09-21T10:05:35.682960+0000","last_clean":"2024-09-21T10:05:35.682960+0000","last_became_active":"2024-09-21T10:01:40.420730+0000","last_became_peered":"2024-09-21T10:01:40.420730+0000","last_unstale":"2024-09-21T10:05:35.682960+0000","last_undegraded":"2024-09-21T10:05:35.682960+0000","last_fullsized":"2024-09-21T10:05:35.682960+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:51.504051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"1209'4893","reported_seq":4925,"reported_epoch":1209,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:46.535140+0000","last_change":"2024-09-21T10:05:33.669178+0000","last_active":"2024-09-21T10:05:46.535140+0000","last_peered":"2024-09-21T10:05:46.535140+0000","last_clean":"2024-09-21T10:05:32.629706+0000","last_became_active":"2024-09-21T10:05:33.669178+0000","last_became_peered":"2024-09-21T10:05:33.669178+0000","last_unstale":"2024-09-21T10:05:46.535140+0000","last_undegraded":"2024-09-21T10:05:33.665488+0000","last_fullsized":"2024-09-21T10:05:33.665368+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4893,"log_dups_size":0,"ondisk_log_size":4893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:36.067832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320667648,"num_objects":4893,"num_object_clones":0,"num_object_copies":14679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4893,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4893,"num_write_kb":313152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":4893}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":1286,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.660701+0000","last_change":"2024-09-21T10:05:33.660416+0000","last_active":"2024-09-21T10:05:33.660701+0000","last_peered":"2024-09-21T10:05:33.660701+0000","last_clean":"2024-09-21T10:05:27.632090+0000","last_became_active":"2024-09-21T10:05:33.660416+0000","last_became_peered":"2024-09-21T10:05:33.660416+0000","last_unstale":"2024-09-21T10:05:33.660701+0000","last_undegraded":"2024-09-21T10:05:33.660701+0000","last_fullsized":"2024-09-21T10:05:33.652219+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:16.913146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:16.913146+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:48:27.919907+0000","scrub_duration":0.001342315,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1815,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.683658+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:05:35.683658+0000","last_peered":"2024-09-21T10:05:35.683658+0000","last_clean":"2024-09-21T10:05:35.683658+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:05:35.683658+0000","last_undegraded":"2024-09-21T10:05:35.683658+0000","last_fullsized":"2024-09-21T10:05:35.683658+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1770,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.675760+0000","last_change":"2024-09-21T10:01:40.414547+0000","last_active":"2024-09-21T10:05:35.675760+0000","last_peered":"2024-09-21T10:05:35.675760+0000","last_clean":"2024-09-21T10:05:35.675760+0000","last_became_active":"2024-09-21T10:01:40.414243+0000","last_became_peered":"2024-09-21T10:01:40.414243+0000","last_unstale":"2024-09-21T10:05:35.675760+0000","last_undegraded":"2024-09-21T10:05:35.675760+0000","last_fullsized":"2024-09-21T10:05:35.675760+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:55:26.044481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":550,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.686877+0000","last_change":"2024-09-21T10:01:40.421427+0000","last_active":"2024-09-21T10:05:35.686877+0000","last_peered":"2024-09-21T10:05:35.686877+0000","last_clean":"2024-09-21T10:05:35.686877+0000","last_became_active":"2024-09-21T10:01:40.421047+0000","last_became_peered":"2024-09-21T10:01:40.421047+0000","last_unstale":"2024-09-21T10:05:35.686877+0000","last_undegraded":"2024-09-21T10:05:35.686877+0000","last_fullsized":"2024-09-21T10:05:35.686877+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:48.640296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.8","version":"1209'4846","reported_seq":4874,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:46.537314+0000","last_change":"2024-09-21T10:04:55.934259+0000","last_active":"2024-09-21T10:05:46.537314+0000","last_peered":"2024-09-21T10:05:46.537314+0000","last_clean":"2024-09-21T10:05:46.537314+0000","last_became_active":"2024-09-21T10:04:55.933861+0000","last_became_peered":"2024-09-21T10:04:55.933861+0000","last_unstale":"2024-09-21T10:05:46.537314+0000","last_undegraded":"2024-09-21T10:05:46.537314+0000","last_fullsized":"2024-09-21T10:05:46.537314+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4846,"log_dups_size":0,"ondisk_log_size":4846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:22:32.497986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317587456,"num_objects":4846,"num_object_clones":0,"num_object_copies":14538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4846,"num_write_kb":310144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":546,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.673720+0000","last_change":"2024-09-21T10:05:24.788288+0000","last_active":"2024-09-21T10:05:35.673720+0000","last_peered":"2024-09-21T10:05:35.673720+0000","last_clean":"2024-09-21T10:05:35.673720+0000","last_became_active":"2024-09-21T09:59:43.371384+0000","last_became_peered":"2024-09-21T09:59:43.371384+0000","last_unstale":"2024-09-21T10:05:35.673720+0000","last_undegraded":"2024-09-21T10:05:35.673720+0000","last_fullsized":"2024-09-21T10:05:35.673720+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:24.785926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:24.785926+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:08:16.075802+0000","scrub_duration":16,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":166,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.654513+0000","last_change":"2024-09-21T10:01:40.426865+0000","last_active":"2024-09-21T10:05:33.654513+0000","last_peered":"2024-09-21T10:05:33.654513+0000","last_clean":"2024-09-21T10:05:33.654513+0000","last_became_active":"2024-09-21T10:01:40.413152+0000","last_became_peered":"2024-09-21T10:01:40.413152+0000","last_unstale":"2024-09-21T10:05:33.654513+0000","last_undegraded":"2024-09-21T10:05:33.654513+0000","last_fullsized":"2024-09-21T10:05:33.654513+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:59.034627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1095,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.673359+0000","last_change":"2024-09-21T10:05:33.663884+0000","last_active":"2024-09-21T10:05:35.673359+0000","last_peered":"2024-09-21T10:05:35.673359+0000","last_clean":"2024-09-21T10:00:01.324252+0000","last_became_active":"2024-09-21T10:05:33.663884+0000","last_became_peered":"2024-09-21T10:05:33.663884+0000","last_unstale":"2024-09-21T10:05:35.673359+0000","last_undegraded":"2024-09-21T10:05:35.673359+0000","last_fullsized":"2024-09-21T10:05:33.658103+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:08.262661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":680,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.683931+0000","last_change":"2024-09-21T10:01:40.421262+0000","last_active":"2024-09-21T10:05:35.683931+0000","last_peered":"2024-09-21T10:05:35.683931+0000","last_clean":"2024-09-21T10:05:35.683931+0000","last_became_active":"2024-09-21T10:01:40.420959+0000","last_became_peered":"2024-09-21T10:01:40.420959+0000","last_unstale":"2024-09-21T10:05:35.683931+0000","last_undegraded":"2024-09-21T10:05:35.683931+0000","last_fullsized":"2024-09-21T10:05:35.683931+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:06.866421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"1209'4785","reported_seq":4817,"reported_epoch":1209,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:44.043268+0000","last_change":"2024-09-21T10:05:33.664012+0000","last_active":"2024-09-21T10:05:44.043268+0000","last_peered":"2024-09-21T10:05:44.043268+0000","last_clean":"2024-09-21T10:05:32.615686+0000","last_became_active":"2024-09-21T10:05:33.664012+0000","last_became_peered":"2024-09-21T10:05:33.664012+0000","last_unstale":"2024-09-21T10:05:44.043268+0000","last_undegraded":"2024-09-21T10:05:33.656301+0000","last_fullsized":"2024-09-21T10:05:33.656171+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4785,"log_dups_size":0,"ondisk_log_size":4785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:06:02.486518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313589760,"num_objects":4785,"num_object_clones":0,"num_object_copies":14355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4785,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4785,"num_write_kb":306240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":4785}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":98,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.656200+0000","last_change":"2024-09-21T10:00:46.494180+0000","last_active":"2024-09-21T10:05:33.656200+0000","last_peered":"2024-09-21T10:05:33.656200+0000","last_clean":"2024-09-21T10:05:33.656200+0000","last_became_active":"2024-09-21T10:00:16.355386+0000","last_became_peered":"2024-09-21T10:00:16.355386+0000","last_unstale":"2024-09-21T10:05:33.656200+0000","last_undegraded":"2024-09-21T10:05:33.656200+0000","last_fullsized":"2024-09-21T10:05:33.656200+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:46.494129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:46.494129+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:11.193593+0000","scrub_duration":0.00183863,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":771,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.679814+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T10:05:35.679814+0000","last_peered":"2024-09-21T10:05:35.679814+0000","last_clean":"2024-09-21T10:05:35.679814+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T10:05:35.679814+0000","last_undegraded":"2024-09-21T10:05:35.679814+0000","last_fullsized":"2024-09-21T10:05:35.679814+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":579,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.687267+0000","last_change":"2024-09-21T10:01:40.418328+0000","last_active":"2024-09-21T10:05:35.687267+0000","last_peered":"2024-09-21T10:05:35.687267+0000","last_clean":"2024-09-21T10:05:35.687267+0000","last_became_active":"2024-09-21T10:01:40.418011+0000","last_became_peered":"2024-09-21T10:01:40.418011+0000","last_unstale":"2024-09-21T10:05:35.687267+0000","last_undegraded":"2024-09-21T10:05:35.687267+0000","last_fullsized":"2024-09-21T10:05:35.687267+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:58.716988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":462,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.676622+0000","last_change":"2024-09-21T10:05:33.693560+0000","last_active":"2024-09-21T10:05:35.676622+0000","last_peered":"2024-09-21T10:05:35.676622+0000","last_clean":"2024-09-21T09:36:44.128763+0000","last_became_active":"2024-09-21T10:05:33.693560+0000","last_became_peered":"2024-09-21T10:05:33.693560+0000","last_unstale":"2024-09-21T10:05:35.676622+0000","last_undegraded":"2024-09-21T10:05:35.676622+0000","last_fullsized":"2024-09-21T10:05:33.658109+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.851473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"1209'4847","reported_seq":4880,"reported_epoch":1209,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:46.524572+0000","last_change":"2024-09-21T10:05:33.669975+0000","last_active":"2024-09-21T10:05:46.524572+0000","last_peered":"2024-09-21T10:05:46.524572+0000","last_clean":"2024-09-21T10:05:32.648270+0000","last_became_active":"2024-09-21T10:05:33.669975+0000","last_became_peered":"2024-09-21T10:05:33.669975+0000","last_unstale":"2024-09-21T10:05:46.524572+0000","last_undegraded":"2024-09-21T10:05:33.662513+0000","last_fullsized":"2024-09-21T10:05:33.662414+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4847,"log_dups_size":0,"ondisk_log_size":4847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:57.231455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317652992,"num_objects":4847,"num_object_clones":0,"num_object_copies":14541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4847,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4847,"num_write_kb":310208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":4847}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":98,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.657857+0000","last_change":"2024-09-21T10:00:20.637858+0000","last_active":"2024-09-21T10:05:33.657857+0000","last_peered":"2024-09-21T10:05:33.657857+0000","last_clean":"2024-09-21T10:05:33.657857+0000","last_became_active":"2024-09-21T10:00:16.353796+0000","last_became_peered":"2024-09-21T10:00:16.353796+0000","last_unstale":"2024-09-21T10:05:33.657857+0000","last_undegraded":"2024-09-21T10:05:33.657857+0000","last_fullsized":"2024-09-21T10:05:33.657857+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:20.637806+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:20.637806+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:12:16.884495+0000","scrub_duration":0.0016753919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":606,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.683768+0000","last_change":"2024-09-21T10:01:40.414536+0000","last_active":"2024-09-21T10:05:35.683768+0000","last_peered":"2024-09-21T10:05:35.683768+0000","last_clean":"2024-09-21T10:05:35.683768+0000","last_became_active":"2024-09-21T10:01:40.413913+0000","last_became_peered":"2024-09-21T10:01:40.413913+0000","last_unstale":"2024-09-21T10:05:35.683768+0000","last_undegraded":"2024-09-21T10:05:35.683768+0000","last_fullsized":"2024-09-21T10:05:35.683768+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:27.994971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1151,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.660475+0000","last_change":"2024-09-21T10:05:33.660221+0000","last_active":"2024-09-21T10:05:33.660475+0000","last_peered":"2024-09-21T10:05:33.660475+0000","last_clean":"2024-09-21T10:05:27.633063+0000","last_became_active":"2024-09-21T10:05:33.660221+0000","last_became_peered":"2024-09-21T10:05:33.660221+0000","last_unstale":"2024-09-21T10:05:33.660475+0000","last_undegraded":"2024-09-21T10:05:33.660475+0000","last_fullsized":"2024-09-21T10:05:33.653704+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:57.456145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1831,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.673602+0000","last_change":"2024-09-21T09:45:05.097646+0000","last_active":"2024-09-21T10:05:35.673602+0000","last_peered":"2024-09-21T10:05:35.673602+0000","last_clean":"2024-09-21T10:05:35.673602+0000","last_became_active":"2024-09-21T09:45:05.096552+0000","last_became_peered":"2024-09-21T09:45:05.096552+0000","last_unstale":"2024-09-21T10:05:35.673602+0000","last_undegraded":"2024-09-21T10:05:35.673602+0000","last_fullsized":"2024-09-21T10:05:35.673602+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:26.763666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"1209'4753","reported_seq":4780,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:44.057345+0000","last_change":"2024-09-21T10:04:55.448819+0000","last_active":"2024-09-21T10:05:44.057345+0000","last_peered":"2024-09-21T10:05:44.057345+0000","last_clean":"2024-09-21T10:05:44.057345+0000","last_became_active":"2024-09-21T10:04:55.448338+0000","last_became_peered":"2024-09-21T10:04:55.448338+0000","last_unstale":"2024-09-21T10:05:44.057345+0000","last_undegraded":"2024-09-21T10:05:44.057345+0000","last_fullsized":"2024-09-21T10:05:44.057345+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4753,"log_dups_size":0,"ondisk_log_size":4753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:58.758191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311492608,"num_objects":4753,"num_object_clones":0,"num_object_copies":14259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4753,"num_write_kb":304192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":13,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.683805+0000","last_change":"2024-09-21T10:05:33.670399+0000","last_active":"2024-09-21T10:05:35.683805+0000","last_peered":"2024-09-21T10:05:35.683805+0000","last_clean":"2024-09-21T09:41:42.235098+0000","last_became_active":"2024-09-21T10:05:33.670399+0000","last_became_peered":"2024-09-21T10:05:33.670399+0000","last_unstale":"2024-09-21T10:05:35.683805+0000","last_undegraded":"2024-09-21T10:05:35.683805+0000","last_fullsized":"2024-09-21T10:05:33.661603+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T09:59:47.767654+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T09:59:47.767654+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:06:34.496848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1766,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.681449+0000","last_change":"2024-09-21T09:46:47.120214+0000","last_active":"2024-09-21T10:05:35.681449+0000","last_peered":"2024-09-21T10:05:35.681449+0000","last_clean":"2024-09-21T10:05:35.681449+0000","last_became_active":"2024-09-21T09:46:47.119878+0000","last_became_peered":"2024-09-21T09:46:47.119878+0000","last_unstale":"2024-09-21T10:05:35.681449+0000","last_undegraded":"2024-09-21T10:05:35.681449+0000","last_fullsized":"2024-09-21T10:05:35.681449+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:22:46.365146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1719,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.669589+0000","last_change":"2024-09-21T10:05:33.669283+0000","last_active":"2024-09-21T10:05:33.669589+0000","last_peered":"2024-09-21T10:05:33.669589+0000","last_clean":"2024-09-21T10:05:27.625682+0000","last_became_active":"2024-09-21T10:05:33.669283+0000","last_became_peered":"2024-09-21T10:05:33.669283+0000","last_unstale":"2024-09-21T10:05:33.669589+0000","last_undegraded":"2024-09-21T10:05:33.669589+0000","last_fullsized":"2024-09-21T10:05:33.656325+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:00:59.990587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1764,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.678178+0000","last_change":"2024-09-21T09:46:47.121078+0000","last_active":"2024-09-21T10:05:35.678178+0000","last_peered":"2024-09-21T10:05:35.678178+0000","last_clean":"2024-09-21T10:05:35.678178+0000","last_became_active":"2024-09-21T09:46:47.118472+0000","last_became_peered":"2024-09-21T09:46:47.118472+0000","last_unstale":"2024-09-21T10:05:35.678178+0000","last_undegraded":"2024-09-21T10:05:35.678178+0000","last_fullsized":"2024-09-21T10:05:35.678178+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:30.277273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1209'4732","reported_seq":4749,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:46.095058+0000","last_change":"2024-09-21T10:04:55.932700+0000","last_active":"2024-09-21T10:05:46.095058+0000","last_peered":"2024-09-21T10:05:46.095058+0000","last_clean":"2024-09-21T10:05:46.095058+0000","last_became_active":"2024-09-21T10:04:55.932416+0000","last_became_peered":"2024-09-21T10:04:55.932416+0000","last_unstale":"2024-09-21T10:05:46.095058+0000","last_undegraded":"2024-09-21T10:05:46.095058+0000","last_fullsized":"2024-09-21T10:05:46.095058+0000","mapping_epoch":1200,"log_start":"0'0","ondisk_log_start":"0'0","created":1200,"last_epoch_clean":1201,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:04:54.431499+0000","last_clean_scrub_stamp":"2024-09-21T10:04:54.431499+0000","objects_scrubbed":0,"log_size":4732,"log_dups_size":0,"ondisk_log_size":4732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.824627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310116352,"num_objects":4732,"num_object_clones":0,"num_object_copies":14196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4732,"num_write_kb":302848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":205,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.680373+0000","last_change":"2024-09-21T10:05:25.468250+0000","last_active":"2024-09-21T10:05:35.680373+0000","last_peered":"2024-09-21T10:05:35.680373+0000","last_clean":"2024-09-21T10:05:35.680373+0000","last_became_active":"2024-09-21T09:59:43.368157+0000","last_became_peered":"2024-09-21T09:59:43.368157+0000","last_unstale":"2024-09-21T10:05:35.680373+0000","last_undegraded":"2024-09-21T10:05:35.680373+0000","last_fullsized":"2024-09-21T10:05:35.680373+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:25.468199+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:25.468199+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:03.775358+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1859,"reported_epoch":1209,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:35.684985+0000","last_change":"2024-09-21T10:05:33.662766+0000","last_active":"2024-09-21T10:05:35.684985+0000","last_peered":"2024-09-21T10:05:35.684985+0000","last_clean":"2024-09-21T10:05:27.631048+0000","last_became_active":"2024-09-21T10:05:33.662766+0000","last_became_peered":"2024-09-21T10:05:33.662766+0000","last_unstale":"2024-09-21T10:05:35.684985+0000","last_undegraded":"2024-09-21T10:05:33.655656+0000","last_fullsized":"2024-09-21T10:05:33.655511+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:23.746125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":88,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.656141+0000","last_change":"2024-09-21T10:00:16.355102+0000","last_active":"2024-09-21T10:05:33.656141+0000","last_peered":"2024-09-21T10:05:33.656141+0000","last_clean":"2024-09-21T10:05:33.656141+0000","last_became_active":"2024-09-21T10:00:16.354473+0000","last_became_peered":"2024-09-21T10:00:16.354473+0000","last_unstale":"2024-09-21T10:05:33.656141+0000","last_undegraded":"2024-09-21T10:05:33.656141+0000","last_fullsized":"2024-09-21T10:05:33.656141+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:49:35.449111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1076,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.692062+0000","last_change":"2024-09-21T10:05:33.691864+0000","last_active":"2024-09-21T10:05:33.692062+0000","last_peered":"2024-09-21T10:05:33.692062+0000","last_clean":"2024-09-21T10:00:01.323468+0000","last_became_active":"2024-09-21T10:05:33.691864+0000","last_became_peered":"2024-09-21T10:05:33.691864+0000","last_unstale":"2024-09-21T10:05:33.692062+0000","last_undegraded":"2024-09-21T10:05:33.692062+0000","last_fullsized":"2024-09-21T10:05:33.655353+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:10.707259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1129,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.690088+0000","last_change":"2024-09-21T10:05:33.689833+0000","last_active":"2024-09-21T10:05:33.690088+0000","last_peered":"2024-09-21T10:05:33.690088+0000","last_clean":"2024-09-21T10:01:36.400750+0000","last_became_active":"2024-09-21T10:05:33.689833+0000","last_became_peered":"2024-09-21T10:05:33.689833+0000","last_unstale":"2024-09-21T10:05:33.690088+0000","last_undegraded":"2024-09-21T10:05:33.690088+0000","last_fullsized":"2024-09-21T10:05:33.657158+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:02.357017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":339,"reported_epoch":1208,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:33.656608+0000","last_change":"2024-09-21T10:05:33.656131+0000","last_active":"2024-09-21T10:05:33.656608+0000","last_peered":"2024-09-21T10:05:33.656608+0000","last_clean":"2024-09-21T09:43:10.269236+0000","last_became_active":"2024-09-21T10:05:33.656131+0000","last_became_peered":"2024-09-21T10:05:33.656131+0000","last_unstale":"2024-09-21T10:05:33.656608+0000","last_undegraded":"2024-09-21T10:05:33.653260+0000","last_fullsized":"2024-09-21T10:05:33.653150+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:04.341826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1713,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.654701+0000","last_change":"2024-09-21T10:01:40.413147+0000","last_active":"2024-09-21T10:05:33.654701+0000","last_peered":"2024-09-21T10:05:33.654701+0000","last_clean":"2024-09-21T10:05:33.654701+0000","last_became_active":"2024-09-21T10:01:40.412833+0000","last_became_peered":"2024-09-21T10:01:40.412833+0000","last_unstale":"2024-09-21T10:05:33.654701+0000","last_undegraded":"2024-09-21T10:05:33.654701+0000","last_fullsized":"2024-09-21T10:05:33.654701+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:12:33.262350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":1142,"reported_epoch":1208,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:33.662790+0000","last_change":"2024-09-21T10:05:33.662522+0000","last_active":"2024-09-21T10:05:33.662790+0000","last_peered":"2024-09-21T10:05:33.662790+0000","last_clean":"2024-09-21T10:05:27.632853+0000","last_became_active":"2024-09-21T10:05:33.662522+0000","last_became_peered":"2024-09-21T10:05:33.662522+0000","last_unstale":"2024-09-21T10:05:33.662790+0000","last_undegraded":"2024-09-21T10:05:33.653248+0000","last_fullsized":"2024-09-21T10:05:33.653119+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:15:39.282711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":589,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.653450+0000","last_change":"2024-09-21T10:01:40.427109+0000","last_active":"2024-09-21T10:05:33.653450+0000","last_peered":"2024-09-21T10:05:33.653450+0000","last_clean":"2024-09-21T10:05:33.653450+0000","last_became_active":"2024-09-21T10:01:40.426821+0000","last_became_peered":"2024-09-21T10:01:40.426821+0000","last_unstale":"2024-09-21T10:05:33.653450+0000","last_undegraded":"2024-09-21T10:05:33.653450+0000","last_fullsized":"2024-09-21T10:05:33.653450+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:21:12.260797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1753,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.678396+0000","last_change":"2024-09-21T10:00:16.357151+0000","last_active":"2024-09-21T10:05:35.678396+0000","last_peered":"2024-09-21T10:05:35.678396+0000","last_clean":"2024-09-21T10:05:35.678396+0000","last_became_active":"2024-09-21T10:00:16.356864+0000","last_became_peered":"2024-09-21T10:00:16.356864+0000","last_unstale":"2024-09-21T10:05:35.678396+0000","last_undegraded":"2024-09-21T10:05:35.678396+0000","last_fullsized":"2024-09-21T10:05:35.678396+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:52.851887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1839,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.687086+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T10:05:35.687086+0000","last_peered":"2024-09-21T10:05:35.687086+0000","last_clean":"2024-09-21T10:05:35.687086+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T10:05:35.687086+0000","last_undegraded":"2024-09-21T10:05:35.687086+0000","last_fullsized":"2024-09-21T10:05:35.687086+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1781,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.680735+0000","last_change":"2024-09-21T10:05:33.671325+0000","last_active":"2024-09-21T10:05:35.680735+0000","last_peered":"2024-09-21T10:05:35.680735+0000","last_clean":"2024-09-21T10:05:27.641203+0000","last_became_active":"2024-09-21T10:05:33.671325+0000","last_became_peered":"2024-09-21T10:05:33.671325+0000","last_unstale":"2024-09-21T10:05:35.680735+0000","last_undegraded":"2024-09-21T10:05:35.680735+0000","last_fullsized":"2024-09-21T10:05:33.656065+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:12.664263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":98,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-09-21T10:05:33.656108+0000","last_change":"2024-09-21T10:01:40.414393+0000","last_active":"2024-09-21T10:05:33.656108+0000","last_peered":"2024-09-21T10:05:33.656108+0000","last_clean":"2024-09-21T10:05:33.656108+0000","last_became_active":"2024-09-21T10:01:40.414096+0000","last_became_peered":"2024-09-21T10:01:40.414096+0000","last_unstale":"2024-09-21T10:05:33.656108+0000","last_undegraded":"2024-09-21T10:05:33.656108+0000","last_fullsized":"2024-09-21T10:05:33.656108+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:32.540479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1780,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.680930+0000","last_change":"2024-09-21T10:01:40.434475+0000","last_active":"2024-09-21T10:05:35.680930+0000","last_peered":"2024-09-21T10:05:35.680930+0000","last_clean":"2024-09-21T10:05:35.680930+0000","last_became_active":"2024-09-21T10:01:40.432013+0000","last_became_peered":"2024-09-21T10:01:40.432013+0000","last_unstale":"2024-09-21T10:05:35.680930+0000","last_undegraded":"2024-09-21T10:05:35.680930+0000","last_fullsized":"2024-09-21T10:05:35.680930+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:44.373378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1843,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.684793+0000","last_change":"2024-09-21T09:46:47.117078+0000","last_active":"2024-09-21T10:05:35.684793+0000","last_peered":"2024-09-21T10:05:35.684793+0000","last_clean":"2024-09-21T10:05:35.684793+0000","last_became_active":"2024-09-21T09:46:47.116483+0000","last_became_peered":"2024-09-21T09:46:47.116483+0000","last_unstale":"2024-09-21T10:05:35.684793+0000","last_undegraded":"2024-09-21T10:05:35.684793+0000","last_fullsized":"2024-09-21T10:05:35.684793+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.189512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":674,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.678742+0000","last_change":"2024-09-21T10:05:33.670060+0000","last_active":"2024-09-21T10:05:35.678742+0000","last_peered":"2024-09-21T10:05:35.678742+0000","last_clean":"2024-09-21T10:05:27.641667+0000","last_became_active":"2024-09-21T10:05:33.670060+0000","last_became_peered":"2024-09-21T10:05:33.670060+0000","last_unstale":"2024-09-21T10:05:35.678742+0000","last_undegraded":"2024-09-21T10:05:35.678742+0000","last_fullsized":"2024-09-21T10:05:33.655383+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:36.170508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1766,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.679097+0000","last_change":"2024-09-21T10:05:33.674726+0000","last_active":"2024-09-21T10:05:35.679097+0000","last_peered":"2024-09-21T10:05:35.679097+0000","last_clean":"2024-09-21T10:05:27.640553+0000","last_became_active":"2024-09-21T10:05:33.674726+0000","last_became_peered":"2024-09-21T10:05:33.674726+0000","last_unstale":"2024-09-21T10:05:35.679097+0000","last_undegraded":"2024-09-21T10:05:35.679097+0000","last_fullsized":"2024-09-21T10:05:33.656417+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:20.309271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":12,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.684480+0000","last_change":"2024-09-21T10:05:33.668729+0000","last_active":"2024-09-21T10:05:35.684480+0000","last_peered":"2024-09-21T10:05:35.684480+0000","last_clean":"2024-09-21T09:36:42.092105+0000","last_became_active":"2024-09-21T10:05:33.668729+0000","last_became_peered":"2024-09-21T10:05:33.668729+0000","last_unstale":"2024-09-21T10:05:35.684480+0000","last_undegraded":"2024-09-21T10:05:35.684480+0000","last_fullsized":"2024-09-21T10:05:33.661752+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:24:37.236990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1800,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.683853+0000","last_change":"2024-09-21T10:01:40.420219+0000","last_active":"2024-09-21T10:05:35.683853+0000","last_peered":"2024-09-21T10:05:35.683853+0000","last_clean":"2024-09-21T10:05:35.683853+0000","last_became_active":"2024-09-21T10:01:40.419839+0000","last_became_peered":"2024-09-21T10:01:40.419839+0000","last_unstale":"2024-09-21T10:05:35.683853+0000","last_undegraded":"2024-09-21T10:05:35.683853+0000","last_fullsized":"2024-09-21T10:05:35.683853+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:59.370554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1836,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.673115+0000","last_change":"2024-09-21T10:05:33.663574+0000","last_active":"2024-09-21T10:05:35.673115+0000","last_peered":"2024-09-21T10:05:35.673115+0000","last_clean":"2024-09-21T10:05:27.629144+0000","last_became_active":"2024-09-21T10:05:33.663574+0000","last_became_peered":"2024-09-21T10:05:33.663574+0000","last_unstale":"2024-09-21T10:05:35.673115+0000","last_undegraded":"2024-09-21T10:05:35.673115+0000","last_fullsized":"2024-09-21T10:05:33.657127+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:27:39.148905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1127,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.681589+0000","last_change":"2024-09-21T10:05:33.669019+0000","last_active":"2024-09-21T10:05:35.681589+0000","last_peered":"2024-09-21T10:05:35.681589+0000","last_clean":"2024-09-21T10:01:36.400682+0000","last_became_active":"2024-09-21T10:05:33.669019+0000","last_became_peered":"2024-09-21T10:05:33.669019+0000","last_unstale":"2024-09-21T10:05:35.681589+0000","last_undegraded":"2024-09-21T10:05:35.681589+0000","last_fullsized":"2024-09-21T10:05:33.664339+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:22.413689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":90,"reported_epoch":1208,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:33.658610+0000","last_change":"2024-09-21T10:05:33.658235+0000","last_active":"2024-09-21T10:05:33.658610+0000","last_peered":"2024-09-21T10:05:33.658610+0000","last_clean":"2024-09-21T10:01:36.400978+0000","last_became_active":"2024-09-21T10:05:33.658235+0000","last_became_peered":"2024-09-21T10:05:33.658235+0000","last_unstale":"2024-09-21T10:05:33.658610+0000","last_undegraded":"2024-09-21T10:05:33.654826+0000","last_fullsized":"2024-09-21T10:05:33.654708+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:27.678596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":552,"reported_epoch":1209,"state":"active+undersized","last_fresh":"2024-09-21T10:05:35.680218+0000","last_change":"2024-09-21T10:05:33.681625+0000","last_active":"2024-09-21T10:05:35.680218+0000","last_peered":"2024-09-21T10:05:35.680218+0000","last_clean":"2024-09-21T09:46:33.086009+0000","last_became_active":"2024-09-21T10:05:33.681625+0000","last_became_peered":"2024-09-21T10:05:33.681625+0000","last_unstale":"2024-09-21T10:05:35.680218+0000","last_undegraded":"2024-09-21T10:05:35.680218+0000","last_fullsized":"2024-09-21T10:05:33.659925+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:02.357625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":78,"reported_epoch":1208,"state":"active+undersized+degraded","last_fresh":"2024-09-21T10:05:33.690433+0000","last_change":"2024-09-21T10:05:33.690176+0000","last_active":"2024-09-21T10:05:33.690433+0000","last_peered":"2024-09-21T10:05:33.690433+0000","last_clean":"2024-09-21T10:00:01.325242+0000","last_became_active":"2024-09-21T10:05:33.690176+0000","last_became_peered":"2024-09-21T10:05:33.690176+0000","last_unstale":"2024-09-21T10:05:33.690433+0000","last_undegraded":"2024-09-21T10:05:33.657993+0000","last_fullsized":"2024-09-21T10:05:33.657884+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:38:21.265182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1765,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.678956+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:05:35.678956+0000","last_peered":"2024-09-21T10:05:35.678956+0000","last_clean":"2024-09-21T10:05:35.678956+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:05:35.678956+0000","last_undegraded":"2024-09-21T10:05:35.678956+0000","last_fullsized":"2024-09-21T10:05:35.678956+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1766,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.679039+0000","last_change":"2024-09-21T10:00:16.361160+0000","last_active":"2024-09-21T10:05:35.679039+0000","last_peered":"2024-09-21T10:05:35.679039+0000","last_clean":"2024-09-21T10:05:35.679039+0000","last_became_active":"2024-09-21T10:00:16.360524+0000","last_became_peered":"2024-09-21T10:00:16.360524+0000","last_unstale":"2024-09-21T10:05:35.679039+0000","last_undegraded":"2024-09-21T10:05:35.679039+0000","last_fullsized":"2024-09-21T10:05:35.679039+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:52.747211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1129,"reported_epoch":1208,"state":"active+undersized","last_fresh":"2024-09-21T10:05:33.669441+0000","last_change":"2024-09-21T10:05:33.669156+0000","last_active":"2024-09-21T10:05:33.669441+0000","last_peered":"2024-09-21T10:05:33.669441+0000","last_clean":"2024-09-21T10:01:36.400247+0000","last_became_active":"2024-09-21T10:05:33.669156+0000","last_became_peered":"2024-09-21T10:05:33.669156+0000","last_unstale":"2024-09-21T10:05:33.669441+0000","last_undegraded":"2024-09-21T10:05:33.669441+0000","last_fullsized":"2024-09-21T10:05:33.654672+0000","mapping_epoch":1207,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:33.863215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1847,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-09-21T10:05:35.687166+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T10:05:35.687166+0000","last_peered":"2024-09-21T10:05:35.687166+0000","last_clean":"2024-09-21T10:05:35.687166+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T10:05:35.687166+0000","last_undegraded":"2024-09-21T10:05:35.687166+0000","last_fullsized":"2024-09-21T10:05:35.687166+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5057871872,"num_objects":77177,"num_object_clones":0,"num_object_copies":231531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33703,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":77177,"num_write_kb":4939328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14729347072,"data_stored":14729347072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77177,"ondisk_log_size":77177,"up":41,"acting":41,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":81,"acting":81,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19598,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":78,"acting":78,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":22,"acting":22,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":68,"num_write":14,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1100,"ondisk_log_size":1100,"up":30,"acting":30,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397879,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2134044,"kb_used_data":1544148,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":91611108,"statfs":{"total":95995035648,"available":93809774592,"internally_reserved":0,"allocated":1581207552,"data_stored":1578534260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594107,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1818644,"kb_used_data":1521292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":91926508,"statfs":{"total":95995035648,"available":94132744192,"internally_reserved":0,"allocated":1557803008,"data_stored":1555137773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790337,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3095312,"kb_used_data":2785480,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":90649840,"statfs":{"total":95995035648,"available":92825436160,"internally_reserved":0,"allocated":2852331520,"data_stored":2849666751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986561,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1935476,"kb_used_data":1625644,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91809676,"statfs":{"total":95995035648,"available":94013108224,"internally_reserved":0,"allocated":1664659456,"data_stored":1662001520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999206,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2799432,"kb_used_data":2203264,"kb_used_omap":25,"kb_used_meta":596134,"kb_avail":90945720,"statfs":{"total":95995035648,"available":93128417280,"internally_reserved":0,"allocated":2256142336,"data_stored":2253485202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":610442183},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111149,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3399052,"kb_used_data":2848260,"kb_used_omap":31,"kb_used_meta":550752,"kb_avail":90346100,"statfs":{"total":95995035648,"available":92514406400,"internally_reserved":0,"allocated":2916618240,"data_stored":2913957532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":563970280},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2203318223092,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335034,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2244428,"kb_used_data":1821636,"kb_used_omap":31,"kb_used_meta":422752,"kb_avail":91500724,"statfs":{"total":95995035648,"available":93696741376,"internally_reserved":0,"allocated":1865355264,"data_stored":1862694019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":432898144},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1870004224,"data_stored":1870004224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2923560960,"data_stored":2923560960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2249523200,"data_stored":2249523200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1674444800,"data_stored":1674444800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2861432832,"data_stored":2861432832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1570570240,"data_stored":1570570240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1579810816,"data_stored":1579810816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:05:53.061 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:05:53.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pg_num 21 2024-09-21T10:05:53.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:53.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:53.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:53.252 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:53.277 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:53.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:53.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:53.426 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:53.527 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:53.560 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:53.566 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:53 smithi017 ceph-mon[102913]: pgmap v1652: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 36525/250800 objects degraded (14.563%) 2024-09-21T10:05:53.566 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:53 smithi017 ceph-mon[105151]: pgmap v1652: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 36525/250800 objects degraded (14.563%) 2024-09-21T10:05:53.654 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 87134 87118 108.876 79.25 0.0108783 0.00917966 2024-09-21T10:05:53.661 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.692 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:53.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:53.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:53 smithi134 ceph-mon[90085]: pgmap v1652: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 36525/250800 objects degraded (14.563%) 2024-09-21T10:05:53.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:53.824 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:53.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:53.925 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:53.954 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:53.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:54.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:54.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:54.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:54.200 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:54.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.235 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:54.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:54.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:54.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:54.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:54.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:54.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:54.529 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:54.530 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:54.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:54.654 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 88405 88389 108.298 79.4375 0.012985 0.00922887 2024-09-21T10:05:54.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:54.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:54.767 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:54.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:54.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:54.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:54.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:54.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:55.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:55.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:55.075 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:05:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:05:54] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.51.0" 2024-09-21T10:05:55.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:55.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:55.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:55.344 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:55.446 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:55.477 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:55.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:55.579 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:55.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:55 smithi017 ceph-mon[102913]: pgmap v1653: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 37253/256341 objects degraded (14.533%) 2024-09-21T10:05:55.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:55.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:55.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:55.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:55 smithi017 ceph-mon[105151]: pgmap v1653: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 37253/256341 objects degraded (14.533%) 2024-09-21T10:05:55.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:55.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:55.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:55.609 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:55.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:55.654 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 89723 89707 107.799 82.375 0.00850734 0.0092719 2024-09-21T10:05:55.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:55 smithi134 ceph-mon[90085]: pgmap v1653: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 37253/256341 objects degraded (14.533%) 2024-09-21T10:05:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:05:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:55.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:55.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:55.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:55.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:55.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:56.000 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:56.030 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:56.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:56.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:56.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:56.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:56.272 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:56.305 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:56.407 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:56.439 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:56.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:56.541 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:56.568 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:56.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:56.654 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 90981 90965 107.249 78.625 0.0068486 0.00931933 2024-09-21T10:05:56.670 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:56.697 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:56.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:56.799 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:56.828 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:56.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:56.867 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:56.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:56.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:56.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:57.077 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:57.106 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:57.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:57.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:57.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:57.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:57.375 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:57.377 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:05:57.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:57.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:57.608 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:57 smithi017 ceph-mon[102913]: pgmap v1654: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s; 37631/261120 objects degraded (14.411%) 2024-09-21T10:05:57.608 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1790050086' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:57.609 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:57 smithi017 ceph-mon[105151]: pgmap v1654: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s; 37631/261120 objects degraded (14.411%) 2024-09-21T10:05:57.609 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1790050086' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:57.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:57.654 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 92256 92240 106.738 79.6875 0.016964 0.00936392 2024-09-21T10:05:57.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:57 smithi134 ceph-mon[90085]: pgmap v1654: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s; 37631/261120 objects degraded (14.411%) 2024-09-21T10:05:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1790050086' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:57.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:57.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:57.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:57.894 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:57.923 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:57.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:58.024 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:58.057 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:58.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:58.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.196 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:58.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:58.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:58.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:58.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:58.434 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:58.460 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:58.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:58.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:58 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:58.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:58 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1790050086' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:58.547 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:58 smithi017 ceph-mon[102913]: osdmap e1210: 8 total, 7 up, 7 in 2024-09-21T10:05:58.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:58 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:58.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:58 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1790050086' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:58.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:58 smithi017 ceph-mon[105151]: osdmap e1210: 8 total, 7 up, 7 in 2024-09-21T10:05:58.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:58.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:58.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:58.654 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 93582 93566 106.304 82.875 0.00668937 0.00940183 2024-09-21T10:05:58.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:58 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:58 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1790050086' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:58 smithi134 ceph-mon[90085]: osdmap e1210: 8 total, 7 up, 7 in 2024-09-21T10:05:58.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:58.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:58.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:58.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:58.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:58.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:59.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:59.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:59.137 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:59.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:59.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:59.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:59.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:59.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.447 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:59.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:59.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:59.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:59.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:59.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:59 smithi017 ceph-mon[102913]: pgmap v1656: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 39130/269085 objects degraded (14.542%) 2024-09-21T10:05:59.617 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:05:59 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 39130/269085 objects degraded (14.542%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:59.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:59 smithi017 ceph-mon[105151]: pgmap v1656: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 39130/269085 objects degraded (14.542%) 2024-09-21T10:05:59.617 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:05:59 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 39130/269085 objects degraded (14.542%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:59.654 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 94915 94899 105.894 83.3125 0.0161259 0.00943874 2024-09-21T10:05:59.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:59.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:05:59.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:59 smithi134 ceph-mon[90085]: pgmap v1656: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 39130/269085 objects degraded (14.542%) 2024-09-21T10:05:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:05:59 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 39130/269085 objects degraded (14.542%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:05:59.826 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:59.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:05:59.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:59.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:59.960 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:05:59.960 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:05:59.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:05:59.960 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:05:59.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.961 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:60915 6.10 6:0e7dbdda:::benchmark_data_smithi017_149547_object60914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1086)", 2024-09-21T10:05:59.961 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:06.422484+0000", 2024-09-21T10:05:59.961 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 533.52750557399997, 2024-09-21T10:05:59.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.042944147, 2024-09-21T10:05:59.961 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:05:59.961 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:05:59.961 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 60915 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422484+0000", 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:05:59.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422484+0000", 2024-09-21T10:05:59.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:05:59.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422487+0000", 2024-09-21T10:05:59.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.931e-06 2024-09-21T10:05:59.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422536+0000", 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9316e-05 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422577+0000", 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1190999999999999e-05 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:05:59.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.422593+0000", 2024-09-21T10:05:59.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5591999999999999e-05 2024-09-21T10:05:59.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.499893+0000", 2024-09-21T10:05:59.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077299781999999997 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:06.499902+0000", 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1400000000000006e-06 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.456197+0000", 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95629477799999996 2024-09-21T10:05:59.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.456846+0000", 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00064964700000000005 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.465294+0000", 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0084478999999999995 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:07.465428+0000", 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013386999999999999 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91404 6.10 6:0e0e7bfc:::benchmark_data_smithi017_149547_object91403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.613932+0000", 2024-09-21T10:05:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 499.33605710099999, 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0513244660000001, 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91404 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.969 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:05:59.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:05:59.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613932+0000", 2024-09-21T10:05:59.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613932+0000", 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613933+0000", 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.34e-07 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:05:59.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613952+0000", 2024-09-21T10:05:59.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9488e-05 2024-09-21T10:05:59.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:05:59.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613959+0000", 2024-09-21T10:05:59.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5089999999999999e-06 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.613964+0000", 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9520000000000002e-06 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.616064+0000", 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00209991 2024-09-21T10:05:59.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.616072+0000", 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1009999999999992e-06 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.647238+0000", 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0311661350000001 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.647363+0000", 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012525599999999999 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.647460+0000", 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7281999999999995e-05 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650199+0000", 2024-09-21T10:05:59.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002738613 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650356+0000", 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015701499999999999 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665141+0000", 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014785400000000001 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665208+0000", 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6427000000000006e-05 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665257+0000", 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8943999999999998e-05 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91409 6.10 6:0d2601f6:::benchmark_data_smithi017_149547_object91408:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:05:59.977 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.621998+0000", 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 499.327991811, 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0433568630000001, 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91409 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.621998+0000", 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.621998+0000", 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.621999+0000", 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.933e-06 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622021+0000", 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1667000000000001e-05 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622034+0000", 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2906e-05 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622042+0000", 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2099999999999993e-06 2024-09-21T10:05:59.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622143+0000", 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100886 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.622154+0000", 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0513000000000001e-05 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.648650+0000", 2024-09-21T10:05:59.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0264962689999999 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.648749+0000", 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9400000000000004e-05 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.648846+0000", 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6937999999999999e-05 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650374+0000", 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015274659999999999 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651976+0000", 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016021449999999999 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665292+0000", 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013315993999999999 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665328+0000", 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6068000000000001e-05 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.665354+0000", 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6468000000000001e-05 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.984 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91437 6.10 6:0f983da6:::benchmark_data_smithi017_149547_object91436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.645165+0000", 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 499.30482463999999, 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0215859970000001, 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91437 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645165+0000", 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645165+0000", 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645166+0000", 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.466e-06 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645190+0000", 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3915999999999999e-05 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645204+0000", 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4166e-05 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645212+0000", 2024-09-21T10:05:59.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1629999999999996e-06 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645357+0000", 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014457399999999999 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.645368+0000", 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1324e-05 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.649427+0000", 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004059067 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.649545+0000", 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011762200000000001 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.649640+0000", 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5396000000000006e-05 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:05:59.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651930+0000", 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022892300000000002 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652014+0000", 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3967000000000004e-05 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.666638+0000", 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014624511 2024-09-21T10:05:59.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.666709+0000", 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0510000000000001e-05 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.666751+0000", 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2085000000000001e-05 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91468 6.10 6:0b811041:::benchmark_data_smithi017_149547_object91467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.668218+0000", 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 499.28177150400001, 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99924510700000002, 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91468 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668218+0000", 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668218+0000", 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:05:59.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668220+0000", 2024-09-21T10:05:59.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.685e-06 2024-09-21T10:05:59.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:05:59.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668247+0000", 2024-09-21T10:05:59.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7639000000000001e-05 2024-09-21T10:05:59.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668278+0000", 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0616e-05 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668287+0000", 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3319999999999998e-06 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668434+0000", 2024-09-21T10:05:59.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014714700000000001 2024-09-21T10:05:59.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:05:59.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.668445+0000", 2024-09-21T10:05:59.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0542e-05 2024-09-21T10:05:59.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:05:59.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650407+0000", 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98196202300000002 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650475+0000", 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8063000000000002e-05 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.650555+0000", 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0555000000000004e-05 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.997 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652024+0000", 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014685810000000001 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652062+0000", 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7450000000000002e-05 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.667339+0000", 2024-09-21T10:05:59.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015277824000000001 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.667412+0000", 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2403000000000005e-05 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.667463+0000", 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1246999999999999e-05 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:05:59.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91476 6.10 6:0f93928d:::benchmark_data_smithi017_149547_object91475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.673753+0000", 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 499.27623660900002, 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99824931400000005, 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:06:00.000 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91476 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673753+0000", 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673753+0000", 2024-09-21T10:06:00.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673754+0000", 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5629999999999999e-06 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673782+0000", 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7636000000000001e-05 2024-09-21T10:06:00.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673805+0000", 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2946999999999999e-05 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673813+0000", 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2700000000000004e-06 2024-09-21T10:06:00.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673946+0000", 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013301900000000001 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.673955+0000", 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1880000000000008e-06 2024-09-21T10:06:00.004 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651172+0000", 2024-09-21T10:06:00.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97721698300000004 2024-09-21T10:06:00.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651289+0000", 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116628 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.651376+0000", 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6612000000000004e-05 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.653572+0000", 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021968230000000001 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.653583+0000", 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0685e-05 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.671796+0000", 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018212424000000001 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.671952+0000", 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000156422 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.672002+0000", 2024-09-21T10:06:00.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0114000000000002e-05 2024-09-21T10:06:00.009 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.009 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.009 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.010 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:91478 6.10 6:083e0ee5:::benchmark_data_smithi017_149547_object91477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:06:00.010 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:57:40.675325+0000", 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 499.27466447099999, 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99692708900000004, 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 91478 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675325+0000", 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675325+0000", 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675326+0000", 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3960000000000001e-06 2024-09-21T10:06:00.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675349+0000", 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3180000000000002e-05 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675367+0000", 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7591000000000001e-05 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675375+0000", 2024-09-21T10:06:00.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5129999999999999e-06 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675438+0000", 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3595000000000004e-05 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:40.675448+0000", 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5659999999999996e-06 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652093+0000", 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97664543400000003 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652161+0000", 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8285000000000001e-05 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.652243+0000", 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1036000000000001e-05 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.655074+0000", 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028315800000000002 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.655124+0000", 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9434999999999997e-05 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.672049+0000", 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016925639999999999 2024-09-21T10:06:00.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.672210+0000", 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016079700000000001 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:57:41.672252+0000", 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2040999999999999e-05 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53003 7.4 7:22671f4e:::benchmark_data_smithi017_180590_object53002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.348891+0000", 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 273.60109876299998, 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004137056, 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53003 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348891+0000", 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348891+0000", 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348891+0000", 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3099999999999997e-07 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348917+0000", 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5942e-05 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348926+0000", 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7260000000000008e-06 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348933+0000", 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6749999999999996e-06 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.351946+0000", 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030134020000000001 2024-09-21T10:06:00.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.351952+0000", 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7289999999999997e-06 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.470485+0000", 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11853295 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.470493+0000", 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3440000000000001e-06 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.352953+0000", 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.882459565 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.353028+0000", 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5092000000000005e-05 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.023 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53008 7.4 7:20f177af:::benchmark_data_smithi017_180590_object53007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.348943+0000", 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 273.60104589000002, 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004518749, 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.024 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53008 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348943+0000", 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348943+0000", 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348944+0000", 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5000000000000003e-07 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348969+0000", 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5337000000000002e-05 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348980+0000", 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.057e-05 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.348986+0000", 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0000000000000002e-06 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.352013+0000", 2024-09-21T10:06:00.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003027222 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.352019+0000", 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1879999999999999e-06 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.476795+0000", 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.124775264 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.476825+0000", 2024-09-21T10:06:00.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0437999999999999e-05 2024-09-21T10:06:00.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.353396+0000", 2024-09-21T10:06:00.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.87657047899999996 2024-09-21T10:06:00.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.353462+0000", 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6700999999999997e-05 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53011 7.6 7:60b3f60f:::benchmark_data_smithi017_180590_object53010:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.393629+0000", 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 273.55636022300001, 2024-09-21T10:06:00.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98845262599999995, 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53011 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.393629+0000", 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.393629+0000", 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.393631+0000", 2024-09-21T10:06:00.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1330000000000002e-06 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.393667+0000", 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5549000000000001e-05 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.415542+0000", 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021875623 2024-09-21T10:06:00.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.415559+0000", 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6694999999999999e-05 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.496749+0000", 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.081189497999999999 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.496760+0000", 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1809e-05 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.354373+0000", 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.85761251199999999 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.354542+0000", 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000168722 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.354731+0000", 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018961499999999999 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.359104+0000", 2024-09-21T10:06:00.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043730399999999999 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.364073+0000", 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049688809999999996 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.381981+0000", 2024-09-21T10:06:00.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017908013 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.382040+0000", 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8689e-05 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.382082+0000", 2024-09-21T10:06:00.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1847000000000001e-05 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72895 7.4 7:27e3b190:::benchmark_data_smithi017_180590_object72894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.669787+0000", 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.28020282700001, 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1010801379999999, 2024-09-21T10:06:00.039 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72895 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669787+0000", 2024-09-21T10:06:00.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669787+0000", 2024-09-21T10:06:00.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669788+0000", 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.739e-06 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669824+0000", 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5379000000000001e-05 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669842+0000", 2024-09-21T10:06:00.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8621e-05 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669851+0000", 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3110000000000006e-06 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669933+0000", 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1909000000000005e-05 2024-09-21T10:06:00.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.669943+0000", 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0488e-05 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.599855+0000", 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92991173400000005 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.599935+0000", 2024-09-21T10:06:00.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0220000000000001e-05 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.600024+0000", 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9548000000000004e-05 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602963+0000", 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029380550000000002 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605565+0000", 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026027089999999999 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770758+0000", 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16519246000000001 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770816+0000", 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8242000000000002e-05 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770867+0000", 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0723000000000002e-05 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72907 7.4 7:254bdb82:::benchmark_data_smithi017_180590_object72906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.677271+0000", 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.27271877999999, 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0938297560000001, 2024-09-21T10:06:00.047 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72907 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677271+0000", 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677271+0000", 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677273+0000", 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4660000000000002e-06 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677299+0000", 2024-09-21T10:06:00.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5531999999999999e-05 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677312+0000", 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3246e-05 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677321+0000", 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7919999999999998e-06 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677392+0000", 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1397999999999996e-05 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.677402+0000", 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0053e-05 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.600705+0000", 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92330275500000003 2024-09-21T10:06:00.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.600807+0000", 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101969 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.600880+0000", 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3143000000000001e-05 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602968+0000", 2024-09-21T10:06:00.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020885119999999998 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605625+0000", 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026569359999999999 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771015+0000", 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16538913199999999 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771063+0000", 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8959999999999999e-05 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771100+0000", 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6862e-05 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72934 7.4 7:21599fe0:::benchmark_data_smithi017_180590_object72933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.054 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.700107+0000", 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.249882321, 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0711450929999999, 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72934 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700107+0000", 2024-09-21T10:06:00.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:00.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:00.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700107+0000", 2024-09-21T10:06:00.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700108+0000", 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6599999999999996e-07 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700128+0000", 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0166e-05 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700135+0000", 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3880000000000002e-06 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700141+0000", 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7010000000000001e-06 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700271+0000", 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130419 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.700280+0000", 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0010000000000007e-06 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.601448+0000", 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90116788199999998 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.601515+0000", 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6563000000000006e-05 2024-09-21T10:06:00.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.601594+0000", 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9702000000000004e-05 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.604156+0000", 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025619150000000001 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605670+0000", 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015133569999999999 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771153+0000", 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.165483349 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771215+0000", 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1717999999999994e-05 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771252+0000", 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7366000000000001e-05 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72952 7.4 7:21f84216:::benchmark_data_smithi017_180590_object72951:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.064 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.715253+0000", 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.234736056, 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0561387069999999, 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72952 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.065 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715253+0000", 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715253+0000", 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715254+0000", 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7400000000000003e-07 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715273+0000", 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9479000000000001e-05 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715281+0000", 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5660000000000001e-06 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.715286+0000", 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3750000000000002e-06 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.716709+0000", 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001422817 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.716719+0000", 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4709999999999996e-06 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602200+0000", 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88548115699999996 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602274+0000", 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3770999999999996e-05 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.602356+0000", 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2385000000000005e-05 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605519+0000", 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031632660000000001 2024-09-21T10:06:00.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.768876+0000", 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16335690999999999 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771315+0000", 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024384879999999999 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771359+0000", 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4230999999999997e-05 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771392+0000", 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3216999999999997e-05 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72954 7.4 7:27352eec:::benchmark_data_smithi017_180590_object72953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.072 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.718051+0000", 2024-09-21T10:06:00.073 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.23193788699999, 2024-09-21T10:06:00.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.053472309, 2024-09-21T10:06:00.073 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.073 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.073 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.073 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.073 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.073 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.073 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72954 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718051+0000", 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718051+0000", 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718053+0000", 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5340000000000001e-06 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718078+0000", 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4752999999999998e-05 2024-09-21T10:06:00.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718092+0000", 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3943e-05 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718099+0000", 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5000000000000002e-06 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718163+0000", 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4077000000000003e-05 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.718172+0000", 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1560000000000001e-06 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.603120+0000", 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88494770199999995 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.603220+0000", 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100022 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.603340+0000", 2024-09-21T10:06:00.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120164 2024-09-21T10:06:00.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.605635+0000", 2024-09-21T10:06:00.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022948489999999998 2024-09-21T10:06:00.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770915+0000", 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16528029599999999 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771430+0000", 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051476500000000004 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771492+0000", 2024-09-21T10:06:00.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1538000000000006e-05 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771524+0000", 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.201e-05 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72980 7.4 7:266036bf:::benchmark_data_smithi017_180590_object72979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.739931+0000", 2024-09-21T10:06:00.081 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.21005828099999, 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.031714531, 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72980 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739931+0000", 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739931+0000", 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739932+0000", 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9100000000000003e-07 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739950+0000", 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7878000000000001e-05 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739960+0000", 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0139999999999999e-05 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.739965+0000", 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1939999999999997e-06 2024-09-21T10:06:00.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.740017+0000", 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1517999999999997e-05 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.740025+0000", 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2859999999999999e-06 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.604188+0000", 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.86416299100000005 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.604288+0000", 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100313 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.604412+0000", 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012408100000000001 2024-09-21T10:06:00.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770710+0000", 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.166298106 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770960+0000", 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024910999999999998 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771561+0000", 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060110899999999995 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771601+0000", 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0813999999999999e-05 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771646+0000", 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4199999999999997e-05 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.088 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72985 7.4 7:22398816:::benchmark_data_smithi017_180590_object72984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.743380+0000", 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.20660900799999, 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.032025556, 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.089 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72985 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743380+0000", 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743380+0000", 2024-09-21T10:06:00.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743382+0000", 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3680000000000001e-06 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743408+0000", 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6115999999999999e-05 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743418+0000", 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0105999999999999e-05 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743425+0000", 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5899999999999996e-06 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743491+0000", 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6229000000000006e-05 2024-09-21T10:06:00.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.743501+0000", 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0304e-05 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.768925+0000", 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.02542402 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.769066+0000", 2024-09-21T10:06:00.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140512 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.769184+0000", 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118606 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.770971+0000", 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001787177 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771686+0000", 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00071413399999999995 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775288+0000", 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036027530000000002 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775354+0000", 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5534e-05 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775406+0000", 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2107000000000001e-05 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:73030 7.4 7:2535c205:::benchmark_data_smithi017_180590_object73029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.787612+0000", 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.16237759399999, 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99312697800000005, 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.096 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 73030 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787612+0000", 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787612+0000", 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787613+0000", 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.609e-06 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787638+0000", 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4691000000000001e-05 2024-09-21T10:06:00.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787650+0000", 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1525999999999999e-05 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787657+0000", 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0979999999999996e-06 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787739+0000", 2024-09-21T10:06:00.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2614999999999994e-05 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.787749+0000", 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2350000000000005e-06 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771717+0000", 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98396802299999997 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771839+0000", 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012282600000000001 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.771997+0000", 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015736600000000001 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775456+0000", 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034597149999999999 2024-09-21T10:06:00.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775500+0000", 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3065999999999997e-05 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780576+0000", 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050762309999999996 2024-09-21T10:06:00.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780695+0000", 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119022 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780739+0000", 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3955000000000002e-05 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:73041 7.4 7:27218752:::benchmark_data_smithi017_180590_object73040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.797618+0000", 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.15237134399999, 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98331477199999995, 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.104 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 73041 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797618+0000", 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797618+0000", 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797620+0000", 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.576e-06 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797646+0000", 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6086000000000001e-05 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797659+0000", 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3589e-05 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.797666+0000", 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0809999999999996e-06 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.798014+0000", 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034764000000000003 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.798028+0000", 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4021e-05 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.773406+0000", 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97537753000000005 2024-09-21T10:06:00.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.773542+0000", 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013645700000000001 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.773716+0000", 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017366899999999999 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780752+0000", 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070358240000000004 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780794+0000", 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2048999999999997e-05 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780848+0000", 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3959000000000003e-05 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780896+0000", 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8223000000000002e-05 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780933+0000", 2024-09-21T10:06:00.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7067999999999998e-05 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:73045 7.4 7:2226a9f9:::benchmark_data_smithi017_180590_object73044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1161)", 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:55.805230+0000", 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 244.14475972899999, 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97914256600000005, 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:00.112 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 73045 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:00.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805230+0000", 2024-09-21T10:06:00.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:00.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805230+0000", 2024-09-21T10:06:00.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:00.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805232+0000", 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9740000000000001e-06 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805258+0000", 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6401e-05 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805272+0000", 2024-09-21T10:06:00.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4212999999999999e-05 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.805280+0000", 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6690000000000006e-06 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.806040+0000", 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075974500000000004 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.806046+0000", 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.212e-06 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775534+0000", 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96948825199999999 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775674+0000", 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013970700000000001 2024-09-21T10:06:00.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.775820+0000", 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014583400000000001 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.780805+0000", 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049853930000000003 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.784242+0000", 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034368900000000002 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.784277+0000", 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5036999999999997e-05 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.784329+0000", 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2142999999999999e-05 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:00.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:00.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:06:00.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:56.784372+0000", 2024-09-21T10:06:00.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3096000000000002e-05 2024-09-21T10:06:00.120 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.120 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.120 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.120 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:00.120 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:00.120 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:06:00.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:00.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:00.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:00.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:00.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:00.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:00.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:00.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:00.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:06:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:06:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:00.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:00.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:00.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:00.655 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 96107 96091 105.343 74.5 0.030612 0.0094847 2024-09-21T10:06:00.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:00.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:00.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:00.896 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:00.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:01.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:06:01.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:01.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:01.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:01.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:01.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:01.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:01.305 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:01.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:01.407 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:01.439 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:01.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:01.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:01.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:01.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:01.655 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 97328 97312 104.842 76.3125 0.00931408 0.00953337 2024-09-21T10:06:01.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:01 smithi017 ceph-mon[105151]: pgmap v1657: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s; 39899/275193 objects degraded (14.499%) 2024-09-21T10:06:01.703 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:01 smithi017 ceph-mon[102913]: pgmap v1657: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s; 39899/275193 objects degraded (14.499%) 2024-09-21T10:06:01.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:01.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:01 smithi134 ceph-mon[90085]: pgmap v1657: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s; 39899/275193 objects degraded (14.499%) 2024-09-21T10:06:01.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:01.851 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:01.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:01.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:01.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:01.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:02.094 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:02.126 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:02.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:02.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:02.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:02.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:02.368 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:02.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.397 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:02.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:02.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:06:02.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:02.587 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:02.657 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 98535 98519 104.343 75.4375 0.024597 0.00957866 2024-09-21T10:06:02.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:02.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:02.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:02.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:02.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:02.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:02.980 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:03.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:03.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:03.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:03.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:03.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:03.258 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:03.290 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:03.392 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:03.421 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:03.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:03.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:03.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:03.655 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:06:03.654578+0000 min lat: 0.00121399 max lat: 1.06457 avg lat: 0.00962642 2024-09-21T10:06:03.655 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:06:03.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 99709 99693 103.827 73.375 0.0102151 0.00962642 2024-09-21T10:06:03.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:03 smithi017 ceph-mon[102913]: pgmap v1658: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 41895/288438 objects degraded (14.525%) 2024-09-21T10:06:03.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:03 smithi017 ceph-mon[105151]: pgmap v1658: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 41895/288438 objects degraded (14.525%) 2024-09-21T10:06:03.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:03 smithi134 ceph-mon[90085]: pgmap v1658: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 41895/288438 objects degraded (14.525%) 2024-09-21T10:06:03.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:03.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:03.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:03.939 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:03.961 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:04.063 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:04.088 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:04.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:04.189 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:04.215 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:04.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:04.344 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:04.446 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:04.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:04 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 41895/288438 objects degraded (14.525%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:04 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 41895/288438 objects degraded (14.525%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:04.481 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:04.583 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:04.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:04.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:04.655 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 100802 100786 103.244 68.3125 0.0240597 0.00968114 2024-09-21T10:06:04.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:04.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:04 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 41895/288438 objects degraded (14.525%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:04.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:04.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:04.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:04.883 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:04.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:04.984 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:05.013 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:05.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:05.115 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:05.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.146 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:05.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:06:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:06:04] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-09-21T10:06:05.247 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:05.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.274 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:05.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:05.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:05.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:05.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:05.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:05.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:05.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:05.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:05 smithi134 ceph-mon[90085]: pgmap v1659: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 42674/294018 objects degraded (14.514%) 2024-09-21T10:06:05.655 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 101869 101853 102.655 66.6875 0.0109757 0.00972602 2024-09-21T10:06:05.666 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:05.684 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:05 smithi017 ceph-mon[102913]: pgmap v1659: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 42674/294018 objects degraded (14.514%) 2024-09-21T10:06:05.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:05 smithi017 ceph-mon[105151]: pgmap v1659: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 42674/294018 objects degraded (14.514%) 2024-09-21T10:06:05.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.692 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:05.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:05.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:05.821 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:05.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:05.923 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:05.956 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:05.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:06.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:06.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:06.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:06.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:06.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:06.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:06.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:06.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:06.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:06.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:06.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:06.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:06.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:06.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:06.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 102636 102620 101.786 47.9375 0.00641873 0.00981973 2024-09-21T10:06:06.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:06.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:06.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:06.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:06.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:06.983 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:07.018 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:07.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:07.120 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:07.149 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:07.250 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:07.278 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:07.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:07.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:07 smithi134 ceph-mon[90085]: pgmap v1660: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s; 43042/298455 objects degraded (14.422%) 2024-09-21T10:06:07.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:07 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/122806587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:07.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:07.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:07.644 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:07.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 103717 103701 101.251 67.5625 0.0100326 0.00987171 2024-09-21T10:06:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.681 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:07.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:07 smithi017 ceph-mon[102913]: pgmap v1660: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s; 43042/298455 objects degraded (14.422%) 2024-09-21T10:06:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:07 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/122806587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:07 smithi017 ceph-mon[105151]: pgmap v1660: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s; 43042/298455 objects degraded (14.422%) 2024-09-21T10:06:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:07 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/122806587' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:07.782 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:07.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.804 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:07.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:07.905 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:07.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:07.957 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:06:07.957 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:06:07.957 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:06:07.957 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:06:07.957 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:06:07.957 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:06:07.958 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:06:07.958 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:07.958 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:07.958 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:07.958 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:07.958 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:06:07.958 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:07.958 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:07.958 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:08.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:08.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:08.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:08.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:08.200 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:08.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:08.246 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:08.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:08.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:08.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:08.431 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:08.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.463 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:08.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:08.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:08.700 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 104802 104786 100.737 67.8125 0.015528 0.00992197 2024-09-21T10:06:08.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:08.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:08.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:08.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:08.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:08.933 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:06:08.934 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:06:08.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:09.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:09.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:09.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:09.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:09.261 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:09.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:09.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:09.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:09.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:09.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:09.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:09.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:09.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:09.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:09.614 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:09 smithi017 ceph-mon[102913]: pgmap v1661: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 44318/305313 objects degraded (14.516%) 2024-09-21T10:06:09.614 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:09 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 44318/305313 objects degraded (14.516%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:09.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:09 smithi017 ceph-mon[105151]: pgmap v1661: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 44318/305313 objects degraded (14.516%) 2024-09-21T10:06:09.615 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:09 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 44318/305313 objects degraded (14.516%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:09.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 105888 105872 100.239 67.875 0.00471739 0.00997139 2024-09-21T10:06:09.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:09 smithi134 ceph-mon[90085]: pgmap v1661: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 44318/305313 objects degraded (14.516%) 2024-09-21T10:06:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:09 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 44318/305313 objects degraded (14.516%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:09.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:09.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:09.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:09.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.899 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:09.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:10.000 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:10.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:10.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:10.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:10.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:10.278 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:10.307 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:10.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:10.401 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pg_num to 21 2024-09-21T10:06:10.409 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:10.430 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:10.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:10.476 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:10 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/404560240' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:06:10.476 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:10.476 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:10 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/404560240' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:06:10.476 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:10 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/404560240' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:06:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:06:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:06:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:10.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:10.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:10.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:10.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 106988 106972 99.7689 68.75 0.0209201 0.0100186 2024-09-21T10:06:10.672 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:10.702 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:10.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:10.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:10.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:10.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:06:10.938 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:10.962 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:10.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:11.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:11.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:11.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:11.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:11.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:11.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:11.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:11.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:11.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:11.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:11.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:11.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:11.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 107993 107977 99.2254 62.8125 0.00744189 0.0100729 2024-09-21T10:06:11.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:11.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:11.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:11 smithi017 ceph-mon[102913]: pgmap v1662: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 45008/310143 objects degraded (14.512%) 2024-09-21T10:06:11.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:11 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/404560240' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-21T10:06:11.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:11 smithi017 ceph-mon[102913]: osdmap e1211: 8 total, 7 up, 7 in 2024-09-21T10:06:11.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:11 smithi017 ceph-mon[105151]: pgmap v1662: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 45008/310143 objects degraded (14.512%) 2024-09-21T10:06:11.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:11 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/404560240' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-21T10:06:11.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:11 smithi017 ceph-mon[105151]: osdmap e1211: 8 total, 7 up, 7 in 2024-09-21T10:06:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:11 smithi134 ceph-mon[90085]: pgmap v1662: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 45008/310143 objects degraded (14.512%) 2024-09-21T10:06:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:11 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/404560240' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-21T10:06:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:11 smithi134 ceph-mon[90085]: osdmap e1211: 8 total, 7 up, 7 in 2024-09-21T10:06:11.772 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:11.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.809 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:11.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:11.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:11.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.951 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:11.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:12.052 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:12.082 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:12.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:12.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:12.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:12.344 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:12.373 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:12.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:12.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:12.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:12.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:12.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:12.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:12.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:12.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 108992 108976 98.6922 62.4375 0.00567246 0.010126 2024-09-21T10:06:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:12 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-21T10:06:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:12 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-21T10:06:12.740 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:12.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:12 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-21T10:06:12.767 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:12.869 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:12.899 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:12.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:13.001 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:13.031 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:13.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:13.133 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:13.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:13.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:13.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:13.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:13.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:13.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:13.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:13.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:13.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:13.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:13.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 110236 110220 98.3929 77.75 0.0101905 0.0101587 2024-09-21T10:06:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:13 smithi017 ceph-mon[102913]: pgmap v1664: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 46704/321141 objects degraded (14.543%) 2024-09-21T10:06:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-21T10:06:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:13 smithi017 ceph-mon[102913]: osdmap e1212: 8 total, 7 up, 7 in 2024-09-21T10:06:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:13 smithi017 ceph-mon[105151]: pgmap v1664: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 46704/321141 objects degraded (14.543%) 2024-09-21T10:06:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-21T10:06:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:13 smithi017 ceph-mon[105151]: osdmap e1212: 8 total, 7 up, 7 in 2024-09-21T10:06:13.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:13 smithi134 ceph-mon[90085]: pgmap v1664: 99 pgs: 29 active+undersized, 16 active+undersized+degraded, 54 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 46704/321141 objects degraded (14.543%) 2024-09-21T10:06:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-21T10:06:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:13 smithi134 ceph-mon[90085]: osdmap e1212: 8 total, 7 up, 7 in 2024-09-21T10:06:13.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:13.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:13.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:13.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:13.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:13.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:14.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:14.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:14.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:14.192 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:14.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:14.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:14.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:14.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:14.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:14.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:14.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:14.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:14.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:14.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 111308 111292 97.9508 67 0.0132319 0.0102045 2024-09-21T10:06:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:14 smithi017 ceph-mon[102913]: osdmap e1213: 8 total, 7 up, 7 in 2024-09-21T10:06:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:14 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 46704/321141 objects degraded (14.543%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1051858075' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:06:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:14 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:06:14.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:14 smithi017 ceph-mon[105151]: osdmap e1213: 8 total, 7 up, 7 in 2024-09-21T10:06:14.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:14 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 46704/321141 objects degraded (14.543%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:14.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1051858075' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:06:14.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:14 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:06:14.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:14 smithi134 ceph-mon[90085]: osdmap e1213: 8 total, 7 up, 7 in 2024-09-21T10:06:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:14 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 46704/321141 objects degraded (14.543%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1051858075' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:06:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:14 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:06:14.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:14.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:14.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:14.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:14.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:14.947 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:06:15.008 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:15.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:15.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:15.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:15.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:15.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:06:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:06:14] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-09-21T10:06:15.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:15.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:15.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:15.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:15.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:15.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:15.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:15.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:15 smithi017 ceph-mon[102913]: pgmap v1667: 109 pgs: 9 unknown, 29 active+undersized, 16 active+undersized+degraded, 55 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 47397/326268 objects degraded (14.527%) 2024-09-21T10:06:15.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:15 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:15.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:15 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:06:15.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:15 smithi017 ceph-mon[102913]: osdmap e1214: 8 total, 7 up, 7 in 2024-09-21T10:06:15.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:15 smithi017 ceph-mon[105151]: pgmap v1667: 109 pgs: 9 unknown, 29 active+undersized, 16 active+undersized+degraded, 55 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 47397/326268 objects degraded (14.527%) 2024-09-21T10:06:15.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:15 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:15.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:15 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:06:15.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:15 smithi017 ceph-mon[105151]: osdmap e1214: 8 total, 7 up, 7 in 2024-09-21T10:06:15.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:15.656 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 112436 112420 97.5695 70.5 0.00847205 0.0102445 2024-09-21T10:06:15.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:15 smithi134 ceph-mon[90085]: pgmap v1667: 109 pgs: 9 unknown, 29 active+undersized, 16 active+undersized+degraded, 55 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 47397/326268 objects degraded (14.527%) 2024-09-21T10:06:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:15 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:15 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:06:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:15 smithi134 ceph-mon[90085]: osdmap e1214: 8 total, 7 up, 7 in 2024-09-21T10:06:15.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:15.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:15.865 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:15.892 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:15.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:15.896 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:06:15.896 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:06:15.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:16.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:16.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:16.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:16.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:16.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:16.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:16.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:16.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:16.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:16.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:16.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:16.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:16.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:16.657 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 113475 113459 97.1224 64.9375 0.0143572 0.0102912 2024-09-21T10:06:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:16 smithi017 ceph-mon[102913]: 1.4 scrub starts 2024-09-21T10:06:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:16 smithi017 ceph-mon[102913]: 1.4 scrub ok 2024-09-21T10:06:16.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:16 smithi017 ceph-mon[105151]: 1.4 scrub starts 2024-09-21T10:06:16.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:16 smithi017 ceph-mon[105151]: 1.4 scrub ok 2024-09-21T10:06:16.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:16 smithi134 ceph-mon[90085]: 1.4 scrub starts 2024-09-21T10:06:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:16 smithi134 ceph-mon[90085]: 1.4 scrub ok 2024-09-21T10:06:16.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:16.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:16.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:16.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.906 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:16.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:17.007 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:17.031 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:17.036 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:17.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:17.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:06:17.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:17.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:17.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:17.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:17.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:17.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:17.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:17.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:17.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:17.603 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:17.632 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:17.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:17.657 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 114638 114622 96.792 72.6875 0.0146695 0.0103265 2024-09-21T10:06:17.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:17 smithi017 ceph-mon[102913]: 1.0 scrub starts 2024-09-21T10:06:17.668 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:17 smithi017 ceph-mon[102913]: 1.0 scrub ok 2024-09-21T10:06:17.668 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:17 smithi017 ceph-mon[102913]: pgmap v1669: 109 pgs: 3 unknown, 32 active+undersized, 16 active+undersized+degraded, 58 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 47725/330474 objects degraded (14.441%) 2024-09-21T10:06:17.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:17 smithi017 ceph-mon[105151]: 1.0 scrub starts 2024-09-21T10:06:17.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:17 smithi017 ceph-mon[105151]: 1.0 scrub ok 2024-09-21T10:06:17.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:17 smithi017 ceph-mon[105151]: pgmap v1669: 109 pgs: 3 unknown, 32 active+undersized, 16 active+undersized+degraded, 58 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 47725/330474 objects degraded (14.441%) 2024-09-21T10:06:17.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:17 smithi134 ceph-mon[90085]: 1.0 scrub starts 2024-09-21T10:06:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:17 smithi134 ceph-mon[90085]: 1.0 scrub ok 2024-09-21T10:06:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:17 smithi134 ceph-mon[90085]: pgmap v1669: 109 pgs: 3 unknown, 32 active+undersized, 16 active+undersized+degraded, 58 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 47725/330474 objects degraded (14.441%) 2024-09-21T10:06:17.761 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:17.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:17.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:18.009 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:18.040 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:18.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:18.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:18.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:18.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:06:18.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:18.304 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:18.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:18.406 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:18.432 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:18.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:18.534 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:18.557 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:18.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:18.657 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 115609 115593 96.3105 60.6875 0.00804551 0.0103785 2024-09-21T10:06:18.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:18.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:18.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:18.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:18.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:18.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:18.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[102913]: 1.14 scrub starts 2024-09-21T10:06:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[102913]: 1.14 scrub ok 2024-09-21T10:06:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:06:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[102913]: 1.7 scrub ok 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[102913]: 1.11 scrub starts 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[102913]: 1.11 scrub ok 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 47725/330474 objects degraded (14.441%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[105151]: 1.14 scrub starts 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[105151]: 1.14 scrub ok 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[105151]: 1.7 scrub ok 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[105151]: 1.11 scrub starts 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[105151]: 1.11 scrub ok 2024-09-21T10:06:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:18 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 47725/330474 objects degraded (14.441%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:18.976 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:18.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:18 smithi134 ceph-mon[90085]: 1.14 scrub starts 2024-09-21T10:06:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:18 smithi134 ceph-mon[90085]: 1.14 scrub ok 2024-09-21T10:06:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:18 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:06:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:18 smithi134 ceph-mon[90085]: 1.7 scrub ok 2024-09-21T10:06:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:18 smithi134 ceph-mon[90085]: 1.11 scrub starts 2024-09-21T10:06:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:18 smithi134 ceph-mon[90085]: 1.11 scrub ok 2024-09-21T10:06:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:18 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 47725/330474 objects degraded (14.441%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:19.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:19.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:19.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:19.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:19.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:19.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:19.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:19.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:19.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:19.398 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 6] out_osds: [1] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2024-09-21T10:06:19.398 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:06:19.398 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-09-21T10:06:19.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.0 'trim stale osdmaps' 2024-09-21T10:06:19.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:19.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:19.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:19.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:19.657 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 116631 116615 95.8832 63.875 0.0131498 0.0104245 2024-09-21T10:06:19.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:19.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:19.781 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:19.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:19.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:19.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:19.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:19 smithi017 ceph-mon[102913]: pgmap v1670: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 903 op/s; 49038/337413 objects degraded (14.534%) 2024-09-21T10:06:19.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:19 smithi017 ceph-mon[102913]: 1.6 scrub starts 2024-09-21T10:06:19.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:19 smithi017 ceph-mon[102913]: 1.6 scrub ok 2024-09-21T10:06:19.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:19 smithi017 ceph-mon[105151]: pgmap v1670: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 903 op/s; 49038/337413 objects degraded (14.534%) 2024-09-21T10:06:19.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:19 smithi017 ceph-mon[105151]: 1.6 scrub starts 2024-09-21T10:06:19.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:19 smithi017 ceph-mon[105151]: 1.6 scrub ok 2024-09-21T10:06:19.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:19.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:19 smithi134 ceph-mon[90085]: pgmap v1670: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 903 op/s; 49038/337413 objects degraded (14.534%) 2024-09-21T10:06:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:19 smithi134 ceph-mon[90085]: 1.6 scrub starts 2024-09-21T10:06:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:19 smithi134 ceph-mon[90085]: 1.6 scrub ok 2024-09-21T10:06:20.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:20.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:20.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:20.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:20.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:20.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:20.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:20.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:06:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:06:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:20.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:20.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:20.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:20.635 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:20.657 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 117864 117848 95.6387 77.0625 0.00423477 0.0104513 2024-09-21T10:06:20.667 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:20.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:20 smithi017 ceph-mon[102913]: 1.1 scrub starts 2024-09-21T10:06:20.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:20 smithi017 ceph-mon[102913]: 1.1 scrub ok 2024-09-21T10:06:20.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:20 smithi017 ceph-mon[105151]: 1.1 scrub starts 2024-09-21T10:06:20.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:20 smithi017 ceph-mon[105151]: 1.1 scrub ok 2024-09-21T10:06:20.769 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:20.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:20.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:20.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:20.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:20 smithi134 ceph-mon[90085]: 1.1 scrub starts 2024-09-21T10:06:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:20 smithi134 ceph-mon[90085]: 1.1 scrub ok 2024-09-21T10:06:21.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:21.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:21.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:21.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:21.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:21.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:21.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:21.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:21.380 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:21.380 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:21.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:21.518 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:21.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:21.620 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:21.650 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:21.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:21.657 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 118993 118977 95.3172 70.5625 0.00910977 0.0104863 2024-09-21T10:06:21.674 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:21 smithi017 ceph-mon[102913]: pgmap v1671: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 57 MiB/s wr, 915 op/s; 49745/342606 objects degraded (14.520%) 2024-09-21T10:06:21.674 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:21 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:06:21.674 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:21 smithi017 ceph-mon[102913]: 1.10 scrub ok 2024-09-21T10:06:21.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:21 smithi017 ceph-mon[105151]: pgmap v1671: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 57 MiB/s wr, 915 op/s; 49745/342606 objects degraded (14.520%) 2024-09-21T10:06:21.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:21 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:06:21.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:21 smithi017 ceph-mon[105151]: 1.10 scrub ok 2024-09-21T10:06:21.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:21.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:21.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:21.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:21.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:21.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:21 smithi134 ceph-mon[90085]: pgmap v1671: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 57 MiB/s wr, 915 op/s; 49745/342606 objects degraded (14.520%) 2024-09-21T10:06:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:21 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:06:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:21 smithi134 ceph-mon[90085]: 1.10 scrub ok 2024-09-21T10:06:22.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:22.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:22.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:22.196 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:22.229 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:22.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:22.331 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:22.357 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:22.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:22.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:22.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:22.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:22.625 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:22.658 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 120194 120178 95.0607 75.0625 0.00589517 0.0105147 2024-09-21T10:06:22.727 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:22.754 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:22.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:22.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:22.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:22.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:22.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:23.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:23.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:23.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:23.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:23.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:23.287 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:23.314 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:23.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:23.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:23.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:23.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:23.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:23.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:23.661 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:06:23.660018+0000 min lat: 0.00121399 max lat: 1.06457 avg lat: 0.0105396 2024-09-21T10:06:23.661 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:06:23.661 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 121426 121410 94.8313 77 0.010867 0.0105396 2024-09-21T10:06:23.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:23.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:23.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:23.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:23 smithi017 ceph-mon[102913]: pgmap v1672: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 51457/353763 objects degraded (14.546%) 2024-09-21T10:06:23.796 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:23 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 51457/353763 objects degraded (14.546%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:23.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:23 smithi017 ceph-mon[105151]: pgmap v1672: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 51457/353763 objects degraded (14.546%) 2024-09-21T10:06:23.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:23 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 51457/353763 objects degraded (14.546%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:23.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:23.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:23.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:23.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:23 smithi134 ceph-mon[90085]: pgmap v1672: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 51457/353763 objects degraded (14.546%) 2024-09-21T10:06:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:23 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 51457/353763 objects degraded (14.546%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:24.002 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:24.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:24.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:24.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:24.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:24.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:24.400 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:24.440 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:24.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:24.550 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:06:24.550 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:06:24.550 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:06:24.550 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:06:24.550 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:06:24.550 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:06:24.551 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:06:24.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:24.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:24.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:24.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:24.551 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:06:24.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:24.551 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:24.552 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:24.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:24.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:24.661 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 122596 122580 94.5632 73.125 0.00438518 0.0105704 2024-09-21T10:06:24.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:24.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:24.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:24.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:24.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:24.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:24 smithi017 ceph-mon[102913]: 1.f scrub starts 2024-09-21T10:06:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:24 smithi017 ceph-mon[102913]: 1.f scrub ok 2024-09-21T10:06:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/331286989' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:24.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:24 smithi017 ceph-mon[105151]: 1.f scrub starts 2024-09-21T10:06:24.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:24 smithi017 ceph-mon[105151]: 1.f scrub ok 2024-09-21T10:06:24.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/331286989' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:24.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:06:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:06:24] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-09-21T10:06:24.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:24 smithi134 ceph-mon[90085]: 1.f scrub starts 2024-09-21T10:06:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:24 smithi134 ceph-mon[90085]: 1.f scrub ok 2024-09-21T10:06:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/331286989' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:25.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:25.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:25.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:25.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:25.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:25.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:25.386 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:25.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:25.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:25.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:25.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:25.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:25.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:25 smithi017 ceph-mon[102913]: pgmap v1673: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 52214/359265 objects degraded (14.534%) 2024-09-21T10:06:25.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:25.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:25 smithi017 ceph-mon[105151]: pgmap v1673: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 52214/359265 objects degraded (14.534%) 2024-09-21T10:06:25.644 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:25.661 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 123789 123773 94.3193 74.5625 0.0076909 0.0105979 2024-09-21T10:06:25.711 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:25.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:25.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:25.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:25.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:25.903 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:06:25.903 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:06:25.903 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:06:25.903 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:06:25.903 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:06:25.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:25 smithi134 ceph-mon[90085]: pgmap v1673: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 52214/359265 objects degraded (14.534%) 2024-09-21T10:06:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:26.020 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:26.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:26.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:26.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:26.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:26.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:26.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:26.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:26.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:26.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:26.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:26.595 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:26.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:26.661 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 124948 124932 94.0556 72.4375 0.00693559 0.0106265 2024-09-21T10:06:26.697 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:26.724 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:26.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:06:26.825 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:06:26.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:26.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:26.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:26.960 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:26.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:26.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:27.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:27.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:27.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:27.227 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:27.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.259 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:27.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:27.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:27.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:27.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:27.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:27.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:27.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:27.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:27.661 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 126185 126169 93.8558 77.3125 0.00791316 0.0106494 2024-09-21T10:06:27.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:27.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:27 smithi017 ceph-mon[102913]: pgmap v1674: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 52591/363678 objects degraded (14.461%) 2024-09-21T10:06:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:27 smithi017 ceph-mon[102913]: 1.e scrub starts 2024-09-21T10:06:27.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:27 smithi017 ceph-mon[102913]: 1.e scrub ok 2024-09-21T10:06:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:27 smithi017 ceph-mon[105151]: pgmap v1674: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 52591/363678 objects degraded (14.461%) 2024-09-21T10:06:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:27 smithi017 ceph-mon[105151]: 1.e scrub starts 2024-09-21T10:06:27.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:27 smithi017 ceph-mon[105151]: 1.e scrub ok 2024-09-21T10:06:27.789 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:27.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:27.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:27.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:06:27.938 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:27.963 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:27.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:27 smithi134 ceph-mon[90085]: pgmap v1674: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 52591/363678 objects degraded (14.461%) 2024-09-21T10:06:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:27 smithi134 ceph-mon[90085]: 1.e scrub starts 2024-09-21T10:06:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:27 smithi134 ceph-mon[90085]: 1.e scrub ok 2024-09-21T10:06:28.065 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:28.090 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:28.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:28.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:28.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:28.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:28.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:28.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:28.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:28.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:28.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:28.637 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:28.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 127307 127291 93.5765 70.125 0.0137826 0.0106815 2024-09-21T10:06:28.666 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:28.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:28 smithi017 ceph-mon[102913]: 1.c scrub starts 2024-09-21T10:06:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:28 smithi017 ceph-mon[102913]: 1.c scrub ok 2024-09-21T10:06:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:28 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 52591/363678 objects degraded (14.461%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:28 smithi017 ceph-mon[105151]: 1.c scrub starts 2024-09-21T10:06:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:28 smithi017 ceph-mon[105151]: 1.c scrub ok 2024-09-21T10:06:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:28 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 52591/363678 objects degraded (14.461%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:28.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:28.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.799 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:28.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:28.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:06:28.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:28.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:28 smithi134 ceph-mon[90085]: 1.c scrub starts 2024-09-21T10:06:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:28 smithi134 ceph-mon[90085]: 1.c scrub ok 2024-09-21T10:06:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:28 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 52591/363678 objects degraded (14.461%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:29.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:29.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.150 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:29.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:29.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:29.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:29.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:29.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:29.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:29.549 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:29.578 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:29.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[102913]: pgmap v1675: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 54109/371502 objects degraded (14.565%) 2024-09-21T10:06:29.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:29.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:29.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:29.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:29.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:29.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[105151]: pgmap v1675: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 54109/371502 objects degraded (14.565%) 2024-09-21T10:06:29.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:29.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:29.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:29.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:29.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:29.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 128252 128236 93.1752 59.0625 0.00914558 0.0107276 2024-09-21T10:06:29.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:29.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:29.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:29.828 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:29.856 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:29.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:29.958 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:29.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:29.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:29 smithi134 ceph-mon[90085]: pgmap v1675: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 54109/371502 objects degraded (14.565%) 2024-09-21T10:06:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:30.087 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:30.115 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:30.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:30.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:30.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:30.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:30.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:30.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:06:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:06:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:30.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:30.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:30.592 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:30.593 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:30.598 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:06:30.634 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:30.660 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:30.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:30.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 129346 129330 92.8901 68.375 0.0197353 0.0107604 2024-09-21T10:06:30.704 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-21T10:06:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-21T10:06:30.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:30.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:30.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:30.944 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:30 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/531656717' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:30 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:30 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/531656717' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:30 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:30 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/531656717' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:30 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:31.046 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:31.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:31.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:31.175 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:31.208 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:31.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:31.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:31.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:31.449 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:31.476 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:31.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:31.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:31.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 130583 130567 92.713 77.3125 0.00470685 0.0107811 2024-09-21T10:06:31.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:31 smithi017 ceph-mon[105151]: pgmap v1676: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 54835/376962 objects degraded (14.547%) 2024-09-21T10:06:31.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:31 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:06:31.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:31 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:31.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:31 smithi017 ceph-mon[105151]: osdmap e1215: 8 total, 7 up, 7 in 2024-09-21T10:06:31.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:31 smithi017 ceph-mon[102913]: pgmap v1676: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 54835/376962 objects degraded (14.547%) 2024-09-21T10:06:31.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:31 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:06:31.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:31 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:31.670 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:31 smithi017 ceph-mon[102913]: osdmap e1215: 8 total, 7 up, 7 in 2024-09-21T10:06:31.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:31.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:31.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:31.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:31.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:31.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:31 smithi134 ceph-mon[90085]: pgmap v1676: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 54835/376962 objects degraded (14.547%) 2024-09-21T10:06:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:31 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:06:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:31 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:31 smithi134 ceph-mon[90085]: osdmap e1215: 8 total, 7 up, 7 in 2024-09-21T10:06:32.011 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:32.037 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:32.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:32.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:32.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:32.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:32.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:32.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:32.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:32.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:32.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:32.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:32.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:32.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:32.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:32.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 131772 131756 92.5062 74.3125 0.0219994 0.0108053 2024-09-21T10:06:32.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:32.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:32.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:32.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:32.865 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:32.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:32.967 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:32.993 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:32.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:06:32.995 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:06:32.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 'trim stale osdmaps' 2024-09-21T10:06:33.095 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:33.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:33.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:33.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:33.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:33.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:33.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:06:33.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.420 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:33.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:33.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:33.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:33.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:33.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0132 2024-09-21T10:06:33.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 132903 2024-09-21T10:06:33.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-21T10:06:33.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-21T10:06:33.662 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 92.2803 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 36.4653 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 234.688 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 1.3125 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1476 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 583.445 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3755 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 21 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0108323 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0149102 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 1.06457 2024-09-21T10:06:33.663 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00121399 2024-09-21T10:06:33.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:33.669 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:06:33.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.723 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:33.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:33.824 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:33.849 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:33 smithi017 ceph-mon[102913]: pgmap v1678: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.9 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 56597/388707 objects degraded (14.560%) 2024-09-21T10:06:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 56597/388707 objects degraded (14.560%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:33 smithi017 ceph-mon[105151]: pgmap v1678: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.9 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 56597/388707 objects degraded (14.560%) 2024-09-21T10:06:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 56597/388707 objects degraded (14.560%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:33.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:33.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:33 smithi134 ceph-mon[90085]: pgmap v1678: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.9 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 56597/388707 objects degraded (14.560%) 2024-09-21T10:06:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 56597/388707 objects degraded (14.560%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T10:06:34.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:34.118 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:34.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:34.221 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:34.251 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:34.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:34.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:34.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:34.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:34.494 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:34.523 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:34.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:34.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:34.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:34.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:34.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:34.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:34.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:34.913 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:34.944 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:34.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:06:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:06:34] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-09-21T10:06:35.046 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:35.084 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:35.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:35.185 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:35.212 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:35.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:35.313 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:35.342 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:35.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:35.470 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:35.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:35.573 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:35.602 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:35.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:35 smithi017 ceph-mon[102913]: pgmap v1679: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 8.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 71 MiB/s wr, 1.16k op/s; 57150/392886 objects degraded (14.546%) 2024-09-21T10:06:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:35 smithi017 ceph-mon[105151]: pgmap v1679: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 8.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 71 MiB/s wr, 1.16k op/s; 57150/392886 objects degraded (14.546%) 2024-09-21T10:06:35.703 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:35.732 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:35.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:35.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:35.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:35.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:35.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:35 smithi134 ceph-mon[90085]: pgmap v1679: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 8.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 71 MiB/s wr, 1.16k op/s; 57150/392886 objects degraded (14.546%) 2024-09-21T10:06:36.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:36.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:36.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:36.135 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:06:36.135 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:06:36.135 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:06:36.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:36.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:36.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:36.152 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:36.153 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:36.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:36.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:36.172 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:06:36.172 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:36.180 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:06:36.204 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:06:36.204 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:36.223 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:06:36.223 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:36.255 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:06:36.255 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:36.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:36.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:36.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:36.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:36.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:36.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:36.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:36.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:36.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:36.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:36.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:36.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:36 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3546291965' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:36.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:36 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3546291965' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:36.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:36 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3546291965' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:37.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:37.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:37.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:37.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:37.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:37.253 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:37.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.275 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:37.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:37.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:37.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:37.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:37.448 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:06:37.448 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:06:37.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:37.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:37.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:37.655 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:37.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:37.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:37.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:37 smithi017 ceph-mon[102913]: pgmap v1680: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 8.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 67 MiB/s wr, 1.15k op/s; 57231/393606 objects degraded (14.540%) 2024-09-21T10:06:37.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:37 smithi017 ceph-mon[105151]: pgmap v1680: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 8.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 67 MiB/s wr, 1.15k op/s; 57231/393606 objects degraded (14.540%) 2024-09-21T10:06:37.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:37 smithi134 ceph-mon[90085]: pgmap v1680: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 8.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 67 MiB/s wr, 1.15k op/s; 57231/393606 objects degraded (14.540%) 2024-09-21T10:06:37.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:37.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:37.934 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:37.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.970 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:37.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:38.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:38.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:38.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:38.234 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:38.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:38.336 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:38.365 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:38.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:38.467 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:38.496 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:38.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:38.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:38.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:38.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:38.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 57231/393606 objects degraded (14.540%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:38.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:38.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:38.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:38.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:38.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 57231/393606 objects degraded (14.540%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:38.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 57231/393606 objects degraded (14.540%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:39.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:39.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:39.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:39.142 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:39.170 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:39.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:39.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:39.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:39.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:39.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:39.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:39.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:39.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.569 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:39.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:39.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:39.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:39.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:39.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:39.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:39.944 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:39.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:39 smithi017 ceph-mon[102913]: pgmap v1681: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 8.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 57 MiB/s wr, 1.16k op/s; 56832/391311 objects degraded (14.523%) 2024-09-21T10:06:39.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:39 smithi017 ceph-mon[105151]: pgmap v1681: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 8.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 57 MiB/s wr, 1.16k op/s; 56832/391311 objects degraded (14.523%) 2024-09-21T10:06:39.976 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:39 smithi134 ceph-mon[90085]: pgmap v1681: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 8.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 57 MiB/s wr, 1.16k op/s; 56832/391311 objects degraded (14.523%) 2024-09-21T10:06:40.078 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:40.102 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:40.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:40.204 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:40.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:40.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:40.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:40.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:40.472 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:40.493 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:06:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:06:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:40.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:40.595 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:40.622 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:40.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:40.724 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:40.751 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:40.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:40.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:40.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:40.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:40.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:41.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:41.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:41.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:41.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:41.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:41.249 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:41.276 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:41.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:41.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:41.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:41.517 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:41.547 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:41.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:41.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:41.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:41 smithi017 ceph-mon[102913]: pgmap v1682: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 45 MiB/s wr, 1.17k op/s; 56048/385611 objects degraded (14.535%) 2024-09-21T10:06:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:41 smithi017 ceph-mon[105151]: pgmap v1682: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 45 MiB/s wr, 1.17k op/s; 56048/385611 objects degraded (14.535%) 2024-09-21T10:06:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:41 smithi134 ceph-mon[90085]: pgmap v1682: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.8 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 102 B/s rd, 45 MiB/s wr, 1.17k op/s; 56048/385611 objects degraded (14.535%) 2024-09-21T10:06:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:41.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:41.818 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:41.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:41.920 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:41.944 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:41.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:42.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:42.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:42.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:42.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:42.222 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:42.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:42.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:42.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:42.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:42.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:42.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:42.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:42.634 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:42.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:42.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:42.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:42.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:42.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:42.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:42.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:43.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:43.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:43.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:06:43.082 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:06:43.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:43.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:43.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:43.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:43.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:43.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:43.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:43.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:43 smithi017 ceph-mon[102913]: pgmap v1683: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 87 B/s rd, 39 MiB/s wr, 1.34k op/s; 54277/373743 objects degraded (14.523%) 2024-09-21T10:06:43.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:43 smithi017 ceph-mon[105151]: pgmap v1683: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 87 B/s rd, 39 MiB/s wr, 1.34k op/s; 54277/373743 objects degraded (14.523%) 2024-09-21T10:06:43.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:43.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:43.563 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:43.584 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:43.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:43.686 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:43.707 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:43.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:43 smithi134 ceph-mon[90085]: pgmap v1683: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 87 B/s rd, 39 MiB/s wr, 1.34k op/s; 54277/373743 objects degraded (14.523%) 2024-09-21T10:06:43.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:43.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:43.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:43.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:43.968 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:43.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:44.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:44.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:44.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:44.228 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:44.258 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:44.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:44.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:44.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:44.502 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:44.513 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:44 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 54277/373743 objects degraded (14.523%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:44.514 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:44 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 54277/373743 objects degraded (14.523%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:44.528 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:44.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:44.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:44.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:44 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 54277/373743 objects degraded (14.523%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:44.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:44.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:44.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:44.890 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:06:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:06:44] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-09-21T10:06:44.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:44.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:44.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:44.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:06:45.055 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:45.078 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:45.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:45.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:06:45.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:45.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:45.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:45.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:45.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:45.518 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:45.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:45.650 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:45.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:45 smithi017 ceph-mon[102913]: pgmap v1684: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 17 MiB/s wr, 1.12k op/s; 53570/368529 objects degraded (14.536%) 2024-09-21T10:06:45.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:45 smithi017 ceph-mon[105151]: pgmap v1684: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 17 MiB/s wr, 1.12k op/s; 53570/368529 objects degraded (14.536%) 2024-09-21T10:06:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:45.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:45 smithi134 ceph-mon[90085]: pgmap v1684: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.5 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 17 MiB/s wr, 1.12k op/s; 53570/368529 objects degraded (14.536%) 2024-09-21T10:06:45.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:45.810 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:45.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:45.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:45.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:45.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:46.048 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:46.074 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:46.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:46.176 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:46.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.210 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:46.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:46.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:46.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.346 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:46.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:46.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:46.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:46.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:46.588 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:46.612 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:46.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:46.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:46.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:46.857 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:46.885 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:46.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:46.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:47.025 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:47.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:47.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:47.154 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:47.180 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:47.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:47.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:47.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:47.403 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:47 smithi017 ceph-mon[102913]: pgmap v1685: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 1.09k op/s; 53238/364455 objects degraded (14.608%) 2024-09-21T10:06:47.403 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:47 smithi017 ceph-mon[105151]: pgmap v1685: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 1.09k op/s; 53238/364455 objects degraded (14.608%) 2024-09-21T10:06:47.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:47.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:47.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:47.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:47.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:47.719 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:47.746 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:47 smithi134 ceph-mon[90085]: pgmap v1685: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 1.09k op/s; 53238/364455 objects degraded (14.608%) 2024-09-21T10:06:47.746 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:47.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:47.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:47.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:47.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:47.935 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-21T10:06:47.935 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-21T10:06:47.935 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-21T10:06:47.935 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-21T10:06:47.935 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-21T10:06:47.936 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-21T10:06:47.936 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-21T10:06:47.936 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:06:47.936 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-21T10:06:47.936 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-21T10:06:47.936 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-21T10:06:47.936 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-21T10:06:47.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:48.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:48.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:48.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:48.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:48.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:48.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:48.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:48.451 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:48.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:48.467 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:48.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:48.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.591 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:48.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:48.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:48.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:48.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:48.835 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:48.864 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:48.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:49.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:49.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:49.068 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:06:49.069 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:06:49.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 'trim stale osdmaps' 2024-09-21T10:06:49.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:49.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:49.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:49.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:49.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:49.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:49.403 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:49.430 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:49.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:49.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:49.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:49.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:49.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:49 smithi017 ceph-mon[102913]: pgmap v1686: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.3 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 5.0 MiB/s wr, 1.17k op/s; 51818/357042 objects degraded (14.513%) 2024-09-21T10:06:49.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:49 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 51818/357042 objects degraded (14.513%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:49.610 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:49 smithi017 ceph-mon[105151]: pgmap v1686: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.3 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 5.0 MiB/s wr, 1.17k op/s; 51818/357042 objects degraded (14.513%) 2024-09-21T10:06:49.611 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:49 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 51818/357042 objects degraded (14.513%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:49.687 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:49.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.721 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:49.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:49 smithi134 ceph-mon[90085]: pgmap v1686: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.3 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 5.0 MiB/s wr, 1.17k op/s; 51818/357042 objects degraded (14.513%) 2024-09-21T10:06:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:49 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 51818/357042 objects degraded (14.513%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:49.822 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:49.852 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:49.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:49.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:49.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:49.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:50.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:50.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:50.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:50.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:50.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:50.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:50.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:50.458 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:06:50.458 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:50.491 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:06:50.493 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:06:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:06:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:50.535 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:50.536 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:06:50.536 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:50.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:50.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:06:50.579 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:50.580 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:06:50.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:50.616 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:06:50.617 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:50.654 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:06:50.655 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:50.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:50.695 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:06:50.695 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:50.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:50.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:50.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:50.871 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:50.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:50.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:51.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:51.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:51.119 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:51.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.149 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:51.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:51.250 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:51.277 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:51.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:51.320 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:06:51.321 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:51 smithi017 ceph-mon[102913]: pgmap v1687: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 51107/351690 objects degraded (14.532%) 2024-09-21T10:06:51.321 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:51 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/792077398' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:51.321 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:51 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/17354234' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:51.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:51 smithi017 ceph-mon[105151]: pgmap v1687: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 51107/351690 objects degraded (14.532%) 2024-09-21T10:06:51.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:51 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/792077398' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:51.321 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:51 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/17354234' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:51.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:51.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:51.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:51.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:51.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:51.659 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:51.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.687 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:51.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:51 smithi134 ceph-mon[90085]: pgmap v1687: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 7.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 51107/351690 objects degraded (14.532%) 2024-09-21T10:06:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:51 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/792077398' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:51 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/17354234' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:51.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:51.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:51.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:51.925 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:51.952 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:51.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:52.053 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:52.078 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:52.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:52.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:52.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:52.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:52.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:52.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:52.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:52.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:52.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:06:52.545 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:06:52.596 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:52.623 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:52.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:52 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:52.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:52 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/17354234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:06:52.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:52 smithi017 ceph-mon[102913]: osdmap e1216: 8 total, 7 up, 7 in 2024-09-21T10:06:52.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:52 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:52.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:52 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/17354234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:06:52.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:52 smithi017 ceph-mon[105151]: osdmap e1216: 8 total, 7 up, 7 in 2024-09-21T10:06:52.724 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:52 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:52 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/17354234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:06:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:52 smithi134 ceph-mon[90085]: osdmap e1216: 8 total, 7 up, 7 in 2024-09-21T10:06:52.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.754 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:52.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:52.854 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:52.881 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:52.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:52.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:53.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:53.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:53.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:53.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:53.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:53.259 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.293 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:53.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:53.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:53.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:53.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:53.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:53.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.590 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:53.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:53.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:53 smithi017 ceph-mon[102913]: pgmap v1689: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 49545/341310 objects degraded (14.516%) 2024-09-21T10:06:53.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:53 smithi017 ceph-mon[105151]: pgmap v1689: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 49545/341310 objects degraded (14.516%) 2024-09-21T10:06:53.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:53.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:53.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:53 smithi134 ceph-mon[90085]: pgmap v1689: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 49545/341310 objects degraded (14.516%) 2024-09-21T10:06:53.829 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:53.855 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:53.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:53.956 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:53.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:53.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:54.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:06:54.087 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:54.113 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:54.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:54.215 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:54.239 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:54.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:54.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:54.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:54.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:54.491 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:54.515 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:54.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:54.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:06:54.616 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:54.643 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:54.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:54.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:54 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 49545/341310 objects degraded (14.516%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:54.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:54 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 49545/341310 objects degraded (14.516%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:54 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 49545/341310 objects degraded (14.516%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:54.744 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:54.775 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:54.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:54.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:55.022 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:06:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:06:54] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-09-21T10:06:55.025 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:55.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:55.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:55.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:55.186 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:158221 8.3 e1216/1200 8:c98e829d:::benchmark_data_smithi017_209235_object25315:head v 1216'9789, mlcod=1216'9787)", 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:55.177700+0000", 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0057930940000000004, 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058214130000000001, 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:55.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.177700+0000", 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.177700+0000", 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.177701+0000", 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1480000000000001e-06 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.177707+0000", 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3400000000000003e-06 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.177724+0000", 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7113e-05 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.177737+0000", 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2741e-05 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181016+0000", 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032792199999999998 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181037+0000", 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0426999999999999e-05 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:158227 8.a 8:5af0f35c:::benchmark_data_smithi017_209235_object25321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1216)", 2024-09-21T10:06:55.190 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:55.180954+0000", 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.002538963, 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026244409999999999, 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 158227 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.180954+0000", 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.180954+0000", 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.180956+0000", 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.154e-06 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.180964+0000", 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5909999999999998e-06 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.180986+0000", 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.232e-05 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.180999+0000", 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2617e-05 2024-09-21T10:06:55.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181150+0000", 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015172499999999999 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181270+0000", 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119585 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181385+0000", 2024-09-21T10:06:55.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115382 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:158218 8.f e1216/1207 8:f9f03608:::benchmark_data_smithi017_209235_object25312:head v 1216'9760, mlcod=1216'9758)", 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:55.181042+0000", 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0024511110000000002, 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026056880000000001, 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:06:55.195 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181042+0000", 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181042+0000", 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181043+0000", 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1099999999999995e-07 2024-09-21T10:06:55.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181049+0000", 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6080000000000001e-06 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181059+0000", 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.499e-06 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181066+0000", 2024-09-21T10:06:55.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9399999999999996e-06 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.182812+0000", 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001745988 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.182837+0000", 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5846e-05 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:158224 8.3 e1216/1200)", 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:55.181540+0000", 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.001952454, 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021507589999999999, 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181540+0000", 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181540+0000", 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181541+0000", 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1900000000000001e-07 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181549+0000", 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3309999999999998e-06 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:06:55.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181561+0000", 2024-09-21T10:06:55.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2316e-05 2024-09-21T10:06:55.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:06:55.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:55.181569+0000", 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6340000000000001e-06 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 4 2024-09-21T10:06:55.202 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:06:55.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:55.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:55.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:55.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:55.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:55.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.519 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:55.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:55.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:55 smithi017 ceph-mon[102913]: pgmap v1690: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 48418/333228 objects degraded (14.530%) 2024-09-21T10:06:55.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:55.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:55.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:55 smithi017 ceph-mon[105151]: pgmap v1690: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 48418/333228 objects degraded (14.530%) 2024-09-21T10:06:55.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:55.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:55.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:55.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:55.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:55 smithi134 ceph-mon[90085]: pgmap v1690: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 48418/333228 objects degraded (14.530%) 2024-09-21T10:06:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:06:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:55.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:55.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:55.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:55.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:55.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:06:55.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:56.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:56.088 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:56.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:56.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:56.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:56.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:56.433 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:56.449 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:56.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:56.528 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:56.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:56.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:56.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:56.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.694 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:56.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:56.794 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:56.824 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:56.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:56.926 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:56.953 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:56.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:57.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:57.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:57.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:57.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:57.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:57.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:57.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:57.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:57.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:57.491 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:57.518 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:57.620 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:57.646 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:57.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:57.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:57 smithi017 ceph-mon[102913]: pgmap v1691: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 48027/328701 objects degraded (14.611%) 2024-09-21T10:06:57.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:57 smithi017 ceph-mon[105151]: pgmap v1691: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 48027/328701 objects degraded (14.611%) 2024-09-21T10:06:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:57 smithi134 ceph-mon[90085]: pgmap v1691: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 48027/328701 objects degraded (14.611%) 2024-09-21T10:06:57.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:57.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:57.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:57.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:57.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:58.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:58.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:58.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:58.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:58.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:58.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:58.344 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:58.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:58.446 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:58.470 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:58.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:58.572 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:58.596 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:58.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:58.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:58.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:58.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:58.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:58.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:58.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:58.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:59.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:59.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:59.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:59.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:59.262 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:59.286 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:59.388 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:59.415 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:59.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:59.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:59.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:59.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:59.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:59.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:59 smithi017 ceph-mon[102913]: pgmap v1692: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 46471/320607 objects degraded (14.495%) 2024-09-21T10:06:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:06:59 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 46471/320607 objects degraded (14.495%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:59 smithi017 ceph-mon[105151]: pgmap v1692: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 46471/320607 objects degraded (14.495%) 2024-09-21T10:06:59.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:06:59 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 46471/320607 objects degraded (14.495%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:59 smithi134 ceph-mon[90085]: pgmap v1692: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.5 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 46471/320607 objects degraded (14.495%) 2024-09-21T10:06:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:06:59 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 46471/320607 objects degraded (14.495%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:59.800 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:59.827 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:06:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:59.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:59.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:06:59.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:00.077 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:00.101 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:00.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:00.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:00.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:00.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:00.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.385 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:00.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:00.486 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:07:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:07:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:00.514 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:00.619 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:00.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:00.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:00.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:00.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:00.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:00.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:01.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:01.079 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:01.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:01.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:01.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:01.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:01.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:01.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:01.464 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:01.493 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:01.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:01.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:01.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.636 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:01.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:01.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:01 smithi017 ceph-mon[105151]: pgmap v1693: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 45702/314910 objects degraded (14.513%) 2024-09-21T10:07:01.688 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:01 smithi017 ceph-mon[102913]: pgmap v1693: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 45702/314910 objects degraded (14.513%) 2024-09-21T10:07:01.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:01 smithi134 ceph-mon[90085]: pgmap v1693: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.4 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 45702/314910 objects degraded (14.513%) 2024-09-21T10:07:01.779 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:01.880 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:01.895 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-21T10:07:01.895 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-21T10:07:01.895 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-21T10:07:01.895 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-21T10:07:01.896 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-21T10:07:01.896 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-21T10:07:01.896 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-21T10:07:01.896 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:07:01.896 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-21T10:07:01.896 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-21T10:07:01.896 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-21T10:07:01.896 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-21T10:07:01.906 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:01.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:02.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:02.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:02.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:02.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:02.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:02.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:02.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:02.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:02.426 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:02.457 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:02.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:02.595 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:02.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:02.642 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:07:02.642 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:07:02.642 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:07:02.642 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:07:02.642 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:07:02.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:02.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:02.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:02.835 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:02.863 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:02.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:02.965 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:02.996 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:03.098 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:03.128 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:03.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:03.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:03.254 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:03.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:03.357 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:03.385 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:03.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:03.487 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:03.513 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:03.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:03.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.643 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:03.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:03.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:03 smithi017 ceph-mon[102913]: pgmap v1694: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s; 44155/304413 objects degraded (14.505%) 2024-09-21T10:07:03.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:03 smithi017 ceph-mon[105151]: pgmap v1694: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s; 44155/304413 objects degraded (14.505%) 2024-09-21T10:07:03.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:03 smithi134 ceph-mon[90085]: pgmap v1694: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s; 44155/304413 objects degraded (14.505%) 2024-09-21T10:07:03.744 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:03.769 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:03.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:03.871 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:03.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.903 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:03.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:04.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:04.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:04.143 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:04.168 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:04.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:04.269 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:04.297 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:04.321 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:07:04.321 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:07:04.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 'trim stale osdmaps' 2024-09-21T10:07:04.370 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:07:04.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:04.438 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:04.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:04.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:04.567 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:04.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:04.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:04 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 44155/304413 objects degraded (14.505%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:04.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:04 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3076579263' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:04.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:04 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:04.656 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:04 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 44155/304413 objects degraded (14.505%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:04.656 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:04 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3076579263' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:04.656 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:04 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:04.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:04.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:04.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:04.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:04 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 44155/304413 objects degraded (14.505%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:04.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:04 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3076579263' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:04.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:04 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:04.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:04.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:04.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:04.928 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:07:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:07:04] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-09-21T10:07:04.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:05.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:05.102 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:05.126 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:05.227 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:05.261 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:05.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:05.363 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:05.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.391 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:05.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:05.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:05.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:05.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:05.628 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:05.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:05 smithi134 ceph-mon[90085]: pgmap v1695: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 43013/296277 objects degraded (14.518%) 2024-09-21T10:07:05.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:05 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:05.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:05 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:05.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:05 smithi134 ceph-mon[90085]: osdmap e1217: 8 total, 7 up, 7 in 2024-09-21T10:07:05.646 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:05 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1135641253' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:05.659 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[102913]: pgmap v1695: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 43013/296277 objects degraded (14.518%) 2024-09-21T10:07:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[102913]: osdmap e1217: 8 total, 7 up, 7 in 2024-09-21T10:07:05.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1135641253' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[105151]: pgmap v1695: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 6.0 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 43013/296277 objects degraded (14.518%) 2024-09-21T10:07:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[105151]: osdmap e1217: 8 total, 7 up, 7 in 2024-09-21T10:07:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:05 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1135641253' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:05.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:05.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:05.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:05.891 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:05.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:05.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:06.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:06.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:06.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:06.162 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:06.182 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:06.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:06.284 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:06.309 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:06.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:06.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:06.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:06.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:06.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:06.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:06.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:06.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:06.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:06.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:06.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:06.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:06.942 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:06.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:06.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:07.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:07.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:07.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:07.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:07.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:07.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:07.353 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:07.380 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:07.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:07.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:07.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:07.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:07.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:07.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:07 smithi017 ceph-mon[102913]: pgmap v1697: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 42630/291858 objects degraded (14.606%) 2024-09-21T10:07:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:07 smithi017 ceph-mon[105151]: pgmap v1697: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 42630/291858 objects degraded (14.606%) 2024-09-21T10:07:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:07 smithi134 ceph-mon[90085]: pgmap v1697: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 42630/291858 objects degraded (14.606%) 2024-09-21T10:07:07.767 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:07.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:07.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:07.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:07.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:07.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:08.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:08.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:08.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:08.172 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:08.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:08.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:08.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:08.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:08.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:08.446 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:08.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.480 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:08.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:08.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:08.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:08.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:08.724 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:08.751 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:08.852 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:08.877 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:08.979 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:09.009 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:09.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:09.110 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:09.131 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:09.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:09.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:07:09.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:07:09.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:07:09.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:07:09.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:07:09.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:07:09.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:07:09.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:09.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:09.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:09.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:09.233 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:07:09.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:09.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:09.233 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:09.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:09.258 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:09.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:09.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:09.392 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:09.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:09.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:09.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:09.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:09.618 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:09.644 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:09.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:09 smithi017 ceph-mon[102913]: pgmap v1698: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 41184/284112 objects degraded (14.496%) 2024-09-21T10:07:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:09 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 41184/284112 objects degraded (14.496%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:09 smithi017 ceph-mon[105151]: pgmap v1698: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 41184/284112 objects degraded (14.496%) 2024-09-21T10:07:09.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:09 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 41184/284112 objects degraded (14.496%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:09 smithi134 ceph-mon[90085]: pgmap v1698: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 41184/284112 objects degraded (14.496%) 2024-09-21T10:07:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:09 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 41184/284112 objects degraded (14.496%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:09.746 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:09.775 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:09.877 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:09.903 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:09.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:10.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:10.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:10.146 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:10.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:10.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:10.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:10.311 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:10.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:10.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:10.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:10.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:07:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:07:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:10.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:10.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:10.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:10.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:10.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:10.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:07:10.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:10.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:10.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:10.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:11.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:11.557 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:07:11.557 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:07:11.558 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:11.657 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:11.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.687 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:11.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:11 smithi017 ceph-mon[102913]: pgmap v1699: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 40433/278505 objects degraded (14.518%) 2024-09-21T10:07:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:11 smithi017 ceph-mon[105151]: pgmap v1699: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 40433/278505 objects degraded (14.518%) 2024-09-21T10:07:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:11 smithi134 ceph-mon[90085]: pgmap v1699: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 40433/278505 objects degraded (14.518%) 2024-09-21T10:07:11.788 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:11.816 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:11.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:11.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:11.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:11.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:12.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:12.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:12.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:12.113 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:12.199 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:12.224 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:12.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:12.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:12.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:12.473 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:12.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:07:12.498 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:12.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:12.600 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:12.625 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:12.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:12.727 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:12.752 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:12.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:12.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:12.944 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:12.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:13.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:13.072 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:13.174 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:13.197 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:13.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:13.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:13.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:13.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:13.426 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:13 smithi017 ceph-mon[102913]: pgmap v1700: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 38993/268467 objects degraded (14.524%) 2024-09-21T10:07:13.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:13 smithi017 ceph-mon[105151]: pgmap v1700: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 38993/268467 objects degraded (14.524%) 2024-09-21T10:07:13.446 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:13.476 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:13.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:07:13.578 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:13.604 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:13.706 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:13.712 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:07:13.728 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:13.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:13 smithi134 ceph-mon[90085]: pgmap v1700: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 38993/268467 objects degraded (14.524%) 2024-09-21T10:07:13.830 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:13.856 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:13.958 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:13.982 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:14.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:14.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:14.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:14.258 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:14.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:14.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:14.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:14.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:14.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:14.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:14.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:14.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:14.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:14 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 38993/268467 objects degraded (14.524%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:14.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:14 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 38993/268467 objects degraded (14.524%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:14.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:14 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 38993/268467 objects degraded (14.524%), 16 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:14.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:14.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:14.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:14.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:14.935 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:14.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:15.037 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:15.064 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:15.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:15.097 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:07:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:07:14] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-09-21T10:07:15.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:15.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:15.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:15.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:15.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:15.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:15.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:15.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:15.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:15.592 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:07:15.592 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:07:15.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:15.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:15 smithi017 ceph-mon[102913]: pgmap v1701: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 37866/260619 objects degraded (14.529%) 2024-09-21T10:07:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:15 smithi017 ceph-mon[105151]: pgmap v1701: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 37866/260619 objects degraded (14.529%) 2024-09-21T10:07:15.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:15 smithi134 ceph-mon[90085]: pgmap v1701: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 37866/260619 objects degraded (14.529%) 2024-09-21T10:07:15.751 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:15.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:15.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:15.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:15.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:15.993 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:16.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.022 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:16.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:16.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:16.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:16.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:16.263 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:16.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:16.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:16.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:16.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:16.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:07:16.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:16.580 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:16.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:16.639 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:16.669 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:07:16.669 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:07:16.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.5 'trim stale osdmaps' 2024-09-21T10:07:16.682 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:16.706 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:16.714 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:16.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:16.872 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:16.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:16.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:17.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:17.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:17.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:17.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:17.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.341 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:17.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:17.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:17.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:17.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:17.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:17 smithi017 ceph-mon[102913]: pgmap v1702: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 37477/256260 objects degraded (14.625%) 2024-09-21T10:07:17.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:17 smithi017 ceph-mon[105151]: pgmap v1702: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 37477/256260 objects degraded (14.625%) 2024-09-21T10:07:17.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:17.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:17.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:17 smithi134 ceph-mon[90085]: pgmap v1702: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 37477/256260 objects degraded (14.625%) 2024-09-21T10:07:17.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:17.863 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:17.890 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:17.991 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:18.018 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:18.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:18.120 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:18.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.150 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:18.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:18.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:18.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:18.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:18.428 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:18.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:18.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:18.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:18.668 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:18.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:18.796 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:18.822 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:18.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:18.925 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:18.955 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:19.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:19.082 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:19.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:19.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:19.210 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:19.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.212 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:19.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:19.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:19.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:19.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:19 smithi017 ceph-mon[102913]: pgmap v1703: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 36041/248631 objects degraded (14.496%) 2024-09-21T10:07:19.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:19 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 36041/248631 objects degraded (14.496%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:19.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:19 smithi017 ceph-mon[105151]: pgmap v1703: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 36041/248631 objects degraded (14.496%) 2024-09-21T10:07:19.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:19 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 36041/248631 objects degraded (14.496%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:19.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:19.473 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:19.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:19.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:19.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:19.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:19.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:19 smithi134 ceph-mon[90085]: pgmap v1703: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 36041/248631 objects degraded (14.496%) 2024-09-21T10:07:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:19 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 36041/248631 objects degraded (14.496%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:19.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:19.802 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:19.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:19.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:19.930 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:20.032 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:20.060 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:20.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:20.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:20.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:20.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:20.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:20.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:20.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:20.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:07:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:07:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:20.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:20.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:20.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:20.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:20.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:20.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:20.899 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:21.001 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:21.023 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:21.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:21.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:21.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:21.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:21.263 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:21.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:21.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:21.438 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:07:21.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:21.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.581 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:21.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:21.682 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:21 smithi017 ceph-mon[102913]: pgmap v1704: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 35270/242784 objects degraded (14.527%) 2024-09-21T10:07:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:21 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3073280819' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:21 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:21 smithi017 ceph-mon[105151]: pgmap v1704: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 35270/242784 objects degraded (14.527%) 2024-09-21T10:07:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:21 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3073280819' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:21.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:21 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:21.709 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:21 smithi134 ceph-mon[90085]: pgmap v1704: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 35270/242784 objects degraded (14.527%) 2024-09-21T10:07:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:21 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3073280819' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:21 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:21.811 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:21.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:21.939 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:21.965 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:21.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:21.980 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:07:21.980 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:22.014 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:07:22.015 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:22.050 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:07:22.051 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:22.067 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:22.092 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:22.092 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:07:22.093 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:22.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:22.094 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:07:22.133 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:07:22.133 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:22.165 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:07:22.165 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:22.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:22.196 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:07:22.196 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:22.243 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:22.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:22.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:22.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:22.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:22.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:22.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:22.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:22.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:22.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:22.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:22 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:22 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:07:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:22 smithi017 ceph-mon[102913]: osdmap e1218: 8 total, 7 up, 7 in 2024-09-21T10:07:22.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:22 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3425367082' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:22 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:22 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:07:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:22 smithi017 ceph-mon[105151]: osdmap e1218: 8 total, 7 up, 7 in 2024-09-21T10:07:22.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:22 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3425367082' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:22 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:22 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:07:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:22 smithi134 ceph-mon[90085]: osdmap e1218: 8 total, 7 up, 7 in 2024-09-21T10:07:22.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:22 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3425367082' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:22.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:22.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:22.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:22.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:22.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:22.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:23.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:23.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:23.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:23.236 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:07:23.236 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:07:23.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:23.344 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:23.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:23.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:23 smithi017 ceph-mon[102913]: pgmap v1706: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 33784/232605 objects degraded (14.524%) 2024-09-21T10:07:23.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:23 smithi017 ceph-mon[105151]: pgmap v1706: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 33784/232605 objects degraded (14.524%) 2024-09-21T10:07:23.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:23.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:23.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:23.618 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:23.646 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:23 smithi134 ceph-mon[90085]: pgmap v1706: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 33784/232605 objects degraded (14.524%) 2024-09-21T10:07:23.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.793 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:23.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:23.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:23.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:24.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:24.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:24.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:24.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:24.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:24.331 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:24.357 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:24.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:24.459 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:24.490 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:24.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:24.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:24.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:24 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 33784/232605 objects degraded (14.524%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:24.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:24 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 33784/232605 objects degraded (14.524%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:24.728 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:24 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 33784/232605 objects degraded (14.524%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:24.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:24.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:24.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:24.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.899 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:24.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:25.000 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:25.027 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:25.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:25.081 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:07:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:07:24] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-09-21T10:07:25.129 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:25.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.160 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:25.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:25.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:07:25.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:25.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:25.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:25.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:25.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:25.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:25.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:25.602 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:25.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:25.608 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-21T10:07:25.609 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-21T10:07:25.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:25 smithi017 ceph-mon[102913]: pgmap v1707: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 33352/230226 objects degraded (14.487%) 2024-09-21T10:07:25.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:25.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:25 smithi017 ceph-mon[105151]: pgmap v1707: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 33352/230226 objects degraded (14.487%) 2024-09-21T10:07:25.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:25.703 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:25.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:25.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:25 smithi134 ceph-mon[90085]: pgmap v1707: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 33352/230226 objects degraded (14.487%) 2024-09-21T10:07:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:25.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:25.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:25.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:25.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:26.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:26.101 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:26.132 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:26.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:26.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:26.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:26.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:26.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:07:26.371 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:26.401 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:26.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:26.502 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:26.527 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:26.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:26.629 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:26.660 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:26.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:26.762 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:26.791 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:26.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:26.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:27.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:27.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:27.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:27.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:27.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:27.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:27.303 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:27.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:27.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:27.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:27.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:27.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:27.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:27.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:27.715 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:27.741 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:27 smithi134 ceph-mon[90085]: pgmap v1708: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 32251/219834 objects degraded (14.671%) 2024-09-21T10:07:27.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:27.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:27.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:27 smithi017 ceph-mon[102913]: pgmap v1708: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 32251/219834 objects degraded (14.671%) 2024-09-21T10:07:27.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:27 smithi017 ceph-mon[105151]: pgmap v1708: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 32251/219834 objects degraded (14.671%) 2024-09-21T10:07:27.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:28.011 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:28.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:28.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:28.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:28.231 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:196944 8.a e1218/1200 8:58ea3d74:::benchmark_data_smithi017_209235_object64038:head v 1218'12548, mlcod=1218'12546)", 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.284677+0000", 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0026288750000000001, 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026555250000000002, 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:07:28.232 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284677+0000", 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284677+0000", 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284678+0000", 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5700000000000002e-07 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284685+0000", 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1550000000000001e-06 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284695+0000", 2024-09-21T10:07:28.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0538e-05 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284701+0000", 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9100000000000002e-06 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284750+0000", 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8741e-05 2024-09-21T10:07:28.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284773+0000", 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2710000000000001e-05 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45620.0:196927 8.f e1218/1207)", 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.284727+0000", 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0025783989999999999, 2024-09-21T10:07:28.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026572810000000001, 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284727+0000", 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284727+0000", 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284728+0000", 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8299999999999998e-07 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284730+0000", 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8669999999999998e-06 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284737+0000", 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5939999999999999e-06 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284742+0000", 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8770000000000001e-06 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45620.0:196932 8.f e1218/1207)", 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.284751+0000", 2024-09-21T10:07:28.239 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.002554859, 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026694689999999998, 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284751+0000", 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284751+0000", 2024-09-21T10:07:28.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284751+0000", 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9999999999999999e-07 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284753+0000", 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1019999999999999e-06 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284760+0000", 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5599999999999999e-06 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284764+0000", 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2339999999999996e-06 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:196927 8.f 8:fa181506:::benchmark_data_smithi017_209235_object64021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:07:28.242 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.266850+0000", 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.020455658000000002, 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020615286, 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 196927 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.266850+0000", 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.266850+0000", 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.266851+0000", 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5069999999999999e-06 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.266856+0000", 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4039999999999997e-06 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.266866+0000", 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9990000000000003e-06 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.266873+0000", 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8920000000000003e-06 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.268659+0000", 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001786588 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.268766+0000", 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010694 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.268876+0000", 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109768 2024-09-21T10:07:28.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.279785+0000", 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010908675 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:196932 8.f 8:fa9e9ee0:::benchmark_data_smithi017_209235_object64026:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.272408+0000", 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.014898003, 2024-09-21T10:07:28.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015127020999999999, 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 196932 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.272408+0000", 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.272408+0000", 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.272409+0000", 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8300000000000001e-06 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.272421+0000", 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1924e-05 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.272433+0000", 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1603999999999999e-05 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.272440+0000", 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2130000000000002e-06 2024-09-21T10:07:28.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.272508+0000", 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7455999999999999e-05 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.272631+0000", 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012324100000000001 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:07:28.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.272749+0000", 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117624 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.285462+0000", 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012713647 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:196938 8.8 8:1a94d025:::benchmark_data_smithi017_209235_object64032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.275169+0000", 2024-09-21T10:07:28.252 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.012136355999999999, 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012429826999999999, 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 196938 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.275169+0000", 2024-09-21T10:07:28.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.275169+0000", 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.275170+0000", 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.048e-06 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.275174+0000", 2024-09-21T10:07:28.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.895e-06 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.275185+0000", 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0349e-05 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.275192+0000", 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.43e-06 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:07:28.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.279946+0000", 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047543890000000004 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280063+0000", 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117019 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280254+0000", 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019058700000000001 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284722+0000", 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044677440000000001 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284763+0000", 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1340000000000001e-05 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:196940 8.0 8:0d4e0e54:::benchmark_data_smithi017_209235_object64034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.278832+0000", 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0084738939999999992, 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0088438409999999999, 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 196940 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.258 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.278832+0000", 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.278832+0000", 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.278833+0000", 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4699999999999999e-06 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.278838+0000", 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6589999999999998e-06 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.278852+0000", 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.428e-05 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.278859+0000", 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9519999999999997e-06 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.283721+0000", 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048616170000000004 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.283831+0000", 2024-09-21T10:07:28.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000110231 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5", 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.283950+0000", 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011900699999999999 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:196945 8.f 8:f5543647:::benchmark_data_smithi017_209235_object64039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.280229+0000", 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0070762780000000001, 2024-09-21T10:07:28.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075096090000000004, 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 196945 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280229+0000", 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280229+0000", 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280230+0000", 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2900000000000002e-07 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280234+0000", 2024-09-21T10:07:28.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0640000000000004e-06 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280244+0000", 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7229999999999999e-06 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280251+0000", 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1770000000000001e-06 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:07:28.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280308+0000", 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6505999999999997e-05 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280427+0000", 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119296 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.282257+0000", 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018295410000000001 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.266 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:196946 8.0 8.9f6c2560 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.280669+0000", 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0066369389999999997, 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071302240000000001, 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 196946 2024-09-21T10:07:28.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280669+0000", 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280669+0000", 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280670+0000", 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8100000000000001e-07 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280674+0000", 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0459999999999999e-06 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280684+0000", 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0168e-05 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280690+0000", 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5300000000000002e-06 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:196947 8.f 8:f8398734:::benchmark_data_smithi017_209235_object64041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.280861+0000", 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.006444773, 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069906609999999996, 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.270 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 196947 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280861+0000", 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280861+0000", 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280862+0000", 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4200000000000005e-07 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280866+0000", 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0210000000000001e-06 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280876+0000", 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8539999999999992e-06 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.280882+0000", 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1920000000000002e-06 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.285490+0000", 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046082880000000003 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.285593+0000", 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103115 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.285715+0000", 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121699 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:196948 8.8 8.927b7928 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:07:28.274 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:27.284638+0000", 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.002667146, 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032765260000000001, 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 196948 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:07:28.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284638+0000", 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284638+0000", 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284640+0000", 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.341e-06 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284644+0000", 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4009999999999999e-06 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284656+0000", 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1755e-05 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:27.284664+0000", 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.763e-06 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:07:28.277 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.278 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:07:28.278 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-21T10:07:28.278 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 11 2024-09-21T10:07:28.278 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:07:28.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:28.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:28.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:28.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:28.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:28.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:28.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:28.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.570 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:28.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:28.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:28.693 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:28.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:28.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:28.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:28.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:28.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:28.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:29.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:29.089 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:29.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:29.191 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:29.216 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:29.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:29.318 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:29.344 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:29.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:29.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:29.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:29.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:29.586 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:29.613 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:29.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:29.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:29 smithi134 ceph-mon[90085]: pgmap v1709: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 30730/211836 objects degraded (14.507%) 2024-09-21T10:07:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:29 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 30730/211836 objects degraded (14.507%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:29.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:29.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:29.856 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:29.888 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:29.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:29.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:29 smithi017 ceph-mon[102913]: pgmap v1709: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 30730/211836 objects degraded (14.507%) 2024-09-21T10:07:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:29 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 30730/211836 objects degraded (14.507%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:29 smithi017 ceph-mon[105151]: pgmap v1709: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 30730/211836 objects degraded (14.507%) 2024-09-21T10:07:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:29 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 30730/211836 objects degraded (14.507%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:29.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:30.028 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:30.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:30.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:30.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:30.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:30.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:30.425 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:30.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:30.479 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:07:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:07:30.214Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:30.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:30.567 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:30.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:30.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:30.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:30.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:30.843 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:30.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:30.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:30.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:30.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:31.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:31.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:31.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:31.256 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:31.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:31.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:31.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:31.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:31.481 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:31.513 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:31.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:31.615 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:31.641 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:31.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:31.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:31.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:31 smithi134 ceph-mon[90085]: pgmap v1710: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 30007/206229 objects degraded (14.550%) 2024-09-21T10:07:31.767 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:31.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:31.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:31.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:31 smithi017 ceph-mon[102913]: pgmap v1710: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 30007/206229 objects degraded (14.550%) 2024-09-21T10:07:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:31 smithi017 ceph-mon[105151]: pgmap v1710: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 30007/206229 objects degraded (14.550%) 2024-09-21T10:07:32.007 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:32.040 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:32.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:32.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:32.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:32.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:32.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:32.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:32.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:32.417 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:32.456 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:32.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:32.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:32.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:32.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:32.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:32.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:32.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:32.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:32.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:32.986 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:33.012 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:33.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:33.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:33.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:33.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:33.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:33.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:33.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:33.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:33.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:33.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:33.572 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:33.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:33.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:33.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:33.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:33 smithi134 ceph-mon[90085]: pgmap v1711: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s; 28656/197061 objects degraded (14.542%) 2024-09-21T10:07:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 28656/197061 objects degraded (14.542%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:33.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:33.844 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:33.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:33 smithi017 ceph-mon[102913]: pgmap v1711: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s; 28656/197061 objects degraded (14.542%) 2024-09-21T10:07:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 28656/197061 objects degraded (14.542%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:33.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:33 smithi017 ceph-mon[105151]: pgmap v1711: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s; 28656/197061 objects degraded (14.542%) 2024-09-21T10:07:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 28656/197061 objects degraded (14.542%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:33.981 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:34.082 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.115 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:34.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:34.216 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:34.251 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:34.352 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:34.387 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:34.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:34.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:34.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:34.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:34.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:07:34.636 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:34.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.664 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:34.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:34.765 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:34.789 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:34.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:34.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:34.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:34.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:07:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:07:34] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-09-21T10:07:35.048 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:07:35.049 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:07:35.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.7 'trim stale osdmaps' 2024-09-21T10:07:35.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:35.125 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:35.138 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:35.184 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:35.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:35.214 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:35.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:35.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.341 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:35.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:35.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:35.494 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:35.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:35.590 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:35 smithi017 ceph-mon[102913]: pgmap v1712: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 28292/194892 objects degraded (14.517%) 2024-09-21T10:07:35.591 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:35 smithi017 ceph-mon[105151]: pgmap v1712: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 28292/194892 objects degraded (14.517%) 2024-09-21T10:07:35.596 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:35.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.619 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:35.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:35.644 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:35 smithi134 ceph-mon[90085]: pgmap v1712: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 28292/194892 objects degraded (14.517%) 2024-09-21T10:07:35.720 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:35.751 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:35.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:35.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:35.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:36.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:36.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:36.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:36.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:36.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:36.292 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:36.320 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:36.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:36.421 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.451 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:36.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:36.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:36.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:36.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:36.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:36.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:36.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:36.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:36.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:36.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:36.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:37.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:37.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:37.122 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:37.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:37.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:37.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:36 smithi017 ceph-mon[105151]: pgmap v1713: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 27178/185010 objects degraded (14.690%) 2024-09-21T10:07:37.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:37.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:37.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:36 smithi017 ceph-mon[102913]: pgmap v1713: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 27178/185010 objects degraded (14.690%) 2024-09-21T10:07:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.150 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:37.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:36 smithi134 ceph-mon[90085]: pgmap v1713: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 27178/185010 objects degraded (14.690%) 2024-09-21T10:07:37.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:37.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:37.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:37.393 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:37.428 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:37.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:37.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:37.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:37.670 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:37.698 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:37.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:37.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:37.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:37.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:37.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:37.967 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:38.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:38.093 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:38.196 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:38.222 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:38.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:38.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:38.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:38.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:38.464 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:38.491 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:38.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:38.593 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:38.619 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:38.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:38.720 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:38.749 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:38.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:38.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:38.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:38.999 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:39.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.026 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:39.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:39.127 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:39.154 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:39.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:39.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:39.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:39.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:39.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:39.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:39.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:39.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:39.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:39.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:39.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:39 smithi017 ceph-mon[105151]: pgmap v1714: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 25799/177492 objects degraded (14.535%) 2024-09-21T10:07:39.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:39 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 25799/177492 objects degraded (14.535%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:39.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:39 smithi017 ceph-mon[102913]: pgmap v1714: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 25799/177492 objects degraded (14.535%) 2024-09-21T10:07:39.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:39 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 25799/177492 objects degraded (14.535%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:39.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:39.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:39 smithi134 ceph-mon[90085]: pgmap v1714: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 25799/177492 objects degraded (14.535%) 2024-09-21T10:07:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:39 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 25799/177492 objects degraded (14.535%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:39.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:39.866 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:39.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.012 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:40.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:40.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:40.142 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:40.244 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.275 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:40.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:40.345 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:07:40.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:40.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:40.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4046839598' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4152173014' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:07:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:07:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:40.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:40.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:40.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:40.652 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:40.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.684 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:40.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4046839598' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4152173014' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4046839598' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4152173014' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:40.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:40.816 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:40.863 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:07:40.863 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:07:40.864 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:40.865 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:40.865 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:40.918 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:40.949 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:41.051 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:41.079 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:41.082 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:41.183 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:41.211 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:41.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:41.313 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:41.338 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:41.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:41.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:41 smithi017 ceph-mon[102913]: pgmap v1715: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 25045/171789 objects degraded (14.579%) 2024-09-21T10:07:41.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:41 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:07:41.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:41 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4152173014' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:41.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:41 smithi017 ceph-mon[102913]: osdmap e1219: 8 total, 7 up, 7 in 2024-09-21T10:07:41.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:41 smithi017 ceph-mon[105151]: pgmap v1715: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 25045/171789 objects degraded (14.579%) 2024-09-21T10:07:41.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:41 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:07:41.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:41 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4152173014' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:41.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:41 smithi017 ceph-mon[105151]: osdmap e1219: 8 total, 7 up, 7 in 2024-09-21T10:07:41.439 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:41.469 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:41.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:41.571 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:41.600 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:41.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:41.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:41.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:41 smithi134 ceph-mon[90085]: pgmap v1715: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 25045/171789 objects degraded (14.579%) 2024-09-21T10:07:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:41 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:07:41.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:41 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4152173014' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:41.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:41 smithi134 ceph-mon[90085]: osdmap e1219: 8 total, 7 up, 7 in 2024-09-21T10:07:41.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:41.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:41.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:41.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:42.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:42.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:42.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:07:42.123 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:07:42.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:42.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:42.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:42.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:42.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:42.433 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:42.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:42.535 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:42.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:42.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:42.667 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:42.695 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:42.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:42.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:42.834 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:42.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:42.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:42.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:42.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:43.080 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:43.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:43.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:43.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:43.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:43.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:43.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:43.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:43.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:43.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:43 smithi017 ceph-mon[102913]: pgmap v1717: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 23420/161277 objects degraded (14.522%) 2024-09-21T10:07:43.420 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:43 smithi017 ceph-mon[105151]: pgmap v1717: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 23420/161277 objects degraded (14.522%) 2024-09-21T10:07:43.507 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:43.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.540 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:43.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:43.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:43.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:43.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:43 smithi134 ceph-mon[90085]: pgmap v1717: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 23420/161277 objects degraded (14.522%) 2024-09-21T10:07:43.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.817 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:43.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:43.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:43.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:43.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:44.060 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:44.085 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:44.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:44.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:07:44.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:44.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:44.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:44.276 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:07:44.276 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:07:44.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:44.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:44.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:44.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:44.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:44.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:44.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:44.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:44 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 23420/161277 objects degraded (14.522%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:44 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 23420/161277 objects degraded (14.522%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:44 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 23420/161277 objects degraded (14.522%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:44.769 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:44.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:44.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:44.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:44.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:44.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:44.993 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:07:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:07:44] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-09-21T10:07:45.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:45.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:45.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:45.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:07:45.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:45.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:45.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:45.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:45.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:45.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:45.491 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:45.514 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:45.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:45.618 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:45.641 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:45 smithi017 ceph-mon[102913]: pgmap v1718: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 23040/158952 objects degraded (14.495%) 2024-09-21T10:07:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:45 smithi017 ceph-mon[105151]: pgmap v1718: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 23040/158952 objects degraded (14.495%) 2024-09-21T10:07:45.743 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:45 smithi134 ceph-mon[90085]: pgmap v1718: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 23040/158952 objects degraded (14.495%) 2024-09-21T10:07:45.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:45.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:45.875 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:45.904 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:45.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:46.005 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:46.032 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:46.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:46.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:46.279 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:46.305 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:46.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:46.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:46.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:46.547 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:46.576 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:46.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:46.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:07:46.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:46.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:46.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:46.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:46.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:46.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-21T10:07:46.915 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-21T10:07:46.916 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-21T10:07:46.916 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-21T10:07:46.994 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:47.023 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:47.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:47.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:47.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:47.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:47.274 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:47.309 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:47.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:47.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:47.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:47.556 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:47.587 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:47.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:47 smithi017 ceph-mon[102913]: pgmap v1719: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 21906/149019 objects degraded (14.700%) 2024-09-21T10:07:47.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:47 smithi017 ceph-mon[105151]: pgmap v1719: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 21906/149019 objects degraded (14.700%) 2024-09-21T10:07:47.689 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:07:47.697 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:07:47.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 'trim stale osdmaps' 2024-09-21T10:07:47.713 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:47.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:47 smithi134 ceph-mon[90085]: pgmap v1719: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s; 21906/149019 objects degraded (14.700%) 2024-09-21T10:07:47.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:47.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:47.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:47.958 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:47.982 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:47.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:48.084 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:48.113 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:48.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:48.155 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:48.161 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:48.214 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:48.240 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:48.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:48.342 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:48.380 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:48.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:48.531 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:48.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:48.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:48.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:48.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:48.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.814 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:48.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:48.915 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:48.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.946 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:48.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:49.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:49.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.095 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:49.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:49.196 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.224 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:49.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:49.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:49.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:49.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:49 smithi017 ceph-mon[102913]: pgmap v1720: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 20555/141798 objects degraded (14.496%) 2024-09-21T10:07:49.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:49 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 20555/141798 objects degraded (14.496%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:49.419 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:49 smithi017 ceph-mon[105151]: pgmap v1720: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 20555/141798 objects degraded (14.496%) 2024-09-21T10:07:49.419 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:49 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 20555/141798 objects degraded (14.496%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:49.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:49.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:49.612 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:49.640 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:49.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:49.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:49.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:49 smithi134 ceph-mon[90085]: pgmap v1720: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 20555/141798 objects degraded (14.496%) 2024-09-21T10:07:49.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:49 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 20555/141798 objects degraded (14.496%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.772 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:49.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:49.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:49.912 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:49.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:49.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:50.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:50.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.046 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:50.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:50.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:50.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:50.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:50.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:50.326 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:50.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:50.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:50.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:07:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:07:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:50.565 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:50.594 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:50.696 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:50.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.727 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:50.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:50.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:50.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:50.965 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:50.994 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:51.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:51.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:51.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:51.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:51.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:51.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:51.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:51.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:51.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:51.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:51.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:51.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:51.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:51 smithi017 ceph-mon[102913]: pgmap v1721: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 19809/136251 objects degraded (14.539%) 2024-09-21T10:07:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:51 smithi017 ceph-mon[105151]: pgmap v1721: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 19809/136251 objects degraded (14.539%) 2024-09-21T10:07:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:51 smithi134 ceph-mon[90085]: pgmap v1721: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 19809/136251 objects degraded (14.539%) 2024-09-21T10:07:51.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:51.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:51.929 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:51.951 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:51.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:52.053 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:52.086 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:52.110 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:52.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:52.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:52.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:52.331 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:52.365 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:52.467 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:52.497 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:52.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:52.629 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:52.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:52.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:52.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:52.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:52.927 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:52.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:53.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:53.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:53.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:53.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:53.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:53.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:53.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:53.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:53.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:53.481 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:53.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:53.565 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:53 smithi017 ceph-mon[102913]: pgmap v1722: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s; 18299/125754 objects degraded (14.551%) 2024-09-21T10:07:53.565 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:53 smithi017 ceph-mon[105151]: pgmap v1722: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s; 18299/125754 objects degraded (14.551%) 2024-09-21T10:07:53.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:53.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:53.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:53.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:53 smithi134 ceph-mon[90085]: pgmap v1722: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s; 18299/125754 objects degraded (14.551%) 2024-09-21T10:07:53.757 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:53.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:53.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:53.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:53.996 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:54.025 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:54.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:54.127 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:54.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:54.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:54.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:54.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:54.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:54.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:54.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:54.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:54.601 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:54.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:54.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:54 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 18299/125754 objects degraded (14.551%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:54.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:54 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 18299/125754 objects degraded (14.551%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:54.704 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:54.734 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:54.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:54 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 18299/125754 objects degraded (14.551%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:54.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:54.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:54.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:54.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:07:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:07:54] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-09-21T10:07:54.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:55.016 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:55.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:55.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:55.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:55.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:55.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:55.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:55.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:55.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:55.434 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:55.536 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:55.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:55.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:55.574 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[102913]: pgmap v1723: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 17972/123663 objects degraded (14.533%) 2024-09-21T10:07:55.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:55.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:55.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:55.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:55.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[105151]: pgmap v1723: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 17972/123663 objects degraded (14.533%) 2024-09-21T10:07:55.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:55.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:55.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:55.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:55.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:55.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:55.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:55 smithi134 ceph-mon[90085]: pgmap v1723: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 17972/123663 objects degraded (14.533%) 2024-09-21T10:07:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:55.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:55.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:55.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:55.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:55.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:56.083 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:56.114 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:56.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:56.216 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:56.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:56.351 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:56.377 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:56.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:56.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:56.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:56.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:56.634 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:56.656 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:56.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:56.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:56.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:56.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:56.905 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:56.934 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:56.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:57.036 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:57.065 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:57.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:57.207 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:57.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:57.309 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:57.340 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:57.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:57.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:57.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:57.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:57.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:57.608 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:57.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:57.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[102913]: pgmap v1724: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s; 16807/113196 objects degraded (14.848%) 2024-09-21T10:07:57.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:07:57.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:07:57.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:57.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1788102249' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:57.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[105151]: pgmap v1724: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s; 16807/113196 objects degraded (14.848%) 2024-09-21T10:07:57.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:07:57.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:07:57.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:57.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1788102249' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:57.687 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:07:57.711 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:57.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:57.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:57 smithi134 ceph-mon[90085]: pgmap v1724: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s; 16807/113196 objects degraded (14.848%) 2024-09-21T10:07:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:07:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:07:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:07:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1788102249' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:57.835 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:57.861 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:57.963 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:57.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:57.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:58.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:58.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:58.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:58.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.247 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:58.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:58.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:58.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.385 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:58.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:58.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:58.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:58.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:58.626 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:58.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.654 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:58.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:58 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:58 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1788102249' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:07:58.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:58 smithi017 ceph-mon[102913]: osdmap e1220: 8 total, 7 up, 7 in 2024-09-21T10:07:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:58 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:58 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1788102249' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:07:58.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:58 smithi017 ceph-mon[105151]: osdmap e1220: 8 total, 7 up, 7 in 2024-09-21T10:07:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:58 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:58 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1788102249' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:07:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:58 smithi134 ceph-mon[90085]: osdmap e1220: 8 total, 7 up, 7 in 2024-09-21T10:07:58.755 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:58.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.781 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:58.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:58.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:58.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:58.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:59.017 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:59.038 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:59.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:59.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:59.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:59.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:59.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:59.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:59.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:59.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:59.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:59.553 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:59.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:07:59.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:59.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:59 smithi017 ceph-mon[102913]: pgmap v1726: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 15314/105126 objects degraded (14.567%) 2024-09-21T10:07:59.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:07:59 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 16807/113196 objects degraded (14.848%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:59 smithi017 ceph-mon[105151]: pgmap v1726: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 15314/105126 objects degraded (14.567%) 2024-09-21T10:07:59.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:07:59 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 16807/113196 objects degraded (14.848%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:59.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:59.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:59 smithi134 ceph-mon[90085]: pgmap v1726: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 15314/105126 objects degraded (14.567%) 2024-09-21T10:07:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:07:59 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 16807/113196 objects degraded (14.848%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:59.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:59.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:59.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:59.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:59.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:07:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:00.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:00.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:00.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:00.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:00.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:00.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:00.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:00.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:00.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:08:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:08:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:00.507 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:00.539 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:00.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:00.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:00.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:00.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:00.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.816 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:00.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:00.917 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:00.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.948 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:00.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:01.049 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:01.078 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:01.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:01.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:01.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:01.322 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:01.350 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:01.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:01 smithi017 ceph-mon[102913]: pgmap v1727: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 14576/99765 objects degraded (14.610%) 2024-09-21T10:08:01.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:01 smithi017 ceph-mon[105151]: pgmap v1727: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 14576/99765 objects degraded (14.610%) 2024-09-21T10:08:01.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:01.481 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:01.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:01.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:01.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:01.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:01.662 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-21T10:08:01.662 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-21T10:08:01.663 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-21T10:08:01.663 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-21T10:08:01.663 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-21T10:08:01.663 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-21T10:08:01.663 INFO:teuthology.orchestra.run.smithi017.stderr:bench [] [] [] [] 2024-09-21T10:08:01.663 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:08:01.663 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-21T10:08:01.664 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-21T10:08:01.664 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore allocator dump block 2024-09-21T10:08:01.664 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: invalid command 2024-09-21T10:08:01.724 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:01 smithi134 ceph-mon[90085]: pgmap v1727: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 2.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 14576/99765 objects degraded (14.610%) 2024-09-21T10:08:01.751 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:01.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:01.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:01.896 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:01.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:02.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:02.140 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:02.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:02.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:02.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:02.272 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:08:02.272 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:02.291 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:02.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:02.313 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:08:02.313 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:02.351 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:08:02.351 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:02.385 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:08:02.385 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:02.386 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:08:02.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:02.415 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:08:02.415 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:02.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:02.435 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:08:02.435 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:02.469 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:08:02.469 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:02.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:02.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:02.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:02.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:02.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3419300885' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:02.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3419300885' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:02.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3419300885' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:02.798 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:02.828 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:02.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:02.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:02.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:03.070 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.101 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:03.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:03.202 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:03.226 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:03.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:03.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:03.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:03.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:03 smithi017 ceph-mon[102913]: pgmap v1728: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 13165/90168 objects degraded (14.601%) 2024-09-21T10:08:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:03 smithi017 ceph-mon[105151]: pgmap v1728: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 13165/90168 objects degraded (14.601%) 2024-09-21T10:08:03.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:03.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:03.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:03.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:03.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:03.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:03.728 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:150250 6.11 6:88bfa6f6:::benchmark_data_smithi017_149547_object31171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:58:39.582468+0000", 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 564.14099290599995, 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.995005574, 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.729 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 150250 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582468+0000", 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582468+0000", 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582470+0000", 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5889999999999999e-06 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582475+0000", 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8679999999999998e-06 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582487+0000", 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2245e-05 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582495+0000", 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7470000000000005e-06 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582596+0000", 2024-09-21T10:08:03.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101345 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:39.582612+0000", 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5900999999999999e-05 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.572675+0000", 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99006340000000004 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.572815+0000", 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140223 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.573194+0000", 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037835199999999999 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577338+0000", 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041438459999999996 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577353+0000", 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5315000000000002e-05 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577387+0000", 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4332999999999998e-05 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577428+0000", 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0550999999999999e-05 2024-09-21T10:08:03.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:58:40.577474+0000", 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5859000000000003e-05 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:204393 6.13 6:cd3bd688:::benchmark_data_smithi017_149547_object85314:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:38.360732+0000", 2024-09-21T10:08:03.736 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 505.362729231, 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.009112421, 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 204393 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360732+0000", 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360732+0000", 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360733+0000", 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.545e-06 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360738+0000", 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5120000000000002e-06 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360747+0000", 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7350000000000002e-06 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.360753+0000", 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.827e-06 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.361569+0000", 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00081525800000000002 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.361578+0000", 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1269999999999992e-06 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.369754+0000", 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.00817614 2024-09-21T10:08:03.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.369844+0000", 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0277000000000003e-05 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.741 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:204403 6.13 6:c9e2c2d4:::benchmark_data_smithi017_149547_object85324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-21T10:08:03.742 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:38.362822+0000", 2024-09-21T10:08:03.742 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 505.36063854899999, 2024-09-21T10:08:03.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0171761130000001, 2024-09-21T10:08:03.742 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.742 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.742 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.742 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 204403 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362822+0000", 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362822+0000", 2024-09-21T10:08:03.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:03.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:03 smithi134 ceph-mon[90085]: pgmap v1728: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 13165/90168 objects degraded (14.601%) 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362823+0000", 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7599999999999996e-07 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362827+0000", 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2370000000000001e-06 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362835+0000", 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7960000000000001e-06 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.362842+0000", 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4989999999999999e-06 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.365952+0000", 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031100170000000001 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.365987+0000", 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5614999999999997e-05 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.370197+0000", 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.004209111 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.370300+0000", 2024-09-21T10:08:03.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010301 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.370725+0000", 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042504000000000001 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379813+0000", 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0090883849999999992 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379869+0000", 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6437999999999997e-05 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379913+0000", 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3151000000000001e-05 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379961+0000", 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8149000000000003e-05 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.379999+0000", 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7789e-05 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:204414 6.13 6:cbe9328b:::benchmark_data_smithi017_149547_object85335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1118)", 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T09:59:38.371877+0000", 2024-09-21T10:08:03.751 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 505.35158406900001, 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.01438549, 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 204414 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371877+0000", 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371877+0000", 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371878+0000", 2024-09-21T10:08:03.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3099999999999996e-07 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371881+0000", 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6720000000000002e-06 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371890+0000", 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0599999999999997e-06 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.371896+0000", 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6409999999999999e-06 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.375801+0000", 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039046380000000002 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:38.375810+0000", 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.448e-06 2024-09-21T10:08:03.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.376959+0000", 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.001148868 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.377067+0000", 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010764000000000001 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.377390+0000", 2024-09-21T10:08:03.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032288500000000002 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.380040+0000", 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002650857 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.386099+0000", 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006058211 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.386160+0000", 2024-09-21T10:08:03.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1499000000000002e-05 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.386219+0000", 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8955000000000001e-05 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T09:59:39.386262+0000", 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3285000000000003e-05 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.758 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.762 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:235488 6.13 6:ce0cbf8e:::benchmark_data_smithi017_149547_object116409:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-09-21T10:08:03.762 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:15.357755+0000", 2024-09-21T10:08:03.762 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 468.36570635999999, 2024-09-21T10:08:03.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.033180937, 2024-09-21T10:08:03.762 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.762 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.762 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 235488 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357755+0000", 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357755+0000", 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357755+0000", 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6800000000000001e-07 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357757+0000", 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.357e-06 2024-09-21T10:08:03.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357765+0000", 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8050000000000007e-06 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.357771+0000", 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5960000000000003e-06 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.365382+0000", 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076113500000000002 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.365391+0000", 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3559999999999993e-06 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.387335+0000", 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0219442320000001 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.387475+0000", 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014022899999999999 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.387625+0000", 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015034 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390759+0000", 2024-09-21T10:08:03.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031338189999999999 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390825+0000", 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6186999999999995e-05 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390843+0000", 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7481000000000001e-05 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390896+0000", 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3182e-05 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390936+0000", 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9535e-05 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:235503 6.13 6:cbfc1a84:::benchmark_data_smithi017_149547_object116424:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:15.377179+0000", 2024-09-21T10:08:03.769 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 468.346282358, 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.019305262, 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 235503 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377179+0000", 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377179+0000", 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377180+0000", 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.623e-06 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377184+0000", 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0369999999999996e-06 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377194+0000", 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0172e-05 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377201+0000", 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5490000000000003e-06 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377430+0000", 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022947899999999999 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.377439+0000", 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3080000000000008e-06 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.390980+0000", 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.013541246 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.391103+0000", 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012309300000000001 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.391228+0000", 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124628 2024-09-21T10:08:03.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396204+0000", 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049765 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396316+0000", 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111854 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396408+0000", 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2206000000000005e-05 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396444+0000", 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5416000000000001e-05 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396484+0000", 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0151000000000002e-05 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.776 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:235510 6.13 6:ce30ca24:::benchmark_data_smithi017_149547_object116431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:15.386849+0000", 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 468.33661220200003, 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.009715766, 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:08:03.777 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 235510 2024-09-21T10:08:03.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.778 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386849+0000", 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386849+0000", 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386850+0000", 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4699999999999999e-06 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386857+0000", 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7499999999999997e-06 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386870+0000", 2024-09-21T10:08:03.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2972999999999999e-05 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386879+0000", 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7069999999999998e-06 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386939+0000", 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0467999999999998e-05 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.386949+0000", 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.611e-06 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.393738+0000", 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.006789538 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.393858+0000", 2024-09-21T10:08:03.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119818 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.393980+0000", 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121723 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396342+0000", 2024-09-21T10:08:03.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023616980000000002 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396395+0000", 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3122999999999999e-05 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396492+0000", 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6954999999999996e-05 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396532+0000", 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9941999999999998e-05 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396565+0000", 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2990000000000001e-05 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45173.0:235513 6.13 6:ccc4fd59:::benchmark_data_smithi017_149547_object116434:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-09-21T10:08:03.785 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:00:15.387276+0000", 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 468.33618489399998, 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.015868797, 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45173", 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2163639450", 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 235513 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.786 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387276+0000", 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387276+0000", 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387277+0000", 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0419999999999999e-06 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387282+0000", 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7600000000000002e-06 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387293+0000", 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1251e-05 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387301+0000", 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9729999999999997e-06 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387888+0000", 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00058712800000000002 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:15.387897+0000", 2024-09-21T10:08:03.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6920000000000008e-06 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396614+0000", 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0087173739999999 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396730+0000", 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011608 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.396890+0000", 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015913200000000001 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.402889+0000", 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059999279999999999 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.402939+0000", 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9267000000000001e-05 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.403024+0000", 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4907999999999994e-05 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.403091+0000", 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7324999999999997e-05 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:00:16.403145+0000", 2024-09-21T10:08:03.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3937000000000002e-05 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72192 7.0 7:034f8e73:::benchmark_data_smithi017_180590_object72191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1160)", 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:54.588801+0000", 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 369.13466001799998, 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0011584870000001, 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.793 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72192 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588801+0000", 2024-09-21T10:08:03.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588801+0000", 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588803+0000", 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6559999999999999e-06 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588827+0000", 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.446e-05 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588839+0000", 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2384e-05 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.588847+0000", 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4189999999999996e-06 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.591832+0000", 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002984675 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.591840+0000", 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8179999999999993e-06 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.584201+0000", 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99236069699999996 2024-09-21T10:08:03.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.584359+0000", 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000157978 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.584534+0000", 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017445500000000001 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589781+0000", 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052477799999999996 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589837+0000", 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6205999999999997e-05 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589852+0000", 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4062e-05 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589911+0000", 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9385000000000001e-05 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.589959+0000", 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8511999999999999e-05 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.800 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:72222 7.0 7:00589716:::benchmark_data_smithi017_180590_object72221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1160)", 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:54.613264+0000", 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 369.110196745, 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98000705700000001, 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72222 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613264+0000", 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613264+0000", 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613266+0000", 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.68e-06 2024-09-21T10:08:03.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613292+0000", 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5613e-05 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613303+0000", 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1161999999999999e-05 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613310+0000", 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5850000000000002e-06 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613375+0000", 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4787000000000002e-05 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:54.613384+0000", 2024-09-21T10:08:03.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4069999999999999e-06 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.586475+0000", 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97309033099999998 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.586636+0000", 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000161075 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:08:03.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.586824+0000", 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018775399999999999 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.590000+0000", 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031762299999999999 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.590102+0000", 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010170699999999999 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.593168+0000", 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030662179999999999 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.593225+0000", 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.681e-05 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:55.593271+0000", 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6697999999999999e-05 2024-09-21T10:08:03.807 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:200312 8.b 8:d9785afd:::benchmark_data_smithi017_209235_object67406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:30.257346+0000", 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.466115252000002, 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15427818200000001, 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.808 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 200312 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.257346+0000", 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.257346+0000", 2024-09-21T10:08:03.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.257346+0000", 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1600000000000001e-07 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.257350+0000", 2024-09-21T10:08:03.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4999999999999999e-06 2024-09-21T10:08:03.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.257360+0000", 2024-09-21T10:08:03.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6800000000000005e-06 2024-09-21T10:08:03.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.257366+0000", 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.409e-06 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.400339+0000", 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.142972499 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.400467+0000", 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128918 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.400669+0000", 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020203699999999999 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.403955+0000", 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032856880000000002 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.411473+0000", 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075177630000000002 2024-09-21T10:08:03.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.411528+0000", 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5547999999999997e-05 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.411624+0000", 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5424e-05 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.814 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:200316 8.b 8:d74374a6:::benchmark_data_smithi017_209235_object67410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:30.260933+0000", 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.462528427999999, 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.156244676, 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 200316 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.260933+0000", 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.260933+0000", 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.260934+0000", 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5489999999999999e-06 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.260939+0000", 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3009999999999997e-06 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.397790+0000", 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13685054899999999 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.397810+0000", 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0219000000000001e-05 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.404388+0000", 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006577566 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.404495+0000", 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000107401 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.404620+0000", 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124428 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.410714+0000", 2024-09-21T10:08:03.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006094319 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.417040+0000", 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063263770000000002 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.417092+0000", 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1552999999999999e-05 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:30.417177+0000", 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5414000000000006e-05 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:211657 8.b 8:df98d0c0:::benchmark_data_smithi017_209235_object78751:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:39.985284+0000", 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 23.738177171, 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.045244614000000002, 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.821 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 211657 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985284+0000", 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985284+0000", 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985285+0000", 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6819999999999999e-06 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985291+0000", 2024-09-21T10:08:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3179999999999997e-06 2024-09-21T10:08:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985304+0000", 2024-09-21T10:08:03.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2768999999999999e-05 2024-09-21T10:08:03.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985311+0000", 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7419999999999997e-06 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.996332+0000", 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011020429999999999 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.996438+0000", 2024-09-21T10:08:03.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010597 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.996552+0000", 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114633 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.016461+0000", 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019908787000000001 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.030349+0000", 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013888312999999999 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.030419+0000", 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9325000000000005e-05 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.030528+0000", 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109645 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.828 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:211659 8.b 8:d44e70ab:::benchmark_data_smithi017_209235_object78753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:39.985329+0000", 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 23.738131968000001, 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.048447496999999999, 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 211659 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.829 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985329+0000", 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985329+0000", 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985330+0000", 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5300000000000004e-07 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985332+0000", 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7669999999999999e-06 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985339+0000", 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.883e-06 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.985345+0000", 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7880000000000003e-06 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.000212+0000", 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014866839999999999 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.000316+0000", 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103743 2024-09-21T10:08:03.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.000422+0000", 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106637 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.016505+0000", 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016082434 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.033639+0000", 2024-09-21T10:08:03.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017134435 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.033686+0000", 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.672e-05 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.033777+0000", 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0697000000000003e-05 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:211663 8.b 8:daae27eb:::benchmark_data_smithi017_209235_object78757:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1218)", 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:07:39.988088+0000", 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 23.735373450000001, 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.045848539000000001, 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.835 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 211663 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.988088+0000", 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.988088+0000", 2024-09-21T10:08:03.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.988089+0000", 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.252e-06 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.988093+0000", 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6940000000000003e-06 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.988109+0000", 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5519000000000002e-05 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:39.988117+0000", 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5129999999999999e-06 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.003395+0000", 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015278781999999999 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.003502+0000", 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106869 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.003626+0000", 2024-09-21T10:08:03.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012422999999999999 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.024112+0000", 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020485818999999999 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.033785+0000", 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0096727549999999999 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.033832+0000", 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7080999999999999e-05 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:07:40.033936+0000", 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010402499999999999 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:238266 8.0 8:04f364bf:::benchmark_data_smithi017_209235_object105360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1220)", 2024-09-21T10:08:03.841 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:02.383536+0000", 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.3399252429999999, 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.042488801999999999, 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 238266 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.383536+0000", 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.383536+0000", 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.383537+0000", 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4399999999999999e-07 2024-09-21T10:08:03.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.383540+0000", 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8460000000000001e-06 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.383550+0000", 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4700000000000008e-06 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.383557+0000", 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0790000000000003e-06 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.390162+0000", 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066046309999999997 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.390265+0000", 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103088 2024-09-21T10:08:03.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5", 2024-09-21T10:08:03.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.390374+0000", 2024-09-21T10:08:03.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109633 2024-09-21T10:08:03.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.422516+0000", 2024-09-21T10:08:03.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.032141550999999997 2024-09-21T10:08:03.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.422843+0000", 2024-09-21T10:08:03.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032734700000000002 2024-09-21T10:08:03.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.425913+0000", 2024-09-21T10:08:03.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030693209999999999 2024-09-21T10:08:03.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.426025+0000", 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011199199999999999 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:238272 8.b 8:d8a68142:::benchmark_data_smithi017_209235_object105366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1220)", 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:02.388805+0000", 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.3346556810000001, 2024-09-21T10:08:03.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.044493215000000003, 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 238272 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.388805+0000", 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.388805+0000", 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.388807+0000", 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7799999999999999e-06 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.388813+0000", 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7679999999999997e-06 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.388825+0000", 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2062000000000001e-05 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.388833+0000", 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6979999999999998e-06 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.388888+0000", 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5878000000000003e-05 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.389005+0000", 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116928 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.389116+0000", 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000110814 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.422478+0000", 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033361546999999998 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.432722+0000", 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010244458999999999 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.433104+0000", 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038153299999999998 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.433299+0000", 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019474800000000001 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.856 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:238276 8.7 8:e9ec2284:::benchmark_data_smithi017_209235_object105370:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1220)", 2024-09-21T10:08:03.856 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:02.391461+0000", 2024-09-21T10:08:03.856 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.332000152, 2024-09-21T10:08:03.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.043510539000000001, 2024-09-21T10:08:03.856 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.856 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:03.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:03.858 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.858 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.858 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.858 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.858 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 238276 2024-09-21T10:08:03.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.858 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.391461+0000", 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.391461+0000", 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.391462+0000", 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1370000000000001e-06 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.391466+0000", 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1570000000000002e-06 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.391475+0000", 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0140000000000004e-06 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.391480+0000", 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1279999999999999e-06 2024-09-21T10:08:03.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.391531+0000", 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0853999999999998e-05 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.391643+0000", 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111828 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:08:03.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.391747+0000", 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010394499999999999 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.424520+0000", 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.032772742000000001 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.424555+0000", 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5043000000000003e-05 2024-09-21T10:08:03.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.434879+0000", 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010324697000000001 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.434929+0000", 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9944999999999997e-05 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.434971+0000", 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2048999999999997e-05 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:238278 8.f 8:fc1072a9:::benchmark_data_smithi017_209235_object105372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1220)", 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:02.393708+0000", 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.3297532599999999, 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.050258408999999997, 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.864 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 238278 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.393708+0000", 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.393708+0000", 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.393708+0000", 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.13e-07 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.393711+0000", 2024-09-21T10:08:03.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.813e-06 2024-09-21T10:08:03.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.393718+0000", 2024-09-21T10:08:03.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2540000000000003e-06 2024-09-21T10:08:03.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.393724+0000", 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7100000000000004e-06 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.394825+0000", 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00110115 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.394930+0000", 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010488399999999999 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.395038+0000", 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010836999999999999 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.434985+0000", 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.039946158000000002 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.443752+0000", 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087671199999999998 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.443806+0000", 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4559999999999999e-05 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.443966+0000", 2024-09-21T10:08:03.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000159877 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:238280 8.b 8:d715bef4:::benchmark_data_smithi017_209235_object105374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1220)", 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:02.395106+0000", 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.3283552949999999, 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.047732205999999999, 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:08:03.871 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 238280 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.395106+0000", 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.395106+0000", 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:08:03.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.395107+0000", 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.347e-06 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.395112+0000", 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.476e-06 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.422294+0000", 2024-09-21T10:08:03.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027182293999999999 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.422304+0000", 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0446e-05 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.422508+0000", 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020340000000000001 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.422797+0000", 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028940500000000001 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.422922+0000", 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012538800000000001 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.432002+0000", 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0090797010000000008 2024-09-21T10:08:03.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.442584+0000", 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010581770000000001 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.442741+0000", 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015692700000000001 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:02.442838+0000", 2024-09-21T10:08:03.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7052000000000005e-05 2024-09-21T10:08:03.877 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.877 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.877 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.877 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:08:03.877 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:08:03.877 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:08:03.959 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:03.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:04.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:04.125 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:04.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:04.226 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:04.256 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:04.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:04.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:04.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:04.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:04.528 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:04.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:04.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:04.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:04.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:04 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 13165/90168 objects degraded (14.601%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:04.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:04 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 13165/90168 objects degraded (14.601%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:04.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:04 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 13165/90168 objects degraded (14.601%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:04.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:04.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.806 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:04.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:04.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:04.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:04.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:04.980 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:08:04.980 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:08:05.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:05.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:05.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:05.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:08:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:08:04] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-09-21T10:08:05.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:05.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:05.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:08:05.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:05.347 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:05.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:05.449 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:05.473 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:05.574 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:05.605 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:05.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:05 smithi017 ceph-mon[102913]: pgmap v1729: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 12815/87993 objects degraded (14.564%) 2024-09-21T10:08:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:05 smithi017 ceph-mon[105151]: pgmap v1729: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 12815/87993 objects degraded (14.564%) 2024-09-21T10:08:05.706 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:05.739 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:05.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:05 smithi134 ceph-mon[90085]: pgmap v1729: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 12815/87993 objects degraded (14.564%) 2024-09-21T10:08:05.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:05.868 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:08:05.868 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:08:05.870 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:05.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:05.972 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:06.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.006 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:06.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:06.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:06.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:06.246 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:06.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.279 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:06.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:06.338 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:08:06.362 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:06.380 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:06.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:06.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:06.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:06.667 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:06.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:06.755 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:06.821 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:06.853 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:06.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:06.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:06.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:06.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:07.089 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:07.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:07.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:07.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:07.256 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:07.358 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:07.393 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:07.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:07.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:07.525 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:07.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:07.626 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:07.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:07.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:07.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:07 smithi017 ceph-mon[102913]: pgmap v1730: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 11656/77736 objects degraded (14.994%) 2024-09-21T10:08:07.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:07 smithi017 ceph-mon[105151]: pgmap v1730: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 11656/77736 objects degraded (14.994%) 2024-09-21T10:08:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:07 smithi134 ceph-mon[90085]: pgmap v1730: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 11656/77736 objects degraded (14.994%) 2024-09-21T10:08:07.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:07.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:07.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:08:07.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:07.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:07.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:08.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:08.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:08.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:08.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:08.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:08.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:08.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:08.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:08.481 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:08.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.513 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:08.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:08.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:08.643 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:08.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:08.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:08.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:08.870 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:08:08.870 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:08:08.870 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.870 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:245922 8.1 e1220/1200 8:83d3c7ea:::benchmark_data_smithi017_209235_object113016:head v 1220'15446, mlcod=1220'15444)", 2024-09-21T10:08:08.870 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:08.863578+0000", 2024-09-21T10:08:08.870 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0029698020000000001, 2024-09-21T10:08:08.870 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029980950000000001, 2024-09-21T10:08:08.870 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863578+0000", 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863578+0000", 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.871 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863577+0000", 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863582+0000", 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2669999999999997e-06 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863592+0000", 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1849999999999993e-06 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.872 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863596+0000", 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4909999999999999e-06 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863649+0000", 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2945000000000002e-05 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863669+0000", 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0129e-05 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:245920 8.0 e1220/1207 8:0a75ef9e:::benchmark_data_smithi017_209235_object113014:head v 1220'15164, mlcod=1220'15162)", 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:08.865506+0000", 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0010411540000000001, 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001130556, 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:08.874 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.865506+0000", 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.865506+0000", 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.865506+0000", 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:08.875 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.865513+0000", 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9260000000000003e-06 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.865525+0000", 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2604e-05 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:08.876 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.865530+0000", 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2530000000000004e-06 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.865608+0000", 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.7514000000000004e-05 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.865628+0000", 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.997e-05 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:08.877 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:245918 8.a e1220/1200 8:56fde1e2:::benchmark_data_smithi017_209235_object113012:head v 1220'15672, mlcod=1220'15670)", 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:08.858897+0000", 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.007650408, 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0077966590000000001, 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.858897+0000", 2024-09-21T10:08:08.878 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.858897+0000", 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.858896+0000", 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.879 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.858903+0000", 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7989999999999996e-06 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.858915+0000", 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1802e-05 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.858921+0000", 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9349999999999999e-06 2024-09-21T10:08:08.880 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.858986+0000", 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5090000000000002e-05 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.859007+0000", 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0618000000000001e-05 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.881 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:245923 8.9 e1220/1200 8:98c1407f:::benchmark_data_smithi017_209235_object113017:head v 1220'15701, mlcod=1220'15699)", 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:08.863289+0000", 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.003258103, 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0034573730000000001, 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863289+0000", 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.882 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863289+0000", 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863289+0000", 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863295+0000", 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0990000000000004e-06 2024-09-21T10:08:08.883 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863310+0000", 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5528000000000001e-05 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863316+0000", 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4070000000000001e-06 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:08.884 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863389+0000", 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3638000000000002e-05 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863407+0000", 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7524999999999999e-05 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:245921 8.a e1220/1200 8:51284d6b:::benchmark_data_smithi017_209235_object113015:head v 1220'15673, mlcod=1220'15670)", 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:08.863573+0000", 2024-09-21T10:08:08.885 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0029741939999999999, 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003227377, 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863573+0000", 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863573+0000", 2024-09-21T10:08:08.886 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863573+0000", 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863579+0000", 2024-09-21T10:08:08.887 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7420000000000003e-06 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863589+0000", 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0253e-05 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863594+0000", 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9860000000000002e-06 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:08.888 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863646+0000", 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2509000000000002e-05 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:08.863663+0000", 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6974999999999999e-05 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: ], 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 5 2024-09-21T10:08:08.889 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:08:08.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:08.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:08.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:09.026 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:09.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:09.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:09.182 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:09.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:09.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:09.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:09.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:09.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:09.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:09.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:09.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:09.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:09.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:09 smithi017 ceph-mon[105151]: pgmap v1731: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 10188/69882 objects degraded (14.579%) 2024-09-21T10:08:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:09 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 10188/69882 objects degraded (14.579%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:09 smithi017 ceph-mon[102913]: pgmap v1731: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 10188/69882 objects degraded (14.579%) 2024-09-21T10:08:09.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:09 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 10188/69882 objects degraded (14.579%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:09.703 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:09 smithi134 ceph-mon[90085]: pgmap v1731: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 10188/69882 objects degraded (14.579%) 2024-09-21T10:08:09.703 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:09 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 10188/69882 objects degraded (14.579%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:09.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:09.736 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:08:09.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:09.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:09.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:09.977 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:09.981 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 6] out_osds: [1] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2024-09-21T10:08:09.981 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:08:09.981 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2024-09-21T10:08:09.981 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2024-09-21T10:08:09.982 DEBUG:teuthology.orchestra.run.smithi134:osd.4> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.4.service 2024-09-21T10:08:09.984 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl start ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.4 2024-09-21T10:08:10.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:10.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:10.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:10.088 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:10.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:10.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.206 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:10.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:10.307 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:10.332 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:10.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:10.384 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:08:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:08:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:10.384 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:10 smithi134 systemd[1]: Starting Ceph osd.4 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:08:10.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:10.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:10.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:10.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:10.614 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:10.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:10.716 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:10.743 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:10.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:10.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:10.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:10.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:10.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:10.994 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:10 smithi134 podman[157654]: 2024-09-21 10:08:10.590687176 +0000 UTC m=+0.108075351 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T10:08:11.026 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:11.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:11.128 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.160 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:11.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:11.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:11.296 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:08:11.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:11.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:11.406 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:11.438 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:11.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:11.528 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:11 smithi134 podman[157654]: 2024-09-21 10:08:11.376309989 +0000 UTC m=+0.893698136 container create cc74485b5381a23a9ae9cb6cf8af787808f05d7b1d89cb6c4079b27fda4cc8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:08:11.528 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:11.528 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:11.528 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:11 smithi134 ceph-mon[90085]: pgmap v1732: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 9429/64281 objects degraded (14.668%) 2024-09-21T10:08:11.529 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:11 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/361231874' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:11.529 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:11 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:11.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:11.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:11.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:11.685 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[102913]: pgmap v1732: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 9429/64281 objects degraded (14.668%) 2024-09-21T10:08:11.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/361231874' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:11.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[105151]: pgmap v1732: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 9429/64281 objects degraded (14.668%) 2024-09-21T10:08:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/361231874' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:11 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:11.716 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:11.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:11.818 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:11.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.851 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:11.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:11.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:11.988 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:11 smithi134 podman[157654]: 2024-09-21 10:08:11.671151068 +0000 UTC m=+1.188539232 container init cc74485b5381a23a9ae9cb6cf8af787808f05d7b1d89cb6c4079b27fda4cc8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate, ceph=True, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T10:08:11.989 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:11 smithi134 podman[157654]: 2024-09-21 10:08:11.679205378 +0000 UTC m=+1.196593524 container start cc74485b5381a23a9ae9cb6cf8af787808f05d7b1d89cb6c4079b27fda4cc8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate, ceph=True, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:08:11.989 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:11 smithi134 podman[157654]: 2024-09-21 10:08:11.779628213 +0000 UTC m=+1.297016359 container attach cc74485b5381a23a9ae9cb6cf8af787808f05d7b1d89cb6c4079b27fda4cc8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T10:08:11.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:11.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:12.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:12.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:12.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:12.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate[157735]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T10:08:12.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 bash[157654]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T10:08:12.245 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate[157735]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:08:12.245 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 bash[157654]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:08:12.245 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate[157735]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:08:12.245 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 bash[157654]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:08:12.245 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate[157735]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:08:12.245 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 bash[157654]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:08:12.245 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate[157735]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T10:08:12.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 bash[157654]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T10:08:12.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate[157735]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T10:08:12.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 bash[157654]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T10:08:12.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate[157735]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T10:08:12.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 bash[157654]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T10:08:12.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:12.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:12.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:12.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:12.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:12.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:12.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:12.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:12 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:12.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:12 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:12.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:12 smithi017 ceph-mon[102913]: osdmap e1221: 8 total, 7 up, 7 in 2024-09-21T10:08:12.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:12 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:12.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:12 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:12.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:12 smithi017 ceph-mon[105151]: osdmap e1221: 8 total, 7 up, 7 in 2024-09-21T10:08:12.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:12.673 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:12.673 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:12.673 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:12 smithi134 ceph-mon[90085]: osdmap e1221: 8 total, 7 up, 7 in 2024-09-21T10:08:12.673 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:12 smithi134 podman[157654]: 2024-09-21 10:08:12.321128644 +0000 UTC m=+1.838516794 container died cc74485b5381a23a9ae9cb6cf8af787808f05d7b1d89cb6c4079b27fda4cc8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:08:12.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:12.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:12.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:12.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:12.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:12.946 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:12.973 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:12.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:13.002 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:08:13.003 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:13.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:08:13.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:13.072 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:08:13.072 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:13.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:13.110 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:08:13.111 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:13.111 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:08:13.116 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:13.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:13.124 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:08:13.124 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:08:13.124 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:08:13.124 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:08:13.124 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:08:13.143 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:08:13.143 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:13.176 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:08:13.177 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:13.212 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:08:13.212 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:13.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:13.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:13.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:13.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:13.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:13.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:13 smithi017 ceph-mon[102913]: pgmap v1734: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 7978/54597 objects degraded (14.613%) 2024-09-21T10:08:13.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:13 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4175555704' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:13 smithi017 ceph-mon[105151]: pgmap v1734: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 7978/54597 objects degraded (14.613%) 2024-09-21T10:08:13.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:13 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4175555704' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:13.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:13.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:13.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:13.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:13.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:13.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:13 smithi134 ceph-mon[90085]: pgmap v1734: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 7978/54597 objects degraded (14.613%) 2024-09-21T10:08:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:13 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4175555704' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:13.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:13.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:13.861 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:08:13.924 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:13.952 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:13.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:14.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:14.090 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:14.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:14.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:14.236 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:14.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:14.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:08:14.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:14.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:14.426 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:14.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:14.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:14.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:14.563 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:14 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7978/54597 objects degraded (14.613%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:14.563 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:14 smithi134 podman[157910]: 2024-09-21 10:08:14.292964829 +0000 UTC m=+2.142791999 container remove cc74485b5381a23a9ae9cb6cf8af787808f05d7b1d89cb6c4079b27fda4cc8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T10:08:14.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:14 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7978/54597 objects degraded (14.613%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:14.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:14 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7978/54597 objects degraded (14.613%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:14.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:14.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:14.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:14.666 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:08:14.666 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:08:14.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:14.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:14.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:14.843 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:14.907 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:14.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.939 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:14.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:08:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:08:14] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.51.0" 2024-09-21T10:08:14.994 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:14 smithi134 podman[158128]: 2024-09-21 10:08:14.485133346 +0000 UTC m=+0.026991477 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T10:08:15.040 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:15.067 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:15.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:15.169 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:15.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:15.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:15.309 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:14 smithi134 podman[158128]: 2024-09-21 10:08:14.994378149 +0000 UTC m=+0.536236271 container create f5ea068205091e9651900bcbecc0fd263ea3b54e86c7837d784fcdca32b3a7e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T10:08:15.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.343 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:15.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:15.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:15.488 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:15.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:15.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:08:15.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:15 smithi134 ceph-mon[90085]: pgmap v1735: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.1 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 7596/52305 objects degraded (14.523%) 2024-09-21T10:08:15.574 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:15 smithi134 podman[158128]: 2024-09-21 10:08:15.306567188 +0000 UTC m=+0.848425323 container init f5ea068205091e9651900bcbecc0fd263ea3b54e86c7837d784fcdca32b3a7e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-21T10:08:15.574 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:15 smithi134 podman[158128]: 2024-09-21 10:08:15.314868456 +0000 UTC m=+0.856726575 container start f5ea068205091e9651900bcbecc0fd263ea3b54e86c7837d784fcdca32b3a7e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T10:08:15.574 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:15 smithi134 ceph-osd[158247]: -- 172.21.15.134:0/3702078582 <== mon.2 v2:172.21.15.17:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x55a77c1c7340 con 0x55a77c04f400 2024-09-21T10:08:15.574 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:15 smithi134 ceph-osd[158247]: -- 172.21.15.134:0/3702078582 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a77c07cd00 con 0x55a77c04f400 2024-09-21T10:08:15.574 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:15 smithi134 bash[158128]: f5ea068205091e9651900bcbecc0fd263ea3b54e86c7837d784fcdca32b3a7e5 2024-09-21T10:08:15.574 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:15 smithi134 systemd[1]: Started Ceph osd.4 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:08:15.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:15.633 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:15.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:15 smithi017 ceph-mon[102913]: pgmap v1735: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.1 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 7596/52305 objects degraded (14.523%) 2024-09-21T10:08:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:15 smithi017 ceph-mon[105151]: pgmap v1735: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 1.1 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 7596/52305 objects degraded (14.523%) 2024-09-21T10:08:15.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:15.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:15.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:15.872 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:15.907 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:16.008 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:16.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.037 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:16.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:16.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:16.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:16.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:16.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:08:15.872+0000 7f9d64d34440 -1 Falling back to public interface 2024-09-21T10:08:16.280 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:16.309 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:16.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:16.410 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:16.438 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:16.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:16.565 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:16.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:16.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:08:16.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:16.808 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:16.836 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:16.918 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:16.938 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:16.975 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:16.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:17.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:17.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:17.150 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:17.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:17.260 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:17.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:17.361 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:17.379 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:08:17.390 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:17.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:17.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:17.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:17.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:17 smithi134 ceph-mon[90085]: pgmap v1736: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 885 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s; 6417/42513 objects degraded (15.094%) 2024-09-21T10:08:17.574 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3255748225' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:17.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:17.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:17 smithi017 ceph-mon[102913]: pgmap v1736: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 885 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s; 6417/42513 objects degraded (15.094%) 2024-09-21T10:08:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3255748225' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:17 smithi017 ceph-mon[105151]: pgmap v1736: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 885 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s; 6417/42513 objects degraded (15.094%) 2024-09-21T10:08:17.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3255748225' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:17.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:17.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:17.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:17.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:17.951 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:17.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:18.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:18.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:18.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:18.175 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:18.175 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:18.194 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:18.230 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:18.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:18.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:18.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:18.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:18.473 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.503 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:18.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:18.596 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:18 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:18.597 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:18 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3255748225' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:08:18.597 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:18 smithi134 ceph-mon[90085]: osdmap e1222: 8 total, 7 up, 7 in 2024-09-21T10:08:18.604 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.644 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:18.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:18 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:18 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3255748225' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:08:18.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:18 smithi017 ceph-mon[102913]: osdmap e1222: 8 total, 7 up, 7 in 2024-09-21T10:08:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:18 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:18 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3255748225' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:08:18.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:18 smithi017 ceph-mon[105151]: osdmap e1222: 8 total, 7 up, 7 in 2024-09-21T10:08:18.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:18.783 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:18.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:18.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:18.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:19.019 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:19.047 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:19.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:19.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:19.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:19.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:19.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:19.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:19.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:19.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:19.413 INFO:teuthology.orchestra.run.smithi134.stderr:no valid command found; 10 closest matches: 2024-09-21T10:08:19.413 INFO:teuthology.orchestra.run.smithi134.stderr:0 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:1 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:2 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:abort 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:assert 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:bluefs files list 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:bluefs stats 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:bluestore bluefs device info [] 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:config diff 2024-09-21T10:08:19.414 INFO:teuthology.orchestra.run.smithi134.stderr:admin_socket: invalid command 2024-09-21T10:08:19.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:19.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:19.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:19.566 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:19.668 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:19 smithi017 ceph-mon[102913]: pgmap v1738: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 732 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 5035/35199 objects degraded (14.304%) 2024-09-21T10:08:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:19 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5035/35199 objects degraded (14.304%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:19 smithi017 ceph-mon[105151]: pgmap v1738: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 732 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 5035/35199 objects degraded (14.304%) 2024-09-21T10:08:19.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:19 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5035/35199 objects degraded (14.304%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:19.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.701 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:19.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:19 smithi134 ceph-mon[90085]: pgmap v1738: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 732 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s; 5035/35199 objects degraded (14.304%) 2024-09-21T10:08:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:19 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5035/35199 objects degraded (14.304%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:19.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:19.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:19.925 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:08:19.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:19.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:20.062 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.096 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:20.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:20.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:08:20.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:20.216 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:08:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:08:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:20.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:20.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:20.350 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:20.372 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:20.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:20.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:20.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:20.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:20.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:20.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:20.752 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:20.786 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:20.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:20.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:20.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:21.024 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:08:21.039 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2024-09-21T10:08:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.066 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:21.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:21.118 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:21.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:21.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:21.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:21.291 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:21.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:21.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:21.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:21.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:21.559 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:21.582 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:21.644 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:21 smithi134 ceph-mon[90085]: pgmap v1739: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 617 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 4314/29670 objects degraded (14.540%) 2024-09-21T10:08:21.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:21.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:21 smithi017 ceph-mon[102913]: pgmap v1739: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 617 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 4314/29670 objects degraded (14.540%) 2024-09-21T10:08:21.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:21 smithi017 ceph-mon[105151]: pgmap v1739: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 617 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 4314/29670 objects degraded (14.540%) 2024-09-21T10:08:21.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:21.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:21.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:21.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:21.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:21.952 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:21.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:21.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:22.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:22.144 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:22.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:22.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:22.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:22.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:22.447 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:22.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:22.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.569 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:22.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:22.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:22.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:22.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:22.744 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:08:22.597+0000 7f9d64d34440 -1 osd.4 1206 log_to_monitors true 2024-09-21T10:08:22.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:22.839 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:22.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:22.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:22.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:22.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:23.060 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:08:23.060 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:23.079 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:23.104 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:08:23.104 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:23.106 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:23.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:23.139 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:08:23.139 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:23.175 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:08:23.176 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:23.176 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-21T10:08:23.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:23.212 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:08:23.212 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:23.246 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:08:23.246 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:23.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:23.282 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:08:23.282 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:23.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:23.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:23.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:23.412 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:23 smithi134 ceph-mon[90085]: pgmap v1740: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 419 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 2943/20178 objects degraded (14.585%) 2024-09-21T10:08:23.412 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:23 smithi134 ceph-mon[90085]: from='osd.4 [v2:172.21.15.134:6800/1500230278,v1:172.21.15.134:6801/1500230278]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:08:23.412 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:23 smithi134 ceph-mon[90085]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:08:23.412 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:23 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3870355538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:262625 8.9 8:9c978548:::benchmark_data_smithi017_209235_object129719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1223)", 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:23.445755+0000", 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.00034689600000000002, 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00037553400000000001, 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:08:23.449 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 262625 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445755+0000", 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:23.450 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445755+0000", 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445753+0000", 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445760+0000", 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8140000000000003e-06 2024-09-21T10:08:23.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445778+0000", 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7289e-05 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445784+0000", 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.742e-06 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:23.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445843+0000", 2024-09-21T10:08:23.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8706999999999999e-05 2024-09-21T10:08:23.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445961+0000", 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011818400000000001 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,5", 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.446070+0000", 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000108341 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:262628 8.9 8.2fb96489 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1223)", 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:23.445820+0000", 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.00028206500000000002, 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000390706, 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "queued for pg", 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 262628 2024-09-21T10:08:23.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445820+0000", 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445820+0000", 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445819+0000", 2024-09-21T10:08:23.456 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445823+0000", 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0609999999999997e-06 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445833+0000", 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5990000000000008e-06 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.457 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:23.458 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.445838+0000", 2024-09-21T10:08:23.458 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0520000000000001e-06 2024-09-21T10:08:23.458 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:23.458 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:23.458 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:23.458 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.458 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.458 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:262620 8.3 8:cfacf938:::benchmark_data_smithi017_209235_object129714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1223)", 2024-09-21T10:08:23.458 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:23.440611+0000", 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0054907569999999998, 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0056323290000000002, 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 262620 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.440611+0000", 2024-09-21T10:08:23.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.440611+0000", 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.440610+0000", 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:23.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.440616+0000", 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0329999999999997e-06 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.440626+0000", 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5349999999999993e-06 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.440631+0000", 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7960000000000001e-06 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:23.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.440678+0000", 2024-09-21T10:08:23.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6461999999999997e-05 2024-09-21T10:08:23.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:23.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.440808+0000", 2024-09-21T10:08:23.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013014799999999999 2024-09-21T10:08:23.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.440943+0000", 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000135 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.444185+0000", 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003241767 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:23.444299+0000", 2024-09-21T10:08:23.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011458799999999999 2024-09-21T10:08:23.464 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:23.464 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:23.464 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:23.464 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:23.464 INFO:teuthology.orchestra.run.smithi134.stdout: ], 2024-09-21T10:08:23.464 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 3 2024-09-21T10:08:23.464 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:08:23.489 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:23.518 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:23.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:23.620 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:23.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.664 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:23.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:23.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:23 smithi017 ceph-mon[102913]: pgmap v1740: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 419 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 2943/20178 objects degraded (14.585%) 2024-09-21T10:08:23.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:23 smithi017 ceph-mon[102913]: from='osd.4 [v2:172.21.15.134:6800/1500230278,v1:172.21.15.134:6801/1500230278]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:08:23.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:23 smithi017 ceph-mon[102913]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:08:23.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:23 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3870355538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:23.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:23 smithi017 ceph-mon[105151]: pgmap v1740: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 419 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s; 2943/20178 objects degraded (14.585%) 2024-09-21T10:08:23.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:23 smithi017 ceph-mon[105151]: from='osd.4 [v2:172.21.15.134:6800/1500230278,v1:172.21.15.134:6801/1500230278]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:08:23.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:23 smithi017 ceph-mon[105151]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:08:23.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:23 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3870355538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:23.744 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:08:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:08:23.432+0000 7f9d5a2db640 -1 osd.4 1206 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:08:23.765 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:23.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:23.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:23.933 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:24.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:24.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:24.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:24.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:24.201 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:08:24.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:24.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:24.328 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:24.369 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:24.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:24.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:24.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:24.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:24.496 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:24 smithi134 ceph-mon[90085]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T10:08:24.496 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:24 smithi134 ceph-mon[90085]: osdmap e1223: 8 total, 7 up, 7 in 2024-09-21T10:08:24.496 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:24 smithi134 ceph-mon[90085]: from='osd.4 [v2:172.21.15.134:6800/1500230278,v1:172.21.15.134:6801/1500230278]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]} : dispatch 2024-09-21T10:08:24.496 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:24 smithi134 ceph-mon[90085]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]} : dispatch 2024-09-21T10:08:24.496 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:24 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 2943/20178 objects degraded (14.585%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:24.538 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:24.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:24.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:24.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:24.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[102913]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T10:08:24.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[102913]: osdmap e1223: 8 total, 7 up, 7 in 2024-09-21T10:08:24.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[102913]: from='osd.4 [v2:172.21.15.134:6800/1500230278,v1:172.21.15.134:6801/1500230278]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]} : dispatch 2024-09-21T10:08:24.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[102913]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]} : dispatch 2024-09-21T10:08:24.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 2943/20178 objects degraded (14.585%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:24.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[105151]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T10:08:24.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[105151]: osdmap e1223: 8 total, 7 up, 7 in 2024-09-21T10:08:24.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[105151]: from='osd.4 [v2:172.21.15.134:6800/1500230278,v1:172.21.15.134:6801/1500230278]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]} : dispatch 2024-09-21T10:08:24.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[105151]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]} : dispatch 2024-09-21T10:08:24.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 2943/20178 objects degraded (14.585%), 16 pgs degraded, 49 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:24.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:24.743 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:24.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:24.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:24.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:24.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:24.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:08:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:08:24] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.51.0" 2024-09-21T10:08:24.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:24.975 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:08:24.975 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:08:24.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:24.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:25.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:25.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.115 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:25.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:25.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:25.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:25.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:25.343 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:25.369 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:25.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:25.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:25.494 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:25.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:25.506 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[102913]: pgmap v1742: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 373 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 2549/17970 objects degraded (14.185%) 2024-09-21T10:08:25.506 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[102913]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:08:25.506 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[102913]: osd.4 [v2:172.21.15.134:6800/1500230278,v1:172.21.15.134:6801/1500230278] boot 2024-09-21T10:08:25.506 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[102913]: osdmap e1224: 8 total, 8 up, 7 in 2024-09-21T10:08:25.506 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T10:08:25.506 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:25.506 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:25.507 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[105151]: pgmap v1742: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 373 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 2549/17970 objects degraded (14.185%) 2024-09-21T10:08:25.507 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[105151]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:08:25.507 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[105151]: osd.4 [v2:172.21.15.134:6800/1500230278,v1:172.21.15.134:6801/1500230278] boot 2024-09-21T10:08:25.507 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[105151]: osdmap e1224: 8 total, 8 up, 7 in 2024-09-21T10:08:25.507 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T10:08:25.507 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:25.507 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:25.597 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:25.618 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:25.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:25.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:25.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:25 smithi134 ceph-mon[90085]: pgmap v1742: 109 pgs: 33 active+undersized, 16 active+undersized+degraded, 60 active+clean; 373 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 2549/17970 objects degraded (14.185%) 2024-09-21T10:08:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:25 smithi134 ceph-mon[90085]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:08:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:25 smithi134 ceph-mon[90085]: osd.4 [v2:172.21.15.134:6800/1500230278,v1:172.21.15.134:6801/1500230278] boot 2024-09-21T10:08:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:25 smithi134 ceph-mon[90085]: osdmap e1224: 8 total, 8 up, 7 in 2024-09-21T10:08:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T10:08:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:25.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:25.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:25.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:25.968 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:25.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.991 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:25.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:26.040 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:08:26.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:26.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:26.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:26.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:26.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:26.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:26.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:26.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:26.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:26.421 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:08:26.467 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:26.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.502 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:26.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:26.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:26.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:26.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[102913]: osdmap e1225: 8 total, 8 up, 7 in 2024-09-21T10:08:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3253738001' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[102913]: osdmap e1226: 8 total, 8 up, 7 in 2024-09-21T10:08:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[105151]: osdmap e1225: 8 total, 8 up, 7 in 2024-09-21T10:08:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3253738001' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:08:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:26 smithi017 ceph-mon[105151]: osdmap e1226: 8 total, 8 up, 7 in 2024-09-21T10:08:26.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:26 smithi134 ceph-mon[90085]: osdmap e1225: 8 total, 8 up, 7 in 2024-09-21T10:08:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:26 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3253738001' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:26 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:26 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:08:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:26 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:26 smithi134 ceph-mon[90085]: osdmap e1226: 8 total, 8 up, 7 in 2024-09-21T10:08:26.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:26.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:26.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:26.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:26.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:26.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:08:27.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:27.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:27.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:27.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:27.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:27.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:27.320 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:27.351 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:27.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:27.396 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:27.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:27.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:27.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:27.533 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: 1.12 scrub starts 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: 1.12 scrub ok 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: pgmap v1745: 109 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 5 peering, 20 active+undersized, 9 active+undersized+degraded, 73 active+clean; 220 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 1696/10620 objects degraded (15.970%) 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: 8.0 starting backfill to osd.4 from (0'0,1206'3745] MAX to 1223'16268 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: 8.2 starting backfill to osd.4 from (0'0,1206'3786] MAX to 1223'16423 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: 8.5 starting backfill to osd.4 from (0'0,1206'3745] MAX to 1223'16307 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: 8.4 starting backfill to osd.4 from (0'0,1206'3871] MAX to 1223'16446 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: 8.f starting backfill to osd.4 from (0'0,1206'3767] MAX to 1223'16217 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: 8.e starting backfill to osd.4 from (0'0,1206'3837] MAX to 1223'16907 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[102913]: 8.b starting backfill to osd.4 from (0'0,1206'3801] MAX to 1223'16495 2024-09-21T10:08:27.534 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: 1.12 scrub starts 2024-09-21T10:08:27.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: 1.12 scrub ok 2024-09-21T10:08:27.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: pgmap v1745: 109 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 5 peering, 20 active+undersized, 9 active+undersized+degraded, 73 active+clean; 220 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 1696/10620 objects degraded (15.970%) 2024-09-21T10:08:27.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: 8.0 starting backfill to osd.4 from (0'0,1206'3745] MAX to 1223'16268 2024-09-21T10:08:27.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: 8.2 starting backfill to osd.4 from (0'0,1206'3786] MAX to 1223'16423 2024-09-21T10:08:27.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: 8.5 starting backfill to osd.4 from (0'0,1206'3745] MAX to 1223'16307 2024-09-21T10:08:27.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: 8.4 starting backfill to osd.4 from (0'0,1206'3871] MAX to 1223'16446 2024-09-21T10:08:27.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: 8.f starting backfill to osd.4 from (0'0,1206'3767] MAX to 1223'16217 2024-09-21T10:08:27.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: 8.e starting backfill to osd.4 from (0'0,1206'3837] MAX to 1223'16907 2024-09-21T10:08:27.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:27 smithi017 ceph-mon[105151]: 8.b starting backfill to osd.4 from (0'0,1206'3801] MAX to 1223'16495 2024-09-21T10:08:27.604 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:27.633 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:27.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:27.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: 1.12 scrub starts 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: 1.12 scrub ok 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: pgmap v1745: 109 pgs: 1 remapped+peering, 1 active+clean+scrubbing, 5 peering, 20 active+undersized, 9 active+undersized+degraded, 73 active+clean; 220 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 1696/10620 objects degraded (15.970%) 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: 8.0 starting backfill to osd.4 from (0'0,1206'3745] MAX to 1223'16268 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: 8.2 starting backfill to osd.4 from (0'0,1206'3786] MAX to 1223'16423 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: 8.5 starting backfill to osd.4 from (0'0,1206'3745] MAX to 1223'16307 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: 8.4 starting backfill to osd.4 from (0'0,1206'3871] MAX to 1223'16446 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: 8.f starting backfill to osd.4 from (0'0,1206'3767] MAX to 1223'16217 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: 8.e starting backfill to osd.4 from (0'0,1206'3837] MAX to 1223'16907 2024-09-21T10:08:27.743 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:27 smithi134 ceph-mon[90085]: 8.b starting backfill to osd.4 from (0'0,1206'3801] MAX to 1223'16495 2024-09-21T10:08:27.772 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:27.874 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:27.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:28.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:28.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:28.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:28.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:28.208 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:28.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:28.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:28.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:28.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:28.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:28.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.480 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:28.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:28.555 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 132903 objects 2024-09-21T10:08:28.555 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :114.885 2024-09-21T10:08:28.556 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:08:28.556 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:08:28.557 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:08:28.557 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:08:28.557 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:08:28.580 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.603 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:28.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:28.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:28.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:28.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:28.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:28.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:28.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:28.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:28.976 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:28.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:29.078 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:29.099 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:29.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:29.201 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:29.219 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:08:29.223 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:29.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:29.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:08:29.276 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:29.276 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:08:29.276 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:29.290 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:08:29.290 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:29.312 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:08:29.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:29.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:29.332 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:08:29.332 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:29.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:29.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:29.349 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:08:29.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:29.366 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:08:29.366 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:29.382 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:08:29.382 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:29.398 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:08:29.398 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:29.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:29.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:29.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:29 smithi134 ceph-mon[90085]: 1.2 scrub starts 2024-09-21T10:08:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:29 smithi134 ceph-mon[90085]: pgmap v1747: 109 pgs: 3 active+undersized+degraded+remapped+backfill_wait, 1 remapped+peering, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 5 peering, 15 active+undersized, 4 active+undersized+degraded, 79 active+clean; 128 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 821/6195 objects degraded (13.253%); 195 KiB/s, 3 objects/s recovering 2024-09-21T10:08:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:29 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 821/6195 objects degraded (13.253%), 8 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:29.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:29 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/971259588' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:29.577 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:29.600 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:29.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:29 smithi017 ceph-mon[102913]: 1.2 scrub starts 2024-09-21T10:08:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:29 smithi017 ceph-mon[102913]: pgmap v1747: 109 pgs: 3 active+undersized+degraded+remapped+backfill_wait, 1 remapped+peering, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 5 peering, 15 active+undersized, 4 active+undersized+degraded, 79 active+clean; 128 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 821/6195 objects degraded (13.253%); 195 KiB/s, 3 objects/s recovering 2024-09-21T10:08:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:29 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 821/6195 objects degraded (13.253%), 8 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:29 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/971259588' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:29 smithi017 ceph-mon[105151]: 1.2 scrub starts 2024-09-21T10:08:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:29 smithi017 ceph-mon[105151]: pgmap v1747: 109 pgs: 3 active+undersized+degraded+remapped+backfill_wait, 1 remapped+peering, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 5 peering, 15 active+undersized, 4 active+undersized+degraded, 79 active+clean; 128 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 821/6195 objects degraded (13.253%); 195 KiB/s, 3 objects/s recovering 2024-09-21T10:08:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:29 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 821/6195 objects degraded (13.253%), 8 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-09-21T10:08:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:29 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/971259588' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:29.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:29.724 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:29.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:29.825 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:29.847 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:29.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:29.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:29.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:29.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:30.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:30.099 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:30.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:30.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:30.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:30.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:30.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:30.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:30.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:30.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:30.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:30.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:30 smithi134 ceph-mon[90085]: osdmap e1227: 8 total, 8 up, 7 in 2024-09-21T10:08:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:08:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:08:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:30.554 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:30.574 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:30.596 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:30.697 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:30.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:30 smithi017 ceph-mon[102913]: osdmap e1227: 8 total, 8 up, 7 in 2024-09-21T10:08:30.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:30 smithi017 ceph-mon[105151]: osdmap e1227: 8 total, 8 up, 7 in 2024-09-21T10:08:30.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:30.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:30.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:30.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:30.875 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-09-21T10:08:30.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-09-21T10:08:30.948 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.970 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:30.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:31.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:31.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:31.149 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:31.149 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:31.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:31.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:31.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:31.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:31.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:31.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:31.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:31.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:31.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:31.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:31.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:31.696 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:31 smithi017 ceph-mon[102913]: 1.2 scrub starts 2024-09-21T10:08:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:31 smithi017 ceph-mon[102913]: pgmap v1749: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 1 remapped+peering, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 101 active+clean; 70 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 823 op/s; 354/3405 objects degraded (10.396%); 164 KiB/s, 3 objects/s recovering 2024-09-21T10:08:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:31 smithi017 ceph-mon[102913]: osdmap e1228: 8 total, 8 up, 7 in 2024-09-21T10:08:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:31 smithi017 ceph-mon[105151]: 1.2 scrub starts 2024-09-21T10:08:31.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:31 smithi017 ceph-mon[105151]: pgmap v1749: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 1 remapped+peering, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 101 active+clean; 70 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 823 op/s; 354/3405 objects degraded (10.396%); 164 KiB/s, 3 objects/s recovering 2024-09-21T10:08:31.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:31 smithi017 ceph-mon[105151]: osdmap e1228: 8 total, 8 up, 7 in 2024-09-21T10:08:31.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:31.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:31 smithi134 ceph-mon[90085]: 1.2 scrub starts 2024-09-21T10:08:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:31 smithi134 ceph-mon[90085]: pgmap v1749: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 1 remapped+peering, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 101 active+clean; 70 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 823 op/s; 354/3405 objects degraded (10.396%); 164 KiB/s, 3 objects/s recovering 2024-09-21T10:08:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:31 smithi134 ceph-mon[90085]: osdmap e1228: 8 total, 8 up, 7 in 2024-09-21T10:08:31.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:31.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:31.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:31.944 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:31.965 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:31.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:32.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:32.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:32.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:32.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:32.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:32.315 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:32.337 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:32.438 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:32.461 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:32.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:32.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:32.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:32.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:32.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:32.709 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:32.736 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:32 smithi134 ceph-mon[90085]: osdmap e1229: 8 total, 8 up, 7 in 2024-09-21T10:08:32.811 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:32.832 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:32.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:32.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:32 smithi017 ceph-mon[102913]: osdmap e1229: 8 total, 8 up, 7 in 2024-09-21T10:08:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:32 smithi017 ceph-mon[105151]: osdmap e1229: 8 total, 8 up, 7 in 2024-09-21T10:08:32.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:33.063 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:33.084 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:33.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:33.120 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:33.186 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:33.208 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:33.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:33.310 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:33.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:33.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:33.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:33.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:33.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:33.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:33.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:33.590 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:33.590 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:33.691 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:33.712 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:33.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:33 smithi134 ceph-mon[90085]: pgmap v1752: 109 pgs: 1 active+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+undersized+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 104 active+clean; 7.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 58/429 objects degraded (13.520%); 575 KiB/s, 9 objects/s recovering 2024-09-21T10:08:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:33 smithi134 ceph-mon[90085]: 1.d scrub starts 2024-09-21T10:08:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:33 smithi134 ceph-mon[90085]: 1.d scrub ok 2024-09-21T10:08:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:33 smithi134 ceph-mon[90085]: 1.13 scrub starts 2024-09-21T10:08:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:33 smithi134 ceph-mon[90085]: 1.13 scrub ok 2024-09-21T10:08:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:33 smithi134 ceph-mon[90085]: osdmap e1230: 8 total, 8 up, 7 in 2024-09-21T10:08:33.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:33.835 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:33.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:33.937 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[102913]: pgmap v1752: 109 pgs: 1 active+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+undersized+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 104 active+clean; 7.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 58/429 objects degraded (13.520%); 575 KiB/s, 9 objects/s recovering 2024-09-21T10:08:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[102913]: 1.d scrub starts 2024-09-21T10:08:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[102913]: 1.d scrub ok 2024-09-21T10:08:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[102913]: 1.13 scrub starts 2024-09-21T10:08:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[102913]: 1.13 scrub ok 2024-09-21T10:08:33.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[102913]: osdmap e1230: 8 total, 8 up, 7 in 2024-09-21T10:08:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[105151]: pgmap v1752: 109 pgs: 1 active+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+undersized+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 104 active+clean; 7.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 58/429 objects degraded (13.520%); 575 KiB/s, 9 objects/s recovering 2024-09-21T10:08:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[105151]: 1.d scrub starts 2024-09-21T10:08:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[105151]: 1.d scrub ok 2024-09-21T10:08:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[105151]: 1.13 scrub starts 2024-09-21T10:08:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[105151]: 1.13 scrub ok 2024-09-21T10:08:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:33 smithi017 ceph-mon[105151]: osdmap e1230: 8 total, 8 up, 7 in 2024-09-21T10:08:33.958 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:34.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:34.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:34.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:34.188 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:34.210 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:34.251 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:08:34.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:34.338 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:34.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:34.439 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:34.461 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:34 smithi134 ceph-mon[90085]: 1.2 scrub starts 2024-09-21T10:08:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:34 smithi134 ceph-mon[90085]: 1.2 scrub ok 2024-09-21T10:08:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:34 smithi134 ceph-mon[90085]: osdmap e1231: 8 total, 8 up, 7 in 2024-09-21T10:08:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:34 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 58/429 objects degraded (13.520%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:08:34.541 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:34.562 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:34.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:34.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:34.685 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:34.706 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:34.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:34.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:34 smithi017 ceph-mon[102913]: 1.2 scrub starts 2024-09-21T10:08:34.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:34 smithi017 ceph-mon[102913]: 1.2 scrub ok 2024-09-21T10:08:34.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:34 smithi017 ceph-mon[102913]: osdmap e1231: 8 total, 8 up, 7 in 2024-09-21T10:08:34.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:34 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 58/429 objects degraded (13.520%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:08:34.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:34 smithi017 ceph-mon[105151]: 1.2 scrub starts 2024-09-21T10:08:34.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:34 smithi017 ceph-mon[105151]: 1.2 scrub ok 2024-09-21T10:08:34.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:34 smithi017 ceph-mon[105151]: osdmap e1231: 8 total, 8 up, 7 in 2024-09-21T10:08:34.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:34 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 58/429 objects degraded (13.520%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:08:34.808 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:34.829 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:34.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:34.930 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:34.952 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:34.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:35.053 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:35.075 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:35.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:35.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:08:35.120 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:08:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:08:34] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.51.0" 2024-09-21T10:08:35.176 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:35.198 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:35.201 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:35.299 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:35.320 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:35.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:35.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:35.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:35.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:35.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:35.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:35 smithi134 ceph-mon[90085]: pgmap v1755: 109 pgs: 1 active+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 1 peering, 106 active+clean; 608 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 275 op/s; 601 KiB/s, 9 objects/s recovering 2024-09-21T10:08:35.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:35 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/429 objects degraded (13.520%), 1 pg degraded) 2024-09-21T10:08:35.560 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:35 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:08:35.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:35.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:35.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:35.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:35.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:35 smithi017 ceph-mon[102913]: pgmap v1755: 109 pgs: 1 active+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 1 peering, 106 active+clean; 608 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 275 op/s; 601 KiB/s, 9 objects/s recovering 2024-09-21T10:08:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:35 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/429 objects degraded (13.520%), 1 pg degraded) 2024-09-21T10:08:35.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:35 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:08:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:35 smithi017 ceph-mon[105151]: pgmap v1755: 109 pgs: 1 active+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 1 peering, 106 active+clean; 608 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 275 op/s; 601 KiB/s, 9 objects/s recovering 2024-09-21T10:08:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:35 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/429 objects degraded (13.520%), 1 pg degraded) 2024-09-21T10:08:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:35 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:08:35.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:35.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:35.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:35.921 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:35.943 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:35.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:36.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:36.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:36.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:36.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:36.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:36.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:36.293 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:36.315 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:36.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:36.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:36.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:36.464 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:36.464 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:36.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:36.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:36.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:36.664 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:36.686 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:36.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:36.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:36.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:36.826 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 6] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:08:36.826 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:08:36.826 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-09-21T10:08:36.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd in 1 2024-09-21T10:08:36.912 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:36.934 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:36.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:37.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:37.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:37.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:37.160 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:37.182 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:37.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:37.251 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:08:37.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:37.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:37.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:37.431 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:37.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:37.471 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:08:37.518 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:37.533 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:37.555 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:37.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:37.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:37.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:37.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:37.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:37 smithi134 ceph-mon[90085]: pgmap v1756: 109 pgs: 1 active+undersized+remapped, 1 peering, 107 active+clean; 608 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 187 op/s; 410 KiB/s, 6 objects/s recovering 2024-09-21T10:08:37.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:37 smithi134 ceph-mon[90085]: 1.5 scrub starts 2024-09-21T10:08:37.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:37 smithi134 ceph-mon[90085]: 1.5 scrub ok 2024-09-21T10:08:37.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:37 smithi134 ceph-mon[90085]: 1.b scrub starts 2024-09-21T10:08:37.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:37 smithi134 ceph-mon[90085]: 1.b scrub ok 2024-09-21T10:08:37.693 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:37 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2685707319' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:37.782 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:37.805 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:37.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:37.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:37.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[102913]: pgmap v1756: 109 pgs: 1 active+undersized+remapped, 1 peering, 107 active+clean; 608 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 187 op/s; 410 KiB/s, 6 objects/s recovering 2024-09-21T10:08:37.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[102913]: 1.5 scrub starts 2024-09-21T10:08:37.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[102913]: 1.5 scrub ok 2024-09-21T10:08:37.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[102913]: 1.b scrub starts 2024-09-21T10:08:37.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[102913]: 1.b scrub ok 2024-09-21T10:08:37.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2685707319' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:37.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[105151]: pgmap v1756: 109 pgs: 1 active+undersized+remapped, 1 peering, 107 active+clean; 608 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 187 op/s; 410 KiB/s, 6 objects/s recovering 2024-09-21T10:08:37.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[105151]: 1.5 scrub starts 2024-09-21T10:08:37.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[105151]: 1.5 scrub ok 2024-09-21T10:08:37.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[105151]: 1.b scrub starts 2024-09-21T10:08:37.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[105151]: 1.b scrub ok 2024-09-21T10:08:37.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:37 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2685707319' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:37.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:37.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:38.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:38.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:38.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:38.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:38.184 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:38.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:38.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.309 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:38.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:38.410 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:38.432 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:38.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:38.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:38.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:38.661 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:38.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:38 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:38.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:38 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2685707319' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:38.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:38 smithi017 ceph-mon[105151]: osdmap e1232: 8 total, 8 up, 7 in 2024-09-21T10:08:38.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:38 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:08:38.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:38 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:38.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:38 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2685707319' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:38.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:38 smithi017 ceph-mon[102913]: osdmap e1232: 8 total, 8 up, 7 in 2024-09-21T10:08:38.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:38 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:08:38.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:38.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:38 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:38 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2685707319' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:38 smithi134 ceph-mon[90085]: osdmap e1232: 8 total, 8 up, 7 in 2024-09-21T10:08:38.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:38 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:08:38.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:38.810 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:38.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:38.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:38.927 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:38.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:39.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:39.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:39.167 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:39.190 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:39.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:39.278 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:39.278 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:39.294 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:39.315 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:39.417 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:39.437 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:39.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:39.489 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:39 smithi134 ceph-mon[90085]: pgmap v1758: 109 pgs: 109 active+clean; 608 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 18 op/s 2024-09-21T10:08:39.489 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:08:39.489 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:39 smithi134 ceph-mon[90085]: osdmap e1233: 8 total, 8 up, 7 in 2024-09-21T10:08:39.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:39.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:39.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:39.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:39.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:39.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:39 smithi017 ceph-mon[102913]: pgmap v1758: 109 pgs: 109 active+clean; 608 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 18 op/s 2024-09-21T10:08:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:08:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:39 smithi017 ceph-mon[102913]: osdmap e1233: 8 total, 8 up, 7 in 2024-09-21T10:08:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:39 smithi017 ceph-mon[105151]: pgmap v1758: 109 pgs: 109 active+clean; 608 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 18 op/s 2024-09-21T10:08:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:08:39.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:39 smithi017 ceph-mon[105151]: osdmap e1233: 8 total, 8 up, 7 in 2024-09-21T10:08:39.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:39.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:39.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:39.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:08:39.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:39.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:39.997 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:40.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:40.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:40.143 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:40.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:40.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:40.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:40.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:40.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:40.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:40.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:40.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:40.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:40.489 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_3' removed 2024-09-21T10:08:40.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:08:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:08:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:40.536 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:40.577 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:40.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:40 smithi017 ceph-mon[102913]: osdmap e1234: 8 total, 8 up, 7 in 2024-09-21T10:08:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/783615224' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:08:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:40 smithi017 ceph-mon[105151]: osdmap e1234: 8 total, 8 up, 7 in 2024-09-21T10:08:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/783615224' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:08:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:40.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:40.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:40.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:40.824 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:40.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:40.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:40.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:40.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:40 smithi134 ceph-mon[90085]: osdmap e1234: 8 total, 8 up, 7 in 2024-09-21T10:08:40.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/783615224' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:08:40.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:40.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:41.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:41.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:41.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:41.173 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:41.194 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:41.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:41.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:41.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:41.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:41.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:41.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:08:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:41.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:41.544 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:41.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.567 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:41.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:41.668 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:41.675 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:41 smithi134 ceph-mon[90085]: pgmap v1761: 109 pgs: 2 peering, 107 active+clean; 599 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-21T10:08:41.675 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:41 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/783615224' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:08:41.675 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:41 smithi134 ceph-mon[90085]: osdmap e1235: 8 total, 8 up, 7 in 2024-09-21T10:08:41.675 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:41 smithi134 ceph-mon[90085]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:08:41.691 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:41.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:41.712 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:08:41.712 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:08:41.713 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-09-21T10:08:41.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_4 16 2024-09-21T10:08:41.735 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:41 smithi017 ceph-mon[102913]: pgmap v1761: 109 pgs: 2 peering, 107 active+clean; 599 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-21T10:08:41.735 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:41 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/783615224' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:08:41.735 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:41 smithi017 ceph-mon[102913]: osdmap e1235: 8 total, 8 up, 7 in 2024-09-21T10:08:41.735 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:41 smithi017 ceph-mon[102913]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:08:41.736 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:41 smithi017 ceph-mon[105151]: pgmap v1761: 109 pgs: 2 peering, 107 active+clean; 599 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail 2024-09-21T10:08:41.736 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:41 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/783615224' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:08:41.736 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:41 smithi017 ceph-mon[105151]: osdmap e1235: 8 total, 8 up, 7 in 2024-09-21T10:08:41.736 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:41 smithi017 ceph-mon[105151]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:08:41.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:41.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:41.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:41.931 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:41.931 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:41.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:42.040 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:42.062 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:42.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:42.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:42.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:42.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:42.290 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:42.313 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:42.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:42.415 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:42.436 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:42.538 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:42.561 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:42.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:42.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:08:42.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:42.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:42.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:42.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:42.809 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:42.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:42.874 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:42.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:42.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:42.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:43.017 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:43.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:43.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:43.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:43.160 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:43.181 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:43.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:43.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:43.409 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:43.430 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:43.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:43.532 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:43.553 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:43.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:43.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:43.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:43.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:43 smithi134 ceph-mon[90085]: pgmap v1763: 93 pgs: 2 peering, 91 active+clean; 669 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 16 KiB/s rd, 6 op/s 2024-09-21T10:08:43.771 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:43 smithi017 ceph-mon[102913]: pgmap v1763: 93 pgs: 2 peering, 91 active+clean; 669 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 16 KiB/s rd, 6 op/s 2024-09-21T10:08:43.771 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:43 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:08:43.712+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1235 definitely_dead 0 2024-09-21T10:08:43.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:43 smithi017 ceph-mon[105151]: pgmap v1763: 93 pgs: 2 peering, 91 active+clean; 669 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 16 KiB/s rd, 6 op/s 2024-09-21T10:08:43.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:43.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:43.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:43.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:43.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:43.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:44.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:44.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:44.154 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:44.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:44.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:44.273 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:08:44.273 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:44.275 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:08:44.275 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:44.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:44.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:08:44.292 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:44.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:44.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:44.309 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:08:44.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:44.325 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:08:44.325 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:44.341 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:08:44.341 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:44.357 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:08:44.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:44.373 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:08:44.373 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:44.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:44.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:44.503 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.1. 2024-09-21T10:08:44.527 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:44.548 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:44.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:44.643 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:08:44.643 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:08:44.643 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 0 2024-09-21T10:08:44.643 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:08:44.643 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/609406513' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:08:44.643 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4185706136' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:44.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:44.672 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/609406513' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:08:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4185706136' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/609406513' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:08:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4185706136' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:44.773 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:44.794 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:44.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:44.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:45.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:45.047 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:45.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:45.073 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:08:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:08:44] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.51.0" 2024-09-21T10:08:45.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:45.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:45.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:45.273 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:45.297 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:45.314 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:08:45.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:45.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:45.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:45.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:45.550 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-09-21T10:08:45.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:45.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:45.609 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:45.656 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:45.678 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:45.730 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:45 smithi134 ceph-mon[90085]: pgmap v1764: 93 pgs: 2 peering, 91 active+clean; 696 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 19 KiB/s rd, 7 op/s 2024-09-21T10:08:45.730 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:45 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/609406513' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:08:45.730 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:45 smithi134 ceph-mon[90085]: osdmap e1236: 8 total, 8 up, 8 in 2024-09-21T10:08:45.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:45.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:45.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:45.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:45 smithi017 ceph-mon[102913]: pgmap v1764: 93 pgs: 2 peering, 91 active+clean; 696 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 19 KiB/s rd, 7 op/s 2024-09-21T10:08:45.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:45 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/609406513' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:08:45.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:45 smithi017 ceph-mon[102913]: osdmap e1236: 8 total, 8 up, 8 in 2024-09-21T10:08:45.836 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:45 smithi017 ceph-mon[105151]: pgmap v1764: 93 pgs: 2 peering, 91 active+clean; 696 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 19 KiB/s rd, 7 op/s 2024-09-21T10:08:45.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:45 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/609406513' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:08:45.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:45 smithi017 ceph-mon[105151]: osdmap e1236: 8 total, 8 up, 8 in 2024-09-21T10:08:45.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:45.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:45.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:46.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:46.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:46.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:46.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:46.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:46.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:46.278 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:46.301 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:46.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:46.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:46.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:46.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.554 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:46.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:46.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:46.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:46.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:46.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:46.803 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:46.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:46.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:46 smithi017 ceph-mon[102913]: osdmap e1237: 8 total, 8 up, 8 in 2024-09-21T10:08:46.832 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:46 smithi017 ceph-mon[105151]: osdmap e1237: 8 total, 8 up, 8 in 2024-09-21T10:08:46.837 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:46 smithi134 ceph-mon[90085]: osdmap e1237: 8 total, 8 up, 8 in 2024-09-21T10:08:46.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:46.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:46.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:47.031 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:47.052 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:47.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:47.090 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:47.154 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:47.175 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:47.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:47.277 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:47.297 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:47.374 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:08:47.374 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:08:47.374 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:08:47.374 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:08:47.374 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:08:47.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:47.420 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:47.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:47.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:47.525 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_4' created 2024-09-21T10:08:47.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.546 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:47.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:47.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:47.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:47.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:47.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:47.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:47.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:47.895 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:47.919 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:47.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:47.929 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:47 smithi134 ceph-mon[90085]: pgmap v1767: 93 pgs: 93 active+clean; 696 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 21 KiB/s rd, 8 op/s 2024-09-21T10:08:47.929 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:47 smithi134 ceph-mon[90085]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:08:47.929 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:47 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/198347040' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:08:47.929 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:47 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:08:47.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:47 smithi017 ceph-mon[102913]: pgmap v1767: 93 pgs: 93 active+clean; 696 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 21 KiB/s rd, 8 op/s 2024-09-21T10:08:47.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:47 smithi017 ceph-mon[102913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:08:47.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:47 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/198347040' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:08:47.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:47 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:08:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:47 smithi017 ceph-mon[105151]: pgmap v1767: 93 pgs: 93 active+clean; 696 KiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 21 KiB/s rd, 8 op/s 2024-09-21T10:08:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:47 smithi017 ceph-mon[105151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:08:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:47 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/198347040' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:08:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:47 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:08:48.019 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:08:48.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:48.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:48.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:48.145 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:48.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:48.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:48.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:48.290 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:48.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:48.308 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:48.391 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:48.466 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:48.572 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:48.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.594 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:48.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:48.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:08:48.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:48 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-21T10:08:48.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:48 smithi017 ceph-mon[102913]: osdmap e1238: 8 total, 8 up, 8 in 2024-09-21T10:08:48.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:48 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-21T10:08:48.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:48 smithi017 ceph-mon[105151]: osdmap e1238: 8 total, 8 up, 8 in 2024-09-21T10:08:48.686 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:48 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-21T10:08:48.686 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:48 smithi134 ceph-mon[90085]: osdmap e1238: 8 total, 8 up, 8 in 2024-09-21T10:08:48.695 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:48.716 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:48.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:48.818 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:48.840 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:48.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:48.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:48.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:48.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:48.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-09-21T10:08:49.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:49.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:49.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:49.200 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:49.222 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:49.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:49.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:49.349 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:49.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:49.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:49.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:49.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:49.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:49.593 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:49.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:49.635 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:49.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:49.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:49.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:49.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:49.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:49.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:49.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:49 smithi017 ceph-mon[102913]: pgmap v1769: 109 pgs: 5 creating+peering, 11 unknown, 93 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 20 KiB/s wr, 5 op/s; 7.3 KiB/s, 0 objects/s recovering 2024-09-21T10:08:49.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:49 smithi017 ceph-mon[102913]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:49.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:49 smithi017 ceph-mon[102913]: osdmap e1239: 8 total, 8 up, 8 in 2024-09-21T10:08:49.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:49 smithi017 ceph-mon[105151]: pgmap v1769: 109 pgs: 5 creating+peering, 11 unknown, 93 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 20 KiB/s wr, 5 op/s; 7.3 KiB/s, 0 objects/s recovering 2024-09-21T10:08:49.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:49 smithi017 ceph-mon[105151]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:49.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:49 smithi017 ceph-mon[105151]: osdmap e1239: 8 total, 8 up, 8 in 2024-09-21T10:08:49.879 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:49 smithi134 ceph-mon[90085]: pgmap v1769: 109 pgs: 5 creating+peering, 11 unknown, 93 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 20 KiB/s wr, 5 op/s; 7.3 KiB/s, 0 objects/s recovering 2024-09-21T10:08:49.879 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:49 smithi134 ceph-mon[90085]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:49.879 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:49 smithi134 ceph-mon[90085]: osdmap e1239: 8 total, 8 up, 8 in 2024-09-21T10:08:49.953 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:49.976 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:49.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:50.077 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:50.100 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:50.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:50.202 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:50.226 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:50.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:50.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:50.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:50.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:50.441 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:08:50.441 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:08:50.441 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:08:50.441 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:08:50.441 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263811 8.4 8.5ca8d1b4 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.443394+0000", 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 25.994205847, 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97607429599999995, 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263811 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.442 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443394+0000", 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443394+0000", 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443393+0000", 2024-09-21T10:08:50.443 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443399+0000", 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9609999999999999e-06 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443411+0000", 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1708e-05 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443418+0000", 2024-09-21T10:08:50.444 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1439999999999997e-06 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443601+0000", 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00018249299999999999 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443606+0000", 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3630000000000002e-06 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.445 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.419459+0000", 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.975852371 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.419468+0000", 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.7180000000000008e-06 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.446 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263812 8.4 8.c9d94834 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.443432+0000", 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 25.994167481000002, 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97606504900000002, 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263812 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.447 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443432+0000", 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443432+0000", 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443432+0000", 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.448 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443436+0000", 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4700000000000004e-06 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443443+0000", 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.063e-06 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443448+0000", 2024-09-21T10:08:50.449 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5460000000000002e-06 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443625+0000", 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017739599999999999 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443630+0000", 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.583e-06 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.450 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.419489+0000", 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97585938500000002 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.419497+0000", 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1659999999999994e-06 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263816 8.2 8.2cca6682 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.443453+0000", 2024-09-21T10:08:50.451 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 25.994146503, 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97134293500000002, 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263816 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443453+0000", 2024-09-21T10:08:50.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443453+0000", 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443453+0000", 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443455+0000", 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4449999999999999e-06 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443461+0000", 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7130000000000002e-06 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443465+0000", 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0980000000000004e-06 2024-09-21T10:08:50.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443533+0000", 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7779000000000003e-05 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443543+0000", 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0209e-05 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.414784+0000", 2024-09-21T10:08:50.456 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97124098699999994 2024-09-21T10:08:50.456 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.456 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.456 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.456 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.414796+0000", 2024-09-21T10:08:50.457 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:50.458 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2169e-05 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263823 8.2 8.34cbd772 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.443476+0000", 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 25.994124231000001, 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97135318000000004, 2024-09-21T10:08:50.459 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263823 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443476+0000", 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443476+0000", 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443475+0000", 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443478+0000", 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.6970000000000002e-06 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443484+0000", 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.314e-06 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443488+0000", 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3100000000000002e-06 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443565+0000", 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6998999999999999e-05 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443572+0000", 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4609999999999996e-06 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.414820+0000", 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97124847199999997 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.414829+0000", 2024-09-21T10:08:50.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3599999999999996e-06 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263843 8.2 8.659c36a2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.462475+0000", 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 25.975125308999999, 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95498026199999997, 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:08:50.464 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263843 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462475+0000", 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462475+0000", 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.465 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462472+0000", 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999971 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462482+0000", 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0180000000000001e-05 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462501+0000", 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8695999999999999e-05 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.466 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462509+0000", 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9389999999999997e-06 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462560+0000", 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0914000000000001e-05 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:08:50.467 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462570+0000", 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0215e-05 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.417442+0000", 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95487207600000001 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.417455+0000", 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2867999999999999e-05 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.468 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:264190 8.5 e1226/1225 8:a422f3db:::benchmark_data_smithi017_209235_object131284:head v 1226'16330, mlcod=1226'16328)", 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:26.927281+0000", 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.510318961999999, 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.028875812000000001, 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927281+0000", 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.469 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927281+0000", 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927280+0000", 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.470 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927289+0000", 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.3379999999999994e-06 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927308+0000", 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9576999999999999e-05 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927316+0000", 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2629999999999997e-06 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.471 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927369+0000", 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3690000000000003e-05 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927395+0000", 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.5262e-05 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.956107+0000", 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.028712766000000001 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.472 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.956157+0000", 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9265999999999999e-05 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:264184 8.f e1226/1225 8:fbbdd327:::benchmark_data_smithi017_209235_object131278:head v 1226'16249, mlcod=1226'16247)", 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:26.927382+0000", 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.510217875999999, 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.027376563999999999, 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.473 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927382+0000", 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927382+0000", 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927381+0000", 2024-09-21T10:08:50.474 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927391+0000", 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.4220000000000006e-06 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927400+0000", 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.6230000000000003e-06 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927404+0000", 2024-09-21T10:08:50.475 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2749999999999997e-06 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927506+0000", 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000102509 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927536+0000", 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9903000000000002e-05 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.954683+0000", 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.027146643000000002 2024-09-21T10:08:50.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.954759+0000", 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5675999999999998e-05 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:264187 8.e e1226/1225 8:7b267ec4:::benchmark_data_smithi017_209235_object131281:head v 1226'16931, mlcod=1226'16929)", 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:26.927435+0000", 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.510165166, 2024-09-21T10:08:50.477 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.032732338, 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927435+0000", 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927435+0000", 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.478 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927434+0000", 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927438+0000", 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.591e-06 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927444+0000", 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3790000000000003e-06 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.479 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927448+0000", 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4340000000000001e-06 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927505+0000", 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7435000000000002e-05 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.927537+0000", 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1843e-05 2024-09-21T10:08:50.480 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.960090+0000", 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.032552771000000001 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.960167+0000", 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.7218000000000005e-05 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "backfill_remove(8.b e1226 8:d1390979:::benchmark_data_smithi017_209235_object28884:head,1204'1806,8:d13ae745:::benchmark_data_smithi017_209235_object26054:head,1204'1650,8:d13b37b0:::benchmark_data_smithi017_209235_object11078:head,1203'687,8:d13b7f19:::benchmark_data_smithi017_209235_object20908:head,1203'1302,8:d13f3d10:::benchmark_data_smithi017_209235_object7090:head,1203'450,8:d140b1e8:::benchmark_data_smithi017_209235_object11915:head,1203'738,8:d140c04e:::benchmark_data_smithi017_209235_object21590:head,1204'1343,8:d14128ef:::benchmark_data_smithi017_209235_object11151:head,1203'693,8:d14276ac:::benchmark_data_smithi017_209235_object44730:head,1204'2801,8:d144d144:::benchmark_data_smithi017_209235_object49481:head,1205'3085,8:d1458fcf:::benchmark_data_smithi017_209235_object39335:head,1204'2459,8:d1463e87:::benchmark_data_smithi017_209235_object36095:head,1204'2251,8:d14868ec:::benchmark_data_smithi017_209235_object48084:head,1205'3015,8:d1486edd:::benchmark_data_smithi017_209235_object56731:head,1206'3563,8:d148b5fb:::benchmark_data_smithi017_209235_object8425:head,1203'523,8:d148bb97:::benchmark_data_smithi017_209235_object51684:head,1205'3232,8:d14a6b5e:::benchmark_data_smithi017_209235_object55709:head,1206'3485,8:d14c1a7c:::benchmark_data_smithi017_209235_object47120:head,1204'2968,8:d14c9865:::benchmark_data_smithi017_209235_object2936:head,1203'193,8:d14d877a:::benchmark_data_smithi017_209235_object1798:head,1203'109,8:d14dfc35:::benchmark_data_smithi017_209235_object13248:head,1203'810,8:d14e7219:::benchmark_data_smithi017_209235_object37502:head,1204'2338,8:d1515ea1:::benchmark_data_smithi017_209235_object35212:head,1204'2206,8:d152cdc7:::benchmark_data_smithi017_209235_object40342:head,1204'2525,8:d1535df4:::benchmark_data_smithi017_209235_object33186:head,1204'2081,8:d154d0ab:::benchmark_data_smithi017_209235_object39902:head,1204'2497,8:d1570783:::benchmark_data_smithi017_209235_object51044:head,1205'3197,8:d1598482:::benchmark_data_smithi017_209235_object49160:head,1205'3068,8:d15b9508:::benchmark_data_smithi017_209235_object47524:head,1205'2989,8:d15bef3a:::benchmark_data_smithi017_209235_object2227:head,1203'141,8:d15da47f:::benchmark_data_smithi017_209235_object52598:head,1205'3285,8:d1601080:::benchmark_data_smithi017_209235_object1543:head,1203'96,8:d163aabb:::benchmark_data_smithi017_209235_object44488:head,1204'2792,8:d16554a7:::benchmark_data_smithi017_209235_object25025:head,1204'1572,8:d1656b94:::benchmark_data_smithi017_209235_object48496:head,1205'3028,8:d1683433:::benchmark_data_smithi017_209235_object17262:head,1203'1065,8:d1699b61:::benchmark_data_smithi017_209235_object57610:head,1206'3607,8:d16e26a9:::benchmark_data_smithi017_209235_object7817:head,1203'493,8:d16efac9:::benchmark_data_smithi017_209235_object30988:head,1204'1947,8:d170fdeb:::benchmark_data_smithi017_209235_object18909:head,1203'1181,8:d172ead0:::benchmark_data_smithi017_209235_object19766:head,1203'1230)", 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:26.930075+0000", 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.507524386, 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.025406941999999998, 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.930075+0000", 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.482 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.930075+0000", 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.930074+0000", 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.930116+0000", 2024-09-21T10:08:50.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2132000000000003e-05 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.930159+0000", 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2287999999999999e-05 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.930194+0000", 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5317000000000001e-05 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.954624+0000", 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.024430261000000002 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.954674+0000", 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9886000000000003e-05 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.955482+0000", 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00080815599999999998 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.485 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "MOSDPGPush(8.b 1226/1225 [PushOp(8:d172f3e9:::benchmark_data_smithi017_209235_object132269:head, version: 1215'8269, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:d172f3e9:::benchmark_data_smithi017_209235_object132269:head@1215'8269, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:26.930244+0000", 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.507355476000001, 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.041170011999999999, 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:08:50.486 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.930244+0000", 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.930244+0000", 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.930244+0000", 2024-09-21T10:08:50.487 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.954572+0000", 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.024328281 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.954595+0000", 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2490999999999999e-05 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.954606+0000", 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1386000000000001e-05 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.971113+0000", 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0165072 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.971125+0000", 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1274000000000001e-05 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.971414+0000", 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00028975000000000002 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:264201 8.5 e1226/1225 8:a286d0e1:::benchmark_data_smithi017_209235_object131295:head v 1226'16331, mlcod=1226'16328)", 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:26.940368+0000", 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.497231370000002, 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.019112666, 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.490 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.940368+0000", 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.940368+0000", 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.940367+0000", 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.940376+0000", 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7819999999999999e-06 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.954705+0000", 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014329333 2024-09-21T10:08:50.492 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.954710+0000", 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9729999999999997e-06 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.956761+0000", 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002050624 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.956781+0000", 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0092e-05 2024-09-21T10:08:50.493 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.959299+0000", 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0025177620000000002 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.959481+0000", 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000182475 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.494 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:264226 8.4 e1226/1225 8:2b68468d:::benchmark_data_smithi017_209235_object131320:head v 1226'16469, mlcod=1226'16465)", 2024-09-21T10:08:50.495 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:26.975153+0000", 2024-09-21T10:08:50.495 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.462446407000002, 2024-09-21T10:08:50.495 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.026378863999999998, 2024-09-21T10:08:50.495 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.495 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.495 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.495 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:08:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:08:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:50.496 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.496 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.975153+0000", 2024-09-21T10:08:50.496 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.496 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.496 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.975153+0000", 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.975152+0000", 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.975161+0000", 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1270000000000003e-06 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.975174+0000", 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3760999999999999e-05 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.975181+0000", 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4239999999999998e-06 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.975278+0000", 2024-09-21T10:08:50.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6884000000000003e-05 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:26.975299+0000", 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1733e-05 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.001485+0000", 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.026185683000000001 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.001532+0000", 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7290999999999998e-05 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:264391 8.4 e1226/1225 8:267f57e3:::benchmark_data_smithi017_209235_object131485:head v 1226'16481, mlcod=1226'16478)", 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:27.155133+0000", 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.282466657000001, 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.058794737, 2024-09-21T10:08:50.500 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.155133+0000", 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.155133+0000", 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.155132+0000", 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.155138+0000", 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.823e-06 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.155147+0000", 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.4610000000000008e-06 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.155151+0000", 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1450000000000001e-06 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.211092+0000", 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.055940746999999999 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.211110+0000", 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.876e-05 2024-09-21T10:08:50.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.213891+0000", 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0027808469999999999 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.213928+0000", 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6776999999999997e-05 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "backfill_remove(8.b e1226 8:d2a7e625:::benchmark_data_smithi017_209235_object35244:head,1204'2209,8:d2a7ed35:::benchmark_data_smithi017_209235_object19694:head,1203'1224,8:d2a95285:::benchmark_data_smithi017_209235_object41172:head,1204'2586,8:d2ac3705:::benchmark_data_smithi017_209235_object4728:head,1203'310,8:d2ac6d36:::benchmark_data_smithi017_209235_object2724:head,1203'174,8:d2adea21:::benchmark_data_smithi017_209235_object20550:head,1203'1280,8:d2ae0ed0:::benchmark_data_smithi017_209235_object14299:head,1203'883,8:d2ae8237:::benchmark_data_smithi017_209235_object12935:head,1203'789,8:d2b1718c:::benchmark_data_smithi017_209235_object20587:head,1203'1283,8:d2b34b3f:::benchmark_data_smithi017_209235_object30070:head,1204'1890,8:d2b3b507:::benchmark_data_smithi017_209235_object35884:head,1204'2237,8:d2b4c317:::benchmark_data_smithi017_209235_object35054:head,1204'2196,8:d2b795ad:::benchmark_data_smithi017_209235_object15287:head,1203'941,8:d2b7ad97:::benchmark_data_smithi017_209235_object49504:head,1205'3087,8:d2b7ee48:::benchmark_data_smithi017_209235_object5804:head,1203'371)", 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:27.159327+0000", 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.278272502, 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.053194037999999999, 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.159327+0000", 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.505 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.159327+0000", 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.159326+0000", 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.506 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.159351+0000", 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4865999999999999e-05 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.210100+0000", 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.050749249000000003 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.210132+0000", 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2570999999999997e-05 2024-09-21T10:08:50.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.212139+0000", 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020070069999999999 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.212179+0000", 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9490000000000003e-05 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.212521+0000", 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00034238500000000001 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:264387 8.5 e1226/1225 8:aea0c166:::benchmark_data_smithi017_209235_object131481:head v 1226'16339, mlcod=1226'16337)", 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:27.196207+0000", 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 23.241392920999999, 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.023252495000000001, 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.509 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.196207+0000", 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.196207+0000", 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.196206+0000", 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.196214+0000", 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.2579999999999995e-06 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.210104+0000", 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013889311999999999 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.511 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.210115+0000", 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1391e-05 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.210161+0000", 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5847000000000003e-05 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.210184+0000", 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3201e-05 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.219425+0000", 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0092411320000000009 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.219459+0000", 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4319999999999997e-05 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.513 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:264976 8.2 e1226/1225 8:4c7dfba5:::benchmark_data_smithi017_209235_object132070:head v 1226'16497, mlcod=1226'16493)", 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:27.746728+0000", 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 22.690872312, 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.019205492000000001, 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.746728+0000", 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.746728+0000", 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.746727+0000", 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.746733+0000", 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.93e-06 2024-09-21T10:08:50.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.746742+0000", 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.0569999999999999e-06 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.746747+0000", 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2070000000000003e-06 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.749547+0000", 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0027993250000000001 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.749568+0000", 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1404999999999999e-05 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.765893+0000", 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.016325207000000001 2024-09-21T10:08:50.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:27.765933+0000", 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9878e-05 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "pg_backfill(progress 8.b e 1226/1226 lb 8:dcda0ad6:::benchmark_data_smithi017_209235_object29968:head)", 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:28.418249+0000", 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 22.019350916, 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.036293367999999999, 2024-09-21T10:08:50.518 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.418249+0000", 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.418249+0000", 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.418248+0000", 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.418259+0000", 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0481e-05 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.418270+0000", 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.097e-05 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.418275+0000", 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6540000000000004e-06 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.454502+0000", 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.036226888999999998 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.454515+0000", 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2995e-05 2024-09-21T10:08:50.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.454542+0000", 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7050999999999999e-05 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:265648 8.b e1226/1225 8:d062ebf7:::benchmark_data_smithi017_209235_object132742:head v 1226'16608, mlcod=1226'16604)", 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:28.427201+0000", 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 22.010399146000001, 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.018050763000000001, 2024-09-21T10:08:50.522 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.427201+0000", 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.427201+0000", 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.427200+0000", 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.427208+0000", 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1620000000000008e-06 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.427219+0000", 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1250000000000001e-05 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.427230+0000", 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0479000000000001e-05 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.437184+0000", 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0099545950000000001 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.437208+0000", 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3578e-05 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.445215+0000", 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0080069930000000004 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.445251+0000", 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6557999999999998e-05 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.526 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:265650 8.b e1226/1225 8:d5d87903:::benchmark_data_smithi017_209235_object132744:head v 1226'16609, mlcod=1226'16604)", 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:28.430624+0000", 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 22.006976156, 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.023976735999999998, 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.430624+0000", 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.430624+0000", 2024-09-21T10:08:50.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.430623+0000", 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.430629+0000", 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.942e-06 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.430644+0000", 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4718e-05 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.430649+0000", 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1699999999999996e-06 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.439445+0000", 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.008796702 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.439468+0000", 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2561999999999998e-05 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.454568+0000", 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.015099865000000001 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.454600+0000", 2024-09-21T10:08:50.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2592000000000002e-05 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45620.0:265687 8.b e1226/1225 8:d2f48c32:::benchmark_data_smithi017_209235_object132781:head v 1226'16610, mlcod=1226'16604)", 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:28.435926+0000", 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 22.001673986, 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.018700435000000001, 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.435926+0000", 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.435926+0000", 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.435925+0000", 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.435936+0000", 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.083e-05 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.435947+0000", 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1178e-05 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.435952+0000", 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3229999999999997e-06 2024-09-21T10:08:50.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.441534+0000", 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0055815320000000002 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.441552+0000", 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8576999999999998e-05 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.454606+0000", 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013053692 2024-09-21T10:08:50.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:28.454626+0000", 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0392000000000001e-05 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:08:50.535 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:08:50.536 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:08:50.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.537 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:50.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:50.552 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 6, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:08:50.552 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:08:50.552 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-21T10:08:50.552 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:08:50.552 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:08:50.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-21T10:08:50.638 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:50.661 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:50.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:50.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:50 smithi134 ceph-mon[90085]: osdmap e1240: 8 total, 8 up, 8 in 2024-09-21T10:08:50.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:50 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1380527902' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:50.764 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:50.784 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:50.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:50.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:50.889 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:50 smithi017 ceph-mon[102913]: osdmap e1240: 8 total, 8 up, 8 in 2024-09-21T10:08:50.890 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:50 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1380527902' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:50.890 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:50 smithi017 ceph-mon[105151]: osdmap e1240: 8 total, 8 up, 8 in 2024-09-21T10:08:50.890 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:50 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1380527902' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:50.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:51.011 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:51.032 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:51.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:51.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:51.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:51.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:51.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:51.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:51.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:51.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:51.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:51.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:08:51.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:51.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:51.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:51.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:51.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:51.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:51 smithi134 ceph-mon[90085]: pgmap v1772: 109 pgs: 5 creating+peering, 8 unknown, 96 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 10 KiB/s wr, 2 op/s; 3.6 KiB/s, 0 objects/s recovering 2024-09-21T10:08:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:51 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:51 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1380527902' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:51 smithi134 ceph-mon[90085]: osdmap e1241: 8 total, 8 up, 8 in 2024-09-21T10:08:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:51.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:51.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:51.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:51.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[102913]: pgmap v1772: 109 pgs: 5 creating+peering, 8 unknown, 96 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 10 KiB/s wr, 2 op/s; 3.6 KiB/s, 0 objects/s recovering 2024-09-21T10:08:51.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:51.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1380527902' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:51.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[102913]: osdmap e1241: 8 total, 8 up, 8 in 2024-09-21T10:08:51.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:51.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[105151]: pgmap v1772: 109 pgs: 5 creating+peering, 8 unknown, 96 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 10 KiB/s wr, 2 op/s; 3.6 KiB/s, 0 objects/s recovering 2024-09-21T10:08:51.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:51.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1380527902' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:51.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[105151]: osdmap e1241: 8 total, 8 up, 8 in 2024-09-21T10:08:51.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:51.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:51.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:51.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:52.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:52.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:52.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:52.171 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:52.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:52.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:52.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:52.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:52.398 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:52.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:52.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:52.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:52.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:52.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:52.622 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:52.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:52.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:52.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:52.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:52.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:52.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:52.903 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:52.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:52.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:53.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:53.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:53.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:53.180 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:53.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:53.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:08:53.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:53.307 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:53.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:53.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:53.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:53.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:53.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:53.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:53.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:53.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:53.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:53.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:53.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:53.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:53 smithi017 ceph-mon[102913]: pgmap v1774: 109 pgs: 5 creating+peering, 104 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 10 KiB/s wr, 2 op/s; 3.6 KiB/s, 0 objects/s recovering 2024-09-21T10:08:53.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:53 smithi017 ceph-mon[105151]: pgmap v1774: 109 pgs: 5 creating+peering, 104 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 10 KiB/s wr, 2 op/s; 3.6 KiB/s, 0 objects/s recovering 2024-09-21T10:08:53.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:53.934 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:53.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:53 smithi134 ceph-mon[90085]: pgmap v1774: 109 pgs: 5 creating+peering, 104 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 10 KiB/s wr, 2 op/s; 3.6 KiB/s, 0 objects/s recovering 2024-09-21T10:08:54.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:54.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:54.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:54.065 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:54.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:54.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:54.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:54.288 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:54.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.315 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:54.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:54.416 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:54.442 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:54.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:54.543 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:54.548 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-09-21T10:08:54.570 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:54.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:54.672 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:54.696 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:54.798 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:54.823 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:54.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:54.870 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:54 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1065019868' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:54.870 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:54 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:54.870 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:54 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1065019868' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:54.870 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:54 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:54.870 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:08:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:08:54] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.51.0" 2024-09-21T10:08:54.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:54.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:54 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1065019868' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:54 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:55.049 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:55.075 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:55.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:55.176 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:55.201 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:55.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:55.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:55.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:55.428 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:55.453 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:55.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:55.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:55.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.579 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:55.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:55.680 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:55.692 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:08:55.692 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "" 2024-09-21T10:08:55.692 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:08:55.703 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:55.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:55.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:55.930 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[105151]: pgmap v1775: 109 pgs: 109 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[105151]: osdmap e1242: 8 total, 8 up, 8 in 2024-09-21T10:08:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:55.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[102913]: pgmap v1775: 109 pgs: 109 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:55.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:55.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[102913]: osdmap e1242: 8 total, 8 up, 8 in 2024-09-21T10:08:55.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:55.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:55.955 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:55 smithi134 ceph-mon[90085]: pgmap v1775: 109 pgs: 109 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:08:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:55 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:55 smithi134 ceph-mon[90085]: osdmap e1242: 8 total, 8 up, 8 in 2024-09-21T10:08:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:08:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:56.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:56.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:56.183 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:56.207 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:56.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:56.228 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:08:56.228 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:56.241 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:08:56.241 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:56.247 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-09-21T10:08:56.250 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:08:56.250 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:08:56.258 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:08:56.259 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:56.275 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:08:56.275 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:56.291 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:08:56.292 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:56.297 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-21T10:08:56.298 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:08:56.298 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_237924 2024-09-21T10:08:56.298 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:08:56.298 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:08:56.309 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:56.321 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:08:56.321 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:56.340 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:56.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:56.354 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:08:56.354 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:56.389 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:08:56.389 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:56.442 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:56.472 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:56.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:56.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:56.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:56.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:56.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:56 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3463768405' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:56.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:56 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3463768405' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:56.716 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:56.747 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:56.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:56.848 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:56.880 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:56.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:56.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:56.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:56 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3463768405' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:57.013 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:57.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:57.115 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:57.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.144 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:57.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:57.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:57.279 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:57.298 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3439 3423 213.909 213.938 0.00322084 0.00465945 2024-09-21T10:08:57.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:57.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:57.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:57.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:57.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:57.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:57 smithi017 ceph-mon[102913]: pgmap v1777: 109 pgs: 109 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-21T10:08:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:57 smithi017 ceph-mon[102913]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:57.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:08:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:57 smithi017 ceph-mon[105151]: pgmap v1777: 109 pgs: 109 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-21T10:08:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:57 smithi017 ceph-mon[105151]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:57.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:08:57.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:57.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:57.805 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:57.848 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:57.857 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:57 smithi134 ceph-mon[90085]: pgmap v1777: 109 pgs: 109 active+clean; 713 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-21T10:08:57.857 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:57 smithi134 ceph-mon[90085]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:57.857 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:08:57.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:57.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:58.088 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:58.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:58.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:58.224 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:58.253 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:58.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:58.298 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 6841 6825 213.262 212.625 0.0039068 0.00467201 2024-09-21T10:08:58.299 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:58.355 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:58.385 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:58.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:58.487 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:58.520 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:58.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:58.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:58.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:58.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:58.767 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:58.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:58.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:58.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:58.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:59.031 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:59.062 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:59.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:59.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:08:59.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:59.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:59.299 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 9897 9881 205.779 191 0.00421483 0.00484997 2024-09-21T10:08:59.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.347 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:59.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:59.448 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:59.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:59.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:59.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:59.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:08:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:59.727 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:08:59 smithi134 ceph-mon[90085]: pgmap v1778: 109 pgs: 109 active+clean; 126 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s wr, 250 op/s 2024-09-21T10:08:59.763 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:59.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:59.824 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:08:59 smithi017 ceph-mon[102913]: pgmap v1778: 109 pgs: 109 active+clean; 126 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s wr, 250 op/s 2024-09-21T10:08:59.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:08:59 smithi017 ceph-mon[105151]: pgmap v1778: 109 pgs: 109 active+clean; 126 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 16 MiB/s wr, 250 op/s 2024-09-21T10:08:59.865 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:59.901 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:08:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:00.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:00.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:00.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:00.147 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:00.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:00.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:00.279 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:00.299 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 12821 12805 200.019 182.75 0.00527592 0.00499348 2024-09-21T10:09:00.315 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:00.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:00.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:00.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:00.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:09:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:09:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:09:00.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:00.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:00.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:00.705 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:00.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:00.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:00.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:00.862 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:09:00.862 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:09:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:00.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:00.981 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:01.014 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:01.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:01.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:01.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:01.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:01.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:01.289 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:01.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:01.299 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 15400 15384 192.251 161.188 0.0054243 0.00517565 2024-09-21T10:09:01.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:01.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:01.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:01.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:01.552 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:01.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:01.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:01.684 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:01.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:01.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:01.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:01 smithi017 ceph-mon[102913]: pgmap v1779: 109 pgs: 109 active+clean; 246 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s wr, 400 op/s 2024-09-21T10:09:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:01 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:01 smithi017 ceph-mon[105151]: pgmap v1779: 109 pgs: 109 active+clean; 246 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s wr, 400 op/s 2024-09-21T10:09:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:01 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:01.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:01 smithi134 ceph-mon[90085]: pgmap v1779: 109 pgs: 109 active+clean; 246 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s wr, 400 op/s 2024-09-21T10:09:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:01 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:02.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:02.099 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:02.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:02.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:02.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:02.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:02.300 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 17847 17831 185.668 152.938 0.0055263 0.00537776 2024-09-21T10:09:02.350 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:02.377 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:02.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:02.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:02.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:02.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:02.606 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:09:02.626 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:02.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:02.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:02.760 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:02.794 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:02.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:02.896 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:02.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:02.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2704457372' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2704457372' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:02.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2704457372' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:03.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:03.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:03.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:03.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:03.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:03.234 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 6, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:09:03.235 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:09:03.235 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 2, 3, 4, 5, 7, 6, 1] 2024-09-21T10:09:03.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd out 7 2024-09-21T10:09:03.300 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 20301 20285 181.054 153.375 0.00615978 0.0055169 2024-09-21T10:09:03.318 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:03.346 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:03.448 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:03.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:03.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:03.579 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:03.606 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:03.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:03.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[102913]: pgmap v1780: 109 pgs: 109 active+clean; 709 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-21T10:09:03.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:03.608 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2704457372' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:09:03.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:03.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:03.844 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:03.877 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:03.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[102913]: osdmap e1243: 8 total, 8 up, 8 in 2024-09-21T10:09:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[105151]: pgmap v1780: 109 pgs: 109 active+clean; 709 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-21T10:09:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2704457372' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:09:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[105151]: osdmap e1243: 8 total, 8 up, 8 in 2024-09-21T10:09:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:03 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:03.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:03 smithi134 ceph-mon[90085]: pgmap v1780: 109 pgs: 109 active+clean; 709 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-21T10:09:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:03 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:03 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2704457372' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:09:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:03 smithi134 ceph-mon[90085]: osdmap e1243: 8 total, 8 up, 8 in 2024-09-21T10:09:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:03 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:04.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:04.116 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:04.148 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:04.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:04.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:04.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:04.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:04.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 22669 22653 176.921 148 0.0060894 0.00564509 2024-09-21T10:09:04.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:09:04.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:04.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:04.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:04.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:04.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:04.679 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:04.712 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:04.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:04.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:04.844 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:04.946 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:04 smithi017 ceph-mon[102913]: 1.e scrub starts 2024-09-21T10:09:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:04 smithi017 ceph-mon[102913]: 1.e scrub ok 2024-09-21T10:09:04.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:04 smithi017 ceph-mon[105151]: 1.e scrub starts 2024-09-21T10:09:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:04 smithi017 ceph-mon[105151]: 1.e scrub ok 2024-09-21T10:09:04.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:09:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:09:04] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-21T10:09:04.977 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:04.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:04 smithi134 ceph-mon[90085]: 1.e scrub starts 2024-09-21T10:09:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:04 smithi134 ceph-mon[90085]: 1.e scrub ok 2024-09-21T10:09:05.079 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:05.109 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:05.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:05.211 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:05.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.243 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:05.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:05.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 24706 24690 171.409 127.312 0.00385085 0.00582855 2024-09-21T10:09:05.345 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:05.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.375 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:05.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:05.476 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:05.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.498 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:05.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:05.547 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:05.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:05.624 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:05.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:05.726 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:05.758 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:05.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:05.877 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:05 smithi017 ceph-mon[105151]: pgmap v1782: 109 pgs: 109 active+clean; 988 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-21T10:09:05.877 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:05 smithi017 ceph-mon[105151]: 1.d scrub starts 2024-09-21T10:09:05.877 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:05 smithi017 ceph-mon[105151]: 1.d scrub ok 2024-09-21T10:09:05.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:05 smithi017 ceph-mon[102913]: pgmap v1782: 109 pgs: 109 active+clean; 988 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-21T10:09:05.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:05 smithi017 ceph-mon[102913]: 1.d scrub starts 2024-09-21T10:09:05.878 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:05 smithi017 ceph-mon[102913]: 1.d scrub ok 2024-09-21T10:09:05.883 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:05.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:05.895 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:05 smithi134 ceph-mon[90085]: pgmap v1782: 109 pgs: 109 active+clean; 988 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-21T10:09:05.895 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:05 smithi134 ceph-mon[90085]: 1.d scrub starts 2024-09-21T10:09:05.895 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:05 smithi134 ceph-mon[90085]: 1.d scrub ok 2024-09-21T10:09:05.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:06.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:06.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:06.134 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:06.168 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:06.270 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:06.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 26630 26614 166.293 120.25 0.00611903 0.00600759 2024-09-21T10:09:06.308 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:06.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:06.410 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:06.426 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:06.444 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:06.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:06.546 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:06.580 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:06.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:06.681 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:06.717 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:06.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:06.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:06.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:06.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:06.964 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:06.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.999 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:06.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:07.101 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:07.134 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:07.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:07.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:07.263 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:07.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:07.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:07.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 28900 28884 164.072 141.875 0.00968879 0.00608822 2024-09-21T10:09:07.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:07.428 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:07.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:07.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:07.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:07.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:07.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:09:07.670 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:07.701 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:07.802 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:07.832 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:07.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:07 smithi017 ceph-mon[102913]: pgmap v1783: 109 pgs: 109 active+clean; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-21T10:09:07.909 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:07 smithi017 ceph-mon[105151]: pgmap v1783: 109 pgs: 109 active+clean; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-21T10:09:07.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:07.976 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:07.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:07 smithi134 ceph-mon[90085]: pgmap v1783: 109 pgs: 109 active+clean; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-21T10:09:08.078 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:08.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:08.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:08.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:08.243 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:08.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:08.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 30976 30960 161.209 129.75 0.0048939 0.0061978 2024-09-21T10:09:08.344 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:08.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:08.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:08.478 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:08.511 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:08.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:08.613 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:08.645 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:08.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:08.746 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.778 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:08.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:08.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:08.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:08.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:09.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:09.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:09.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:09.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:09.194 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:09.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:09.296 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:09.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 33174 33158 159.375 137.375 0.00197561 0.00626607 2024-09-21T10:09:09.316 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:09.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:09.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:09.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:09.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:09.581 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:09.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:09.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:09.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:09.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:09.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:09.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:09 smithi017 ceph-mon[102913]: pgmap v1784: 109 pgs: 109 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-09-21T10:09:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:09 smithi017 ceph-mon[105151]: pgmap v1784: 109 pgs: 109 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-09-21T10:09:09.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:09 smithi134 ceph-mon[90085]: pgmap v1784: 109 pgs: 109 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-09-21T10:09:10.001 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:10.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:10.103 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:10.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:10.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:10.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:10.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:10.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 34674 34658 154.688 93.75 0.00470801 0.00645861 2024-09-21T10:09:10.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:10.389 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:10.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:10.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:09:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:09:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:09:10.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:10.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:10.629 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:10.659 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:10.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:10.761 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:10.782 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:10.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:10.884 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:10.916 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:10.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:11.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:11.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:11.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:11.144 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:11.175 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:11.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:11.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:11.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 36422 36406 151.659 109.25 0.00673406 0.00658854 2024-09-21T10:09:11.306 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:11.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:11.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:11.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:11.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:11.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:11.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:11.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:11.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:11 smithi017 ceph-mon[102913]: pgmap v1785: 109 pgs: 109 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-21T10:09:11.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:11 smithi017 ceph-mon[105151]: pgmap v1785: 109 pgs: 109 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-21T10:09:11.717 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:11 smithi134 ceph-mon[90085]: pgmap v1785: 109 pgs: 109 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-21T10:09:11.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:11.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:11.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:12.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:12.104 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:12.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:12.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:12.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:12.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:12.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:12.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 38158 38142 148.961 108.5 0.00588212 0.00670823 2024-09-21T10:09:12.364 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:12.396 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:12.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:12.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:12.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:12.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:12.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:12.760 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:12.787 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:12.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:12.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:12.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:12.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:13.028 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:13.058 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:13.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:13.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:13.285 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:13.301 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 39361 39345 144.621 75.1875 0.00236614 0.0068319 2024-09-21T10:09:13.306 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:13.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:13.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:13.436 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:13.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:13.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:13.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:13.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:13.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:13 smithi017 ceph-mon[102913]: pgmap v1786: 109 pgs: 109 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-21T10:09:13.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:13 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1202081027' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:13.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:13 smithi017 ceph-mon[105151]: pgmap v1786: 109 pgs: 109 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-21T10:09:13.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:13 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1202081027' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:13.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:13.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:13 smithi134 ceph-mon[90085]: pgmap v1786: 109 pgs: 109 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-21T10:09:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:13 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1202081027' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:13.823 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:13.848 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:13.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:13.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:13.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:14.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:14.125 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:14.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:14.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:14.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:14.302 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 41016 41000 142.333 103.438 0.00512258 0.00702063 2024-09-21T10:09:14.351 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:14.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.387 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:14.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:14.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:14.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:14.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:14.626 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:14.655 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:14.757 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:14.789 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:14.891 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:14.916 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:09:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:09:14] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-09-21T10:09:15.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:15.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:15.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:15.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:15.185 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:15.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:15.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:15.302 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 42698 42682 140.374 105.125 0.00416885 0.00711908 2024-09-21T10:09:15.317 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:15.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:15.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:15.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:15.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:15.543 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:15.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.576 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:15.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:15.677 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:15 smithi017 ceph-mon[102913]: pgmap v1787: 109 pgs: 109 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-21T10:09:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:15 smithi017 ceph-mon[105151]: pgmap v1787: 109 pgs: 109 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-21T10:09:15.709 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:15.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:15.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:15 smithi134 ceph-mon[90085]: pgmap v1787: 109 pgs: 109 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-21T10:09:15.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:15.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:15.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:15.935 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:15.956 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:16.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:16.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:16.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:16.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:16.302 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:09:16.301054+0000 min lat: 0.00130567 max lat: 0.348666 avg lat: 0.00727279 2024-09-21T10:09:16.302 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:09:16.302 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 43989 43973 137.39 80.6875 0.00559409 0.00727279 2024-09-21T10:09:16.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:16.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:16.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:16.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:16.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:16.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:16.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:16.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.630 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:16.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:16.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:16.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:16.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:16.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:16.915 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:16.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:17.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:17.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:17.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:17.159 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:17.191 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:17.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:17.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:17.303 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 45690 45674 135.903 106.312 0.0106829 0.00735275 2024-09-21T10:09:17.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:17.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:17.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:17.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:17.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:17.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:17.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:17.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:17.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:17 smithi017 ceph-mon[102913]: pgmap v1788: 109 pgs: 109 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-21T10:09:17.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:17 smithi017 ceph-mon[105151]: pgmap v1788: 109 pgs: 109 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-21T10:09:17.705 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:17.737 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:17.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:17 smithi134 ceph-mon[90085]: pgmap v1788: 109 pgs: 109 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-21T10:09:17.838 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:17.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.870 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:17.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:17.970 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:18.002 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:18.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:18.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:18.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:18.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:18.244 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:18.274 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:18.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:18.303 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 47384 47368 134.538 105.875 0.00475496 0.00742777 2024-09-21T10:09:18.373 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:09:18.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:09:18.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:09:18.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:09:18.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:09:18.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:09:18.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:09:18.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:09:18.374 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:09:18.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:09:18.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:09:18.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:09:18.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:09:18.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:09:18.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:09:18.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:09:18.377 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:18.408 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:18.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:18.509 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:18.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:18.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:18.643 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:18.674 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:18.775 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:18.804 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:18.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:18.905 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:18.937 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:19.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:19.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:19.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:19.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:19.303 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 49139 49123 133.457 109.688 0.0120063 0.00748808 2024-09-21T10:09:19.320 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:19.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.353 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:19.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:19.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:19.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:19.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:19.578 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:19.611 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:19.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:19.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:19 smithi017 ceph-mon[102913]: pgmap v1789: 109 pgs: 109 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-09-21T10:09:19.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:19 smithi017 ceph-mon[105151]: pgmap v1789: 109 pgs: 109 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-09-21T10:09:19.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:19.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:19.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:19.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:19 smithi134 ceph-mon[90085]: pgmap v1789: 109 pgs: 109 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2024-09-21T10:09:19.838 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:19.869 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:19.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:19.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:20.007 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:20.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:20.109 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.137 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:20.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:20.238 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:20.266 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:20.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:20.303 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 50673 50657 131.891 95.875 0.00806906 0.00757659 2024-09-21T10:09:20.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:20.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:09:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:09:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:09:20.506 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:20.536 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:20.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:20.637 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:20.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:20.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:20.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:20.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:20.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:20.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:20.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:21.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:21.066 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:21.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:21.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:21.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:21.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:21.303 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 52373 52357 130.865 106.25 0.00786129 0.0076352 2024-09-21T10:09:21.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:21.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:21.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:21.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:21.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:21.559 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:21 smithi017 ceph-mon[102913]: pgmap v1790: 109 pgs: 109 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:09:21.559 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:21 smithi017 ceph-mon[105151]: pgmap v1790: 109 pgs: 109 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:09:21.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:21.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:21.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:21.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:21.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:21 smithi134 ceph-mon[90085]: pgmap v1790: 109 pgs: 109 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:09:21.773 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:21.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:21.875 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:21.909 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:21.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:22.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:22.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:22.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:22.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:22.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:22.294 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:22.303 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 53972 53956 129.675 99.9375 0.00858303 0.00770657 2024-09-21T10:09:22.330 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:22.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:22.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:22.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:22.577 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:22.609 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:22.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:22.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:22.729 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:22.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:22.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:22.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:22.908 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:22.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:23.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:23.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:23.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:23.210 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:23.304 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 55556 55540 128.539 99 0.0103719 0.00777418 2024-09-21T10:09:23.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:23.343 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:23.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:23.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:23.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:23.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:23.587 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:23.610 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:23.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:23.662 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:09:23.662 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:09:23.662 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:09:23.662 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:09:23.662 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:09:23.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:23 smithi017 ceph-mon[102913]: pgmap v1791: 109 pgs: 109 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-21T10:09:23.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:23 smithi017 ceph-mon[105151]: pgmap v1791: 109 pgs: 109 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-21T10:09:23.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:23 smithi134 ceph-mon[90085]: pgmap v1791: 109 pgs: 109 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-21T10:09:23.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:23.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:23.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:23.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:23.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:23.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:24.024 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:24.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:24.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:24.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:24.271 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:24.304 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 57076 57060 127.341 95 0.0154675 0.00784804 2024-09-21T10:09:24.308 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:24.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:24.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:24.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:09:23.950+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1243 definitely_dead 0 2024-09-21T10:09:24.438 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.7. 2024-09-21T10:09:24.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:24.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:24.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:09:24.560 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:24.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:24.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:24.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2026081445' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-21T10:09:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:24 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-21T10:09:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:24 smithi017 ceph-mon[105151]: Client client.admin marked osd.7 out, while it was still marked up 2024-09-21T10:09:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2026081445' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-21T10:09:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:24 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-21T10:09:24.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:24 smithi017 ceph-mon[102913]: Client client.admin marked osd.7 out, while it was still marked up 2024-09-21T10:09:24.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:09:24.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:24.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2026081445' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-21T10:09:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:24 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-21T10:09:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:24 smithi134 ceph-mon[90085]: Client client.admin marked osd.7 out, while it was still marked up 2024-09-21T10:09:24.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:24.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:24.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:24.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:24.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:25.059 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:25.082 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:25.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:25.126 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:09:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:09:24] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-09-21T10:09:25.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:25.207 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:25.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:25.304 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 57301 57285 123.435 14.0625 0.00291432 0.00785656 2024-09-21T10:09:25.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:25.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:25.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:25.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:25.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:25.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:25.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:25 smithi017 ceph-mon[102913]: pgmap v1792: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:09:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:25 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-09-21T10:09:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:25 smithi017 ceph-mon[102913]: osdmap e1244: 8 total, 8 up, 7 in 2024-09-21T10:09:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:25 smithi017 ceph-mon[105151]: pgmap v1792: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:09:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:25 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-09-21T10:09:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:25 smithi017 ceph-mon[105151]: osdmap e1244: 8 total, 8 up, 7 in 2024-09-21T10:09:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:25.713 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:25.738 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:25 smithi134 ceph-mon[90085]: pgmap v1792: 109 pgs: 109 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:09:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:25 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-09-21T10:09:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:25 smithi134 ceph-mon[90085]: osdmap e1244: 8 total, 8 up, 7 in 2024-09-21T10:09:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:25.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:25.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:25.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:25.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:25.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:26.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:26.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:26.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:26.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:26.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:26.304 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 57301 57285 119.321 0 - 0.00785656 2024-09-21T10:09:26.337 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:26.363 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:26.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:26.465 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:26.491 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:26.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.634 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:26.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[102913]: osdmap e1245: 8 total, 8 up, 7 in 2024-09-21T10:09:26.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[102913]: 1.10 scrub ok 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[102913]: 1.9 scrub starts 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[102913]: 1.9 scrub ok 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[102913]: osdmap e1246: 8 total, 8 up, 7 in 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[105151]: osdmap e1245: 8 total, 8 up, 7 in 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[105151]: 1.10 scrub ok 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[105151]: 1.9 scrub starts 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[105151]: 1.9 scrub ok 2024-09-21T10:09:26.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:26 smithi017 ceph-mon[105151]: osdmap e1246: 8 total, 8 up, 7 in 2024-09-21T10:09:26.736 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:26 smithi134 ceph-mon[90085]: osdmap e1245: 8 total, 8 up, 7 in 2024-09-21T10:09:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:26 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:09:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:26 smithi134 ceph-mon[90085]: 1.10 scrub ok 2024-09-21T10:09:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:26 smithi134 ceph-mon[90085]: 1.9 scrub starts 2024-09-21T10:09:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:26 smithi134 ceph-mon[90085]: 1.9 scrub ok 2024-09-21T10:09:26.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:26 smithi134 ceph-mon[90085]: osdmap e1246: 8 total, 8 up, 7 in 2024-09-21T10:09:26.759 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:26.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:26.861 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:26.883 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:26.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:26.985 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:27.008 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:27.110 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:27.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:27.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:27.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:27.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:27.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:27.304 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 57715 57699 116.306 12.9375 0.0176844 0.00858729 2024-09-21T10:09:27.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:27.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:27.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:27.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:27.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:27.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:27 smithi017 ceph-mon[105151]: pgmap v1795: 109 pgs: 10 peering, 99 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 8.4 KiB/s rd, 77 MiB/s wr, 1.24k op/s; 6.5 KiB/s, 0 objects/s recovering 2024-09-21T10:09:27.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:27 smithi017 ceph-mon[105151]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:09:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:27 smithi017 ceph-mon[102913]: pgmap v1795: 109 pgs: 10 peering, 99 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 8.4 KiB/s rd, 77 MiB/s wr, 1.24k op/s; 6.5 KiB/s, 0 objects/s recovering 2024-09-21T10:09:27.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:27 smithi017 ceph-mon[102913]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:09:27.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:27 smithi134 ceph-mon[90085]: pgmap v1795: 109 pgs: 10 peering, 99 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 8.4 KiB/s rd, 77 MiB/s wr, 1.24k op/s; 6.5 KiB/s, 0 objects/s recovering 2024-09-21T10:09:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:27 smithi134 ceph-mon[90085]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:09:27.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:27.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:27.954 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:27.979 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:27.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:28.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:28.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:28.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:28.204 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:28.228 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:28.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:28.304 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 58499 58483 114.203 49 0.0272696 0.00875017 2024-09-21T10:09:28.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:28.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:28.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:28.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:28.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:28.610 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:28.633 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:28.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:28 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 21373/172065 objects degraded (12.421%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:28.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:28 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 21373/172065 objects degraded (12.421%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:28.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:28.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:28.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:28.853 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:28.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:28.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:28.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:28 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 21373/172065 objects degraded (12.421%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:29.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:29.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:29.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:29.231 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:29.256 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:29.304 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 59257 59241 112.178 47.375 0.0486236 0.00890592 2024-09-21T10:09:29.357 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.382 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:29.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:29.483 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:29.508 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:29.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:29.655 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:29.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:29 smithi017 ceph-mon[102913]: pgmap v1797: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 10 peering, 93 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 21373/172065 objects degraded (12.421%); 30 KiB/s, 0 objects/s recovering 2024-09-21T10:09:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:29 smithi017 ceph-mon[105151]: pgmap v1797: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 10 peering, 93 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 21373/172065 objects degraded (12.421%); 30 KiB/s, 0 objects/s recovering 2024-09-21T10:09:29.757 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:29.783 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:29.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:29.885 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:29.909 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:29.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:29 smithi134 ceph-mon[90085]: pgmap v1797: 109 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 10 peering, 93 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 21373/172065 objects degraded (12.421%); 30 KiB/s, 0 objects/s recovering 2024-09-21T10:09:30.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:30.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:30.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:30.152 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:09:30.152 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:09:30.152 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 2, 3, 4, 5, 6, 1] 2024-09-21T10:09:30.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd out 6 2024-09-21T10:09:30.153 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:30.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:30.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:30.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:30.305 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 60000 59984 110.242 46.4375 0.0405887 0.00906156 2024-09-21T10:09:30.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:30.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:30.449 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:30.474 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:09:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:09:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:09:30.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:30.599 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:30.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:30.700 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:30.726 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:30.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:30.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:30.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:30 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1590564143' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:30.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:30 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:30.845 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:30 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1590564143' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:30.845 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:30 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:30.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:30.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:30.945 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:09:30.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:30 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1590564143' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:30 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:31.013 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:31.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:31.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:31.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:31.265 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:31.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:31.306 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 60755 60739 108.44 47.1875 0.0109262 0.00921399 2024-09-21T10:09:31.332 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:31.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:31.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:31.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:31.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:31.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:31.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:31.723 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:31.746 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:31.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[102913]: pgmap v1798: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11 KiB/s rd, 57 MiB/s wr, 911 op/s; 24973/174546 objects degraded (14.307%); 6946/174546 objects misplaced (3.979%); 1.8 MiB/s, 1 keys/s, 30 objects/s recovering 2024-09-21T10:09:31.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[102913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-09-21T10:09:31.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:09:31.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:31.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[102913]: osdmap e1247: 8 total, 8 up, 7 in 2024-09-21T10:09:31.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[105151]: pgmap v1798: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11 KiB/s rd, 57 MiB/s wr, 911 op/s; 24973/174546 objects degraded (14.307%); 6946/174546 objects misplaced (3.979%); 1.8 MiB/s, 1 keys/s, 30 objects/s recovering 2024-09-21T10:09:31.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[105151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-09-21T10:09:31.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:09:31.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:31.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:31 smithi017 ceph-mon[105151]: osdmap e1247: 8 total, 8 up, 7 in 2024-09-21T10:09:31.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:31.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:31.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:31 smithi134 ceph-mon[90085]: pgmap v1798: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 11 KiB/s rd, 57 MiB/s wr, 911 op/s; 24973/174546 objects degraded (14.307%); 6946/174546 objects misplaced (3.979%); 1.8 MiB/s, 1 keys/s, 30 objects/s recovering 2024-09-21T10:09:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:31 smithi134 ceph-mon[90085]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-09-21T10:09:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:31 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:09:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:31 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:31 smithi134 ceph-mon[90085]: osdmap e1247: 8 total, 8 up, 7 in 2024-09-21T10:09:32.024 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:32.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:32.151 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:32.175 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:32.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:32.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:32.299 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:32.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:32.305 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 61492 61476 106.707 46.0625 0.0641889 0.00936205 2024-09-21T10:09:32.401 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:32.425 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:32.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:32.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:32.566 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:32.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:32.668 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:32.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:32.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:32.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:32.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:32.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:09:32.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:09:32.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:32.720 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/722106045' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:32.720 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:32.721 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:32.721 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:09:32.721 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:09:32.721 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:32.721 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:32 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/722106045' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:32.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:32.812 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:09:32.812 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:09:32.813 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:09:32.814 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:09:32.814 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:09:32.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:32.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:32.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:32.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:32.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:09:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:09:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:32 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/722106045' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:33.077 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:33.103 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:33.205 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:33.231 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:33.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:33.306 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 62153 62137 104.94 41.3125 0.00627448 0.00952026 2024-09-21T10:09:33.333 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:33.358 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:33.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:33.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:33.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:33.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:33.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:33.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:33.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:33.871 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:33.895 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:33.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:33 smithi017 ceph-mon[102913]: pgmap v1800: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s; 24973/178797 objects degraded (13.967%); 6946/178797 objects misplaced (3.885%); 1.6 MiB/s, 1 keys/s, 26 objects/s recovering 2024-09-21T10:09:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 24973/178797 objects degraded (13.967%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:33 smithi017 ceph-mon[105151]: pgmap v1800: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s; 24973/178797 objects degraded (13.967%); 6946/178797 objects misplaced (3.885%); 1.6 MiB/s, 1 keys/s, 26 objects/s recovering 2024-09-21T10:09:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 24973/178797 objects degraded (13.967%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:33 smithi134 ceph-mon[90085]: pgmap v1800: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s; 24973/178797 objects degraded (13.967%); 6946/178797 objects misplaced (3.885%); 1.6 MiB/s, 1 keys/s, 26 objects/s recovering 2024-09-21T10:09:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 24973/178797 objects degraded (13.967%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:33.997 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:34.023 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:34.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:34.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:34.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:34.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:34.270 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:34.292 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:34.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:34.305 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 62887 62871 103.386 45.875 0.00924564 0.0096649 2024-09-21T10:09:34.394 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:34.416 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:34.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:34.518 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:34.544 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:34.646 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:34.668 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:34.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:34.770 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:34.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:34.895 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:34.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:34.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:35.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:35.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:35.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:35.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:09:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:09:34] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.51.0" 2024-09-21T10:09:35.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:35.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:35.306 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 63608 63592 101.89 45.0625 0.013046 0.00980778 2024-09-21T10:09:35.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:35.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:35.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:35.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:35.494 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:35.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:35.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:35.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:35.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:35.747 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:35.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.771 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:35.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:35.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:35.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:35.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:35 smithi017 ceph-mon[102913]: pgmap v1801: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 24973/183588 objects degraded (13.603%); 6688/183588 objects misplaced (3.643%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2024-09-21T10:09:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:35 smithi017 ceph-mon[105151]: pgmap v1801: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 24973/183588 objects degraded (13.603%); 6688/183588 objects misplaced (3.643%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2024-09-21T10:09:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:35 smithi134 ceph-mon[90085]: pgmap v1801: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 24973/183588 objects degraded (13.603%); 6688/183588 objects misplaced (3.643%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2024-09-21T10:09:36.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:36.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:36.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:36.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:36.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:36.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:36.306 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:09:36.304989+0000 min lat: 0.00130567 max lat: 2.35412 avg lat: 0.0099336 2024-09-21T10:09:36.306 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:09:36.306 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 64404 64388 100.587 49.75 0.00823528 0.0099336 2024-09-21T10:09:36.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:36.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:36.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:36.443 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:36.469 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:36.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:36.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:36.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:36.717 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:36.743 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:36.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:36.844 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:36.875 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:36.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:36.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:37.004 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:37.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:37.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:37.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:37.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:37.257 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:37.281 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:37.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:37.306 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 65235 65219 99.4003 51.9375 0.0446675 0.0100536 2024-09-21T10:09:37.383 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:37.410 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:37.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:37.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:37.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:37.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:37.654 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:37.678 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:37.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:37.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:37.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:37.923 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:37.947 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:37.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:37 smithi134 ceph-mon[90085]: pgmap v1802: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 48 MiB/s wr, 765 op/s; 24973/185571 objects degraded (13.457%); 6579/185571 objects misplaced (3.545%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2024-09-21T10:09:38.049 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:38.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.075 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:38.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:38.176 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:37 smithi017 ceph-mon[102913]: pgmap v1802: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 48 MiB/s wr, 765 op/s; 24973/185571 objects degraded (13.457%); 6579/185571 objects misplaced (3.545%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2024-09-21T10:09:38.200 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:37 smithi017 ceph-mon[105151]: pgmap v1802: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 48 MiB/s wr, 765 op/s; 24973/185571 objects degraded (13.457%); 6579/185571 objects misplaced (3.545%); 3.4 MiB/s, 1 keys/s, 55 objects/s recovering 2024-09-21T10:09:38.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:38.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:38.307 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 66049 66033 98.2421 50.875 0.0305743 0.0101725 2024-09-21T10:09:38.366 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:38.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:38.382 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:38.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:38.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:38.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:38.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:38.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:38.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:38.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:38.795 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:38.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:38.811 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 24973/185571 objects degraded (13.457%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:38.811 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 24973/185571 objects degraded (13.457%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:38.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:38.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:38.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:38.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:09:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 24973/185571 objects degraded (13.457%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:39.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:39.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.093 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:39.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:39.194 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:39.216 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:39.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:39.311 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 66753 66737 96.9807 44 0.0108596 0.0103027 2024-09-21T10:09:39.317 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:39.342 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:39.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:39.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:39.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:39.591 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:39.617 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:39.617 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:09:39.618 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:09:39.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:39.719 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:39.743 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:39.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:39.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:39.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:39.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:09:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:39 smithi017 ceph-mon[102913]: pgmap v1803: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s; 24973/194220 objects degraded (12.858%); 6579/194220 objects misplaced (3.387%); 3.4 MiB/s, 0 keys/s, 54 objects/s recovering 2024-09-21T10:09:39.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:39 smithi017 ceph-mon[105151]: pgmap v1803: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s; 24973/194220 objects degraded (12.858%); 6579/194220 objects misplaced (3.387%); 3.4 MiB/s, 0 keys/s, 54 objects/s recovering 2024-09-21T10:09:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:39 smithi134 ceph-mon[90085]: pgmap v1803: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s; 24973/194220 objects degraded (12.858%); 6579/194220 objects misplaced (3.387%); 3.4 MiB/s, 0 keys/s, 54 objects/s recovering 2024-09-21T10:09:40.000 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:40.022 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:40.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:40.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:40.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:40.266 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:40.288 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:40.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:40.308 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 67526 67510 95.8746 48.3125 0.0134203 0.0104231 2024-09-21T10:09:40.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:40.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:40.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:40.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:09:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:09:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:09:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:09:40.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:40.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:40.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:40.731 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:40.755 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:40.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:40.857 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:40.883 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:40.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:40.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:41.001 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:41.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:41.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:41.091 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:41.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:41.224 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:41.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:41.308 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 68205 68189 94.6852 42.4375 0.0266014 0.0105519 2024-09-21T10:09:41.326 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:41.350 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:41.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:41.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:41 smithi017 ceph-mon[102913]: pgmap v1804: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 47 MiB/s wr, 750 op/s; 24973/197085 objects degraded (12.671%); 6191/197085 objects misplaced (3.141%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:41.457 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:41 smithi017 ceph-mon[105151]: pgmap v1804: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 47 MiB/s wr, 750 op/s; 24973/197085 objects degraded (12.671%); 6191/197085 objects misplaced (3.141%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:09:41.476 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:41.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:41.577 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:41.602 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:41.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:41.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:41.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:41 smithi134 ceph-mon[90085]: pgmap v1804: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 47 MiB/s wr, 750 op/s; 24973/197085 objects degraded (12.671%); 6191/197085 objects misplaced (3.141%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:09:41.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:41.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:41.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:41.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:41.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:42.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:42.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:42.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:42.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:42.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:42.311 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 68935 68919 93.6131 45.625 0.0451669 0.0106751 2024-09-21T10:09:42.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:42.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:42.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:42.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:42.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:42.653 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:42.755 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:42.778 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:42.880 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:42.905 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:42.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:43.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:43.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:43.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:43.113 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:43.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:43.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:43.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:43.314 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 69615 69599 92.5257 42.5 0.0140627 0.0108016 2024-09-21T10:09:43.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:43.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:43.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:43.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:43.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:43.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:43.607 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:43.629 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:43 smithi017 ceph-mon[102913]: pgmap v1805: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 24973/201231 objects degraded (12.410%); 6191/201231 objects misplaced (3.077%); 4.0 MiB/s, 63 objects/s recovering 2024-09-21T10:09:43.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:43 smithi017 ceph-mon[105151]: pgmap v1805: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 24973/201231 objects degraded (12.410%); 6191/201231 objects misplaced (3.077%); 4.0 MiB/s, 63 objects/s recovering 2024-09-21T10:09:43.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:43 smithi134 ceph-mon[90085]: pgmap v1805: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 24973/201231 objects degraded (12.410%); 6191/201231 objects misplaced (3.077%); 4.0 MiB/s, 63 objects/s recovering 2024-09-21T10:09:43.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:43.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:43.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:43.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:43.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:44.026 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:44.050 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:44.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:44.152 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:44.174 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:44.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:44.312 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 70265 70249 91.4446 40.625 0.0208214 0.0109259 2024-09-21T10:09:44.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:44.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:44.424 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:44.447 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:44.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:44.549 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:44.573 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:44.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:44.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:44.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:44 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 24973/201231 objects degraded (12.410%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:44.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:09:44.433+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1247 definitely_dead 0 2024-09-21T10:09:44.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:44 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 24973/201231 objects degraded (12.410%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:44.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:44.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:44 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 24973/201231 objects degraded (12.410%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:44.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:44.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:44.945 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:09:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:09:44] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.51.0" 2024-09-21T10:09:44.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:45.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:45.109 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:45.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:45.237 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:45.260 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:45.312 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 70960 70944 90.4649 43.4375 0.0520984 0.0110471 2024-09-21T10:09:45.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:45.386 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:45.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:45.439 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.6. 2024-09-21T10:09:45.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:45 smithi017 ceph-mon[102913]: pgmap v1806: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 47 MiB/s wr, 755 op/s; 24973/206022 objects degraded (12.122%); 5938/206022 objects misplaced (2.882%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:09:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:45 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3755771090' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T10:09:45.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:45 smithi017 ceph-mon[102913]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T10:09:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:45 smithi017 ceph-mon[105151]: pgmap v1806: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 47 MiB/s wr, 755 op/s; 24973/206022 objects degraded (12.122%); 5938/206022 objects misplaced (2.882%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:09:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:45 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3755771090' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T10:09:45.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:45 smithi017 ceph-mon[105151]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T10:09:45.487 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:45.508 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:45.610 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:45.631 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:45.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:45.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:09:45.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:09:45.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:09:45.657 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:09:45.657 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:09:45.657 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:09:45.660 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:09:45.660 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:09:45.678 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:09:45.679 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:09:45.695 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:09:45.695 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:09:45.725 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:09:45.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:09:45.732 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:45 smithi134 ceph-mon[90085]: pgmap v1806: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 47 MiB/s wr, 755 op/s; 24973/206022 objects degraded (12.122%); 5938/206022 objects misplaced (2.882%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:09:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:45 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3755771090' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T10:09:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:45 smithi134 ceph-mon[90085]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T10:09:45.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:09:45.747 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:09:45.754 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:45.856 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:45.879 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:45.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:45.981 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:46.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:46.104 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:46.126 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:46.228 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:46.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.252 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:46.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:46.312 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 71121 71105 88.8572 10.0625 0.00843166 0.0110756 2024-09-21T10:09:46.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:46.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:46.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:46.441 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:09:46.480 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:46.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.504 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:46.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:46.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:46.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3755771090' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T10:09:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:46 smithi017 ceph-mon[102913]: osdmap e1248: 8 total, 8 up, 6 in 2024-09-21T10:09:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1686781244' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/926182184' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:09:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3755771090' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T10:09:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:46 smithi017 ceph-mon[105151]: osdmap e1248: 8 total, 8 up, 6 in 2024-09-21T10:09:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1686781244' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:46.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/926182184' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:09:46.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3755771090' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T10:09:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:46 smithi134 ceph-mon[90085]: osdmap e1248: 8 total, 8 up, 6 in 2024-09-21T10:09:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1686781244' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:46.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/926182184' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:09:46.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:46.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:46.857 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:46.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.881 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:46.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:46.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:47.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:47.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:47.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:47.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:47.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:47.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:47.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:47.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:47.312 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 71121 71105 87.1151 0 - 0.0110756 2024-09-21T10:09:47.357 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:47.380 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:47.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:47.563 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:47.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:47.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:47 smithi017 ceph-mon[105151]: pgmap v1808: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 47 MiB/s wr, 748 op/s; 24973/208026 objects degraded (12.005%); 5827/208026 objects misplaced (2.801%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:09:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:47 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:47 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/926182184' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:09:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:47 smithi017 ceph-mon[105151]: osdmap e1249: 8 total, 8 up, 6 in 2024-09-21T10:09:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:47 smithi017 ceph-mon[102913]: pgmap v1808: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 47 MiB/s wr, 748 op/s; 24973/208026 objects degraded (12.005%); 5827/208026 objects misplaced (2.801%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:09:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:47 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:47 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/926182184' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:09:47.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:47 smithi017 ceph-mon[102913]: osdmap e1249: 8 total, 8 up, 6 in 2024-09-21T10:09:47.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:47.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:47 smithi134 ceph-mon[90085]: pgmap v1808: 109 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 47 MiB/s wr, 748 op/s; 24973/208026 objects degraded (12.005%); 5827/208026 objects misplaced (2.801%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:09:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:47 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:47 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/926182184' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:09:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:47 smithi134 ceph-mon[90085]: osdmap e1249: 8 total, 8 up, 6 in 2024-09-21T10:09:47.817 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:47.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.840 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:47.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:47.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:47.963 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:47.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:48.064 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:48.087 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:48.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:48.189 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:48.211 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:48.312 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 71428 71412 85.809 9.59375 0.0202587 0.0116459 2024-09-21T10:09:48.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:48.333 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:48.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:48.435 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:48.457 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:48.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:48.606 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:48.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:48 smithi017 ceph-mon[102913]: osdmap e1250: 8 total, 8 up, 6 in 2024-09-21T10:09:48.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:48 smithi017 ceph-mon[105151]: osdmap e1250: 8 total, 8 up, 6 in 2024-09-21T10:09:48.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:48.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:48 smithi134 ceph-mon[90085]: osdmap e1250: 8 total, 8 up, 6 in 2024-09-21T10:09:48.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:48.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:48.885 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:48.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:48.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:49.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:49.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:49.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:49.184 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:49.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:49.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:49.308 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:49.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:49.312 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 72054 72038 84.9283 39.125 0.0318495 0.0117665 2024-09-21T10:09:49.409 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:49.431 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:49.533 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:49.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:09:49.555 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:49.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:49.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:49.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:49.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:49 smithi134 ceph-mon[90085]: pgmap v1811: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 5.7 KiB/s rd, 40 MiB/s wr, 641 op/s; 18849/212736 objects degraded (8.860%); 12753/212736 objects misplaced (5.995%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:09:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:49 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 18849/212736 objects degraded (8.860%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:49.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:49.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:49.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:49 smithi017 ceph-mon[102913]: pgmap v1811: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 5.7 KiB/s rd, 40 MiB/s wr, 641 op/s; 18849/212736 objects degraded (8.860%); 12753/212736 objects misplaced (5.995%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:09:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:49 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 18849/212736 objects degraded (8.860%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:49 smithi017 ceph-mon[105151]: pgmap v1811: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 5.7 KiB/s rd, 40 MiB/s wr, 641 op/s; 18849/212736 objects degraded (8.860%); 12753/212736 objects misplaced (5.995%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:09:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:49 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 18849/212736 objects degraded (8.860%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:49.977 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:49.999 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:50.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:50.162 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:50.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:50.277 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:50.301 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:50.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:50.313 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 72635 72619 84.0275 36.3125 0.0402786 0.0118935 2024-09-21T10:09:50.403 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:50.425 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:50.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:09:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:09:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:09:50.527 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:50.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:50.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:50.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:50.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:50.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:50.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:09:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:50.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:50.919 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:09:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:09:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.919 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45851.0:72974 9.4 e1250/1238)", 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.901512+0000", 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00012141700000000001, 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014349499999999999, 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901512+0000", 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901512+0000", 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901513+0000", 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8900000000000001e-07 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901515+0000", 2024-09-21T10:09:50.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5720000000000001e-06 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901521+0000", 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.241e-06 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901527+0000", 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2039999999999996e-06 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:72968 9.2 e1250/1238 9:4ec889b8:::benchmark_data_smithi017_237924_object72967:head v 1250'4530, mlcod=1250'4528)", 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.877617+0000", 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.024016525, 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.024091205000000001, 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.877617+0000", 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.877617+0000", 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.877618+0000", 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.84e-07 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.877654+0000", 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6185e-05 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.877662+0000", 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1410000000000005e-06 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.877668+0000", 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1700000000000002e-06 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894505+0000", 2024-09-21T10:09:50.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016837250000000002 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894525+0000", 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9177999999999998e-05 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:72971 9.4 9:251b3a36:::benchmark_data_smithi017_237924_object72970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:09:50.926 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.882829+0000", 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.018804293999999999, 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018937101000000001, 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72971 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.882829+0000", 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.882829+0000", 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.882831+0000", 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.497e-06 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.882862+0000", 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1438e-05 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884396+0000", 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001534101 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884413+0000", 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6259000000000001e-05 2024-09-21T10:09:50.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884476+0000", 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2916000000000005e-05 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884619+0000", 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000143008 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884774+0000", 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015489400000000001 2024-09-21T10:09:50.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894353+0000", 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095790049999999998 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.895731+0000", 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001378911 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:50.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901681+0000", 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059498420000000003 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:72974 9.4 9:26030d2e:::benchmark_data_smithi017_237924_object72973:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.884437+0000", 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.017196844999999999, 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017415882000000001, 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:09:50.932 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72974 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884437+0000", 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884437+0000", 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884438+0000", 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2410000000000001e-06 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884467+0000", 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8844e-05 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884477+0000", 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0366999999999999e-05 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.884483+0000", 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5180000000000004e-06 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890689+0000", 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0062065530000000001 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890834+0000", 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014493399999999999 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890991+0000", 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000156747 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.895739+0000", 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047475420000000004 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:72973 9.5 e1250/1245 9:a6b4a239:::benchmark_data_smithi017_237924_object72972:head v 1250'4613, mlcod=1242'1)", 2024-09-21T10:09:50.936 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.885368+0000", 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.016265788999999999, 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016550339000000001, 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.885368+0000", 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.885368+0000", 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.885369+0000", 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6199999999999996e-07 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.885391+0000", 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2436999999999999e-05 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.885410+0000", 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9040000000000001e-05 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.885417+0000", 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.063e-06 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.885779+0000", 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036118800000000003 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.885834+0000", 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5893999999999997e-05 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:72978 9.e e1250/1245 9:7206c339:::benchmark_data_smithi017_237924_object72977:head v 1250'4429, mlcod=1243'1172)", 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.890648+0000", 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.010985767, 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011320264, 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.940 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890648+0000", 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890648+0000", 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890649+0000", 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4410000000000001e-06 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890707+0000", 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7760999999999998e-05 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890721+0000", 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4102e-05 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890730+0000", 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4740000000000005e-06 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890906+0000", 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017636400000000001 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890933+0000", 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6829e-05 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.943 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:72976 9.6 e1250/1238 9:6e67611c:::benchmark_data_smithi017_237924_object72975:head v 1250'4588, mlcod=1250'4586)", 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.890791+0000", 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.010842569, 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011230633, 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890791+0000", 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890791+0000", 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890792+0000", 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4499999999999997e-07 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890816+0000", 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.425e-05 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890824+0000", 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8189999999999992e-06 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.890829+0000", 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5350000000000004e-06 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.893364+0000", 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025350809999999998 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.893384+0000", 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9664e-05 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.947 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:72979 9.e e1250/1245)", 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.894648+0000", 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0069857909999999999, 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074245149999999996, 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894648+0000", 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894648+0000", 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894649+0000", 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.392e-06 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894684+0000", 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4822e-05 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894696+0000", 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1755e-05 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894703+0000", 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.883e-06 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:72977 9.6 e1250/1238)", 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.894772+0000", 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00686157, 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073410660000000003, 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894772+0000", 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894772+0000", 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894773+0000", 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.37e-07 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894794+0000", 2024-09-21T10:09:50.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1373999999999999e-05 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894806+0000", 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1685000000000001e-05 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.894811+0000", 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8169999999999998e-06 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPushReply(9.a 1250/1245 [PushReplyOp(9:5fb87abf:::benchmark_data_smithi017_237924_object3204:head)])", 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.901352+0000", 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00028210900000000001, 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00080547799999999999, 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901352+0000", 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901352+0000", 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901353+0000", 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2839999999999999e-06 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901413+0000", 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0495000000000003e-05 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.955 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901436+0000", 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3070000000000001e-05 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.901448+0000", 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1073999999999999e-05 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:72959 9.0 9:0680a9b7:::benchmark_data_smithi017_237924_object72958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:50.843982+0000", 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.057651734000000003, 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.058224453000000002, 2024-09-21T10:09:50.956 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 72959 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.843982+0000", 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.843982+0000", 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.843983+0000", 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.114e-06 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.844003+0000", 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9828999999999999e-05 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:09:50.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.844015+0000", 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1790999999999999e-05 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.844022+0000", 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5970000000000003e-06 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.867251+0000", 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023228641000000001 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.867406+0000", 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015512800000000001 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.867567+0000", 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016100899999999999 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.880212+0000", 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012644857000000001 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:09:50.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:50.880328+0000", 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116419 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 11 2024-09-21T10:09:50.961 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:09:50.961 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:50.982 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:51.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:51.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:51.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:51.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:51.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:51.314 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 73185 73169 83.1219 34.375 0.050405 0.0120235 2024-09-21T10:09:51.398 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:51.420 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:51.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:51.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:51.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:51.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:51.670 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:51.693 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:51.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:51 smithi017 ceph-mon[102913]: pgmap v1812: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 17 KiB/s rd, 28 MiB/s wr, 456 op/s; 28842/214074 objects degraded (13.473%); 6926/214074 objects misplaced (3.235%); 7.2 MiB/s, 1 keys/s, 115 objects/s recovering 2024-09-21T10:09:51.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:51 smithi017 ceph-mon[105151]: pgmap v1812: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 17 KiB/s rd, 28 MiB/s wr, 456 op/s; 28842/214074 objects degraded (13.473%); 6926/214074 objects misplaced (3.235%); 7.2 MiB/s, 1 keys/s, 115 objects/s recovering 2024-09-21T10:09:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:51 smithi134 ceph-mon[90085]: pgmap v1812: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 17 KiB/s rd, 28 MiB/s wr, 456 op/s; 28842/214074 objects degraded (13.473%); 6926/214074 objects misplaced (3.235%); 7.2 MiB/s, 1 keys/s, 115 objects/s recovering 2024-09-21T10:09:51.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:51.818 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:51.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:51.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:51.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:51.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:52.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:52.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:52.220 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:52.244 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:52.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:52.315 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 73806 73790 82.3308 38.8125 0.055649 0.012138 2024-09-21T10:09:52.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:52.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:52.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:52.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:09:52.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:52.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:52.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:52.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:52.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:52.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:52.706 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1] out_osds: [7, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:09:52.706 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:09:52.707 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-09-21T10:09:52.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd in 6 2024-09-21T10:09:52.802 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:52.824 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:52.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:53.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:53.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:53.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:53.137 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:53.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:53.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:53.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 74311 74295 81.4373 31.5625 0.0107372 0.0122704 2024-09-21T10:09:53.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:53.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:53.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:53.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:53.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:53.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:53.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:53.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:53.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:53.737 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:53 smithi017 ceph-mon[102913]: pgmap v1813: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 15 KiB/s rd, 28 MiB/s wr, 453 op/s; 36634/217242 objects degraded (16.863%); 6775/217242 objects misplaced (3.119%); 6.6 MiB/s, 1 keys/s, 106 objects/s recovering 2024-09-21T10:09:53.737 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:53 smithi017 ceph-mon[105151]: pgmap v1813: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 15 KiB/s rd, 28 MiB/s wr, 453 op/s; 36634/217242 objects degraded (16.863%); 6775/217242 objects misplaced (3.119%); 6.6 MiB/s, 1 keys/s, 106 objects/s recovering 2024-09-21T10:09:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:53 smithi134 ceph-mon[90085]: pgmap v1813: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 15 KiB/s rd, 28 MiB/s wr, 453 op/s; 36634/217242 objects degraded (16.863%); 6775/217242 objects misplaced (3.119%); 6.6 MiB/s, 1 keys/s, 106 objects/s recovering 2024-09-21T10:09:53.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:53.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:53.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:53.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:53.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:53.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:54.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:54.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:54.195 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:54.218 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:54.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:54.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 74879 74863 80.6454 35.5 0.0278848 0.0123918 2024-09-21T10:09:54.320 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:54.342 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:54.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:54.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:54.467 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:54.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:54.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:54.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:54.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:54.712 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:54.734 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:54.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:54 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 36634/217242 objects degraded (16.863%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:54.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:54.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:54.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:54 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 36634/217242 objects degraded (16.863%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:54.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:09:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:09:54] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.51.0" 2024-09-21T10:09:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:54 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 36634/217242 objects degraded (16.863%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:54.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:55.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:55.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:55.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:55.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:55.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:55.269 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:55.291 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:55.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 75301 75285 79.7257 26.375 0.0225231 0.0125284 2024-09-21T10:09:55.392 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:55.415 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:55.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:55.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:55.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:55.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:55.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:55.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:55.817 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:55.839 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:55.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:55 smithi017 ceph-mon[102913]: pgmap v1814: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 13 KiB/s rd, 34 MiB/s wr, 547 op/s; 36634/220998 objects degraded (16.577%); 6645/220998 objects misplaced (3.007%); 6.7 MiB/s, 1 keys/s, 108 objects/s recovering 2024-09-21T10:09:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:55.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:55 smithi017 ceph-mon[105151]: pgmap v1814: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 13 KiB/s rd, 34 MiB/s wr, 547 op/s; 36634/220998 objects degraded (16.577%); 6645/220998 objects misplaced (3.007%); 6.7 MiB/s, 1 keys/s, 108 objects/s recovering 2024-09-21T10:09:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:55.963 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:55.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:55 smithi134 ceph-mon[90085]: pgmap v1814: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 13 KiB/s rd, 34 MiB/s wr, 547 op/s; 36634/220998 objects degraded (16.577%); 6645/220998 objects misplaced (3.007%); 6.7 MiB/s, 1 keys/s, 108 objects/s recovering 2024-09-21T10:09:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:09:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:56.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:56.088 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:56.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:56.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:56.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:56.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:56.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:09:56.316165+0000 min lat: 0.00130567 max lat: 2.46167 avg lat: 0.0126527 2024-09-21T10:09:56.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:09:56.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 75870 75854 78.9898 35.5625 0.0498239 0.0126527 2024-09-21T10:09:56.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:56.387 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:56.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:56.489 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:56.512 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:56.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:56.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:56.637 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:56.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:56.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.783 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:56.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:56.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:56.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:56.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:57.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:57.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:57.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:57.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:57.200 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:57.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:57.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:57.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 76352 76336 78.1888 30.125 0.036513 0.0127826 2024-09-21T10:09:57.324 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:57.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:57.426 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:57.449 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:57.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:57.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:57.595 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:57.697 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:57.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:57.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:57.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:57.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:57.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:57 smithi017 ceph-mon[102913]: pgmap v1815: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 10 KiB/s rd, 30 MiB/s wr, 492 op/s; 36634/222483 objects degraded (16.466%); 6645/222483 objects misplaced (2.987%); 5.4 MiB/s, 1 keys/s, 87 objects/s recovering 2024-09-21T10:09:57.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:57 smithi017 ceph-mon[105151]: pgmap v1815: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 10 KiB/s rd, 30 MiB/s wr, 492 op/s; 36634/222483 objects degraded (16.466%); 6645/222483 objects misplaced (2.987%); 5.4 MiB/s, 1 keys/s, 87 objects/s recovering 2024-09-21T10:09:57.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:57 smithi134 ceph-mon[90085]: pgmap v1815: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 10 KiB/s rd, 30 MiB/s wr, 492 op/s; 36634/222483 objects degraded (16.466%); 6645/222483 objects misplaced (2.987%); 5.4 MiB/s, 1 keys/s, 87 objects/s recovering 2024-09-21T10:09:58.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.012 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:58.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:58.112 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:58.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:58.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:58.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:58.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:58.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:58.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 76919 76903 77.4994 35.4375 0.0124785 0.0128958 2024-09-21T10:09:58.390 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:58.411 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:58.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:58.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:58.557 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:58.659 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:58.681 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:58.783 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:58.805 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:58.907 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:58.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:58.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:58.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:58 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 36634/222483 objects degraded (16.466%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:58.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:58 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 36634/222483 objects degraded (16.466%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:58 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 36634/222483 objects degraded (16.466%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:09:59.031 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:59.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:59.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:59.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:59.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:59.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:59.319 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 15 77462 77447 76.8076 34 0.0331868 0.0130127 2024-09-21T10:09:59.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:59.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:59.435 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:59.457 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:59.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:59.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:59.682 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:59.707 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:09:59.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:59.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:59.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:59.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:09:59 smithi017 ceph-mon[102913]: pgmap v1816: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 6.3 KiB/s rd, 34 MiB/s wr, 548 op/s; 36634/228237 objects degraded (16.051%); 6441/228237 objects misplaced (2.822%); 5.7 MiB/s, 0 keys/s, 91 objects/s recovering 2024-09-21T10:09:59.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:09:59 smithi017 ceph-mon[105151]: pgmap v1816: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 6.3 KiB/s rd, 34 MiB/s wr, 548 op/s; 36634/228237 objects degraded (16.051%); 6441/228237 objects misplaced (2.822%); 5.7 MiB/s, 0 keys/s, 91 objects/s recovering 2024-09-21T10:09:59.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:59.993 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:09:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:09:59 smithi134 ceph-mon[90085]: pgmap v1816: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 6.3 KiB/s rd, 34 MiB/s wr, 548 op/s; 36634/228237 objects degraded (16.051%); 6441/228237 objects misplaced (2.822%); 5.7 MiB/s, 0 keys/s, 91 objects/s recovering 2024-09-21T10:09:59.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:00.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:00.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:00.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:00.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:00.265 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:00.319 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 77986 77970 76.1183 32.6875 0.0243767 0.0131298 2024-09-21T10:10:00.366 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:00.387 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:00.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:00.489 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:10:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:10:00.512 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:00.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:00.589 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:00.594 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:00.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:00.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:00.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:00.764 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:00.785 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:00.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:00.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:00.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:00.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 36634/228237 objects degraded (16.051%), 12 pgs degraded 2024-09-21T10:10:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: [WRN] PG_DEGRADED: Degraded data redundancy: 36634/228237 objects degraded (16.051%), 12 pgs degraded 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 1.c is active+recovery_wait+degraded, acting [3,0,4] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 2.2 is active+recovery_wait+degraded, acting [1,5,0] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.0 is active+recovery_wait+undersized+degraded+remapped, acting [1,2] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.5 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.b is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.c is active+recovering+undersized+degraded+remapped, acting [3,4] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.d is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2024-09-21T10:10:00.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[102913]: pg 9.e is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 36634/228237 objects degraded (16.051%), 12 pgs degraded 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: [WRN] PG_DEGRADED: Degraded data redundancy: 36634/228237 objects degraded (16.051%), 12 pgs degraded 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 1.c is active+recovery_wait+degraded, acting [3,0,4] 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 2.2 is active+recovery_wait+degraded, acting [1,5,0] 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.0 is active+recovery_wait+undersized+degraded+remapped, acting [1,2] 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.5 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:10:00.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.b is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:10:00.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.c is active+recovering+undersized+degraded+remapped, acting [3,4] 2024-09-21T10:10:00.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.d is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2024-09-21T10:10:00.954 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:00 smithi017 ceph-mon[105151]: pg 9.e is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2024-09-21T10:10:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 36634/228237 objects degraded (16.051%), 12 pgs degraded 2024-09-21T10:10:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-21T10:10:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: [WRN] PG_DEGRADED: Degraded data redundancy: 36634/228237 objects degraded (16.051%), 12 pgs degraded 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 1.c is active+recovery_wait+degraded, acting [3,0,4] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 2.2 is active+recovery_wait+degraded, acting [1,5,0] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.0 is active+recovery_wait+undersized+degraded+remapped, acting [1,2] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.5 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.b is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.c is active+recovering+undersized+degraded+remapped, acting [3,4] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.d is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2024-09-21T10:10:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:00 smithi134 ceph-mon[90085]: pg 9.e is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2024-09-21T10:10:01.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:01.056 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:01.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:01.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:01.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:01.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:01.296 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:01.319 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 78516 78500 75.457 33.125 0.051011 0.0132463 2024-09-21T10:10:01.319 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:01.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:01.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:01.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:01.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:01.586 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:01.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:01.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:01.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:01.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:01.832 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:01.854 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:01.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:01 smithi017 ceph-mon[102913]: pgmap v1817: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 5.7 KiB/s rd, 30 MiB/s wr, 486 op/s; 36634/230148 objects degraded (15.918%); 6441/230148 objects misplaced (2.799%); 4.6 MiB/s, 0 keys/s, 73 objects/s recovering 2024-09-21T10:10:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:01 smithi017 ceph-mon[105151]: pgmap v1817: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 5.7 KiB/s rd, 30 MiB/s wr, 486 op/s; 36634/230148 objects degraded (15.918%); 6441/230148 objects misplaced (2.799%); 4.6 MiB/s, 0 keys/s, 73 objects/s recovering 2024-09-21T10:10:01.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:01.976 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:01.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:01 smithi134 ceph-mon[90085]: pgmap v1817: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 5.7 KiB/s rd, 30 MiB/s wr, 486 op/s; 36634/230148 objects degraded (15.918%); 6441/230148 objects misplaced (2.799%); 4.6 MiB/s, 0 keys/s, 73 objects/s recovering 2024-09-21T10:10:02.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:02.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:02.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:02.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:02.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:02.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:02.319 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 79079 79063 74.8469 35.1875 0.011118 0.0133528 2024-09-21T10:10:02.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:02.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:02.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:02.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:02.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:02.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:02.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:02.741 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:02.763 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:02.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:02.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:02.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:03.010 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:03.031 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:03.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:03.133 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:03.154 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:03.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:03.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:03.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:03.319 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 79433 79417 74.0602 22.125 0.028174 0.0134942 2024-09-21T10:10:03.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:03.427 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:03.529 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:03.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:03.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:03.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:03.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:03.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:03.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:03.801 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:03.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:03.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:03.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:03.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:03 smithi017 ceph-mon[102913]: pgmap v1818: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 33 MiB/s wr, 534 op/s; 36634/233319 objects degraded (15.701%); 6330/233319 objects misplaced (2.713%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:10:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:03 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 36634/233319 objects degraded (15.701%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:03 smithi017 ceph-mon[105151]: pgmap v1818: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 33 MiB/s wr, 534 op/s; 36634/233319 objects degraded (15.701%); 6330/233319 objects misplaced (2.713%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:10:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:03 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 36634/233319 objects degraded (15.701%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:03 smithi134 ceph-mon[90085]: pgmap v1818: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 33 MiB/s wr, 534 op/s; 36634/233319 objects degraded (15.701%); 6330/233319 objects misplaced (2.713%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:10:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:03 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 36634/233319 objects degraded (15.701%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:04.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:04.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:04.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:04.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:04.178 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:04.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:04.280 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:04.301 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:04.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:04.319 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 79959 79943 73.4546 32.875 0.0484503 0.0136062 2024-09-21T10:10:04.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:04.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:04.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:04.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:04.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:04.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.727 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:04.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:04.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:04.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:04.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:04.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:05.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:05.119 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:05.141 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:05.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:10:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:10:04] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.51.0" 2024-09-21T10:10:05.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:05.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:05.319 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 80498 80482 72.8783 33.6875 0.0513862 0.0137145 2024-09-21T10:10:05.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:05.418 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:05.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:05.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:05.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:05.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:05.671 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:05.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:05.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:05.844 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:05.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:05.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:05.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:05 smithi017 ceph-mon[105151]: pgmap v1819: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 536 op/s; 36634/236565 objects degraded (15.486%); 6173/236565 objects misplaced (2.609%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:10:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:05 smithi017 ceph-mon[102913]: pgmap v1819: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 536 op/s; 36634/236565 objects degraded (15.486%); 6173/236565 objects misplaced (2.609%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:10:05.967 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:05.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:05 smithi134 ceph-mon[90085]: pgmap v1819: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 34 MiB/s wr, 536 op/s; 36634/236565 objects degraded (15.486%); 6173/236565 objects misplaced (2.609%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T10:10:06.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:06.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:06.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:06.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:06.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:06.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:06.319 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 80982 80966 72.2694 30.25 0.0156878 0.0138287 2024-09-21T10:10:06.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:06.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:06.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:06.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:06.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:06.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:06.668 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:06.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:06.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:06.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:06.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:06.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:06.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:07.060 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:07.083 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:07.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:07.185 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:07.208 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:07.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:07.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:07.320 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 81573 81557 71.7707 36.9375 0.0542021 0.0139263 2024-09-21T10:10:07.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:07.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:07.465 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:07.487 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:07.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:07.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:07.649 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:07.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:07.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:07.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:07.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:07 smithi017 ceph-mon[102913]: pgmap v1820: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 29 MiB/s wr, 471 op/s; 36634/237969 objects degraded (15.394%); 6173/237969 objects misplaced (2.594%); 2.5 MiB/s, 39 objects/s recovering 2024-09-21T10:10:07.860 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:07 smithi017 ceph-mon[105151]: pgmap v1820: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 29 MiB/s wr, 471 op/s; 36634/237969 objects degraded (15.394%); 6173/237969 objects misplaced (2.594%); 2.5 MiB/s, 39 objects/s recovering 2024-09-21T10:10:07.902 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:07.924 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:07.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:07 smithi134 ceph-mon[90085]: pgmap v1820: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 29 MiB/s wr, 471 op/s; 36634/237969 objects degraded (15.394%); 6173/237969 objects misplaced (2.594%); 2.5 MiB/s, 39 objects/s recovering 2024-09-21T10:10:08.026 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:08.049 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:08.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:08.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:08.173 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:08.275 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:08.298 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:08.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:08.321 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 82117 82101 71.2462 34 0.0298383 0.0140283 2024-09-21T10:10:08.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:08.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:08.549 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:08.572 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:08.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:08.586 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:10:08.675 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:08.698 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:08.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:08 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3635208053' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:08.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:08 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:08.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:08 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 36634/237969 objects degraded (15.394%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:08 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3635208053' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:08 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:08.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:08 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 36634/237969 objects degraded (15.394%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:08.800 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:08.823 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:08.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:08.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:08.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:08.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:08 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3635208053' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:08 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:08 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 36634/237969 objects degraded (15.394%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:09.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:09.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:09.216 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:09.238 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:09.321 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 82706 82690 70.7745 36.8125 0.0312175 0.0141216 2024-09-21T10:10:09.340 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:09.362 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:09.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:09.464 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:09.487 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:09.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:09.589 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:09.610 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:09.712 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:09.734 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:09.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:09.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:09.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:09.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:09.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:09 smithi017 ceph-mon[102913]: pgmap v1821: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 37 MiB/s wr, 590 op/s; 36634/243750 objects degraded (15.029%); 5924/243750 objects misplaced (2.430%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:10:09.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:09 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:09.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:09 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:09.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:09 smithi017 ceph-mon[102913]: osdmap e1251: 8 total, 8 up, 6 in 2024-09-21T10:10:09.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:09 smithi017 ceph-mon[105151]: pgmap v1821: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 37 MiB/s wr, 590 op/s; 36634/243750 objects degraded (15.029%); 5924/243750 objects misplaced (2.430%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:10:09.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:09 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:09.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:09 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:09.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:09 smithi017 ceph-mon[105151]: osdmap e1251: 8 total, 8 up, 6 in 2024-09-21T10:10:09.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:09 smithi134 ceph-mon[90085]: pgmap v1821: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 37 MiB/s wr, 590 op/s; 36634/243750 objects degraded (15.029%); 5924/243750 objects misplaced (2.430%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:10:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:09 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:09 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:09 smithi134 ceph-mon[90085]: osdmap e1251: 8 total, 8 up, 6 in 2024-09-21T10:10:10.011 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:10.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:10.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:10.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:10.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:10.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:10.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:10.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:10.320 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 83305 83289 70.3241 37.4375 0.0313234 0.0142125 2024-09-21T10:10:10.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:10.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:10.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:10:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:10:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:10:10.494 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:10:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=cleanup t=2024-09-21T10:10:10.299025415Z level=info msg="Completed cleanup jobs" duration=12.591142ms 2024-09-21T10:10:10.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:10.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:10.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:10.675 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:10.697 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:10.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:10.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:10.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:10.942 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:10.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:11.065 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:11.088 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:11.190 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:11.213 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:11.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:11.315 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:11.321 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 83881 83865 69.8665 36 0.0197579 0.0143045 2024-09-21T10:10:11.336 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:11.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:11.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:11.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:11.634 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:11.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:11.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:11.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:11.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:11.890 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:11 smithi017 ceph-mon[102913]: pgmap v1823: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 33 MiB/s wr, 520 op/s; 36634/245760 objects degraded (14.906%); 5924/245760 objects misplaced (2.410%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:10:11.891 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:11 smithi017 ceph-mon[105151]: pgmap v1823: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 33 MiB/s wr, 520 op/s; 36634/245760 objects degraded (14.906%); 5924/245760 objects misplaced (2.410%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:10:11.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:11.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:11.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:11 smithi134 ceph-mon[90085]: pgmap v1823: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 33 MiB/s wr, 520 op/s; 36634/245760 objects degraded (14.906%); 5924/245760 objects misplaced (2.410%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:10:12.046 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:12.068 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:12.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:12.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:12.321 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 84361 84345 69.342 30 0.0253938 0.0144112 2024-09-21T10:10:12.328 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:12.350 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:12.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:12.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:12.473 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:12.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:12.521 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:10:12.521 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:10:12.521 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:10:12.521 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:12.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:12.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:12.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:12.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:12.626 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.6. 2024-09-21T10:10:12.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:12 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:10:12.461+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1251 definitely_dead 0 2024-09-21T10:10:12.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:12 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/742522944' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:12.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:12 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/693240784' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:10:12.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:12 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:10:12.675 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:12 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/742522944' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:12.675 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:12 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/693240784' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:10:12.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:12 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:10:12.720 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:12.742 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:12.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:12.843 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:12.864 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:12.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:12.966 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:12.988 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:12.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:12.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:12 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/742522944' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:12.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:12 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/693240784' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:10:12.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:12 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:10:13.091 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:13.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:13.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:13.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:13.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:13.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:13.321 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 84613 84597 68.6461 15.75 0.00372928 0.0144619 2024-09-21T10:10:13.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:13.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:10:13.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:13.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:13.470 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:13.492 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:13.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:13.594 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.619 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:13.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:13.720 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:13.741 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:13.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:13.843 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:13.864 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:13.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:13 smithi017 ceph-mon[102913]: pgmap v1824: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 36634/245760 objects degraded (14.906%); 5924/245760 objects misplaced (2.410%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:10:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:13 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:10:13.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:13 smithi017 ceph-mon[102913]: osdmap e1252: 8 total, 8 up, 7 in 2024-09-21T10:10:13.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:13 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 36634/245760 objects degraded (14.906%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:13 smithi017 ceph-mon[105151]: pgmap v1824: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 36634/245760 objects degraded (14.906%); 5924/245760 objects misplaced (2.410%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:10:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:13 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:10:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:13 smithi017 ceph-mon[105151]: osdmap e1252: 8 total, 8 up, 7 in 2024-09-21T10:10:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:13 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 36634/245760 objects degraded (14.906%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:13.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:13 smithi134 ceph-mon[90085]: pgmap v1824: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 36634/245760 objects degraded (14.906%); 5924/245760 objects misplaced (2.410%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:10:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:13 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:10:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:13 smithi134 ceph-mon[90085]: osdmap e1252: 8 total, 8 up, 7 in 2024-09-21T10:10:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:13 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 36634/245760 objects degraded (14.906%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:14.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:14.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:14.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:14.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:14.232 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-09-21T10:10:14.262 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:14.286 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:14.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:14.321 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 84906 84890 68.0009 18.3125 0.0180403 0.0146995 2024-09-21T10:10:14.387 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:14.411 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:14.440 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:10:14.440 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:10:14.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:14.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:14.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:14.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:14.693 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:14 smithi017 ceph-mon[102913]: osdmap e1253: 8 total, 8 up, 7 in 2024-09-21T10:10:14.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:14 smithi017 ceph-mon[105151]: osdmap e1253: 8 total, 8 up, 7 in 2024-09-21T10:10:14.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:14.817 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:14.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:14.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:14.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:14 smithi134 ceph-mon[90085]: osdmap e1253: 8 total, 8 up, 7 in 2024-09-21T10:10:15.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:15.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:15.151 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:10:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:10:14] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-09-21T10:10:15.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:10:15.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:15.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:15.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:15.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 85635 85619 67.7158 45.5625 0.0285051 0.0147614 2024-09-21T10:10:15.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:15.423 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:15.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:15.608 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:15.629 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:15.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:15.693 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:15.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:15.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:15.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:15.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:15 smithi017 ceph-mon[102913]: pgmap v1827: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 18 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 31 MiB/s wr, 488 op/s; 32166/252543 objects degraded (12.737%); 5671/252543 objects misplaced (2.246%); 2.6 MiB/s, 42 objects/s recovering 2024-09-21T10:10:15.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:15 smithi017 ceph-mon[105151]: pgmap v1827: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 18 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 31 MiB/s wr, 488 op/s; 32166/252543 objects degraded (12.737%); 5671/252543 objects misplaced (2.246%); 2.6 MiB/s, 42 objects/s recovering 2024-09-21T10:10:15.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:15.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:15.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:15 smithi134 ceph-mon[90085]: pgmap v1827: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 18 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 31 MiB/s wr, 488 op/s; 32166/252543 objects degraded (12.737%); 5671/252543 objects misplaced (2.246%); 2.6 MiB/s, 42 objects/s recovering 2024-09-21T10:10:16.015 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:16.024 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:16.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:16.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:16.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:16.174 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:16.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:16.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:16.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:16.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:16.323 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:10:16.322169+0000 min lat: 0.00130567 max lat: 2.46167 avg lat: 0.014827 2024-09-21T10:10:16.324 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:10:16.324 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 86324 86308 67.4072 43.0625 0.00843774 0.014827 2024-09-21T10:10:16.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:16.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:16.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:16.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:16.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:16.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:16.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:16.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:16.879 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:16.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:16.981 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:17.006 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:17.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:17.108 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:17.131 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:17.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:17.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:17.323 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 86977 86961 67.0789 40.8125 0.0635261 0.014899 2024-09-21T10:10:17.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:17.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:17.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:17.511 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:17.536 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:17.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:17.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:17.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:10:17.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:17.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:17.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:17.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:17 smithi017 ceph-mon[102913]: pgmap v1828: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 18 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 26 MiB/s wr, 412 op/s; 28416/253863 objects degraded (11.193%); 5671/253863 objects misplaced (2.234%); 2.0 MiB/s, 33 objects/s recovering 2024-09-21T10:10:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:17 smithi017 ceph-mon[105151]: pgmap v1828: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 18 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 26 MiB/s wr, 412 op/s; 28416/253863 objects degraded (11.193%); 5671/253863 objects misplaced (2.234%); 2.0 MiB/s, 33 objects/s recovering 2024-09-21T10:10:17.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:17.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:17 smithi134 ceph-mon[90085]: pgmap v1828: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 18 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 26 MiB/s wr, 412 op/s; 28416/253863 objects degraded (11.193%); 5671/253863 objects misplaced (2.234%); 2.0 MiB/s, 33 objects/s recovering 2024-09-21T10:10:18.071 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:18.095 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:18.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:18.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:18.323 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 87593 87577 66.7304 38.5 0.0186624 0.0149794 2024-09-21T10:10:18.340 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:18.363 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:18.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:18.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:18.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:18.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:18.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:18.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:18.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:18.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:18.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:18.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:18.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:18.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:18 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 28416/253863 objects degraded (11.193%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:18.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:18 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 28416/253863 objects degraded (11.193%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:18 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 28416/253863 objects degraded (11.193%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:19.010 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:19.034 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:19.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:19.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:19.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:19.233 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:10:19.233 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:10:19.234 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2024-09-21T10:10:19.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:10:19.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:19.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:19.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:19.323 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 88210 88194 66.391 38.5625 0.0278244 0.0150555 2024-09-21T10:10:19.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:19.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:19.569 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:19.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.594 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:19.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:19.695 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:19.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:19.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:19.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:19 smithi017 ceph-mon[102913]: pgmap v1829: 109 pgs: 1 active+recovering+undersized+remapped, 11 peering, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 588 op/s; 27746/259905 objects degraded (10.675%); 3482/259905 objects misplaced (1.340%); 4.2 MiB/s, 67 objects/s recovering 2024-09-21T10:10:19.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:19 smithi017 ceph-mon[105151]: pgmap v1829: 109 pgs: 1 active+recovering+undersized+remapped, 11 peering, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 588 op/s; 27746/259905 objects degraded (10.675%); 3482/259905 objects misplaced (1.340%); 4.2 MiB/s, 67 objects/s recovering 2024-09-21T10:10:19.956 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:19.981 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:19.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:19 smithi134 ceph-mon[90085]: pgmap v1829: 109 pgs: 1 active+recovering+undersized+remapped, 11 peering, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 37 MiB/s wr, 588 op/s; 27746/259905 objects degraded (10.675%); 3482/259905 objects misplaced (1.340%); 4.2 MiB/s, 67 objects/s recovering 2024-09-21T10:10:20.082 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:20.106 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:20.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:20.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:20.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:20.323 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 88848 88832 66.0753 39.875 0.0172267 0.0151226 2024-09-21T10:10:20.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:20.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:20.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:20.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:10:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:10:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:10:20.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:20.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:20.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:20.655 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:20.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:20.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:20.780 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:20.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:20.882 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:20.906 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:21.008 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:21.033 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:21.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:21.135 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:21.157 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:21.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:21.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:21.323 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 89497 89481 65.7752 40.5625 0.040673 0.0151966 2024-09-21T10:10:21.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:21.451 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:21.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:21.552 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:21.576 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:21.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:21.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:21.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:21.801 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:21.824 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:21.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:21.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:21 smithi017 ceph-mon[102913]: pgmap v1830: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s; 27746/262269 objects degraded (10.579%); 3482/262269 objects misplaced (1.328%); 4.2 MiB/s, 1 keys/s, 68 objects/s recovering 2024-09-21T10:10:21.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:21 smithi017 ceph-mon[105151]: pgmap v1830: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s; 27746/262269 objects degraded (10.579%); 3482/262269 objects misplaced (1.328%); 4.2 MiB/s, 1 keys/s, 68 objects/s recovering 2024-09-21T10:10:21.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:21.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:21 smithi134 ceph-mon[90085]: pgmap v1830: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 687 op/s; 27746/262269 objects degraded (10.579%); 3482/262269 objects misplaced (1.328%); 4.2 MiB/s, 1 keys/s, 68 objects/s recovering 2024-09-21T10:10:22.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:22.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:22.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:22.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.238 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:22.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:22.324 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 90191 90175 65.5147 43.375 0.0750938 0.015255 2024-09-21T10:10:22.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:22.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.378 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:22.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:22.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:22.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:22.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:22.624 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:22.650 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:22.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:22.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:22.807 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:22.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:22.909 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:22.934 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:22.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:23.036 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:23.061 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:23.163 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:23.187 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:23.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:23.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:23.313 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:23.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:23.324 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 15 90927 90912 65.2907 46.0625 0.00882567 0.0153086 2024-09-21T10:10:23.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:23.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:23.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:23.579 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:23.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:23.681 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:23.705 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:23.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:23.807 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:23.828 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:23.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:23.849 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:23.930 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:23.936 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:23 smithi017 ceph-mon[102913]: pgmap v1831: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 28 MiB/s wr, 451 op/s; 27746/262269 objects degraded (10.579%); 3482/262269 objects misplaced (1.328%); 2.6 MiB/s, 1 keys/s, 43 objects/s recovering 2024-09-21T10:10:23.936 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:23 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 27746/262269 objects degraded (10.579%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:23.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:23 smithi017 ceph-mon[105151]: pgmap v1831: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 28 MiB/s wr, 451 op/s; 27746/262269 objects degraded (10.579%); 3482/262269 objects misplaced (1.328%); 2.6 MiB/s, 1 keys/s, 43 objects/s recovering 2024-09-21T10:10:23.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:23 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 27746/262269 objects degraded (10.579%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:23.954 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:23.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:23 smithi134 ceph-mon[90085]: pgmap v1831: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 28 MiB/s wr, 451 op/s; 27746/262269 objects degraded (10.579%); 3482/262269 objects misplaced (1.328%); 2.6 MiB/s, 1 keys/s, 43 objects/s recovering 2024-09-21T10:10:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:23 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 27746/262269 objects degraded (10.579%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:24.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:24.082 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:24.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:24.115 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:24.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:24.231 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:24.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:24.324 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 91646 91630 65.0587 44.875 0.0119368 0.015365 2024-09-21T10:10:24.333 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:24.356 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:24.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:24.458 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:24.481 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:24.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:24.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:24.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:24.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:24.635 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:10:24.635 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:10:24.730 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:24.756 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:24.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:24.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:24.885 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:24.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:24.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:10:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:10:24] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-09-21T10:10:24.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:25.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:25.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:25.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:25.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:25.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:25.278 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:25.301 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:25.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:25.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 92339 92323 64.8143 43.3125 0.0158255 0.0154222 2024-09-21T10:10:25.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:25.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:25.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:25.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:10:25.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:25.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:25.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:25.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:25.770 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:25 smithi017 ceph-mon[102913]: pgmap v1832: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 548 op/s; 27746/270237 objects degraded (10.267%); 3344/270237 objects misplaced (1.237%); 2.4 MiB/s, 0 keys/s, 40 objects/s recovering 2024-09-21T10:10:25.771 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:25.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:25 smithi017 ceph-mon[105151]: pgmap v1832: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 548 op/s; 27746/270237 objects degraded (10.267%); 3344/270237 objects misplaced (1.237%); 2.4 MiB/s, 0 keys/s, 40 objects/s recovering 2024-09-21T10:10:25.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:25.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:25.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:25.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:25.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:25 smithi134 ceph-mon[90085]: pgmap v1832: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 548 op/s; 27746/270237 objects degraded (10.267%); 3344/270237 objects misplaced (1.237%); 2.4 MiB/s, 0 keys/s, 40 objects/s recovering 2024-09-21T10:10:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:26.058 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:26.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:26.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:26.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:26.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:26.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0167 2024-09-21T10:10:26.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 93001 2024-09-21T10:10:26.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-21T10:10:26.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-21T10:10:26.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 64.5721 2024-09-21T10:10:26.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 48.603 2024-09-21T10:10:26.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 213.938 2024-09-21T10:10:26.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:10:26.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1033 2024-09-21T10:10:26.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 777.655 2024-09-21T10:10:26.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3423 2024-09-21T10:10:26.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2024-09-21T10:10:26.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0154808 2024-09-21T10:10:26.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.047016 2024-09-21T10:10:26.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 2.46167 2024-09-21T10:10:26.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00130567 2024-09-21T10:10:26.339 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:10:26.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:26.396 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:26.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:26.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:26.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:26.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:26.644 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:26.668 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:26.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:26.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:26.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:26.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:26.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:27.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:27.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:27.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:27.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:27.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:27.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:27.308 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:27.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:27.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:27.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:27.486 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:27.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:27.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:27.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.630 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:27.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:27.731 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.756 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:27.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:27.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:27.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:27 smithi017 ceph-mon[102913]: pgmap v1833: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 547 op/s; 27746/272262 objects degraded (10.191%); 3344/272262 objects misplaced (1.228%); 2.2 MiB/s, 0 keys/s, 35 objects/s recovering 2024-09-21T10:10:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:27 smithi017 ceph-mon[105151]: pgmap v1833: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 547 op/s; 27746/272262 objects degraded (10.191%); 3344/272262 objects misplaced (1.228%); 2.2 MiB/s, 0 keys/s, 35 objects/s recovering 2024-09-21T10:10:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:27 smithi134 ceph-mon[90085]: pgmap v1833: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 547 op/s; 27746/272262 objects degraded (10.191%); 3344/272262 objects misplaced (1.228%); 2.2 MiB/s, 0 keys/s, 35 objects/s recovering 2024-09-21T10:10:28.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:28.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:28.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:28.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:28.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:28.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:28.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:28.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:28.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:28.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:28.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:28.626 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:28.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:28.711 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:10:28.727 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:28.749 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:28.837 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:10:28.837 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:28.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:28 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1509389665' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:28.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:28 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:28.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:28 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 27746/272262 objects degraded (10.191%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:28.838 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:28 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1509389665' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:28.838 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:28 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:28.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:28 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 27746/272262 objects degraded (10.191%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:28.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:28.880 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:10:28.881 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:28.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:28.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:28.917 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:10:28.917 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:28.955 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:10:28.955 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:28.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:10:28.984 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:28.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:28 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1509389665' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:28 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:28 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 27746/272262 objects degraded (10.191%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:10:29.012 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:29.013 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:10:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:29.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:29.029 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:29.047 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:10:29.048 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:29.078 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:10:29.079 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:29.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:29.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:29.270 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:29.293 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:29.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:29.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:29.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:29.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:29.535 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:29.558 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:29.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:29.659 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:29.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:29.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:29.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:29.806 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:29.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:29.908 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:29.932 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:29.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:29 smithi134 ceph-mon[90085]: pgmap v1834: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 85 B/s rd, 42 MiB/s wr, 724 op/s; 27699/276189 objects degraded (10.029%); 3178/276189 objects misplaced (1.151%); 3.3 MiB/s, 0 keys/s, 53 objects/s recovering 2024-09-21T10:10:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:29 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:29 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:10:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:29 smithi134 ceph-mon[90085]: osdmap e1254: 8 total, 8 up, 7 in 2024-09-21T10:10:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:29 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4131452382' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:30.022 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:10:30.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[102913]: pgmap v1834: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 85 B/s rd, 42 MiB/s wr, 724 op/s; 27699/276189 objects degraded (10.029%); 3178/276189 objects misplaced (1.151%); 3.3 MiB/s, 0 keys/s, 53 objects/s recovering 2024-09-21T10:10:30.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:30.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:10:30.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[102913]: osdmap e1254: 8 total, 8 up, 7 in 2024-09-21T10:10:30.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4131452382' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:30.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[105151]: pgmap v1834: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 85 B/s rd, 42 MiB/s wr, 724 op/s; 27699/276189 objects degraded (10.029%); 3178/276189 objects misplaced (1.151%); 3.3 MiB/s, 0 keys/s, 53 objects/s recovering 2024-09-21T10:10:30.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:30.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:10:30.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[105151]: osdmap e1254: 8 total, 8 up, 7 in 2024-09-21T10:10:30.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:29 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4131452382' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:30.026 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:10:30.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:30.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:30.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:30.181 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:30.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:30.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:30.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:30.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:30.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:30.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:30.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:10:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:10:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:10:30.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:30.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:30.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:30.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:30.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:30.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:30.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.891 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:30.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:30.992 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:31.018 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:31.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:31.121 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:31.143 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:31.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:31.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:31.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:31.387 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:31.411 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:31.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:31.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:31.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:31.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:31.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:31.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:31.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:31.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:31.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:31.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:31.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:31.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:31.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:31 smithi017 ceph-mon[105151]: from='client.45914 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:10:31.979 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:31 smithi017 ceph-mon[105151]: pgmap v1836: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 35 MiB/s wr, 669 op/s; 27699/275805 objects degraded (10.043%); 3178/275805 objects misplaced (1.152%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:10:31.979 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:31 smithi017 ceph-mon[102913]: from='client.45914 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:10:31.980 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:31 smithi017 ceph-mon[102913]: pgmap v1836: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 35 MiB/s wr, 669 op/s; 27699/275805 objects degraded (10.043%); 3178/275805 objects misplaced (1.152%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:10:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:31 smithi134 ceph-mon[90085]: from='client.45914 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:10:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:31 smithi134 ceph-mon[90085]: pgmap v1836: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 35 MiB/s wr, 669 op/s; 27699/275805 objects degraded (10.043%); 3178/275805 objects misplaced (1.152%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:10:32.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:32.089 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:32.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:32.191 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:32.215 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:32.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:32.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:32.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:32.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:32.461 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:32.483 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:32.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:32.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:32.610 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:32.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:32.712 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:32.736 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:32.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:32.855 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1835,"stamp":"2024-09-21T10:10:28.705127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6036610684,"num_objects":92127,"num_object_clones":0,"num_object_copies":276381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27699,"num_objects_misplaced":3178,"num_objects_unfound":0,"num_objects_dirty":92127,"num_whiteouts":0,"num_read":108,"num_read_kb":273,"num_write":93613,"num_write_kb":5939632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":246846085,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81454,"ondisk_log_size":81454,"up":327,"acting":316,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20079592,"kb_used_data":15988144,"kb_used_omap":409,"kb_used_meta":4090854,"kb_avail":636136472,"statfs":{"total":671965249536,"available":651403747328,"internally_reserved":0,"allocated":16371859456,"data_stored":16351829026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":419286,"internal_metadata":4189035050},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":359858200,"num_objects":5492,"num_object_clones":0,"num_object_copies":16476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-47,"num_objects_misplaced":-304,"num_objects_unfound":0,"num_objects_dirty":5492,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6888,"num_write_kb":396097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":358,"num_bytes_recovered":23007914,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.345277"},"pg_stats":[{"pgid":"1.11","version":"288'100","reported_seq":659,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.634302+0000","last_change":"2024-09-21T10:08:39.493413+0000","last_active":"2024-09-21T10:10:13.634302+0000","last_peered":"2024-09-21T10:10:13.634302+0000","last_clean":"2024-09-21T10:10:13.634302+0000","last_became_active":"2024-09-21T10:08:39.493004+0000","last_became_peered":"2024-09-21T10:08:39.493004+0000","last_unstale":"2024-09-21T10:10:13.634302+0000","last_undegraded":"2024-09-21T10:10:13.634302+0000","last_fullsized":"2024-09-21T10:10:13.634302+0000","mapping_epoch":1233,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1234,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.888350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.888350+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:43:15.229147+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1867,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.651448+0000","last_change":"2024-09-21T10:09:25.458014+0000","last_active":"2024-09-21T10:10:13.651448+0000","last_peered":"2024-09-21T10:10:13.651448+0000","last_clean":"2024-09-21T10:10:13.651448+0000","last_became_active":"2024-09-21T10:09:25.452008+0000","last_became_peered":"2024-09-21T10:09:25.452008+0000","last_unstale":"2024-09-21T10:10:13.651448+0000","last_undegraded":"2024-09-21T10:10:13.651448+0000","last_fullsized":"2024-09-21T10:10:13.651448+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:25.502757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":69,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.637483+0000","last_change":"2024-09-21T10:10:13.637483+0000","last_active":"2024-09-21T10:10:13.637483+0000","last_peered":"2024-09-21T10:10:13.637483+0000","last_clean":"2024-09-21T10:10:13.637483+0000","last_became_active":"2024-09-21T10:10:13.622491+0000","last_became_peered":"2024-09-21T10:10:13.622491+0000","last_unstale":"2024-09-21T10:10:13.637483+0000","last_undegraded":"2024-09-21T10:10:13.637483+0000","last_fullsized":"2024-09-21T10:10:13.637483+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:02.511869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"288'100","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.624385+0000","last_change":"2024-09-21T10:10:13.624385+0000","last_active":"2024-09-21T10:10:13.624385+0000","last_peered":"2024-09-21T10:10:13.624385+0000","last_clean":"2024-09-21T10:10:13.624385+0000","last_became_active":"2024-09-21T10:10:13.624038+0000","last_became_peered":"2024-09-21T10:10:13.624038+0000","last_unstale":"2024-09-21T10:10:13.624385+0000","last_undegraded":"2024-09-21T10:10:13.624385+0000","last_fullsized":"2024-09-21T10:10:13.624385+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:25.533587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:25.533587+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":35,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":568,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.661015+0000","last_change":"2024-09-21T10:10:13.660936+0000","last_active":"2024-09-21T10:10:13.661015+0000","last_peered":"2024-09-21T10:10:13.661015+0000","last_clean":"2024-09-21T10:10:13.661015+0000","last_became_active":"2024-09-21T10:10:13.660573+0000","last_became_peered":"2024-09-21T10:10:13.660573+0000","last_unstale":"2024-09-21T10:10:13.661015+0000","last_undegraded":"2024-09-21T10:10:13.661015+0000","last_fullsized":"2024-09-21T10:10:13.661015+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:11:28.164079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1936,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.628437+0000","last_change":"2024-09-21T10:09:25.457058+0000","last_active":"2024-09-21T10:10:13.628437+0000","last_peered":"2024-09-21T10:10:13.628437+0000","last_clean":"2024-09-21T10:10:13.628437+0000","last_became_active":"2024-09-21T10:09:25.456748+0000","last_became_peered":"2024-09-21T10:09:25.456748+0000","last_unstale":"2024-09-21T10:10:13.628437+0000","last_undegraded":"2024-09-21T10:10:13.628437+0000","last_fullsized":"2024-09-21T10:10:13.628437+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:33.730086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"288'100","reported_seq":44,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.629403+0000","last_change":"2024-09-21T10:08:45.523904+0000","last_active":"2024-09-21T10:10:13.629403+0000","last_peered":"2024-09-21T10:10:13.629403+0000","last_clean":"2024-09-21T10:10:13.629403+0000","last_became_active":"2024-09-21T10:08:45.521658+0000","last_became_peered":"2024-09-21T10:08:45.521658+0000","last_unstale":"2024-09-21T10:10:13.629403+0000","last_undegraded":"2024-09-21T10:10:13.629403+0000","last_fullsized":"2024-09-21T10:10:13.629403+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:32.451823+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:08:32.451823+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":23,"num_write":5,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1938,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.631148+0000","last_change":"2024-09-21T10:09:25.461488+0000","last_active":"2024-09-21T10:10:13.631148+0000","last_peered":"2024-09-21T10:10:13.631148+0000","last_clean":"2024-09-21T10:10:13.631148+0000","last_became_active":"2024-09-21T10:09:25.461153+0000","last_became_peered":"2024-09-21T10:09:25.461153+0000","last_unstale":"2024-09-21T10:10:13.631148+0000","last_undegraded":"2024-09-21T10:10:13.631148+0000","last_fullsized":"2024-09-21T10:10:13.631148+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:58:09.052499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1200,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.642239+0000","last_change":"2024-09-21T10:10:13.642239+0000","last_active":"2024-09-21T10:10:13.642239+0000","last_peered":"2024-09-21T10:10:13.642239+0000","last_clean":"2024-09-21T10:10:13.642239+0000","last_became_active":"2024-09-21T10:10:13.641938+0000","last_became_peered":"2024-09-21T10:10:13.641938+0000","last_unstale":"2024-09-21T10:10:13.642239+0000","last_undegraded":"2024-09-21T10:10:13.642239+0000","last_fullsized":"2024-09-21T10:10:13.642239+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:20:48.026420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"288'100","reported_seq":58,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.635817+0000","last_change":"2024-09-21T10:10:13.635736+0000","last_active":"2024-09-21T10:10:13.635817+0000","last_peered":"2024-09-21T10:10:13.635817+0000","last_clean":"2024-09-21T10:10:13.635817+0000","last_became_active":"2024-09-21T10:10:13.634663+0000","last_became_peered":"2024-09-21T10:10:13.634663+0000","last_unstale":"2024-09-21T10:10:13.635817+0000","last_undegraded":"2024-09-21T10:10:13.635817+0000","last_fullsized":"2024-09-21T10:10:13.635817+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:26.189995+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:08:26.189995+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":14,"num_read_kb":34,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1866,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.622838+0000","last_change":"2024-09-21T10:09:25.451536+0000","last_active":"2024-09-21T10:10:13.622838+0000","last_peered":"2024-09-21T10:10:13.622838+0000","last_clean":"2024-09-21T10:10:13.622838+0000","last_became_active":"2024-09-21T10:09:25.451190+0000","last_became_peered":"2024-09-21T10:09:25.451190+0000","last_unstale":"2024-09-21T10:10:13.622838+0000","last_undegraded":"2024-09-21T10:10:13.622838+0000","last_fullsized":"2024-09-21T10:10:13.622838+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:15.455299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":20,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.683037+0000","last_change":"2024-09-21T10:10:13.683037+0000","last_active":"2024-09-21T10:10:13.683037+0000","last_peered":"2024-09-21T10:10:13.683037+0000","last_clean":"2024-09-21T10:10:13.683037+0000","last_became_active":"2024-09-21T10:10:13.656692+0000","last_became_peered":"2024-09-21T10:10:13.656692+0000","last_unstale":"2024-09-21T10:10:13.683037+0000","last_undegraded":"2024-09-21T10:10:13.683037+0000","last_fullsized":"2024-09-21T10:10:13.683037+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:04.431952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"1253'5780","reported_seq":5828,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:24.718799+0000","last_change":"2024-09-21T10:09:46.504815+0000","last_active":"2024-09-21T10:10:24.718799+0000","last_peered":"2024-09-21T10:10:24.718799+0000","last_clean":"2024-09-21T10:09:24.410184+0000","last_became_active":"2024-09-21T10:09:26.732088+0000","last_became_peered":"2024-09-21T10:09:26.732088+0000","last_unstale":"2024-09-21T10:10:24.718799+0000","last_undegraded":"2024-09-21T10:09:46.501925+0000","last_fullsized":"2024-09-21T10:09:26.423099+0000","mapping_epoch":1245,"log_start":"1242'300","ondisk_log_start":"1242'300","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378798080,"num_objects":5780,"num_object_clones":0,"num_object_copies":17340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3599,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5780,"num_write_kb":369920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":2181},{"shards":"0,5,7","objects":3599}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"288'100","reported_seq":132,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.614203+0000","last_change":"2024-09-21T10:09:25.449704+0000","last_active":"2024-09-21T10:10:12.614203+0000","last_peered":"2024-09-21T10:10:12.614203+0000","last_clean":"2024-09-21T10:10:12.614203+0000","last_became_active":"2024-09-21T10:09:25.449217+0000","last_became_peered":"2024-09-21T10:09:25.449217+0000","last_unstale":"2024-09-21T10:10:12.614203+0000","last_undegraded":"2024-09-21T10:10:12.614203+0000","last_fullsized":"2024-09-21T10:10:12.614203+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:04.877986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:04.877986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:47.175383+0000","scrub_duration":0.0015289520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":35,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.613452+0000","last_change":"2024-09-21T10:09:25.481340+0000","last_active":"2024-09-21T10:10:12.613452+0000","last_peered":"2024-09-21T10:10:12.613452+0000","last_clean":"2024-09-21T10:10:12.613452+0000","last_became_active":"2024-09-21T10:09:25.449394+0000","last_became_peered":"2024-09-21T10:09:25.449394+0000","last_unstale":"2024-09-21T10:10:12.613452+0000","last_undegraded":"2024-09-21T10:10:12.613452+0000","last_fullsized":"2024-09-21T10:10:12.613452+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:46.731808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.623868+0000","last_change":"2024-09-21T10:10:13.623868+0000","last_active":"2024-09-21T10:10:13.623868+0000","last_peered":"2024-09-21T10:10:13.623868+0000","last_clean":"2024-09-21T10:10:13.623868+0000","last_became_active":"2024-09-21T10:10:13.623304+0000","last_became_peered":"2024-09-21T10:10:13.623304+0000","last_unstale":"2024-09-21T10:10:13.623868+0000","last_undegraded":"2024-09-21T10:10:13.623868+0000","last_fullsized":"2024-09-21T10:10:13.623868+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:56:35.829980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"1253'5942","reported_seq":5983,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:27.725426+0000","last_change":"2024-09-21T10:08:48.537849+0000","last_active":"2024-09-21T10:10:27.725426+0000","last_peered":"2024-09-21T10:10:27.725426+0000","last_clean":"2024-09-21T10:10:27.725426+0000","last_became_active":"2024-09-21T10:08:48.537409+0000","last_became_peered":"2024-09-21T10:08:48.537409+0000","last_unstale":"2024-09-21T10:10:27.725426+0000","last_undegraded":"2024-09-21T10:10:27.725426+0000","last_fullsized":"2024-09-21T10:10:27.725426+0000","mapping_epoch":1238,"log_start":"1243'1300","ondisk_log_start":"1243'1300","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4642,"log_dups_size":0,"ondisk_log_size":4642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:39:30.928555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380633088,"num_objects":5808,"num_object_clones":0,"num_object_copies":17424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5942,"num_write_kb":376000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"288'100","reported_seq":335,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.678965+0000","last_change":"2024-09-21T10:10:13.678895+0000","last_active":"2024-09-21T10:10:13.678965+0000","last_peered":"2024-09-21T10:10:13.678965+0000","last_clean":"2024-09-21T10:10:13.678965+0000","last_became_active":"2024-09-21T10:10:13.678219+0000","last_became_peered":"2024-09-21T10:10:13.678219+0000","last_unstale":"2024-09-21T10:10:13.678965+0000","last_undegraded":"2024-09-21T10:10:13.678965+0000","last_fullsized":"2024-09-21T10:10:13.678965+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:27.395700+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:27.395700+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:26.476978+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":35,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.613785+0000","last_change":"2024-09-21T10:09:25.483409+0000","last_active":"2024-09-21T10:10:12.613785+0000","last_peered":"2024-09-21T10:10:12.613785+0000","last_clean":"2024-09-21T10:10:12.613785+0000","last_became_active":"2024-09-21T10:09:25.462337+0000","last_became_peered":"2024-09-21T10:09:25.462337+0000","last_unstale":"2024-09-21T10:10:12.613785+0000","last_undegraded":"2024-09-21T10:10:12.613785+0000","last_fullsized":"2024-09-21T10:10:12.613785+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:59.639319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1218,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.613098+0000","last_change":"2024-09-21T10:08:45.515858+0000","last_active":"2024-09-21T10:10:12.613098+0000","last_peered":"2024-09-21T10:10:12.613098+0000","last_clean":"2024-09-21T10:10:12.613098+0000","last_became_active":"2024-09-21T10:08:45.515543+0000","last_became_peered":"2024-09-21T10:08:45.515543+0000","last_unstale":"2024-09-21T10:10:12.613098+0000","last_undegraded":"2024-09-21T10:10:12.613098+0000","last_fullsized":"2024-09-21T10:10:12.613098+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:29.565740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1253'5893","reported_seq":5932,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:27.380347+0000","last_change":"2024-09-21T10:10:13.683511+0000","last_active":"2024-09-21T10:10:27.380347+0000","last_peered":"2024-09-21T10:10:27.380347+0000","last_clean":"2024-09-21T10:09:24.407827+0000","last_became_active":"2024-09-21T10:09:26.758116+0000","last_became_peered":"2024-09-21T10:09:26.758116+0000","last_unstale":"2024-09-21T10:10:27.380347+0000","last_undegraded":"2024-09-21T10:10:13.678384+0000","last_fullsized":"2024-09-21T10:09:26.423305+0000","mapping_epoch":1245,"log_start":"1242'1200","ondisk_log_start":"1242'1200","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4693,"log_dups_size":0,"ondisk_log_size":4693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381616128,"num_objects":5823,"num_object_clones":0,"num_object_copies":17469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3651,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5893,"num_write_kb":374912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5","objects":2172},{"shards":"3,5,7","objects":3616}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"288'100","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.624076+0000","last_change":"2024-09-21T10:10:13.624076+0000","last_active":"2024-09-21T10:10:13.624076+0000","last_peered":"2024-09-21T10:10:13.624076+0000","last_clean":"2024-09-21T10:10:13.624076+0000","last_became_active":"2024-09-21T10:10:13.623709+0000","last_became_peered":"2024-09-21T10:10:13.623709+0000","last_unstale":"2024-09-21T10:10:13.624076+0000","last_undegraded":"2024-09-21T10:10:13.624076+0000","last_fullsized":"2024-09-21T10:10:13.624076+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:23.746751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:23.746751+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:22.068163+0000","scrub_duration":0.0021986229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":14,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.675582+0000","last_change":"2024-09-21T10:10:13.675582+0000","last_active":"2024-09-21T10:10:13.675582+0000","last_peered":"2024-09-21T10:10:13.675582+0000","last_clean":"2024-09-21T10:10:13.675582+0000","last_became_active":"2024-09-21T10:10:13.648834+0000","last_became_peered":"2024-09-21T10:10:13.648834+0000","last_unstale":"2024-09-21T10:10:13.675582+0000","last_undegraded":"2024-09-21T10:10:13.675582+0000","last_fullsized":"2024-09-21T10:10:13.675582+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:50:40.852175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.636010+0000","last_change":"2024-09-21T10:10:13.636010+0000","last_active":"2024-09-21T10:10:13.636010+0000","last_peered":"2024-09-21T10:10:13.636010+0000","last_clean":"2024-09-21T10:10:13.636010+0000","last_became_active":"2024-09-21T10:10:13.635697+0000","last_became_peered":"2024-09-21T10:10:13.635697+0000","last_unstale":"2024-09-21T10:10:13.636010+0000","last_undegraded":"2024-09-21T10:10:13.636010+0000","last_fullsized":"2024-09-21T10:10:13.636010+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:55:05.695643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1253'5838","reported_seq":5879,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:27.650990+0000","last_change":"2024-09-21T10:08:48.536593+0000","last_active":"2024-09-21T10:10:27.650990+0000","last_peered":"2024-09-21T10:10:27.650990+0000","last_clean":"2024-09-21T10:10:27.650990+0000","last_became_active":"2024-09-21T10:08:48.536286+0000","last_became_peered":"2024-09-21T10:08:48.536286+0000","last_unstale":"2024-09-21T10:10:27.650990+0000","last_undegraded":"2024-09-21T10:10:27.650990+0000","last_fullsized":"2024-09-21T10:10:27.650990+0000","mapping_epoch":1238,"log_start":"0'0","ondisk_log_start":"0'0","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:48:54.655713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375390208,"num_objects":5728,"num_object_clones":0,"num_object_copies":17184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5838,"num_write_kb":370112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"288'100","reported_seq":49,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.613700+0000","last_change":"2024-09-21T10:09:25.437099+0000","last_active":"2024-09-21T10:10:12.613700+0000","last_peered":"2024-09-21T10:10:12.613700+0000","last_clean":"2024-09-21T10:10:12.613700+0000","last_became_active":"2024-09-21T10:09:25.436542+0000","last_became_peered":"2024-09-21T10:09:25.436542+0000","last_unstale":"2024-09-21T10:10:12.613700+0000","last_undegraded":"2024-09-21T10:10:12.613700+0000","last_fullsized":"2024-09-21T10:10:12.613700+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:03.581657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:03.581657+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:18.627778+0000","scrub_duration":0.0038308109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":873,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.678649+0000","last_change":"2024-09-21T10:10:13.678563+0000","last_active":"2024-09-21T10:10:13.678649+0000","last_peered":"2024-09-21T10:10:13.678649+0000","last_clean":"2024-09-21T10:10:13.678649+0000","last_became_active":"2024-09-21T10:10:13.676594+0000","last_became_peered":"2024-09-21T10:10:13.676594+0000","last_unstale":"2024-09-21T10:10:13.678649+0000","last_undegraded":"2024-09-21T10:10:13.678649+0000","last_fullsized":"2024-09-21T10:10:13.678649+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:34.275762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1957,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.629823+0000","last_change":"2024-09-21T10:08:45.528426+0000","last_active":"2024-09-21T10:10:13.629823+0000","last_peered":"2024-09-21T10:10:13.629823+0000","last_clean":"2024-09-21T10:10:13.629823+0000","last_became_active":"2024-09-21T10:08:45.528049+0000","last_became_peered":"2024-09-21T10:08:45.528049+0000","last_unstale":"2024-09-21T10:10:13.629823+0000","last_undegraded":"2024-09-21T10:10:13.629823+0000","last_fullsized":"2024-09-21T10:10:13.629823+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:10:10.687040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"1253'5799","reported_seq":5946,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:27.378426+0000","last_change":"2024-09-21T10:10:13.865315+0000","last_active":"2024-09-21T10:10:27.378426+0000","last_peered":"2024-09-21T10:10:27.378426+0000","last_clean":"2024-09-21T10:09:24.416711+0000","last_became_active":"2024-09-21T10:10:13.814696+0000","last_became_peered":"2024-09-21T10:10:13.814696+0000","last_unstale":"2024-09-21T10:10:27.378426+0000","last_undegraded":"2024-09-21T10:10:13.798377+0000","last_fullsized":"2024-09-21T10:10:13.622503+0000","mapping_epoch":1252,"log_start":"1242'1100","ondisk_log_start":"1242'1100","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4699,"log_dups_size":0,"ondisk_log_size":4699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374013952,"num_objects":5707,"num_object_clones":0,"num_object_copies":17121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4325,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5799,"num_write_kb":368192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":3145728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,4],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"2,3,4","objects":892},{"shards":"3,4","objects":1382},{"shards":"3,6,7","objects":3432}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1209,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.624412+0000","last_change":"2024-09-21T10:09:25.602575+0000","last_active":"2024-09-21T10:10:13.624412+0000","last_peered":"2024-09-21T10:10:13.624412+0000","last_clean":"2024-09-21T10:10:13.624412+0000","last_became_active":"2024-09-21T10:09:25.451591+0000","last_became_peered":"2024-09-21T10:09:25.451591+0000","last_unstale":"2024-09-21T10:10:13.624412+0000","last_undegraded":"2024-09-21T10:10:13.624412+0000","last_fullsized":"2024-09-21T10:10:13.624412+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:25.602524+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:25.602524+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:24.338561+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1867,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.620135+0000","last_change":"2024-09-21T10:08:25.436371+0000","last_active":"2024-09-21T10:10:13.620135+0000","last_peered":"2024-09-21T10:10:13.620135+0000","last_clean":"2024-09-21T10:10:13.620135+0000","last_became_active":"2024-09-21T10:08:25.435921+0000","last_became_peered":"2024-09-21T10:08:25.435921+0000","last_unstale":"2024-09-21T10:10:13.620135+0000","last_undegraded":"2024-09-21T10:10:13.620135+0000","last_fullsized":"2024-09-21T10:10:13.620135+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:27.612386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1195,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.661328+0000","last_change":"2024-09-21T10:10:13.661328+0000","last_active":"2024-09-21T10:10:13.661328+0000","last_peered":"2024-09-21T10:10:13.661328+0000","last_clean":"2024-09-21T10:10:13.661328+0000","last_became_active":"2024-09-21T10:10:13.661028+0000","last_became_peered":"2024-09-21T10:10:13.661028+0000","last_unstale":"2024-09-21T10:10:13.661328+0000","last_undegraded":"2024-09-21T10:10:13.661328+0000","last_fullsized":"2024-09-21T10:10:13.661328+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:36.965214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1253'5885","reported_seq":5933,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:27.765799+0000","last_change":"2024-09-21T10:10:13.725956+0000","last_active":"2024-09-21T10:10:27.765799+0000","last_peered":"2024-09-21T10:10:27.765799+0000","last_clean":"2024-09-21T10:09:45.416832+0000","last_became_active":"2024-09-21T10:10:13.721727+0000","last_became_peered":"2024-09-21T10:10:13.721727+0000","last_unstale":"2024-09-21T10:10:27.765799+0000","last_undegraded":"2024-09-21T10:10:13.632913+0000","last_fullsized":"2024-09-21T10:10:13.613387+0000","mapping_epoch":1252,"log_start":"1243'1300","ondisk_log_start":"1243'1300","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4585,"log_dups_size":0,"ondisk_log_size":4585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378208256,"num_objects":5771,"num_object_clones":0,"num_object_copies":17313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":841,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5885,"num_write_kb":372992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":841},{"shards":"1,2","objects":4930}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":302,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.619023+0000","last_change":"2024-09-21T10:00:15.524812+0000","last_active":"2024-09-21T10:10:13.619023+0000","last_peered":"2024-09-21T10:10:13.619023+0000","last_clean":"2024-09-21T10:10:13.619023+0000","last_became_active":"2024-09-21T09:59:43.361751+0000","last_became_peered":"2024-09-21T09:59:43.361751+0000","last_unstale":"2024-09-21T10:10:13.619023+0000","last_undegraded":"2024-09-21T10:10:13.619023+0000","last_fullsized":"2024-09-21T10:10:13.619023+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:13.513881+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.636008+0000","last_change":"2024-09-21T10:10:13.636008+0000","last_active":"2024-09-21T10:10:13.636008+0000","last_peered":"2024-09-21T10:10:13.636008+0000","last_clean":"2024-09-21T10:10:13.636008+0000","last_became_active":"2024-09-21T10:10:13.635632+0000","last_became_peered":"2024-09-21T10:10:13.635632+0000","last_unstale":"2024-09-21T10:10:13.636008+0000","last_undegraded":"2024-09-21T10:10:13.636008+0000","last_fullsized":"2024-09-21T10:10:13.636008+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:28.265954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1218,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.615716+0000","last_change":"2024-09-21T10:08:45.524238+0000","last_active":"2024-09-21T10:10:12.615716+0000","last_peered":"2024-09-21T10:10:12.615716+0000","last_clean":"2024-09-21T10:10:12.615716+0000","last_became_active":"2024-09-21T10:08:45.523517+0000","last_became_peered":"2024-09-21T10:08:45.523517+0000","last_unstale":"2024-09-21T10:10:12.615716+0000","last_undegraded":"2024-09-21T10:10:12.615716+0000","last_fullsized":"2024-09-21T10:10:12.615716+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:13.958459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1253'5939","reported_seq":5983,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:27.358693+0000","last_change":"2024-09-21T10:10:13.676115+0000","last_active":"2024-09-21T10:10:27.358693+0000","last_peered":"2024-09-21T10:10:27.358693+0000","last_clean":"2024-09-21T10:09:24.411450+0000","last_became_active":"2024-09-21T10:09:26.644140+0000","last_became_peered":"2024-09-21T10:09:26.644140+0000","last_unstale":"2024-09-21T10:10:27.358693+0000","last_undegraded":"2024-09-21T10:10:13.672461+0000","last_fullsized":"2024-09-21T10:09:26.425499+0000","mapping_epoch":1245,"log_start":"1243'1200","ondisk_log_start":"1243'1200","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4739,"log_dups_size":0,"ondisk_log_size":4739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382926848,"num_objects":5843,"num_object_clones":0,"num_object_copies":17529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3537,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5939,"num_write_kb":377024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":2306},{"shards":"1,3,7","objects":3490}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"288'100","reported_seq":44,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.623090+0000","last_change":"2024-09-21T10:08:45.524802+0000","last_active":"2024-09-21T10:10:13.623090+0000","last_peered":"2024-09-21T10:10:13.623090+0000","last_clean":"2024-09-21T10:10:13.623090+0000","last_became_active":"2024-09-21T10:08:45.524408+0000","last_became_peered":"2024-09-21T10:08:45.524408+0000","last_unstale":"2024-09-21T10:10:13.623090+0000","last_undegraded":"2024-09-21T10:10:13.623090+0000","last_fullsized":"2024-09-21T10:10:13.623090+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:36.488660+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:08:36.488660+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":23,"num_write":5,"num_write_kb":50,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1199,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.611199+0000","last_change":"2024-09-21T10:09:25.449815+0000","last_active":"2024-09-21T10:10:12.611199+0000","last_peered":"2024-09-21T10:10:12.611199+0000","last_clean":"2024-09-21T10:10:12.611199+0000","last_became_active":"2024-09-21T10:09:25.449325+0000","last_became_peered":"2024-09-21T10:09:25.449325+0000","last_unstale":"2024-09-21T10:10:12.611199+0000","last_undegraded":"2024-09-21T10:10:12.611199+0000","last_fullsized":"2024-09-21T10:10:12.611199+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:12.890694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1241,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.641608+0000","last_change":"2024-09-21T10:10:13.641608+0000","last_active":"2024-09-21T10:10:13.641608+0000","last_peered":"2024-09-21T10:10:13.641608+0000","last_clean":"2024-09-21T10:10:13.641608+0000","last_became_active":"2024-09-21T10:10:13.640299+0000","last_became_peered":"2024-09-21T10:10:13.640299+0000","last_unstale":"2024-09-21T10:10:13.641608+0000","last_undegraded":"2024-09-21T10:10:13.641608+0000","last_fullsized":"2024-09-21T10:10:13.641608+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:46.978168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1253'5803","reported_seq":5844,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:27.357769+0000","last_change":"2024-09-21T10:08:48.535207+0000","last_active":"2024-09-21T10:10:27.357769+0000","last_peered":"2024-09-21T10:10:27.357769+0000","last_clean":"2024-09-21T10:10:27.357769+0000","last_became_active":"2024-09-21T10:08:48.534901+0000","last_became_peered":"2024-09-21T10:08:48.534901+0000","last_unstale":"2024-09-21T10:10:27.357769+0000","last_undegraded":"2024-09-21T10:10:27.357769+0000","last_fullsized":"2024-09-21T10:10:27.357769+0000","mapping_epoch":1238,"log_start":"1242'1100","ondisk_log_start":"1242'1100","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4703,"log_dups_size":0,"ondisk_log_size":4703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:38.336438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374538240,"num_objects":5715,"num_object_clones":0,"num_object_copies":17145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5803,"num_write_kb":368576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":314,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.622637+0000","last_change":"2024-09-21T10:10:13.622553+0000","last_active":"2024-09-21T10:10:13.622637+0000","last_peered":"2024-09-21T10:10:13.622637+0000","last_clean":"2024-09-21T10:10:13.622637+0000","last_became_active":"2024-09-21T10:10:13.622268+0000","last_became_peered":"2024-09-21T10:10:13.622268+0000","last_unstale":"2024-09-21T10:10:13.622637+0000","last_undegraded":"2024-09-21T10:10:13.622637+0000","last_fullsized":"2024-09-21T10:10:13.622637+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:24.686892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:24.686892+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:55.474818+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1879,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.624274+0000","last_change":"2024-09-21T10:09:25.455306+0000","last_active":"2024-09-21T10:10:13.624274+0000","last_peered":"2024-09-21T10:10:13.624274+0000","last_clean":"2024-09-21T10:10:13.624274+0000","last_became_active":"2024-09-21T10:09:25.454568+0000","last_became_peered":"2024-09-21T10:09:25.454568+0000","last_unstale":"2024-09-21T10:10:13.624274+0000","last_undegraded":"2024-09-21T10:10:13.624274+0000","last_fullsized":"2024-09-21T10:10:13.624274+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:25.499158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":44,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.632588+0000","last_change":"2024-09-21T10:08:45.523818+0000","last_active":"2024-09-21T10:10:13.632588+0000","last_peered":"2024-09-21T10:10:13.632588+0000","last_clean":"2024-09-21T10:10:13.632588+0000","last_became_active":"2024-09-21T10:08:45.523507+0000","last_became_peered":"2024-09-21T10:08:45.523507+0000","last_unstale":"2024-09-21T10:10:13.632588+0000","last_undegraded":"2024-09-21T10:10:13.632588+0000","last_fullsized":"2024-09-21T10:10:13.632588+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:32.923210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1253'5929","reported_seq":5970,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:27.379858+0000","last_change":"2024-09-21T10:08:48.535314+0000","last_active":"2024-09-21T10:10:27.379858+0000","last_peered":"2024-09-21T10:10:27.379858+0000","last_clean":"2024-09-21T10:10:27.379858+0000","last_became_active":"2024-09-21T10:08:48.535036+0000","last_became_peered":"2024-09-21T10:08:48.535036+0000","last_unstale":"2024-09-21T10:10:27.379858+0000","last_undegraded":"2024-09-21T10:10:27.379858+0000","last_fullsized":"2024-09-21T10:10:27.379858+0000","mapping_epoch":1238,"log_start":"1243'1200","ondisk_log_start":"1243'1200","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4729,"log_dups_size":0,"ondisk_log_size":4729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:25:00.397920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381353984,"num_objects":5819,"num_object_clones":0,"num_object_copies":17457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5929,"num_write_kb":375936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":388,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.652398+0000","last_change":"2024-09-21T10:10:13.652304+0000","last_active":"2024-09-21T10:10:13.652398+0000","last_peered":"2024-09-21T10:10:13.652398+0000","last_clean":"2024-09-21T10:10:13.652398+0000","last_became_active":"2024-09-21T10:10:13.648459+0000","last_became_peered":"2024-09-21T10:10:13.648459+0000","last_unstale":"2024-09-21T10:10:13.652398+0000","last_undegraded":"2024-09-21T10:10:13.652398+0000","last_fullsized":"2024-09-21T10:10:13.652398+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.705004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.705004+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":33,"num_write":6,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26834,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":344,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.620077+0000","last_change":"2024-09-21T10:09:25.471697+0000","last_active":"2024-09-21T10:10:13.620077+0000","last_peered":"2024-09-21T10:10:13.620077+0000","last_clean":"2024-09-21T10:10:13.620077+0000","last_became_active":"2024-09-21T10:09:25.458890+0000","last_became_peered":"2024-09-21T10:09:25.458890+0000","last_unstale":"2024-09-21T10:10:13.620077+0000","last_undegraded":"2024-09-21T10:10:13.620077+0000","last_fullsized":"2024-09-21T10:10:13.620077+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:07.347060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1858,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.664548+0000","last_change":"2024-09-21T10:10:13.664494+0000","last_active":"2024-09-21T10:10:13.664548+0000","last_peered":"2024-09-21T10:10:13.664548+0000","last_clean":"2024-09-21T10:10:13.664548+0000","last_became_active":"2024-09-21T10:10:13.647871+0000","last_became_peered":"2024-09-21T10:10:13.647871+0000","last_unstale":"2024-09-21T10:10:13.664548+0000","last_undegraded":"2024-09-21T10:10:13.664548+0000","last_fullsized":"2024-09-21T10:10:13.664548+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:13.390934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.642176+0000","last_change":"2024-09-21T10:10:13.642176+0000","last_active":"2024-09-21T10:10:13.642176+0000","last_peered":"2024-09-21T10:10:13.642176+0000","last_clean":"2024-09-21T10:10:13.642176+0000","last_became_active":"2024-09-21T10:10:13.641997+0000","last_became_peered":"2024-09-21T10:10:13.641997+0000","last_unstale":"2024-09-21T10:10:13.642176+0000","last_undegraded":"2024-09-21T10:10:13.642176+0000","last_fullsized":"2024-09-21T10:10:13.642176+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:46.940248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1253'5801","reported_seq":5852,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:24.747330+0000","last_change":"2024-09-21T10:10:13.695829+0000","last_active":"2024-09-21T10:10:24.747330+0000","last_peered":"2024-09-21T10:10:24.747330+0000","last_clean":"2024-09-21T10:09:45.420818+0000","last_became_active":"2024-09-21T10:10:13.689557+0000","last_became_peered":"2024-09-21T10:10:13.689557+0000","last_unstale":"2024-09-21T10:10:24.747330+0000","last_undegraded":"2024-09-21T10:10:13.680588+0000","last_fullsized":"2024-09-21T10:10:13.628840+0000","mapping_epoch":1252,"log_start":"1242'300","ondisk_log_start":"1242'300","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":5501,"log_dups_size":0,"ondisk_log_size":5501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380174336,"num_objects":5801,"num_object_clones":0,"num_object_copies":17403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":805,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5801,"num_write_kb":371264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,2,5","objects":805},{"shards":"0,5","objects":4996}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":58,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.637731+0000","last_change":"2024-09-21T10:10:13.637642+0000","last_active":"2024-09-21T10:10:13.637731+0000","last_peered":"2024-09-21T10:10:13.637731+0000","last_clean":"2024-09-21T10:10:13.637731+0000","last_became_active":"2024-09-21T10:10:13.637267+0000","last_became_peered":"2024-09-21T10:10:13.637267+0000","last_unstale":"2024-09-21T10:10:13.637731+0000","last_undegraded":"2024-09-21T10:10:13.637731+0000","last_fullsized":"2024-09-21T10:10:13.637731+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:04:35.993739+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":22,"num_write":4,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1229,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.615099+0000","last_change":"2024-09-21T10:09:25.465784+0000","last_active":"2024-09-21T10:10:13.615099+0000","last_peered":"2024-09-21T10:10:13.615099+0000","last_clean":"2024-09-21T10:10:13.615099+0000","last_became_active":"2024-09-21T10:09:25.465445+0000","last_became_peered":"2024-09-21T10:09:25.465445+0000","last_unstale":"2024-09-21T10:10:13.615099+0000","last_undegraded":"2024-09-21T10:10:13.615099+0000","last_fullsized":"2024-09-21T10:10:13.615099+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:42:02.016170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1942,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.634202+0000","last_change":"2024-09-21T10:08:25.429287+0000","last_active":"2024-09-21T10:10:13.634202+0000","last_peered":"2024-09-21T10:10:13.634202+0000","last_clean":"2024-09-21T10:10:13.634202+0000","last_became_active":"2024-09-21T10:08:25.428923+0000","last_became_peered":"2024-09-21T10:08:25.428923+0000","last_unstale":"2024-09-21T10:10:13.634202+0000","last_undegraded":"2024-09-21T10:10:13.634202+0000","last_fullsized":"2024-09-21T10:10:13.634202+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:09.255665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1957,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.630849+0000","last_change":"2024-09-21T10:08:45.532253+0000","last_active":"2024-09-21T10:10:13.630849+0000","last_peered":"2024-09-21T10:10:13.630849+0000","last_clean":"2024-09-21T10:10:13.630849+0000","last_became_active":"2024-09-21T10:08:45.531519+0000","last_became_peered":"2024-09-21T10:08:45.531519+0000","last_unstale":"2024-09-21T10:10:13.630849+0000","last_undegraded":"2024-09-21T10:10:13.630849+0000","last_fullsized":"2024-09-21T10:10:13.630849+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:25.378783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"1253'5901","reported_seq":7684,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:27.763821+0000","last_change":"2024-09-21T10:10:13.643523+0000","last_active":"2024-09-21T10:10:27.763821+0000","last_peered":"2024-09-21T10:10:27.763821+0000","last_clean":"2024-09-21T10:09:24.391963+0000","last_became_active":"2024-09-21T10:09:26.716693+0000","last_became_peered":"2024-09-21T10:09:26.716693+0000","last_unstale":"2024-09-21T10:10:27.763821+0000","last_undegraded":"2024-09-21T10:10:13.639837+0000","last_fullsized":"2024-09-21T10:09:26.423890+0000","mapping_epoch":1245,"log_start":"1243'1300","ondisk_log_start":"1243'1300","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4601,"log_dups_size":0,"ondisk_log_size":4601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377028608,"num_objects":5753,"num_object_clones":0,"num_object_copies":17259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2762,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5901,"num_write_kb":372928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":868,"num_bytes_recovered":56885248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5","objects":2991},{"shards":"1,5,7","objects":2762}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":374,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.628257+0000","last_change":"2024-09-21T10:09:25.450579+0000","last_active":"2024-09-21T10:10:13.628257+0000","last_peered":"2024-09-21T10:10:13.628257+0000","last_clean":"2024-09-21T10:10:13.628257+0000","last_became_active":"2024-09-21T10:09:25.445022+0000","last_became_peered":"2024-09-21T10:09:25.445022+0000","last_unstale":"2024-09-21T10:10:13.628257+0000","last_undegraded":"2024-09-21T10:10:13.628257+0000","last_fullsized":"2024-09-21T10:10:13.628257+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:33.068697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:08:33.068697+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":34,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1905,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.620671+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:10:13.620671+0000","last_peered":"2024-09-21T10:10:13.620671+0000","last_clean":"2024-09-21T10:10:13.620671+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:10:13.620671+0000","last_undegraded":"2024-09-21T10:10:13.620671+0000","last_fullsized":"2024-09-21T10:10:13.620671+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":58,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.635659+0000","last_change":"2024-09-21T10:10:13.635570+0000","last_active":"2024-09-21T10:10:13.635659+0000","last_peered":"2024-09-21T10:10:13.635659+0000","last_clean":"2024-09-21T10:10:13.635659+0000","last_became_active":"2024-09-21T10:10:13.635099+0000","last_became_peered":"2024-09-21T10:10:13.635099+0000","last_unstale":"2024-09-21T10:10:13.635659+0000","last_undegraded":"2024-09-21T10:10:13.635659+0000","last_fullsized":"2024-09-21T10:10:13.635659+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:43.686232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":44,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.632499+0000","last_change":"2024-09-21T10:08:45.525625+0000","last_active":"2024-09-21T10:10:13.632499+0000","last_peered":"2024-09-21T10:10:13.632499+0000","last_clean":"2024-09-21T10:10:13.632499+0000","last_became_active":"2024-09-21T10:08:45.525299+0000","last_became_peered":"2024-09-21T10:08:45.525299+0000","last_unstale":"2024-09-21T10:10:13.632499+0000","last_undegraded":"2024-09-21T10:10:13.632499+0000","last_fullsized":"2024-09-21T10:10:13.632499+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:08.084485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"1253'5821","reported_seq":6884,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:24.734767+0000","last_change":"2024-09-21T10:09:46.486729+0000","last_active":"2024-09-21T10:10:24.734767+0000","last_peered":"2024-09-21T10:10:24.734767+0000","last_clean":"2024-09-21T10:09:24.421724+0000","last_became_active":"2024-09-21T10:09:26.681479+0000","last_became_peered":"2024-09-21T10:09:26.681479+0000","last_unstale":"2024-09-21T10:10:24.734767+0000","last_undegraded":"2024-09-21T10:09:46.484067+0000","last_fullsized":"2024-09-21T10:09:26.423258+0000","mapping_epoch":1245,"log_start":"1242'300","ondisk_log_start":"1242'300","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381485056,"num_objects":5821,"num_object_clones":0,"num_object_copies":17463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3133,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5821,"num_write_kb":372544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":508,"num_bytes_recovered":33292288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":2688},{"shards":"0,5,7","objects":3133}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":654,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.618602+0000","last_change":"2024-09-21T10:06:19.901581+0000","last_active":"2024-09-21T10:10:13.618602+0000","last_peered":"2024-09-21T10:10:13.618602+0000","last_clean":"2024-09-21T10:10:13.618602+0000","last_became_active":"2024-09-21T10:06:13.443630+0000","last_became_peered":"2024-09-21T10:06:13.443630+0000","last_unstale":"2024-09-21T10:10:13.618602+0000","last_undegraded":"2024-09-21T10:10:13.618602+0000","last_fullsized":"2024-09-21T10:10:13.618602+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1213,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:19.901507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:19.901507+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:41.572099+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":59,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.636132+0000","last_change":"2024-09-21T10:10:13.636061+0000","last_active":"2024-09-21T10:10:13.636132+0000","last_peered":"2024-09-21T10:10:13.636132+0000","last_clean":"2024-09-21T10:10:13.636132+0000","last_became_active":"2024-09-21T10:10:13.635006+0000","last_became_peered":"2024-09-21T10:10:13.635006+0000","last_unstale":"2024-09-21T10:10:13.636132+0000","last_undegraded":"2024-09-21T10:10:13.636132+0000","last_fullsized":"2024-09-21T10:10:13.636132+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:24.272055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1196,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.686243+0000","last_change":"2024-09-21T10:10:13.686243+0000","last_active":"2024-09-21T10:10:13.686243+0000","last_peered":"2024-09-21T10:10:13.686243+0000","last_clean":"2024-09-21T10:10:13.686243+0000","last_became_active":"2024-09-21T10:10:13.685894+0000","last_became_peered":"2024-09-21T10:10:13.685894+0000","last_unstale":"2024-09-21T10:10:13.686243+0000","last_undegraded":"2024-09-21T10:10:13.686243+0000","last_fullsized":"2024-09-21T10:10:13.686243+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:12.394732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":44,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.631899+0000","last_change":"2024-09-21T10:08:45.525546+0000","last_active":"2024-09-21T10:10:13.631899+0000","last_peered":"2024-09-21T10:10:13.631899+0000","last_clean":"2024-09-21T10:10:13.631899+0000","last_became_active":"2024-09-21T10:08:45.525116+0000","last_became_peered":"2024-09-21T10:08:45.525116+0000","last_unstale":"2024-09-21T10:10:13.631899+0000","last_undegraded":"2024-09-21T10:10:13.631899+0000","last_fullsized":"2024-09-21T10:10:13.631899+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:58.739744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"1253'5954","reported_seq":5995,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:27.755995+0000","last_change":"2024-09-21T10:08:48.537781+0000","last_active":"2024-09-21T10:10:27.755995+0000","last_peered":"2024-09-21T10:10:27.755995+0000","last_clean":"2024-09-21T10:10:27.755995+0000","last_became_active":"2024-09-21T10:08:48.537433+0000","last_became_peered":"2024-09-21T10:08:48.537433+0000","last_unstale":"2024-09-21T10:10:27.755995+0000","last_undegraded":"2024-09-21T10:10:27.755995+0000","last_fullsized":"2024-09-21T10:10:27.755995+0000","mapping_epoch":1238,"log_start":"1243'1400","ondisk_log_start":"1243'1400","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4554,"log_dups_size":0,"ondisk_log_size":4554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:31.756528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381157376,"num_objects":5816,"num_object_clones":0,"num_object_copies":17448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5954,"num_write_kb":376640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.649092+0000","last_change":"2024-09-21T10:10:13.649092+0000","last_active":"2024-09-21T10:10:13.649092+0000","last_peered":"2024-09-21T10:10:13.649092+0000","last_clean":"2024-09-21T10:10:13.649092+0000","last_became_active":"2024-09-21T10:10:13.648610+0000","last_became_peered":"2024-09-21T10:10:13.648610+0000","last_unstale":"2024-09-21T10:10:13.649092+0000","last_undegraded":"2024-09-21T10:10:13.649092+0000","last_fullsized":"2024-09-21T10:10:13.649092+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.730973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.730973+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":34,"num_write":6,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":862,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.617311+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T10:10:13.617311+0000","last_peered":"2024-09-21T10:10:13.617311+0000","last_clean":"2024-09-21T10:10:13.617311+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T10:10:13.617311+0000","last_undegraded":"2024-09-21T10:10:13.617311+0000","last_fullsized":"2024-09-21T10:10:13.617311+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":51,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.631754+0000","last_change":"2024-09-21T10:09:25.453087+0000","last_active":"2024-09-21T10:10:13.631754+0000","last_peered":"2024-09-21T10:10:13.631754+0000","last_clean":"2024-09-21T10:10:13.631754+0000","last_became_active":"2024-09-21T10:09:25.449388+0000","last_became_peered":"2024-09-21T10:09:25.449388+0000","last_unstale":"2024-09-21T10:10:13.631754+0000","last_undegraded":"2024-09-21T10:10:13.631754+0000","last_fullsized":"2024-09-21T10:10:13.631754+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:05.107434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.615735+0000","last_change":"2024-09-21T10:08:45.523135+0000","last_active":"2024-09-21T10:10:13.615735+0000","last_peered":"2024-09-21T10:10:13.615735+0000","last_clean":"2024-09-21T10:10:13.615735+0000","last_became_active":"2024-09-21T10:08:45.522774+0000","last_became_peered":"2024-09-21T10:08:45.522774+0000","last_unstale":"2024-09-21T10:10:13.615735+0000","last_undegraded":"2024-09-21T10:10:13.615735+0000","last_fullsized":"2024-09-21T10:10:13.615735+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:19:19.243019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"1253'5701","reported_seq":8080,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:28.434192+0000","last_change":"2024-09-21T10:09:46.459618+0000","last_active":"2024-09-21T10:10:28.434192+0000","last_peered":"2024-09-21T10:10:28.434192+0000","last_clean":"2024-09-21T10:09:24.391727+0000","last_became_active":"2024-09-21T10:09:26.567025+0000","last_became_peered":"2024-09-21T10:09:26.567025+0000","last_unstale":"2024-09-21T10:10:28.434192+0000","last_undegraded":"2024-09-21T10:09:46.458461+0000","last_fullsized":"2024-09-21T10:09:26.423498+0000","mapping_epoch":1245,"log_start":"0'0","ondisk_log_start":"0'0","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:06:55.852645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364707840,"num_objects":5565,"num_object_clones":0,"num_object_copies":16695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2306,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5701,"num_write_kb":360512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1172,"num_bytes_recovered":76808192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4","objects":3259},{"shards":"1,4,7","objects":2306}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.649029+0000","last_change":"2024-09-21T10:10:13.649029+0000","last_active":"2024-09-21T10:10:13.649029+0000","last_peered":"2024-09-21T10:10:13.649029+0000","last_clean":"2024-09-21T10:10:13.649029+0000","last_became_active":"2024-09-21T10:10:13.648466+0000","last_became_peered":"2024-09-21T10:10:13.648466+0000","last_unstale":"2024-09-21T10:10:13.649029+0000","last_undegraded":"2024-09-21T10:10:13.649029+0000","last_fullsized":"2024-09-21T10:10:13.649029+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:18.749159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:18.749159+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":34,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":44,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.631558+0000","last_change":"2024-09-21T10:08:45.531369+0000","last_active":"2024-09-21T10:10:13.631558+0000","last_peered":"2024-09-21T10:10:13.631558+0000","last_clean":"2024-09-21T10:10:13.631558+0000","last_became_active":"2024-09-21T10:08:45.530951+0000","last_became_peered":"2024-09-21T10:08:45.530951+0000","last_unstale":"2024-09-21T10:10:13.631558+0000","last_undegraded":"2024-09-21T10:10:13.631558+0000","last_fullsized":"2024-09-21T10:10:13.631558+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:27:50.056329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1197,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.641431+0000","last_change":"2024-09-21T10:09:25.453642+0000","last_active":"2024-09-21T10:10:13.641431+0000","last_peered":"2024-09-21T10:10:13.641431+0000","last_clean":"2024-09-21T10:10:13.641431+0000","last_became_active":"2024-09-21T10:09:25.453318+0000","last_became_peered":"2024-09-21T10:09:25.453318+0000","last_unstale":"2024-09-21T10:10:13.641431+0000","last_undegraded":"2024-09-21T10:10:13.641431+0000","last_fullsized":"2024-09-21T10:10:13.641431+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:05.373220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1929,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.629921+0000","last_change":"2024-09-21T10:09:25.452807+0000","last_active":"2024-09-21T10:10:13.629921+0000","last_peered":"2024-09-21T10:10:13.629921+0000","last_clean":"2024-09-21T10:10:13.629921+0000","last_became_active":"2024-09-21T10:09:25.451745+0000","last_became_peered":"2024-09-21T10:09:25.451745+0000","last_unstale":"2024-09-21T10:10:13.629921+0000","last_undegraded":"2024-09-21T10:10:13.629921+0000","last_fullsized":"2024-09-21T10:10:13.629921+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:33:16.553471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"1253'5808","reported_seq":6669,"reported_epoch":1253,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-21T10:10:27.794509+0000","last_change":"2024-09-21T10:10:13.703165+0000","last_active":"2024-09-21T10:10:27.794509+0000","last_peered":"2024-09-21T10:10:27.794509+0000","last_clean":"2024-09-21T10:09:24.420829+0000","last_became_active":"2024-09-21T10:09:26.729464+0000","last_became_peered":"2024-09-21T10:09:26.729464+0000","last_unstale":"2024-09-21T10:10:27.794509+0000","last_undegraded":"2024-09-21T10:10:27.794509+0000","last_fullsized":"2024-09-21T10:09:26.425369+0000","mapping_epoch":1245,"log_start":"1243'1200","ondisk_log_start":"1243'1200","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4608,"log_dups_size":0,"ondisk_log_size":4608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373751832,"num_objects":5704,"num_object_clones":0,"num_object_copies":17112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3178,"num_objects_unfound":0,"num_objects_dirty":5704,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5808,"num_write_kb":368321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":412,"num_bytes_recovered":27000832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5","objects":2526},{"shards":"1,5,7","objects":3178}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":153,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.623475+0000","last_change":"2024-09-21T10:10:13.623475+0000","last_active":"2024-09-21T10:10:13.623475+0000","last_peered":"2024-09-21T10:10:13.623475+0000","last_clean":"2024-09-21T10:10:13.623475+0000","last_became_active":"2024-09-21T10:10:13.622580+0000","last_became_peered":"2024-09-21T10:10:13.622580+0000","last_unstale":"2024-09-21T10:10:13.623475+0000","last_undegraded":"2024-09-21T10:10:13.623475+0000","last_fullsized":"2024-09-21T10:10:13.623475+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:36.469530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:08:36.469530+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:47.757854+0000","scrub_duration":0.001636942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1867,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.623042+0000","last_change":"2024-09-21T10:09:25.477103+0000","last_active":"2024-09-21T10:10:13.623042+0000","last_peered":"2024-09-21T10:10:13.623042+0000","last_clean":"2024-09-21T10:10:13.623042+0000","last_became_active":"2024-09-21T10:09:25.439742+0000","last_became_peered":"2024-09-21T10:09:25.439742+0000","last_unstale":"2024-09-21T10:10:13.623042+0000","last_undegraded":"2024-09-21T10:10:13.623042+0000","last_fullsized":"2024-09-21T10:10:13.623042+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:19.911474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":58,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.637316+0000","last_change":"2024-09-21T10:10:13.637236+0000","last_active":"2024-09-21T10:10:13.637316+0000","last_peered":"2024-09-21T10:10:13.637316+0000","last_clean":"2024-09-21T10:10:13.637316+0000","last_became_active":"2024-09-21T10:10:13.636907+0000","last_became_peered":"2024-09-21T10:10:13.636907+0000","last_unstale":"2024-09-21T10:10:13.637316+0000","last_undegraded":"2024-09-21T10:10:13.637316+0000","last_fullsized":"2024-09-21T10:10:13.637316+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:33.507370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1861,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.623990+0000","last_change":"2024-09-21T10:09:25.455618+0000","last_active":"2024-09-21T10:10:13.623990+0000","last_peered":"2024-09-21T10:10:13.623990+0000","last_clean":"2024-09-21T10:10:13.623990+0000","last_became_active":"2024-09-21T10:09:25.454991+0000","last_became_peered":"2024-09-21T10:09:25.454991+0000","last_unstale":"2024-09-21T10:10:13.623990+0000","last_undegraded":"2024-09-21T10:10:13.623990+0000","last_fullsized":"2024-09-21T10:10:13.623990+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:03.365552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1253'5702","reported_seq":7238,"reported_epoch":1253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:10:27.323731+0000","last_change":"2024-09-21T10:10:13.677528+0000","last_active":"2024-09-21T10:10:27.323731+0000","last_peered":"2024-09-21T10:10:27.323731+0000","last_clean":"2024-09-21T10:09:24.399521+0000","last_became_active":"2024-09-21T10:09:26.604841+0000","last_became_peered":"2024-09-21T10:09:26.604841+0000","last_unstale":"2024-09-21T10:10:27.323731+0000","last_undegraded":"2024-09-21T10:09:26.493324+0000","last_fullsized":"2024-09-21T10:09:26.422905+0000","mapping_epoch":1245,"log_start":"1242'1000","ondisk_log_start":"1242'1000","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4702,"log_dups_size":0,"ondisk_log_size":4702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370016256,"num_objects":5646,"num_object_clones":0,"num_object_copies":16938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2740,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5702,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":746,"num_bytes_recovered":48889856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4","objects":2906},{"shards":"3,4,7","objects":2739}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":334,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.677249+0000","last_change":"2024-09-21T10:10:13.677133+0000","last_active":"2024-09-21T10:10:13.677249+0000","last_peered":"2024-09-21T10:10:13.677249+0000","last_clean":"2024-09-21T10:10:13.677249+0000","last_became_active":"2024-09-21T10:10:13.670149+0000","last_became_peered":"2024-09-21T10:10:13.670149+0000","last_unstale":"2024-09-21T10:10:13.677249+0000","last_undegraded":"2024-09-21T10:10:13.677249+0000","last_fullsized":"2024-09-21T10:10:13.677249+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.423664+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.423664+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:11:34.650236+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1967,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.631186+0000","last_change":"2024-09-21T10:08:45.540447+0000","last_active":"2024-09-21T10:10:13.631186+0000","last_peered":"2024-09-21T10:10:13.631186+0000","last_clean":"2024-09-21T10:10:13.631186+0000","last_became_active":"2024-09-21T10:08:45.528704+0000","last_became_peered":"2024-09-21T10:08:45.528704+0000","last_unstale":"2024-09-21T10:10:13.631186+0000","last_undegraded":"2024-09-21T10:10:13.631186+0000","last_fullsized":"2024-09-21T10:10:13.631186+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:09.404941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.648679+0000","last_change":"2024-09-21T10:10:13.648679+0000","last_active":"2024-09-21T10:10:13.648679+0000","last_peered":"2024-09-21T10:10:13.648679+0000","last_clean":"2024-09-21T10:10:13.648679+0000","last_became_active":"2024-09-21T10:10:13.648229+0000","last_became_peered":"2024-09-21T10:10:13.648229+0000","last_unstale":"2024-09-21T10:10:13.648679+0000","last_undegraded":"2024-09-21T10:10:13.648679+0000","last_fullsized":"2024-09-21T10:10:13.648679+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:08.922979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1178,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.621002+0000","last_change":"2024-09-21T10:10:13.621002+0000","last_active":"2024-09-21T10:10:13.621002+0000","last_peered":"2024-09-21T10:10:13.621002+0000","last_clean":"2024-09-21T10:10:13.621002+0000","last_became_active":"2024-09-21T10:10:13.620575+0000","last_became_peered":"2024-09-21T10:10:13.620575+0000","last_unstale":"2024-09-21T10:10:13.621002+0000","last_undegraded":"2024-09-21T10:10:13.621002+0000","last_fullsized":"2024-09-21T10:10:13.621002+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:06:32.926448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1241,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.640158+0000","last_change":"2024-09-21T10:10:13.640158+0000","last_active":"2024-09-21T10:10:13.640158+0000","last_peered":"2024-09-21T10:10:13.640158+0000","last_clean":"2024-09-21T10:10:13.640158+0000","last_became_active":"2024-09-21T10:10:13.639801+0000","last_became_peered":"2024-09-21T10:10:13.639801+0000","last_unstale":"2024-09-21T10:10:13.640158+0000","last_undegraded":"2024-09-21T10:10:13.640158+0000","last_fullsized":"2024-09-21T10:10:13.640158+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:25.153985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1261,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.631294+0000","last_change":"2024-09-21T10:10:13.631294+0000","last_active":"2024-09-21T10:10:13.631294+0000","last_peered":"2024-09-21T10:10:13.631294+0000","last_clean":"2024-09-21T10:10:13.631294+0000","last_became_active":"2024-09-21T10:10:13.622468+0000","last_became_peered":"2024-09-21T10:10:13.622468+0000","last_unstale":"2024-09-21T10:10:13.631294+0000","last_undegraded":"2024-09-21T10:10:13.631294+0000","last_fullsized":"2024-09-21T10:10:13.631294+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:53.404968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18976,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":58,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.635522+0000","last_change":"2024-09-21T10:10:13.635438+0000","last_active":"2024-09-21T10:10:13.635522+0000","last_peered":"2024-09-21T10:10:13.635522+0000","last_clean":"2024-09-21T10:10:13.635522+0000","last_became_active":"2024-09-21T10:10:13.634268+0000","last_became_peered":"2024-09-21T10:10:13.634268+0000","last_unstale":"2024-09-21T10:10:13.635522+0000","last_undegraded":"2024-09-21T10:10:13.635522+0000","last_fullsized":"2024-09-21T10:10:13.635522+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:00:07.541446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":45,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.617211+0000","last_change":"2024-09-21T10:09:25.473376+0000","last_active":"2024-09-21T10:10:13.617211+0000","last_peered":"2024-09-21T10:10:13.617211+0000","last_clean":"2024-09-21T10:10:13.617211+0000","last_became_active":"2024-09-21T10:09:25.441146+0000","last_became_peered":"2024-09-21T10:09:25.441146+0000","last_unstale":"2024-09-21T10:10:13.617211+0000","last_undegraded":"2024-09-21T10:10:13.617211+0000","last_fullsized":"2024-09-21T10:10:13.617211+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:50:08.490080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"288'100","reported_seq":57,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.619939+0000","last_change":"2024-09-21T10:09:25.453332+0000","last_active":"2024-09-21T10:10:13.619939+0000","last_peered":"2024-09-21T10:10:13.619939+0000","last_clean":"2024-09-21T10:10:13.619939+0000","last_became_active":"2024-09-21T10:09:25.452984+0000","last_became_peered":"2024-09-21T10:09:25.452984+0000","last_unstale":"2024-09-21T10:10:13.619939+0000","last_undegraded":"2024-09-21T10:10:13.619939+0000","last_fullsized":"2024-09-21T10:10:13.619939+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.389965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.389965+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:24.624741+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":647,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.633213+0000","last_change":"2024-09-21T10:09:25.463949+0000","last_active":"2024-09-21T10:10:13.633213+0000","last_peered":"2024-09-21T10:10:13.633213+0000","last_clean":"2024-09-21T10:10:13.633213+0000","last_became_active":"2024-09-21T10:09:25.449976+0000","last_became_peered":"2024-09-21T10:09:25.449976+0000","last_unstale":"2024-09-21T10:10:13.633213+0000","last_undegraded":"2024-09-21T10:10:13.633213+0000","last_fullsized":"2024-09-21T10:10:13.633213+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:09.931834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1864,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.825141+0000","last_change":"2024-09-21T10:10:13.825052+0000","last_active":"2024-09-21T10:10:13.825141+0000","last_peered":"2024-09-21T10:10:13.825141+0000","last_clean":"2024-09-21T10:10:13.825141+0000","last_became_active":"2024-09-21T10:10:13.809124+0000","last_became_peered":"2024-09-21T10:10:13.809124+0000","last_unstale":"2024-09-21T10:10:13.825141+0000","last_undegraded":"2024-09-21T10:10:13.825141+0000","last_fullsized":"2024-09-21T10:10:13.825141+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:49:32.388321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1929,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.633988+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T10:10:13.633988+0000","last_peered":"2024-09-21T10:10:13.633988+0000","last_clean":"2024-09-21T10:10:13.633988+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T10:10:13.633988+0000","last_undegraded":"2024-09-21T10:10:13.633988+0000","last_fullsized":"2024-09-21T10:10:13.633988+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1899,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.656119+0000","last_change":"2024-09-21T10:10:13.656027+0000","last_active":"2024-09-21T10:10:13.656119+0000","last_peered":"2024-09-21T10:10:13.656119+0000","last_clean":"2024-09-21T10:10:13.656119+0000","last_became_active":"2024-09-21T10:10:13.652862+0000","last_became_peered":"2024-09-21T10:10:13.652862+0000","last_unstale":"2024-09-21T10:10:13.656119+0000","last_undegraded":"2024-09-21T10:10:13.656119+0000","last_fullsized":"2024-09-21T10:10:13.656119+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:16.108044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":10,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.634975+0000","last_change":"2024-09-21T10:10:13.634975+0000","last_active":"2024-09-21T10:10:13.634975+0000","last_peered":"2024-09-21T10:10:13.634975+0000","last_clean":"2024-09-21T10:10:13.634975+0000","last_became_active":"2024-09-21T10:10:13.634667+0000","last_became_peered":"2024-09-21T10:10:13.634667+0000","last_unstale":"2024-09-21T10:10:13.634975+0000","last_undegraded":"2024-09-21T10:10:13.634975+0000","last_fullsized":"2024-09-21T10:10:13.634975+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:52.296990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1877,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.624178+0000","last_change":"2024-09-21T10:08:45.520524+0000","last_active":"2024-09-21T10:10:13.624178+0000","last_peered":"2024-09-21T10:10:13.624178+0000","last_clean":"2024-09-21T10:10:13.624178+0000","last_became_active":"2024-09-21T10:08:45.520119+0000","last_became_peered":"2024-09-21T10:08:45.520119+0000","last_unstale":"2024-09-21T10:10:13.624178+0000","last_undegraded":"2024-09-21T10:10:13.624178+0000","last_fullsized":"2024-09-21T10:10:13.624178+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:55.559438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1940,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.631025+0000","last_change":"2024-09-21T10:09:25.457529+0000","last_active":"2024-09-21T10:10:13.631025+0000","last_peered":"2024-09-21T10:10:13.631025+0000","last_clean":"2024-09-21T10:10:13.631025+0000","last_became_active":"2024-09-21T10:09:25.457214+0000","last_became_peered":"2024-09-21T10:09:25.457214+0000","last_unstale":"2024-09-21T10:10:13.631025+0000","last_undegraded":"2024-09-21T10:10:13.631025+0000","last_fullsized":"2024-09-21T10:10:13.631025+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:37.912673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":44,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.613533+0000","last_change":"2024-09-21T10:08:45.523144+0000","last_active":"2024-09-21T10:10:13.613533+0000","last_peered":"2024-09-21T10:10:13.613533+0000","last_clean":"2024-09-21T10:10:13.613533+0000","last_became_active":"2024-09-21T10:08:45.522526+0000","last_became_peered":"2024-09-21T10:08:45.522526+0000","last_unstale":"2024-09-21T10:10:13.613533+0000","last_undegraded":"2024-09-21T10:10:13.613533+0000","last_fullsized":"2024-09-21T10:10:13.613533+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:45.926968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1870,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.616772+0000","last_change":"2024-09-21T10:08:45.515262+0000","last_active":"2024-09-21T10:10:13.616772+0000","last_peered":"2024-09-21T10:10:13.616772+0000","last_clean":"2024-09-21T10:10:13.616772+0000","last_became_active":"2024-09-21T10:08:45.514926+0000","last_became_peered":"2024-09-21T10:08:45.514926+0000","last_unstale":"2024-09-21T10:10:13.616772+0000","last_undegraded":"2024-09-21T10:10:13.616772+0000","last_fullsized":"2024-09-21T10:10:13.616772+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:49.954351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1220,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.611975+0000","last_change":"2024-09-21T10:08:45.524864+0000","last_active":"2024-09-21T10:10:12.611975+0000","last_peered":"2024-09-21T10:10:12.611975+0000","last_clean":"2024-09-21T10:10:12.611975+0000","last_became_active":"2024-09-21T10:08:45.524533+0000","last_became_peered":"2024-09-21T10:08:45.524533+0000","last_unstale":"2024-09-21T10:10:12.611975+0000","last_undegraded":"2024-09-21T10:10:12.611975+0000","last_fullsized":"2024-09-21T10:10:12.611975+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:52.547302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1897,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.620104+0000","last_change":"2024-09-21T10:08:45.525340+0000","last_active":"2024-09-21T10:10:13.620104+0000","last_peered":"2024-09-21T10:10:13.620104+0000","last_clean":"2024-09-21T10:10:13.620104+0000","last_became_active":"2024-09-21T10:08:45.525023+0000","last_became_peered":"2024-09-21T10:08:45.525023+0000","last_unstale":"2024-09-21T10:10:13.620104+0000","last_undegraded":"2024-09-21T10:10:13.620104+0000","last_fullsized":"2024-09-21T10:10:13.620104+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:37.674092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1940,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.630036+0000","last_change":"2024-09-21T10:09:25.453096+0000","last_active":"2024-09-21T10:10:13.630036+0000","last_peered":"2024-09-21T10:10:13.630036+0000","last_clean":"2024-09-21T10:10:13.630036+0000","last_became_active":"2024-09-21T10:09:25.452401+0000","last_became_peered":"2024-09-21T10:09:25.452401+0000","last_unstale":"2024-09-21T10:10:13.630036+0000","last_undegraded":"2024-09-21T10:10:13.630036+0000","last_fullsized":"2024-09-21T10:10:13.630036+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.053333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1218,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.611627+0000","last_change":"2024-09-21T10:08:45.516048+0000","last_active":"2024-09-21T10:10:12.611627+0000","last_peered":"2024-09-21T10:10:12.611627+0000","last_clean":"2024-09-21T10:10:12.611627+0000","last_became_active":"2024-09-21T10:08:45.515656+0000","last_became_peered":"2024-09-21T10:08:45.515656+0000","last_unstale":"2024-09-21T10:10:12.611627+0000","last_undegraded":"2024-09-21T10:10:12.611627+0000","last_fullsized":"2024-09-21T10:10:12.611627+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:22.491802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":190,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.612995+0000","last_change":"2024-09-21T10:08:45.535971+0000","last_active":"2024-09-21T10:10:12.612995+0000","last_peered":"2024-09-21T10:10:12.612995+0000","last_clean":"2024-09-21T10:10:12.612995+0000","last_became_active":"2024-09-21T10:08:45.530341+0000","last_became_peered":"2024-09-21T10:08:45.530341+0000","last_unstale":"2024-09-21T10:10:12.612995+0000","last_undegraded":"2024-09-21T10:10:12.612995+0000","last_fullsized":"2024-09-21T10:10:12.612995+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:58:34.894434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1198,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-09-21T10:10:12.611796+0000","last_change":"2024-09-21T10:08:25.430667+0000","last_active":"2024-09-21T10:10:12.611796+0000","last_peered":"2024-09-21T10:10:12.611796+0000","last_clean":"2024-09-21T10:10:12.611796+0000","last_became_active":"2024-09-21T10:08:25.430357+0000","last_became_peered":"2024-09-21T10:08:25.430357+0000","last_unstale":"2024-09-21T10:10:12.611796+0000","last_undegraded":"2024-09-21T10:10:12.611796+0000","last_fullsized":"2024-09-21T10:10:12.611796+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:19.972020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":192,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.639231+0000","last_change":"2024-09-21T10:10:13.639231+0000","last_active":"2024-09-21T10:10:13.639231+0000","last_peered":"2024-09-21T10:10:13.639231+0000","last_clean":"2024-09-21T10:10:13.639231+0000","last_became_active":"2024-09-21T10:10:13.622497+0000","last_became_peered":"2024-09-21T10:10:13.622497+0000","last_unstale":"2024-09-21T10:10:13.639231+0000","last_undegraded":"2024-09-21T10:10:13.639231+0000","last_fullsized":"2024-09-21T10:10:13.639231+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:31:46.263003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1855,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.624883+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:10:13.624883+0000","last_peered":"2024-09-21T10:10:13.624883+0000","last_clean":"2024-09-21T10:10:13.624883+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:10:13.624883+0000","last_undegraded":"2024-09-21T10:10:13.624883+0000","last_fullsized":"2024-09-21T10:10:13.624883+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1870,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.651687+0000","last_change":"2024-09-21T10:10:13.651599+0000","last_active":"2024-09-21T10:10:13.651687+0000","last_peered":"2024-09-21T10:10:13.651687+0000","last_clean":"2024-09-21T10:10:13.651687+0000","last_became_active":"2024-09-21T10:10:13.651250+0000","last_became_peered":"2024-09-21T10:10:13.651250+0000","last_unstale":"2024-09-21T10:10:13.651687+0000","last_undegraded":"2024-09-21T10:10:13.651687+0000","last_fullsized":"2024-09-21T10:10:13.651687+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:34.100056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1241,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.639970+0000","last_change":"2024-09-21T10:10:13.639970+0000","last_active":"2024-09-21T10:10:13.639970+0000","last_peered":"2024-09-21T10:10:13.639970+0000","last_clean":"2024-09-21T10:10:13.639970+0000","last_became_active":"2024-09-21T10:10:13.639671+0000","last_became_peered":"2024-09-21T10:10:13.639671+0000","last_unstale":"2024-09-21T10:10:13.639970+0000","last_undegraded":"2024-09-21T10:10:13.639970+0000","last_fullsized":"2024-09-21T10:10:13.639970+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:53.941605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1938,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-09-21T10:10:13.634089+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T10:10:13.634089+0000","last_peered":"2024-09-21T10:10:13.634089+0000","last_clean":"2024-09-21T10:10:13.634089+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T10:10:13.634089+0000","last_undegraded":"2024-09-21T10:10:13.634089+0000","last_fullsized":"2024-09-21T10:10:13.634089+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6035800088,"num_objects":92100,"num_object_clones":0,"num_object_copies":276300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27699,"num_objects_misplaced":3178,"num_objects_unfound":0,"num_objects_dirty":92100,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":93496,"num_write_kb":5939009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3758,"num_bytes_recovered":246284288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18397470720,"data_stored":18397462576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79296,"ondisk_log_size":79296,"up":48,"acting":37,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":19400,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34026,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":805047,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":107,"num_read_kb":272,"num_write":54,"num_write_kb":592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":542359,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453397935,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594163,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1265792,"kb_used_data":955960,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92479360,"statfs":{"total":95995035648,"available":94698864640,"internally_reserved":0,"allocated":978903040,"data_stored":976039594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790393,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4399992,"kb_used_data":3505840,"kb_used_omap":1,"kb_used_meta":894142,"kb_avail":89345160,"statfs":{"total":95995035648,"available":91489443840,"internally_reserved":0,"allocated":3589980160,"data_stored":3587121242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915601752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1224,"seq":5257039970330,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1747332,"kb_used_data":1406652,"kb_used_omap":290,"kb_used_meta":340381,"kb_avail":91997820,"statfs":{"total":95995035648,"available":94205767680,"internally_reserved":0,"allocated":1440411648,"data_stored":1437544153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":297377,"internal_metadata":348550751},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999262,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4191220,"kb_used_data":3351020,"kb_used_omap":25,"kb_used_meta":840166,"kb_avail":89553932,"statfs":{"total":95995035648,"available":91703226368,"internally_reserved":0,"allocated":3431444480,"data_stored":3428589179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":860330951},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111205,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2114048,"kb_used_data":1363640,"kb_used_omap":31,"kb_used_meta":750368,"kb_avail":91631104,"statfs":{"total":95995035648,"available":93830250496,"internally_reserved":0,"allocated":1396367360,"data_stored":1393504959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":768377064},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318223148,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3833524,"kb_used_data":3474540,"kb_used_omap":26,"kb_used_meta":358821,"kb_avail":89911628,"statfs":{"total":95995035648,"available":92069507072,"internally_reserved":0,"allocated":3557928960,"data_stored":3555073020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":367432716},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335090,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2527684,"kb_used_data":1930492,"kb_used_omap":31,"kb_used_meta":597152,"kb_avail":91217468,"statfs":{"total":95995035648,"available":93406687232,"internally_reserved":0,"allocated":1976823808,"data_stored":1973956879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":611483744},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1977548800,"data_stored":1977548800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3534098432,"data_stored":3534094360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1383858176,"data_stored":1383858176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3403153408,"data_stored":3403153408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1425608704,"data_stored":1425604632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3597271040,"data_stored":3597271040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":972750848,"data_stored":972750848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2103181312,"data_stored":2103181312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:10:32.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_4 pgp_num 16 2024-09-21T10:10:32.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:32.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:32.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:32 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:10:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:10:33.009 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:33.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:32 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:10:33.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:10:33.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:32 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:10:33.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:10:33.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:10:33.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:33.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:33.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:33.352 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:33.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:33.478 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:33.500 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:33.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:33.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:33.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:33.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:33.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:33.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:33.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:10:33.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:33.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.922 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:33.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:33 smithi134 ceph-mon[90085]: pgmap v1837: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 35 MiB/s wr, 669 op/s; 27699/275805 objects degraded (10.043%); 3178/275805 objects misplaced (1.152%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:10:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 27699/275805 objects degraded (10.043%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:34.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.073 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:34.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:34.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:33 smithi017 ceph-mon[105151]: pgmap v1837: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 35 MiB/s wr, 669 op/s; 27699/275805 objects degraded (10.043%); 3178/275805 objects misplaced (1.152%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:10:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 27699/275805 objects degraded (10.043%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:33 smithi017 ceph-mon[102913]: pgmap v1837: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 35 MiB/s wr, 669 op/s; 27699/275805 objects degraded (10.043%); 3178/275805 objects misplaced (1.152%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T10:10:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 27699/275805 objects degraded (10.043%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:34.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:34.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:34.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:34.348 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:34.450 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:34.473 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:34.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:34.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:34.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:34.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:34.721 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:34.745 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:34.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:34.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:34.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:34.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:35.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:35.083 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:10:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:10:34] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-09-21T10:10:35.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:35.202 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:35.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:35.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:35.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:35.395 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:35.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:35.497 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:35.524 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:35.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:35.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:35.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:35.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:35.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:35.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:35.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:35.936 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:35 smithi134 ceph-mon[90085]: pgmap v1838: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 671 op/s; 27514/267801 objects degraded (10.274%); 2994/267801 objects misplaced (1.118%); 3.6 MiB/s, 58 objects/s recovering 2024-09-21T10:10:36.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:36.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:36.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:36.173 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:36.197 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:35 smithi017 ceph-mon[102913]: pgmap v1838: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 671 op/s; 27514/267801 objects degraded (10.274%); 2994/267801 objects misplaced (1.118%); 3.6 MiB/s, 58 objects/s recovering 2024-09-21T10:10:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:35 smithi017 ceph-mon[105151]: pgmap v1838: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 671 op/s; 27514/267801 objects degraded (10.274%); 2994/267801 objects misplaced (1.118%); 3.6 MiB/s, 58 objects/s recovering 2024-09-21T10:10:36.298 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:36.325 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:36.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:36.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:36.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:36.553 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:36.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:36.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:36.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:36.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:36.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:36.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:36.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:36.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:36.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:36 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:10:37.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:37.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:37.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:37.157 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:37.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:36 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:10:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:36 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:10:37.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:37.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:37.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:37.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:37.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:37.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:37.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:37.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:37.676 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:37.705 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:37.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:37.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:37.832 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:37.934 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:37.960 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:37.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:37 smithi134 ceph-mon[90085]: pgmap v1839: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 664 op/s; 27514/265965 objects degraded (10.345%); 2994/265965 objects misplaced (1.126%); 3.6 MiB/s, 58 objects/s recovering 2024-09-21T10:10:38.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:38.083 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:38.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:38.185 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:37 smithi017 ceph-mon[102913]: pgmap v1839: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 664 op/s; 27514/265965 objects degraded (10.345%); 2994/265965 objects misplaced (1.126%); 3.6 MiB/s, 58 objects/s recovering 2024-09-21T10:10:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:37 smithi017 ceph-mon[105151]: pgmap v1839: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 14 MiB/s wr, 664 op/s; 27514/265965 objects degraded (10.345%); 2994/265965 objects misplaced (1.126%); 3.6 MiB/s, 58 objects/s recovering 2024-09-21T10:10:38.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.214 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:38.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:38.315 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:38.335 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:38.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:38.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:38.577 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:38.599 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:38.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:38.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:38.727 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:38.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:38.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:38.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:38.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:39.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:39.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:39.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:39.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:39.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[102913]: 1.6 scrub starts 2024-09-21T10:10:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[102913]: 1.6 scrub ok 2024-09-21T10:10:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:10:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:10:39.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 27514/265965 objects degraded (10.345%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[105151]: 1.6 scrub starts 2024-09-21T10:10:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[105151]: 1.6 scrub ok 2024-09-21T10:10:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:10:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:10:39.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 27514/265965 objects degraded (10.345%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:38 smithi134 ceph-mon[90085]: 1.6 scrub starts 2024-09-21T10:10:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:38 smithi134 ceph-mon[90085]: 1.6 scrub ok 2024-09-21T10:10:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:38 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:10:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:38 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:10:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 27514/265965 objects degraded (10.345%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:39.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:39.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:39.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:39.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:39.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:39.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:39.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:39.569 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:39.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:39.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:39.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:39.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:39.808 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:39.831 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:39.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:39.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:39.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:40.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:40.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:40.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:39 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:10:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:39 smithi017 ceph-mon[102913]: pgmap v1840: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.1 MiB/s wr, 637 op/s; 27321/259059 objects degraded (10.546%); 2793/259059 objects misplaced (1.078%); 4.8 MiB/s, 76 objects/s recovering 2024-09-21T10:10:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:39 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:10:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:39 smithi017 ceph-mon[105151]: pgmap v1840: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.1 MiB/s wr, 637 op/s; 27321/259059 objects degraded (10.546%); 2793/259059 objects misplaced (1.078%); 4.8 MiB/s, 76 objects/s recovering 2024-09-21T10:10:40.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:39 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:10:40.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:39 smithi134 ceph-mon[90085]: pgmap v1840: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.1 MiB/s wr, 637 op/s; 27321/259059 objects degraded (10.546%); 2793/259059 objects misplaced (1.078%); 4.8 MiB/s, 76 objects/s recovering 2024-09-21T10:10:40.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:40.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:40.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:40.368 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:40.391 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:40.424 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:10:40.424 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:10:40.424 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-21T10:10:40.424 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:10:40.435 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:40.442 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:40.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:10:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:10:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:10:40.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:40.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:40.652 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:40.675 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:40.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:40.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:40.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:40.923 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:40.947 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:40.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:41.049 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:41.074 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:41.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:41.176 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:41.201 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:41.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:41.303 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:41.325 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:41.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:41.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:41.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:41.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:41.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:41.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:41.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:41.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:10:41.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:41.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:41.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:41.882 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:41.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:41 smithi017 ceph-mon[102913]: pgmap v1841: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.1 MiB/s wr, 595 op/s; 27321/256779 objects degraded (10.640%); 2793/256779 objects misplaced (1.088%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T10:10:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:41 smithi017 ceph-mon[105151]: pgmap v1841: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.1 MiB/s wr, 595 op/s; 27321/256779 objects degraded (10.640%); 2793/256779 objects misplaced (1.088%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T10:10:41.983 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:42.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:42.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:42.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:42.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:42.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:41 smithi134 ceph-mon[90085]: pgmap v1841: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.1 MiB/s wr, 595 op/s; 27321/256779 objects degraded (10.640%); 2793/256779 objects misplaced (1.088%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T10:10:42.258 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:42.282 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:42.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:42.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:42.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:42.508 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:42.531 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:42.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:42.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:42.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:42.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:42.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:42.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:42.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:42.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:43.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:43.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:43.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:43.197 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:43.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:42 smithi017 ceph-mon[102913]: pgmap v1842: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 27321/256779 objects degraded (10.640%); 2793/256779 objects misplaced (1.088%); 4.0 MiB/s, 63 objects/s recovering 2024-09-21T10:10:43.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:42 smithi017 ceph-mon[105151]: pgmap v1842: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 27321/256779 objects degraded (10.640%); 2793/256779 objects misplaced (1.088%); 4.0 MiB/s, 63 objects/s recovering 2024-09-21T10:10:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:42 smithi134 ceph-mon[90085]: pgmap v1842: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 27321/256779 objects degraded (10.640%); 2793/256779 objects misplaced (1.088%); 4.0 MiB/s, 63 objects/s recovering 2024-09-21T10:10:43.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:43.335 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:43.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:43.480 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:43.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:43.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:43.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:43.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:43.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:43.756 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:43.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:43.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:43.883 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:43.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:43.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.030 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:44.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:44.131 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:44.154 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:44.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:44.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:43 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 27321/256779 objects degraded (10.640%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:44.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:43 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 27321/256779 objects degraded (10.640%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:43 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 27321/256779 objects degraded (10.640%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:44.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:44.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:44.410 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:44.435 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:44.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:44.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.561 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:44.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:44.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:44.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:44.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:44.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:44.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:44.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:44.848 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:10:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:10:44] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-09-21T10:10:44.943 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:44.967 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:44.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:45.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:45.086 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:10:45.086 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:10:45.087 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:10:45.087 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:10:45.087 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:10:45.087 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:10:45.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:45.088 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:45.088 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:45.088 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:45.088 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:45.088 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:45.089 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:10:45.089 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:45.091 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:45.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:45.124 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:10:45.124 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:45.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:44 smithi017 ceph-mon[102913]: pgmap v1843: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 27112/249009 objects degraded (10.888%); 2593/249009 objects misplaced (1.041%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:10:45.125 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4044858208' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:45.125 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:44 smithi017 ceph-mon[105151]: pgmap v1843: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 27112/249009 objects degraded (10.888%); 2593/249009 objects misplaced (1.041%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:10:45.125 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4044858208' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:45.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:45.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:45.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:44 smithi134 ceph-mon[90085]: pgmap v1843: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 743 op/s; 27112/249009 objects degraded (10.888%); 2593/249009 objects misplaced (1.041%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:10:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4044858208' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:45.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:45.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:45.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:45.493 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:45.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.517 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:45.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:45.617 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:45.641 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:45.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:45.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.767 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:45.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:45.868 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:45.891 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:45.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:45.993 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:46.016 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:46.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:46.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:46.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:46.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:46.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:46.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:46.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:46.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:46 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:10:46.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:46 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:10:46.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:46.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:46.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:46 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:10:46.569 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:46.594 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:46.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:46.696 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:46.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:46.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:46.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:46.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:46.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:46.960 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:46.983 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:46.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:47.020 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:47.085 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:47.107 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:47.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:47.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:47.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:47.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:47.363 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:47.387 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:47 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:10:47.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:47 smithi017 ceph-mon[102913]: pgmap v1844: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 27112/247146 objects degraded (10.970%); 2593/247146 objects misplaced (1.049%); 4.2 MiB/s, 66 objects/s recovering 2024-09-21T10:10:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:47 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:10:47.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:47 smithi017 ceph-mon[105151]: pgmap v1844: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 27112/247146 objects degraded (10.970%); 2593/247146 objects misplaced (1.049%); 4.2 MiB/s, 66 objects/s recovering 2024-09-21T10:10:47.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:47 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:10:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:47 smithi134 ceph-mon[90085]: pgmap v1844: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 27112/247146 objects degraded (10.970%); 2593/247146 objects misplaced (1.049%); 4.2 MiB/s, 66 objects/s recovering 2024-09-21T10:10:47.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:47.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:47.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:47.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:47.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:47.801 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:47.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:47.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:47.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:48.052 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:48.075 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:48.176 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:48.197 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:48.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:48.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:48.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.339 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:48.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:48.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:48.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:48.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:48.576 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:48.599 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:48.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:48.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:48.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:48.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:48.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:48.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:49.012 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:49.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:49.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:49.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:49.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:49.237 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:49.260 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:49.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:49.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:49.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:49.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:49.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:49.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:49.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:49.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:10:49.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:49.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:49 smithi017 ceph-mon[102913]: pgmap v1845: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 26923/239859 objects degraded (11.225%); 2390/239859 objects misplaced (0.996%); 6.2 MiB/s, 99 objects/s recovering 2024-09-21T10:10:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:49 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 26923/239859 objects degraded (11.225%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:49 smithi017 ceph-mon[105151]: pgmap v1845: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 26923/239859 objects degraded (11.225%); 2390/239859 objects misplaced (0.996%); 6.2 MiB/s, 99 objects/s recovering 2024-09-21T10:10:49.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:49 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 26923/239859 objects degraded (11.225%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:49 smithi134 ceph-mon[90085]: pgmap v1845: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 724 op/s; 26923/239859 objects degraded (11.225%); 2390/239859 objects misplaced (0.996%); 6.2 MiB/s, 99 objects/s recovering 2024-09-21T10:10:49.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:49 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 26923/239859 objects degraded (11.225%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:49.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:49.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:49.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:49.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:49.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.980 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:49.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:50.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:50.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:50.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:50.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:50.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:50.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:50.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:50.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:50.442 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 9 pgp_num to 16 2024-09-21T10:10:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:50 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4226684404' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:10:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:10:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:10:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:10:50.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:50.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:50.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:50.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:50.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:50 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4226684404' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:10:50.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:50 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4226684404' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:10:50.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:50.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:50.811 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:50.834 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:50.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:50.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:51.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:51.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:51.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:51.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:51.246 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:51.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:51.372 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:51.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:51.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:51.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:51.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:51.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:51.677 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:51 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:10:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:51 smithi017 ceph-mon[102913]: pgmap v1846: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 26923/237498 objects degraded (11.336%); 2390/237498 objects misplaced (1.006%); 4.2 MiB/s, 66 objects/s recovering 2024-09-21T10:10:51.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:51 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4226684404' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-21T10:10:51.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:51 smithi017 ceph-mon[102913]: osdmap e1255: 8 total, 8 up, 7 in 2024-09-21T10:10:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:51 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:10:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:51 smithi017 ceph-mon[105151]: pgmap v1846: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 26923/237498 objects degraded (11.336%); 2390/237498 objects misplaced (1.006%); 4.2 MiB/s, 66 objects/s recovering 2024-09-21T10:10:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:51 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4226684404' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-21T10:10:51.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:51 smithi017 ceph-mon[105151]: osdmap e1255: 8 total, 8 up, 7 in 2024-09-21T10:10:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:51 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:10:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:51 smithi134 ceph-mon[90085]: pgmap v1846: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 26923/237498 objects degraded (11.336%); 2390/237498 objects misplaced (1.006%); 4.2 MiB/s, 66 objects/s recovering 2024-09-21T10:10:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:51 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4226684404' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-21T10:10:51.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:51 smithi134 ceph-mon[90085]: osdmap e1255: 8 total, 8 up, 7 in 2024-09-21T10:10:51.780 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:51.802 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:51.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:51.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:51.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:51.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:52.067 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:52.091 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:52.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:52.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:52.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:52.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:52.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:52.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:52.465 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:10:52.506 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:52.529 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:52.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:52.654 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:52 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:10:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:52 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2232533911' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:52 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:52 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:10:52.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:52 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2232533911' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:52 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:52 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:10:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:52 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2232533911' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:52 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:52.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:52.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:52.889 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:52.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:52.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:53.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:53.038 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:53.141 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:53.163 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:53.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:53.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:53.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:53.405 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:53.428 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:53.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:53.529 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:53.554 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:53.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:53.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:53.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:53 smithi134 ceph-mon[90085]: pgmap v1848: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 26923/237498 objects degraded (11.336%); 2390/237498 objects misplaced (1.006%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T10:10:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:53 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:10:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:53 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:53.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:53 smithi134 ceph-mon[90085]: osdmap e1256: 8 total, 8 up, 7 in 2024-09-21T10:10:53.761 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:53.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:53.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:53.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:53.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:53 smithi017 ceph-mon[102913]: pgmap v1848: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 26923/237498 objects degraded (11.336%); 2390/237498 objects misplaced (1.006%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T10:10:53.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:53 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:10:53.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:53 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:53.869 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:53 smithi017 ceph-mon[102913]: osdmap e1256: 8 total, 8 up, 7 in 2024-09-21T10:10:53.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:53 smithi017 ceph-mon[105151]: pgmap v1848: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 26923/237498 objects degraded (11.336%); 2390/237498 objects misplaced (1.006%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T10:10:53.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:53 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:10:53.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:53 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:53.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:53 smithi017 ceph-mon[105151]: osdmap e1256: 8 total, 8 up, 7 in 2024-09-21T10:10:53.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:53.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:53.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:54.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:54.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:54.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:54.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:54.258 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:54.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:54.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:54.383 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:54.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:54.485 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:54.509 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:54.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:54.606 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:10:54.606 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:10:54.611 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:54.635 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:54.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:54.730 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:54 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 26923/237498 objects degraded (11.336%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:54.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:54 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 26923/237498 objects degraded (11.336%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:54.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:54 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 26923/237498 objects degraded (11.336%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:54.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:54.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:54.897 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:54.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:54.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:55.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:55.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:55.077 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:10:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:10:54] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-09-21T10:10:55.148 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:55.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:55.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:55.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:55.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:55.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:55.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:55.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:55.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:10:55.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:55.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:55.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:55.740 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:55 smithi017 ceph-mon[102913]: pgmap v1850: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 26739/229869 objects degraded (11.632%); 2176/229869 objects misplaced (0.947%); 6.2 MiB/s, 98 objects/s recovering 2024-09-21T10:10:55.740 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:10:55.740 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:55.740 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:55 smithi017 ceph-mon[105151]: pgmap v1850: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 26739/229869 objects degraded (11.632%); 2176/229869 objects misplaced (0.947%); 6.2 MiB/s, 98 objects/s recovering 2024-09-21T10:10:55.741 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:10:55.741 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:55 smithi134 ceph-mon[90085]: pgmap v1850: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 26739/229869 objects degraded (11.632%); 2176/229869 objects misplaced (0.947%); 6.2 MiB/s, 98 objects/s recovering 2024-09-21T10:10:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:10:55.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:55.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:55.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:55.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:55.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:56.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:56.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:56.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:56.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:56.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:56.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:56.493 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:56.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:10:56.518 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:56.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:56.620 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:56.642 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:56.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:56.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:56 smithi134 ceph-mon[90085]: 1.7 deep-scrub starts 2024-09-21T10:10:56.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:56.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:56.896 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:56.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.918 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:56.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:56 smithi017 ceph-mon[102913]: 1.7 deep-scrub starts 2024-09-21T10:10:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:56 smithi017 ceph-mon[105151]: 1.7 deep-scrub starts 2024-09-21T10:10:57.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:57.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:57.166 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:57.187 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:57.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:57.312 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:57.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:57.413 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:57.438 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:57.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:57.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:57.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:57.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:57.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:57.692 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:57.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:57 smithi134 ceph-mon[90085]: pgmap v1851: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 26739/228075 objects degraded (11.724%); 2176/228075 objects misplaced (0.954%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:10:57.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:57 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:10:57.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:57.839 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:57.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:57.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:57 smithi017 ceph-mon[102913]: pgmap v1851: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 26739/228075 objects degraded (11.724%); 2176/228075 objects misplaced (0.954%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:10:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:57 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:10:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:57 smithi017 ceph-mon[105151]: pgmap v1851: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 26739/228075 objects degraded (11.724%); 2176/228075 objects misplaced (0.954%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:10:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:57 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:10:57.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:57.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:58.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:58.090 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:58.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:58.180 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:10:58.180 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:10:58.181 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-21T10:10:58.181 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:10:58.181 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:10:58.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-21T10:10:58.191 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:58.214 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:58.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:58.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:58.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:58.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:58.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:58.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:58.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:58.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:58.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:58.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:58.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:58.802 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:58.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:58.927 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:59.028 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:59.053 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:59.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:59.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:59.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:59.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:59.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:59.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:59.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:59.423 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:59.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.446 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:10:59.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:59.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:59.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:59.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:59.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:59 smithi134 ceph-mon[90085]: pgmap v1852: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 26530/220473 objects degraded (12.033%); 1916/220473 objects misplaced (0.869%); 6.8 MiB/s, 108 objects/s recovering 2024-09-21T10:10:59.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:10:59 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 26530/220473 objects degraded (12.033%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:59.761 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:59 smithi017 ceph-mon[102913]: pgmap v1852: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 26530/220473 objects degraded (12.033%); 1916/220473 objects misplaced (0.869%); 6.8 MiB/s, 108 objects/s recovering 2024-09-21T10:10:59.761 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:10:59 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 26530/220473 objects degraded (12.033%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:59.762 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:59 smithi017 ceph-mon[105151]: pgmap v1852: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 26530/220473 objects degraded (12.033%); 1916/220473 objects misplaced (0.869%); 6.8 MiB/s, 108 objects/s recovering 2024-09-21T10:10:59.762 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:10:59 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 26530/220473 objects degraded (12.033%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:10:59.835 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:10:59.836 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:59.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:59.874 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:10:59.874 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:59.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:10:59.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:59.909 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:10:59.909 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:10:59.953 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:59.976 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:10:59.977 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:59.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:10:59.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:11:00.024 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:00.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:00.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:00.055 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:11:00.056 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:00.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:00.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:00.281 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:00.303 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:00.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:00.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:00.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:00.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:11:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:11:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:11:00.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:00.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:00.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:00.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:00.808 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:00 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/443200367' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:00.808 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:00 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/443200367' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:00.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:00.862 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:00.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:00.964 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:00.988 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:00.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:00.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:00 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/443200367' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:01.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:01.113 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:01.215 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:01.237 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:01.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:01.339 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:01.365 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:01.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:01.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:01.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:01.655 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:01.663 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:01.676 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:01.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:01.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:01.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:01.814 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:01 smithi017 ceph-mon[102913]: pgmap v1853: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 26530/217980 objects degraded (12.171%); 1916/217980 objects misplaced (0.879%); 5.5 MiB/s, 87 objects/s recovering 2024-09-21T10:11:01.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:01 smithi017 ceph-mon[105151]: pgmap v1853: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 26530/217980 objects degraded (12.171%); 1916/217980 objects misplaced (0.879%); 5.5 MiB/s, 87 objects/s recovering 2024-09-21T10:11:01.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:01.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:01 smithi134 ceph-mon[90085]: pgmap v1853: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 26530/217980 objects degraded (12.171%); 1916/217980 objects misplaced (0.879%); 5.5 MiB/s, 87 objects/s recovering 2024-09-21T10:11:02.018 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:02.043 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:02.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:02.145 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:02.168 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:02.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:02.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:02.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:02.457 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:02.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:02.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:02.694 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:02.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:02.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:02.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:02.845 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:02.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:02.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:02.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:02.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:03.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:03.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:03.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:03.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:03.269 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:03.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:03.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:03.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:03.517 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:03.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:03.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:03.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:03.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:11:03.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:03.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:03.792 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:03.815 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:03.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:03.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:03.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:03 smithi017 ceph-mon[102913]: pgmap v1854: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 26530/217980 objects degraded (12.171%); 1916/217980 objects misplaced (0.879%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:11:03.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:03 smithi017 ceph-mon[105151]: pgmap v1854: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 26530/217980 objects degraded (12.171%); 1916/217980 objects misplaced (0.879%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:11:03.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:03 smithi134 ceph-mon[90085]: pgmap v1854: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 26530/217980 objects degraded (12.171%); 1916/217980 objects misplaced (0.879%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:11:04.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.120 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:04.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:04.221 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:04.243 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:04.345 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:04.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.370 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:04.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:04.471 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:04.493 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:04.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:04.595 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:04.618 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:04.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:04.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:04.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:04.901 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:04.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:04 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 26530/217980 objects degraded (12.171%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:04.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:04 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:11:04.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:11:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:11:04] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.51.0" 2024-09-21T10:11:04.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:04 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 26530/217980 objects degraded (12.171%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:04.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:04 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:11:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:04 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 26530/217980 objects degraded (12.171%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:04 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:11:05.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:05.063 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:05.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:05.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:05.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:05.311 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:05.335 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:05.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:05.437 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:05.460 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:05.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:05.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:05.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:05.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:05.712 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:05.713 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:05 smithi017 ceph-mon[102913]: pgmap v1855: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 26339/209646 objects degraded (12.564%); 1665/209646 objects misplaced (0.794%); 6.9 MiB/s, 109 objects/s recovering 2024-09-21T10:11:05.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:05 smithi017 ceph-mon[105151]: pgmap v1855: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 26339/209646 objects degraded (12.564%); 1665/209646 objects misplaced (0.794%); 6.9 MiB/s, 109 objects/s recovering 2024-09-21T10:11:05.734 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:05.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:05.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:05.860 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:05.896 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:05 smithi134 ceph-mon[90085]: pgmap v1855: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 26339/209646 objects degraded (12.564%); 1665/209646 objects misplaced (0.794%); 6.9 MiB/s, 109 objects/s recovering 2024-09-21T10:11:05.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:06.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:06.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:06.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:06.257 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:06.283 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:06.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:06.385 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:06.406 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:06.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:06.508 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:06.532 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:06.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:06.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:06.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:06.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:06.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:06.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:06.922 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:06.949 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:06.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:07.051 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:07.077 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:07.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:07.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:07.306 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:07.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:07.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:07.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:07.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:07.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:07.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:07.597 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:07.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:07.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:07.776 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:07 smithi134 ceph-mon[90085]: pgmap v1856: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 26339/207579 objects degraded (12.689%); 1665/207579 objects misplaced (0.802%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:11:07.776 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:07 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:11:07.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:07.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:07.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:07 smithi017 ceph-mon[102913]: pgmap v1856: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 26339/207579 objects degraded (12.689%); 1665/207579 objects misplaced (0.802%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:11:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:07 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:11:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:07 smithi017 ceph-mon[105151]: pgmap v1856: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 26339/207579 objects degraded (12.689%); 1665/207579 objects misplaced (0.802%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:11:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:07 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:11:07.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:08.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:08.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:08.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:08.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:08.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:08.250 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:08.273 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:08.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:08.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:08.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:08.523 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:08.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.547 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:08.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:08.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:08.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.673 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:08.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:08.774 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:08.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:08.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:08.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:08.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:08.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:08 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:11:08.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:08 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:11:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:08 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:11:09.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:09.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:09.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:09.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:09.194 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:09.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:09.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:09.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:09.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:09.436 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.459 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:09.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:09.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:09.601 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:09.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:09.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:09.727 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:09.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:09.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:09.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:09.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:09 smithi017 ceph-mon[102913]: pgmap v1857: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 26134/199935 objects degraded (13.071%); 1401/199935 objects misplaced (0.701%); 7.2 MiB/s, 114 objects/s recovering 2024-09-21T10:11:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:09 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 26134/199935 objects degraded (13.071%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:09 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:11:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:09 smithi017 ceph-mon[105151]: pgmap v1857: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 26134/199935 objects degraded (13.071%); 1401/199935 objects misplaced (0.701%); 7.2 MiB/s, 114 objects/s recovering 2024-09-21T10:11:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:09 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 26134/199935 objects degraded (13.071%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:09 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:11:09.968 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:09.993 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:09.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:09 smithi134 ceph-mon[90085]: pgmap v1857: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 26134/199935 objects degraded (13.071%); 1401/199935 objects misplaced (0.701%); 7.2 MiB/s, 114 objects/s recovering 2024-09-21T10:11:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:09 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 26134/199935 objects degraded (13.071%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:09 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:11:10.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:10.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:10.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:10.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:10.268 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:10.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:10.367 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:11:10.367 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.35520.0:53007 7.18 7:1c743eac:::benchmark_data_smithi017_180590_object53006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:01:26.345705+0000", 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 579.30164942600004, 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0290580650000001, 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.368 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.35520", 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/110887640", 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53007 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345705+0000", 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345705+0000", 2024-09-21T10:11:10.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345706+0000", 2024-09-21T10:11:10.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.232e-06 2024-09-21T10:11:10.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345731+0000", 2024-09-21T10:11:10.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4842e-05 2024-09-21T10:11:10.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345741+0000", 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0114e-05 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.345748+0000", 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5699999999999998e-06 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.469569+0000", 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.12382101 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:26.469577+0000", 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8490000000000006e-06 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.370274+0000", 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.900697466 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.370408+0000", 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133802 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.370543+0000", 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013473900000000001 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374294+0000", 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003751739 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374333+0000", 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8186999999999999e-05 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374685+0000", 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035275299999999997 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374728+0000", 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3074999999999997e-05 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:01:27.374763+0000", 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4687000000000002e-05 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35520.0:152147 7.10 e1176/1156 7:0e36332e:::benchmark_data_smithi017_180590_object42587:head v 1176'6272, mlcod=1146'2571)", 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:03:13.407693+0000", 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 472.23966106199998, 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34189436699999998, 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407693+0000", 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407693+0000", 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407694+0000", 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.048e-06 2024-09-21T10:11:10.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407702+0000", 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.345e-06 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407712+0000", 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0546e-05 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407720+0000", 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.25e-06 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407775+0000", 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5878999999999998e-05 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.407801+0000", 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5588e-05 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411581+0000", 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037798039999999999 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749588+0000", 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33800690700000002 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35520.0:152175 7.5 e1176/1148 7:a0037212:::benchmark_data_smithi017_180590_object42615:head v 1176'6406, mlcod=1176'6404)", 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:03:13.410900+0000", 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 472.236454315, 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34129278699999999, 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410900+0000", 2024-09-21T10:11:10.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410900+0000", 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410901+0000", 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6899999999999996e-07 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410908+0000", 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8179999999999998e-06 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410917+0000", 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0380000000000005e-06 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.410923+0000", 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0889999999999996e-06 2024-09-21T10:11:10.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749002+0000", 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.338079081 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749031+0000", 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8997e-05 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.752149+0000", 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031175489999999998 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.752193+0000", 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4246000000000003e-05 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.35520.0:152169 7.a e1176/1142 7:50705924:::benchmark_data_smithi017_180590_object42609:head v 1176'9589, mlcod=1176'9586)", 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:03:13.411045+0000", 2024-09-21T10:11:10.385 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 472.23630911399999, 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34690485799999998, 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411045+0000", 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411045+0000", 2024-09-21T10:11:10.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411046+0000", 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9000000000000005e-07 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.411053+0000", 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8859999999999998e-06 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.748947+0000", 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33789343100000002 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.748966+0000", 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9604000000000001e-05 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749042+0000", 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6174000000000006e-05 2024-09-21T10:11:10.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749065+0000", 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2821999999999999e-05 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.757905+0000", 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0088397830000000004 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.757950+0000", 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5167999999999999e-05 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPush(7.b 1176/1156 [PushOp(7:d7bc6bf0:::benchmark_data_smithi017_180590_object56111:head, version: 1149'3496, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:d7bc6bf0:::benchmark_data_smithi017_180590_object56111:head@1149'3496, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:03:13.412303+0000", 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 472.23505116400003, 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34021348600000001, 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.390 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.412303+0000", 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.412303+0000", 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.412304+0000", 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8299999999999996e-07 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.412341+0000", 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7271000000000003e-05 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.748967+0000", 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33662544999999999 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749005+0000", 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8247999999999997e-05 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749112+0000", 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010668299999999999 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.749126+0000", 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4139e-05 2024-09-21T10:11:10.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:03:13.752517+0000", 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033910120000000001 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:102464 8.e e1210/1207 8:757cb6da:::benchmark_data_smithi017_209235_object102463:head v 1210'6466, mlcod=1210'6463)", 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:06.338447+0000", 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 299.30890699899999, 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17267859999999999, 2024-09-21T10:11:10.394 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338447+0000", 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338447+0000", 2024-09-21T10:11:10.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338449+0000", 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1799999999999999e-06 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338509+0000", 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9942000000000003e-05 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338521+0000", 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2500999999999999e-05 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338528+0000", 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4290000000000004e-06 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.339096+0000", 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00056747899999999997 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.339118+0000", 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.175e-05 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511051+0000", 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.171932801 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511126+0000", 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5518000000000004e-05 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:102478 8.3 e1210/1200 8:c8ad84b4:::benchmark_data_smithi017_209235_object102477:head v 1210'6349, mlcod=1210'6347)", 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:06.338847+0000", 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 299.30850778899998, 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18747862500000001, 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338847+0000", 2024-09-21T10:11:10.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338847+0000", 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338847+0000", 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3e-07 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338876+0000", 2024-09-21T10:11:10.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8527999999999999e-05 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338887+0000", 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1355999999999999e-05 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338894+0000", 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4659999999999996e-06 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.339766+0000", 2024-09-21T10:11:10.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00087272700000000001 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.339786+0000", 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9544000000000001e-05 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.526275+0000", 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.186489404 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.526325+0000", 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9870000000000002e-05 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:102470 8.e e1210/1207 8:7275dc67:::benchmark_data_smithi017_209235_object102469:head v 1210'6467, mlcod=1210'6463)", 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:06.341381+0000", 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 299.305973596, 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.173598646, 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.403 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.341381+0000", 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.341381+0000", 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.341382+0000", 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.463e-06 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.341429+0000", 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6417e-05 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511034+0000", 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16960576699999999 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511052+0000", 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7125e-05 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511152+0000", 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100776 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511170+0000", 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7308000000000001e-05 2024-09-21T10:11:10.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.514915+0000", 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003744837 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.514979+0000", 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4953e-05 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:102484 8.b e1210/1207 8:d7bbbcc9:::benchmark_data_smithi017_209235_object102483:head v 1210'6385, mlcod=1210'6383)", 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:06.345830+0000", 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 299.30152472399999, 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.180438399, 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.345830+0000", 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.345830+0000", 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.345831+0000", 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5850000000000001e-06 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.345858+0000", 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6689000000000001e-05 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511035+0000", 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16517723600000001 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511052+0000", 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6837000000000002e-05 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511118+0000", 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6198999999999994e-05 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511160+0000", 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1279999999999998e-05 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.526215+0000", 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015055821 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.526268+0000", 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2751999999999999e-05 2024-09-21T10:11:10.411 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:102491 8.d e1210/1200 8:b991a1da:::benchmark_data_smithi017_209235_object102490:head v 1210'6282, mlcod=1210'6280)", 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:06.351475+0000", 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 299.29587908500002, 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16636495200000001, 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.351475+0000", 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.351475+0000", 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.351477+0000", 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0040000000000002e-06 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.351518+0000", 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0460000000000002e-05 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511428+0000", 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15991032099999999 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511438+0000", 2024-09-21T10:11:10.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0181e-05 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511496+0000", 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8038000000000002e-05 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.511517+0000", 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0866999999999999e-05 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.517747+0000", 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0062300319999999999 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.517840+0000", 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3048999999999996e-05 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263811 8.4 8:2d8b153a:::benchmark_data_smithi017_209235_object130905:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1225)", 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:25.416982+0000", 2024-09-21T10:11:10.416 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 160.230371953, 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.216450821, 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263811 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.416982+0000", 2024-09-21T10:11:10.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.416982+0000", 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.416985+0000", 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2989999999999999e-06 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.416994+0000", 2024-09-21T10:11:10.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.867e-06 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417020+0000", 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5885999999999999e-05 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417032+0000", 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2258e-05 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417137+0000", 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000104911 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417145+0000", 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.653e-06 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421071+0000", 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039252009999999997 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421080+0000", 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8889999999999997e-06 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.633364+0000", 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2122831169999999 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.633433+0000", 2024-09-21T10:11:10.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9740000000000007e-05 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263812 8.4 8:2c129b93:::benchmark_data_smithi017_209235_object130906:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1225)", 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:25.417057+0000", 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 160.23029720100001, 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.216718411, 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:10.422 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263812 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417057+0000", 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417057+0000", 2024-09-21T10:11:10.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417058+0000", 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8500000000000001e-07 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417061+0000", 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4429999999999999e-06 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417070+0000", 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7260000000000008e-06 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417077+0000", 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6719999999999998e-06 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417205+0000", 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012882900000000001 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417212+0000", 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3929999999999996e-06 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421109+0000", 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038974330000000001 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421117+0000", 2024-09-21T10:11:10.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0020000000000006e-06 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.633668+0000", 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.212550883 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.633776+0000", 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000107445 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.427 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.428 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.429 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:75073 9.6 9:6aa94b22:::benchmark_data_smithi017_237924_object75072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:11:10.429 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:54.659681+0000", 2024-09-21T10:11:10.429 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.987673696000002, 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18124369600000001, 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 75073 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.430 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659681+0000", 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659681+0000", 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659682+0000", 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4589999999999999e-06 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659731+0000", 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8841999999999998e-05 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659742+0000", 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1379000000000001e-05 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659749+0000", 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0729999999999999e-06 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659872+0000", 2024-09-21T10:11:10.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122776 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.660030+0000", 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000157269 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.660168+0000", 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013878799999999999 2024-09-21T10:11:10.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.667039+0000", 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068708140000000003 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.668555+0000", 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001515567 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.840475+0000", 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.171919828 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.840758+0000", 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028381400000000002 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.840924+0000", 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016608700000000001 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:75077 9.6 9:6d1e6353:::benchmark_data_smithi017_237924_object75076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:54.659954+0000", 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 70.987400704999999, 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.181617579, 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.437 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 75077 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659954+0000", 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659954+0000", 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659955+0000", 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9400000000000004e-07 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659999+0000", 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3992999999999997e-05 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.660009+0000", 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0696000000000001e-05 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.660016+0000", 2024-09-21T10:11:10.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0500000000000003e-06 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.663210+0000", 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031933560000000001 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.663385+0000", 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017536799999999999 2024-09-21T10:11:10.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.663528+0000", 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014314800000000001 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.668589+0000", 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050608199999999997 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.668615+0000", 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6037000000000002e-05 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.841152+0000", 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.172536825 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.841415+0000", 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000263007 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.841571+0000", 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015648499999999999 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80760 9.b e1250/1249 9:d9fd3bc4:::benchmark_data_smithi017_237924_object80759:head v 1250'5132, mlcod=0'0)", 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.820464+0000", 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 59.826890091999999, 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.208616776, 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820464+0000", 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820464+0000", 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820466+0000", 2024-09-21T10:11:10.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7799999999999999e-06 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820499+0000", 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2573000000000001e-05 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820516+0000", 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7314999999999999e-05 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820526+0000", 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0151e-05 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820634+0000", 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010801 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820662+0000", 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7682999999999999e-05 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.023957+0000", 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20329536100000001 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029081+0000", 2024-09-21T10:11:10.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005123903 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80770 9.7 e1250/1245 9:eee90d88:::benchmark_data_smithi017_237924_object80769:head v 1250'5097, mlcod=0'0)", 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.829175+0000", 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 59.818179008000001, 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19592934400000001, 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.450 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829175+0000", 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829175+0000", 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829177+0000", 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2219999999999999e-06 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829211+0000", 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3881000000000003e-05 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829227+0000", 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6685999999999999e-05 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829237+0000", 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0144e-05 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.830053+0000", 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00081578499999999997 2024-09-21T10:11:10.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.830080+0000", 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6486999999999999e-05 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.025043+0000", 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19496342 2024-09-21T10:11:10.454 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.025105+0000", 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1718999999999996e-05 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80775 9.7 e1250/1245 9:e0ae36b0:::benchmark_data_smithi017_237924_object80774:head v 1250'5098, mlcod=0'0)", 2024-09-21T10:11:10.455 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.829263+0000", 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 59.818091803000002, 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20849586000000001, 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829263+0000", 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829263+0000", 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829263+0000", 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1899999999999998e-07 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829328+0000", 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4927999999999999e-05 2024-09-21T10:11:10.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829337+0000", 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0310000000000004e-06 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829343+0000", 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8669999999999999e-06 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.011152+0000", 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18180923600000001 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.011181+0000", 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9369e-05 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.037655+0000", 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026473577000000002 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.037758+0000", 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103433 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.460 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80769 9.b e1250/1249 9:df051d48:::benchmark_data_smithi017_237924_object80768:head v 1250'5133, mlcod=0'0)", 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.829522+0000", 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 59.817832465999999, 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.199615032, 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829522+0000", 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829522+0000", 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829523+0000", 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6099999999999999e-07 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829546+0000", 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3050000000000001e-05 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829557+0000", 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1068e-05 2024-09-21T10:11:10.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829563+0000", 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4910000000000002e-06 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829648+0000", 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4617000000000007e-05 2024-09-21T10:11:10.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829672+0000", 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3853e-05 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029090+0000", 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19941814999999999 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029137+0000", 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6941999999999999e-05 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80777 9.1 e1250/1249 9:812de87f:::benchmark_data_smithi017_237924_object80776:head v 1250'4961, mlcod=0'0)", 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.833168+0000", 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 59.814186542000002, 2024-09-21T10:11:10.466 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.193889176, 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.833168+0000", 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.833168+0000", 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.833169+0000", 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.215e-06 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.833193+0000", 2024-09-21T10:11:10.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3935e-05 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.012562+0000", 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17936929200000001 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.012572+0000", 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5880000000000004e-06 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.013907+0000", 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013347509999999999 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.013974+0000", 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7248000000000004e-05 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.027004+0000", 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013029924999999999 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.027057+0000", 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3222e-05 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80781 9.b e1250/1249 9:d7a6da48:::benchmark_data_smithi017_237924_object80780:head v 1250'5134, mlcod=0'0)", 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.835569+0000", 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 59.811785102000002, 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19390075200000001, 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:10.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.835569+0000", 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.835569+0000", 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.835571+0000", 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3009999999999999e-06 2024-09-21T10:11:10.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.835599+0000", 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8442999999999999e-05 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.011337+0000", 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.175737857 2024-09-21T10:11:10.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.011351+0000", 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.36e-05 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.014656+0000", 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033057830000000001 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.014681+0000", 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.461e-05 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029141+0000", 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014459830999999999 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029470+0000", 2024-09-21T10:11:10.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032932700000000003 2024-09-21T10:11:10.477 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.477 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.477 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.477 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:10.477 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:10.477 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:11:10.477 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:11:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:11:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:11:10.501 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:10.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:10.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:10.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:10.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:10.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:10.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:10.747 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:10.772 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:10.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:10.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:10.895 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:10.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:10.997 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:11.020 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:11.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:11.122 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:11.144 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:11.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:11.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:11.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:11.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:11.395 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:11.421 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:11.523 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:11.545 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:11.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:11.647 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:11.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:11.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:11.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:11.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:11.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:11.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:11.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:11 smithi017 ceph-mon[102913]: pgmap v1858: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 26134/197166 objects degraded (13.255%); 1401/197166 objects misplaced (0.711%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:11:11.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:11 smithi017 ceph-mon[105151]: pgmap v1858: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 26134/197166 objects degraded (13.255%); 1401/197166 objects misplaced (0.711%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:11:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:11 smithi134 ceph-mon[90085]: pgmap v1858: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 26134/197166 objects degraded (13.255%); 1401/197166 objects misplaced (0.711%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:11:12.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:12.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:12.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:12.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:12.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:12.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:12.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:12.432 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:12.455 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:12.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:12.556 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:12.582 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:12.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:12.722 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:12.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:12.824 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:12.848 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:12.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:12.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:12.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:13.098 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:13.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:13.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:13.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.264 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:13.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:13.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:13.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:13.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:13.508 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.531 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:13.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:13.632 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:13.656 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:13.727 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:13 smithi017 ceph-mon[102913]: pgmap v1859: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 26134/197166 objects degraded (13.255%); 1401/197166 objects misplaced (0.711%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:11:13.727 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:13 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 26134/197166 objects degraded (13.255%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:13.728 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:13 smithi017 ceph-mon[105151]: pgmap v1859: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 26134/197166 objects degraded (13.255%); 1401/197166 objects misplaced (0.711%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:11:13.728 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:13 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 26134/197166 objects degraded (13.255%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:13.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:13.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:13.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:13.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:13.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:13.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:13 smithi134 ceph-mon[90085]: pgmap v1859: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 26134/197166 objects degraded (13.255%); 1401/197166 objects misplaced (0.711%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:11:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:13 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 26134/197166 objects degraded (13.255%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:14.023 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:14.047 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:14.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:14.137 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:14.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:14.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.203 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:14.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:14.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:14.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:14.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:14.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:14.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:14.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:14.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:14 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:11:14.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:14 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:11:14.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:14.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:14.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:14.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:14.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:14 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:11:15.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:15.046 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:11:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:11:14] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-09-21T10:11:15.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:15.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:15.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:11:15.200 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:15.224 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:15.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:15.326 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:15.349 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:15.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:15.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:15.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:15 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:11:15.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:15 smithi017 ceph-mon[102913]: pgmap v1860: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 25910/188865 objects degraded (13.719%); 1084/188865 objects misplaced (0.574%); 7.6 MiB/s, 120 objects/s recovering 2024-09-21T10:11:15.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:15 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:11:15.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:15 smithi017 ceph-mon[105151]: pgmap v1860: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 25910/188865 objects degraded (13.719%); 1084/188865 objects misplaced (0.574%); 7.6 MiB/s, 120 objects/s recovering 2024-09-21T10:11:15.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:15.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:15.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:15.752 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:15.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:15.876 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:15.901 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:15.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:15 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:11:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:15 smithi134 ceph-mon[90085]: pgmap v1860: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 25910/188865 objects degraded (13.719%); 1084/188865 objects misplaced (0.574%); 7.6 MiB/s, 120 objects/s recovering 2024-09-21T10:11:16.003 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:16.028 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:16.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:16.130 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:16.153 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:16.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:16.255 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:16.276 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:16.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:16.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:16.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:16.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:16.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:16.588 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:11:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:16.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:16.690 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:16.713 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:16.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:16.788 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:11:16.788 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "" 2024-09-21T10:11:16.788 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:11:16.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:16 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2016249495' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:16.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:16 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2016249495' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:16.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:16.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:16.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:16.956 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:16.980 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:16 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2016249495' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:17.081 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:17.104 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:17.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:17.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:17.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:17.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:17.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:17.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:17.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:17.520 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:17.622 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:17.647 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:17.749 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:17.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:17.851 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:17 smithi134 ceph-mon[90085]: pgmap v1861: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 25910/186927 objects degraded (13.861%); 1084/186927 objects misplaced (0.580%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:11:17.852 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:17 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:17.852 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2016249495' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:17.852 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:17 smithi134 ceph-mon[90085]: osdmap e1257: 8 total, 8 up, 7 in 2024-09-21T10:11:17.876 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:17.901 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:17 smithi017 ceph-mon[102913]: pgmap v1861: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 25910/186927 objects degraded (13.861%); 1084/186927 objects misplaced (0.580%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:11:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:17 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2016249495' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:17 smithi017 ceph-mon[102913]: osdmap e1257: 8 total, 8 up, 7 in 2024-09-21T10:11:17.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:17 smithi017 ceph-mon[105151]: pgmap v1861: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 25910/186927 objects degraded (13.861%); 1084/186927 objects misplaced (0.580%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:11:17.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:17 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:17.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2016249495' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:17.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:17 smithi017 ceph-mon[105151]: osdmap e1257: 8 total, 8 up, 7 in 2024-09-21T10:11:18.002 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:18.026 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:18.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:18.127 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:18.151 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:18.253 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:18.275 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:18.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:18.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:18.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:18.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:18.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:18.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:18.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:18.661 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:18.686 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:18.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:18.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:18.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.828 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:18.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:18.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:18 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 25910/186927 objects degraded (13.861%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:18.878 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:18 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 25910/186927 objects degraded (13.861%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:18.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:18.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:18.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:18 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 25910/186927 objects degraded (13.861%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:19.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:19.139 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:19.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:19.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:19.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:19.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:19.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:19.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:19.566 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:19.592 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:19.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:19.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:19.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:19.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:19.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:19.854 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:19 smithi017 ceph-mon[102913]: pgmap v1863: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 25710/179616 objects degraded (14.314%); 720/179616 objects misplaced (0.401%); 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:11:19.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:19 smithi017 ceph-mon[105151]: pgmap v1863: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 25710/179616 objects degraded (14.314%); 720/179616 objects misplaced (0.401%); 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:11:19.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:19.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:19.980 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:19 smithi134 ceph-mon[90085]: pgmap v1863: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 25710/179616 objects degraded (14.314%); 720/179616 objects misplaced (0.401%); 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:11:20.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:20.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:20.105 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:20.129 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:20.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:20.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:20.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:20.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:20.350 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:20.381 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:11:20.381 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:20.400 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:20.422 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:20.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:20.437 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:11:20.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:20.481 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:11:20.481 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:11:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:11:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:11:20.519 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:11:20.519 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:20.524 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:20.547 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:11:20.548 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:20.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:20.578 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:11:20.578 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:20.613 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:11:20.613 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:20.647 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:11:20.647 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:20.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:20.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:20.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:20.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:20.830 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:20.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:20.931 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:20.954 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:20.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:21.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:21.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:21.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:21.210 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:21.237 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:21.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:21.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:21.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:21.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:21.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:11:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2590250895' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[102913]: pgmap v1864: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 25710/177171 objects degraded (14.511%); 720/177171 objects misplaced (0.406%); 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:11:21.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:11:21.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:11:21.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:21.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:21.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:21.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2590250895' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:21.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[105151]: pgmap v1864: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 25710/177171 objects degraded (14.511%); 720/177171 objects misplaced (0.406%); 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:11:21.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:11:21.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:11:21.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:21.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:21 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2590250895' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:21 smithi134 ceph-mon[90085]: pgmap v1864: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 25710/177171 objects degraded (14.511%); 720/177171 objects misplaced (0.406%); 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:11:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:11:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:11:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:21.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:21.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:21.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:21.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:21.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:21.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:21.802 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:21.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:21.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:21.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:21.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:22.030 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:22.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:22.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:22.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:22.182 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:22.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:22.284 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:22.305 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:22.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:22.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:22.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:22.539 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:22.562 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:22.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:22.664 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:22.687 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:22.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:22.721 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:11:22.721 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:11:22.722 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2024-09-21T10:11:22.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:11:22.789 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:22.813 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:22.915 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:22.936 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:22.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:23.038 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:23.063 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:23.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:23.164 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:23.187 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:23.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:23.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:23.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:23.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:23.429 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:23.452 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:23.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:23.577 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:23.678 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:23.705 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:23.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:23.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:23 smithi134 ceph-mon[90085]: pgmap v1865: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 25710/177171 objects degraded (14.511%); 720/177171 objects misplaced (0.406%); 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:11:23.807 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:23.831 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:23.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:23.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:23 smithi017 ceph-mon[105151]: pgmap v1865: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 25710/177171 objects degraded (14.511%); 720/177171 objects misplaced (0.406%); 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:11:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:23 smithi017 ceph-mon[102913]: pgmap v1865: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 25710/177171 objects degraded (14.511%); 720/177171 objects misplaced (0.406%); 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:11:23.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:23.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:24.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:24.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:24.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:24.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:24.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:24.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:24.361 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:24.386 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:24.488 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:24.513 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:24.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:24.614 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:24 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 25710/177171 objects degraded (14.511%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:24.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:24 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 25710/177171 objects degraded (14.511%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:24.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:24.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:24.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:24.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:24 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 25710/177171 objects degraded (14.511%), 10 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:24.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:24.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:24.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:24.881 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:11:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:11:24] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-09-21T10:11:24.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:24.939 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:24.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:25.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:25.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:25.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:25.189 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:25.215 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:25.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:25.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:25.340 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:25.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:25.441 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:25.466 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:25.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:25.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:25.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:25.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:25.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:25.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:25 smithi017 ceph-mon[102913]: pgmap v1866: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 25505/168807 objects degraded (15.109%); 272/168807 objects misplaced (0.161%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:11:25.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:25.728 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:25.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:25 smithi017 ceph-mon[105151]: pgmap v1866: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 25505/168807 objects degraded (15.109%); 272/168807 objects misplaced (0.161%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:11:25.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:25.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:25 smithi134 ceph-mon[90085]: pgmap v1866: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 25505/168807 objects degraded (15.109%); 272/168807 objects misplaced (0.161%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:11:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:11:25.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:25.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:25.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:25.856 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:25.866 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:25.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:25.884 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:25.884 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:25.930 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:25.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:26.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:26.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:26.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:26.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:26.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:26.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:26.329 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:26.352 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:26.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:26.454 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:26.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:26.580 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:26.604 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:26.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:26.705 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:26.730 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:26.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:26.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:26.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:26.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:26.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:27.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:27.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:27.102 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:27.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:27.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:27.226 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:27.248 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:27.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:27.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:11:27.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:27.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:27.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:27.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:27.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:27.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:27.604 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:27.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:11:27.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.630 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:27.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:27.731 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:27.753 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:27.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:27.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:27 smithi017 ceph-mon[102913]: pgmap v1867: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 25505/166926 objects degraded (15.279%); 272/166926 objects misplaced (0.163%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:11:27.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:27 smithi017 ceph-mon[102913]: osdmap e1258: 8 total, 8 up, 7 in 2024-09-21T10:11:27.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:27 smithi017 ceph-mon[105151]: pgmap v1867: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 25505/166926 objects degraded (15.279%); 272/166926 objects misplaced (0.163%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:11:27.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:27 smithi017 ceph-mon[105151]: osdmap e1258: 8 total, 8 up, 7 in 2024-09-21T10:11:27.855 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:27.878 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:27.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:27 smithi134 ceph-mon[90085]: pgmap v1867: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 25505/166926 objects degraded (15.279%); 272/166926 objects misplaced (0.163%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:11:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:27 smithi134 ceph-mon[90085]: osdmap e1258: 8 total, 8 up, 7 in 2024-09-21T10:11:28.028 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:28.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:28.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:28.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:28.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:28.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.308 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:28.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:28.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:28.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:28.535 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:28.557 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:28.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:28.659 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:28.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.682 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:28.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:28.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:28.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:28.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:28 smithi017 ceph-mon[102913]: osdmap e1259: 8 total, 8 up, 7 in 2024-09-21T10:11:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:28 smithi017 ceph-mon[105151]: osdmap e1259: 8 total, 8 up, 7 in 2024-09-21T10:11:28.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:28.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:28 smithi134 ceph-mon[90085]: osdmap e1259: 8 total, 8 up, 7 in 2024-09-21T10:11:29.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:29.091 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:29.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:29.193 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:29.217 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:29.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:29.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:29.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:29.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:29.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:29.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.492 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:29.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:29.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:29.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:29.733 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:29.756 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:29.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:29.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:29.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:29.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:29.930 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:29 smithi017 ceph-mon[102913]: pgmap v1870: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 726 op/s; 22238/159726 objects degraded (13.923%); 2856/159726 objects misplaced (1.788%); 10 MiB/s, 167 objects/s recovering 2024-09-21T10:11:29.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:29 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 22238/159726 objects degraded (13.923%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:29.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:29 smithi017 ceph-mon[105151]: pgmap v1870: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 726 op/s; 22238/159726 objects degraded (13.923%); 2856/159726 objects misplaced (1.788%); 10 MiB/s, 167 objects/s recovering 2024-09-21T10:11:29.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:29 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 22238/159726 objects degraded (13.923%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:29 smithi134 ceph-mon[90085]: pgmap v1870: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 726 op/s; 22238/159726 objects degraded (13.923%); 2856/159726 objects misplaced (1.788%); 10 MiB/s, 167 objects/s recovering 2024-09-21T10:11:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:29 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 22238/159726 objects degraded (13.923%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:30.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:30.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:30.162 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:30.186 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:30.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:30.349 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:30.371 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:30.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:30.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:30.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:11:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:11:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:11:30.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:30.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:30.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:30.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:30.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:30.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:30.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:30.921 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:30.943 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:31.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:31.068 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:31.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:31.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:31.193 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:31.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:31.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:31.438 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:31.462 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:31.563 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:31.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:31.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:31.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:31.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:31.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:31.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:31.860 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:31.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:31 smithi017 ceph-mon[102913]: pgmap v1871: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 22238/157428 objects degraded (14.126%); 2856/157428 objects misplaced (1.814%); 10 MiB/s, 167 objects/s recovering 2024-09-21T10:11:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:31 smithi017 ceph-mon[105151]: pgmap v1871: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 22238/157428 objects degraded (14.126%); 2856/157428 objects misplaced (1.814%); 10 MiB/s, 167 objects/s recovering 2024-09-21T10:11:31.962 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:31.986 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:31.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:31 smithi134 ceph-mon[90085]: pgmap v1871: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 22238/157428 objects degraded (14.126%); 2856/157428 objects misplaced (1.814%); 10 MiB/s, 167 objects/s recovering 2024-09-21T10:11:32.088 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:32.109 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:32.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:32.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:32.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:32.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:32.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:32.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:32.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:32.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:32.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:32.664 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:32.685 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:32.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:32.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:32.809 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:32.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:32.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:32.934 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:32.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:33.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:33.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:33.160 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:33.183 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:33.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:33.285 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:33.309 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:33.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:33.410 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:33.433 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:33.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:33.535 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:33.558 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:33.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:33.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:33.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:33.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:33.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:33.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:33.888 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:33 smithi017 ceph-mon[102913]: pgmap v1872: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 22238/157428 objects degraded (14.126%); 2856/157428 objects misplaced (1.814%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:11:33.888 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:33 smithi017 ceph-mon[105151]: pgmap v1872: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 22238/157428 objects degraded (14.126%); 2856/157428 objects misplaced (1.814%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:11:33.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:33.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:33.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:33 smithi134 ceph-mon[90085]: pgmap v1872: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 22238/157428 objects degraded (14.126%); 2856/157428 objects misplaced (1.814%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:11:34.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:34.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:34.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:34.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:34.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:34.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:34.392 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:34.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:34.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:34.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:34.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:34.682 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:34.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:34.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:34.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:34.927 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:34 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 22238/157428 objects degraded (14.126%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:34.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:34 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 22238/157428 objects degraded (14.126%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:34.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:11:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:11:34] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.51.0" 2024-09-21T10:11:34.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.952 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:34.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:34 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 22238/157428 objects degraded (14.126%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:35.052 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:35.075 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:35.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:35.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:35.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:35.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:35.341 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:35.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:35.443 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:35.467 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:35.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:35.569 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:35.592 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:35.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:35.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:35.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:35.818 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.841 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:35.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:35.895 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:35 smithi134 ceph-mon[90085]: from='client.55480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:11:35.895 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:35 smithi134 ceph-mon[90085]: pgmap v1873: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 22238/148800 objects degraded (14.945%); 2478/148800 objects misplaced (1.665%); 8.3 MiB/s, 132 objects/s recovering 2024-09-21T10:11:35.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:35 smithi017 ceph-mon[102913]: from='client.55480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:11:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:35 smithi017 ceph-mon[102913]: pgmap v1873: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 22238/148800 objects degraded (14.945%); 2478/148800 objects misplaced (1.665%); 8.3 MiB/s, 132 objects/s recovering 2024-09-21T10:11:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:35 smithi017 ceph-mon[105151]: from='client.55480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:11:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:35 smithi017 ceph-mon[105151]: pgmap v1873: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 22238/148800 objects degraded (14.945%); 2478/148800 objects misplaced (1.665%); 8.3 MiB/s, 132 objects/s recovering 2024-09-21T10:11:35.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:36.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:36.114 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:36.216 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:36.237 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:36.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:36.339 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:36.363 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:36.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:36.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:36.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:36.623 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:36.646 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:36.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:36.748 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:36.773 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:36.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:36.875 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:36.898 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:36.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:36.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:37.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:37.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:37.061 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:11:37.066 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:11:37.143 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:37.167 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:37.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:37.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:37.292 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:37.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:37.394 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:37.416 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:37.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:37.518 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:37.540 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:37.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:37.642 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:37.666 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:37.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:37.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:37.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:37.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:37.894 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:37.916 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:37.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:37 smithi017 ceph-mon[102913]: pgmap v1874: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 22238/146628 objects degraded (15.166%); 2478/146628 objects misplaced (1.690%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:11:37.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:37 smithi017 ceph-mon[105151]: pgmap v1874: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 22238/146628 objects degraded (15.166%); 2478/146628 objects misplaced (1.690%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:11:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:37 smithi134 ceph-mon[90085]: pgmap v1874: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 22238/146628 objects degraded (15.166%); 2478/146628 objects misplaced (1.690%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:11:38.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:38.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:38.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:38.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:38.206 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:38.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:38.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:38.353 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:38.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:38.455 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:38.479 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:38.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:38.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:38.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:38.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:38.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:38.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:38.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:38.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:38.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 22238/146628 objects degraded (15.166%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:38.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 22238/146628 objects degraded (15.166%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:38.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 22238/146628 objects degraded (15.166%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:39.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:39.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:39.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:39.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:39.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:39.322 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:39.345 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:39.446 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1872,"stamp":"2024-09-21T10:11:32.401397+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3438015913,"num_objects":52476,"num_object_clones":0,"num_object_copies":157428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22238,"num_objects_misplaced":2856,"num_objects_unfound":0,"num_objects_dirty":52476,"num_whiteouts":0,"num_read":95,"num_read_kb":239,"num_write":133665,"num_write_kb":5952614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9594,"num_bytes_recovered":625813938,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84613,"ondisk_log_size":84613,"up":327,"acting":317,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14027504,"kb_used_data":9578084,"kb_used_omap":409,"kb_used_meta":4448934,"kb_avail":642188560,"statfs":{"total":671965249536,"available":657601085440,"internally_reserved":0,"allocated":9807958016,"data_stored":9787617554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":419286,"internal_metadata":4555708970},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-248578048,"num_objects":-3793,"num_object_clones":0,"num_object_copies":-11379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3267,"num_objects_misplaced":2584,"num_objects_unfound":0,"num_objects_dirty":-3793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3793,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":684,"num_bytes_recovered":44761088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005447"},"pg_stats":[{"pgid":"1.11","version":"288'100","reported_seq":672,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.554929+0000","last_change":"2024-09-21T10:08:39.493413+0000","last_active":"2024-09-21T10:11:27.554929+0000","last_peered":"2024-09-21T10:11:27.554929+0000","last_clean":"2024-09-21T10:11:27.554929+0000","last_became_active":"2024-09-21T10:08:39.493004+0000","last_became_peered":"2024-09-21T10:08:39.493004+0000","last_unstale":"2024-09-21T10:11:27.554929+0000","last_undegraded":"2024-09-21T10:11:27.554929+0000","last_fullsized":"2024-09-21T10:11:27.554929+0000","mapping_epoch":1233,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1234,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.888350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.888350+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:43:15.229147+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1879,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.512960+0000","last_change":"2024-09-21T10:09:25.458014+0000","last_active":"2024-09-21T10:11:27.512960+0000","last_peered":"2024-09-21T10:11:27.512960+0000","last_clean":"2024-09-21T10:11:27.512960+0000","last_became_active":"2024-09-21T10:09:25.452008+0000","last_became_peered":"2024-09-21T10:09:25.452008+0000","last_unstale":"2024-09-21T10:11:27.512960+0000","last_undegraded":"2024-09-21T10:11:27.512960+0000","last_fullsized":"2024-09-21T10:11:27.512960+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:25.502757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":75,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501964+0000","last_change":"2024-09-21T10:10:13.637483+0000","last_active":"2024-09-21T10:11:26.501964+0000","last_peered":"2024-09-21T10:11:26.501964+0000","last_clean":"2024-09-21T10:11:26.501964+0000","last_became_active":"2024-09-21T10:10:13.622491+0000","last_became_peered":"2024-09-21T10:10:13.622491+0000","last_unstale":"2024-09-21T10:11:26.501964+0000","last_undegraded":"2024-09-21T10:11:26.501964+0000","last_fullsized":"2024-09-21T10:11:26.501964+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:05:02.511869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"288'100","reported_seq":40,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524047+0000","last_change":"2024-09-21T10:11:14.145310+0000","last_active":"2024-09-21T10:11:26.524047+0000","last_peered":"2024-09-21T10:11:26.524047+0000","last_clean":"2024-09-21T10:11:26.524047+0000","last_became_active":"2024-09-21T10:10:13.624038+0000","last_became_peered":"2024-09-21T10:10:13.624038+0000","last_unstale":"2024-09-21T10:11:26.524047+0000","last_undegraded":"2024-09-21T10:11:26.524047+0000","last_fullsized":"2024-09-21T10:11:26.524047+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:25.533587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:25.533587+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":35,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":580,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.510912+0000","last_change":"2024-09-21T10:10:13.660936+0000","last_active":"2024-09-21T10:11:27.510912+0000","last_peered":"2024-09-21T10:11:27.510912+0000","last_clean":"2024-09-21T10:11:27.510912+0000","last_became_active":"2024-09-21T10:10:13.660573+0000","last_became_peered":"2024-09-21T10:10:13.660573+0000","last_unstale":"2024-09-21T10:11:27.510912+0000","last_undegraded":"2024-09-21T10:11:27.510912+0000","last_fullsized":"2024-09-21T10:11:27.510912+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:11:28.164079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1949,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553268+0000","last_change":"2024-09-21T10:09:25.457058+0000","last_active":"2024-09-21T10:11:27.553268+0000","last_peered":"2024-09-21T10:11:27.553268+0000","last_clean":"2024-09-21T10:11:27.553268+0000","last_became_active":"2024-09-21T10:09:25.456748+0000","last_became_peered":"2024-09-21T10:09:25.456748+0000","last_unstale":"2024-09-21T10:11:27.553268+0000","last_undegraded":"2024-09-21T10:11:27.553268+0000","last_fullsized":"2024-09-21T10:11:27.553268+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:33.730086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"288'100","reported_seq":57,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.505008+0000","last_change":"2024-09-21T10:08:45.523904+0000","last_active":"2024-09-21T10:11:27.505008+0000","last_peered":"2024-09-21T10:11:27.505008+0000","last_clean":"2024-09-21T10:11:27.505008+0000","last_became_active":"2024-09-21T10:08:45.521658+0000","last_became_peered":"2024-09-21T10:08:45.521658+0000","last_unstale":"2024-09-21T10:11:27.505008+0000","last_undegraded":"2024-09-21T10:11:27.505008+0000","last_fullsized":"2024-09-21T10:11:27.505008+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:32.451823+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:08:32.451823+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":23,"num_write":5,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1950,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.558887+0000","last_change":"2024-09-21T10:09:25.461488+0000","last_active":"2024-09-21T10:11:27.558887+0000","last_peered":"2024-09-21T10:11:27.558887+0000","last_clean":"2024-09-21T10:11:27.558887+0000","last_became_active":"2024-09-21T10:09:25.461153+0000","last_became_peered":"2024-09-21T10:09:25.461153+0000","last_unstale":"2024-09-21T10:11:27.558887+0000","last_undegraded":"2024-09-21T10:11:27.558887+0000","last_fullsized":"2024-09-21T10:11:27.558887+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:58:09.052499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1206,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501611+0000","last_change":"2024-09-21T10:10:13.642239+0000","last_active":"2024-09-21T10:11:26.501611+0000","last_peered":"2024-09-21T10:11:26.501611+0000","last_clean":"2024-09-21T10:11:26.501611+0000","last_became_active":"2024-09-21T10:10:13.641938+0000","last_became_peered":"2024-09-21T10:10:13.641938+0000","last_unstale":"2024-09-21T10:11:26.501611+0000","last_undegraded":"2024-09-21T10:11:26.501611+0000","last_fullsized":"2024-09-21T10:11:26.501611+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:20:48.026420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"288'100","reported_seq":71,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.505229+0000","last_change":"2024-09-21T10:10:13.635736+0000","last_active":"2024-09-21T10:11:27.505229+0000","last_peered":"2024-09-21T10:11:27.505229+0000","last_clean":"2024-09-21T10:11:27.505229+0000","last_became_active":"2024-09-21T10:10:13.634663+0000","last_became_peered":"2024-09-21T10:10:13.634663+0000","last_unstale":"2024-09-21T10:11:27.505229+0000","last_undegraded":"2024-09-21T10:11:27.505229+0000","last_fullsized":"2024-09-21T10:11:27.505229+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:26.189995+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:08:26.189995+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":14,"num_read_kb":34,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1879,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.511981+0000","last_change":"2024-09-21T10:09:25.451536+0000","last_active":"2024-09-21T10:11:27.511981+0000","last_peered":"2024-09-21T10:11:27.511981+0000","last_clean":"2024-09-21T10:11:27.511981+0000","last_became_active":"2024-09-21T10:09:25.451190+0000","last_became_peered":"2024-09-21T10:09:25.451190+0000","last_unstale":"2024-09-21T10:11:27.511981+0000","last_undegraded":"2024-09-21T10:11:27.511981+0000","last_fullsized":"2024-09-21T10:11:27.511981+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:15.455299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":26,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.523954+0000","last_change":"2024-09-21T10:10:13.683037+0000","last_active":"2024-09-21T10:11:26.523954+0000","last_peered":"2024-09-21T10:11:26.523954+0000","last_clean":"2024-09-21T10:11:26.523954+0000","last_became_active":"2024-09-21T10:10:13.656692+0000","last_became_peered":"2024-09-21T10:10:13.656692+0000","last_unstale":"2024-09-21T10:11:26.523954+0000","last_undegraded":"2024-09-21T10:11:26.523954+0000","last_fullsized":"2024-09-21T10:11:26.523954+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:04.431952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"1259'8416","reported_seq":8478,"reported_epoch":1259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:29.832829+0000","last_change":"2024-09-21T10:09:46.504815+0000","last_active":"2024-09-21T10:11:29.832829+0000","last_peered":"2024-09-21T10:11:29.832829+0000","last_clean":"2024-09-21T10:09:24.410184+0000","last_became_active":"2024-09-21T10:09:26.732088+0000","last_became_peered":"2024-09-21T10:09:26.732088+0000","last_unstale":"2024-09-21T10:11:29.832829+0000","last_undegraded":"2024-09-21T10:09:46.501925+0000","last_fullsized":"2024-09-21T10:09:26.423099+0000","mapping_epoch":1245,"log_start":"1243'2900","ondisk_log_start":"1243'2900","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":5516,"log_dups_size":0,"ondisk_log_size":5516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214827008,"num_objects":3278,"num_object_clones":0,"num_object_copies":9834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3599,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8416,"num_write_kb":374208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":-321},{"shards":"0,5,7","objects":1031}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"288'100","reported_seq":138,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.502249+0000","last_change":"2024-09-21T10:09:25.449704+0000","last_active":"2024-09-21T10:11:26.502249+0000","last_peered":"2024-09-21T10:11:26.502249+0000","last_clean":"2024-09-21T10:11:26.502249+0000","last_became_active":"2024-09-21T10:09:25.449217+0000","last_became_peered":"2024-09-21T10:09:25.449217+0000","last_unstale":"2024-09-21T10:11:26.502249+0000","last_undegraded":"2024-09-21T10:11:26.502249+0000","last_fullsized":"2024-09-21T10:11:26.502249+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:04.877986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:04.877986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:47.175383+0000","scrub_duration":0.0015289520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":41,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.502015+0000","last_change":"2024-09-21T10:09:25.481340+0000","last_active":"2024-09-21T10:11:26.502015+0000","last_peered":"2024-09-21T10:11:26.502015+0000","last_clean":"2024-09-21T10:11:26.502015+0000","last_became_active":"2024-09-21T10:09:25.449394+0000","last_became_peered":"2024-09-21T10:09:25.449394+0000","last_unstale":"2024-09-21T10:11:26.502015+0000","last_undegraded":"2024-09-21T10:11:26.502015+0000","last_fullsized":"2024-09-21T10:11:26.502015+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:46.731808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":16,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524137+0000","last_change":"2024-09-21T10:10:13.623868+0000","last_active":"2024-09-21T10:11:26.524137+0000","last_peered":"2024-09-21T10:11:26.524137+0000","last_clean":"2024-09-21T10:11:26.524137+0000","last_became_active":"2024-09-21T10:10:13.623304+0000","last_became_peered":"2024-09-21T10:10:13.623304+0000","last_unstale":"2024-09-21T10:11:26.524137+0000","last_undegraded":"2024-09-21T10:11:26.524137+0000","last_fullsized":"2024-09-21T10:11:26.524137+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:56:35.829980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"1259'8419","reported_seq":8474,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:28.049832+0000","last_change":"2024-09-21T10:08:48.537849+0000","last_active":"2024-09-21T10:11:28.049832+0000","last_peered":"2024-09-21T10:11:28.049832+0000","last_clean":"2024-09-21T10:11:28.049832+0000","last_became_active":"2024-09-21T10:08:48.537409+0000","last_became_peered":"2024-09-21T10:08:48.537409+0000","last_unstale":"2024-09-21T10:11:28.049832+0000","last_undegraded":"2024-09-21T10:11:28.049832+0000","last_fullsized":"2024-09-21T10:11:28.049832+0000","mapping_epoch":1238,"log_start":"1247'3800","ondisk_log_start":"1247'3800","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4619,"log_dups_size":0,"ondisk_log_size":4619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:45.083646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218300416,"num_objects":3331,"num_object_clones":0,"num_object_copies":9993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8419,"num_write_kb":376000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"288'100","reported_seq":348,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.512836+0000","last_change":"2024-09-21T10:10:13.678895+0000","last_active":"2024-09-21T10:11:27.512836+0000","last_peered":"2024-09-21T10:11:27.512836+0000","last_clean":"2024-09-21T10:11:27.512836+0000","last_became_active":"2024-09-21T10:10:13.678219+0000","last_became_peered":"2024-09-21T10:10:13.678219+0000","last_unstale":"2024-09-21T10:11:27.512836+0000","last_undegraded":"2024-09-21T10:11:27.512836+0000","last_fullsized":"2024-09-21T10:11:27.512836+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:27.395700+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:27.395700+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:26.476978+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":41,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.502112+0000","last_change":"2024-09-21T10:09:25.483409+0000","last_active":"2024-09-21T10:11:26.502112+0000","last_peered":"2024-09-21T10:11:26.502112+0000","last_clean":"2024-09-21T10:11:26.502112+0000","last_became_active":"2024-09-21T10:09:25.462337+0000","last_became_peered":"2024-09-21T10:09:25.462337+0000","last_unstale":"2024-09-21T10:11:26.502112+0000","last_undegraded":"2024-09-21T10:11:26.502112+0000","last_fullsized":"2024-09-21T10:11:26.502112+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:11:59.639319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1224,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501772+0000","last_change":"2024-09-21T10:08:45.515858+0000","last_active":"2024-09-21T10:11:26.501772+0000","last_peered":"2024-09-21T10:11:26.501772+0000","last_clean":"2024-09-21T10:11:26.501772+0000","last_became_active":"2024-09-21T10:08:45.515543+0000","last_became_peered":"2024-09-21T10:08:45.515543+0000","last_unstale":"2024-09-21T10:11:26.501772+0000","last_undegraded":"2024-09-21T10:11:26.501772+0000","last_fullsized":"2024-09-21T10:11:26.501772+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:29.565740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1259'8402","reported_seq":8453,"reported_epoch":1259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:27.661426+0000","last_change":"2024-09-21T10:10:13.683511+0000","last_active":"2024-09-21T10:11:27.661426+0000","last_peered":"2024-09-21T10:11:27.661426+0000","last_clean":"2024-09-21T10:09:24.407827+0000","last_became_active":"2024-09-21T10:09:26.758116+0000","last_became_peered":"2024-09-21T10:09:26.758116+0000","last_unstale":"2024-09-21T10:11:27.661426+0000","last_undegraded":"2024-09-21T10:10:13.678384+0000","last_fullsized":"2024-09-21T10:09:26.423305+0000","mapping_epoch":1245,"log_start":"1246'3700","ondisk_log_start":"1246'3700","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4702,"log_dups_size":0,"ondisk_log_size":4702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217186304,"num_objects":3314,"num_object_clones":0,"num_object_copies":9942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3651,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8402,"num_write_kb":374912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5","objects":-337},{"shards":"3,5,7","objects":1107}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.f","version":"288'100","reported_seq":16,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524468+0000","last_change":"2024-09-21T10:10:13.624076+0000","last_active":"2024-09-21T10:11:26.524468+0000","last_peered":"2024-09-21T10:11:26.524468+0000","last_clean":"2024-09-21T10:11:26.524468+0000","last_became_active":"2024-09-21T10:10:13.623709+0000","last_became_peered":"2024-09-21T10:10:13.623709+0000","last_unstale":"2024-09-21T10:11:26.524468+0000","last_undegraded":"2024-09-21T10:11:26.524468+0000","last_fullsized":"2024-09-21T10:11:26.524468+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:23.746751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:23.746751+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:22.068163+0000","scrub_duration":0.0021986229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":20,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524161+0000","last_change":"2024-09-21T10:10:13.675582+0000","last_active":"2024-09-21T10:11:26.524161+0000","last_peered":"2024-09-21T10:11:26.524161+0000","last_clean":"2024-09-21T10:11:26.524161+0000","last_became_active":"2024-09-21T10:10:13.648834+0000","last_became_peered":"2024-09-21T10:10:13.648834+0000","last_unstale":"2024-09-21T10:11:26.524161+0000","last_undegraded":"2024-09-21T10:11:26.524161+0000","last_fullsized":"2024-09-21T10:11:26.524161+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:50:40.852175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":16,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.523924+0000","last_change":"2024-09-21T10:10:13.636010+0000","last_active":"2024-09-21T10:11:26.523924+0000","last_peered":"2024-09-21T10:11:26.523924+0000","last_clean":"2024-09-21T10:11:26.523924+0000","last_became_active":"2024-09-21T10:10:13.635697+0000","last_became_peered":"2024-09-21T10:10:13.635697+0000","last_unstale":"2024-09-21T10:11:26.523924+0000","last_undegraded":"2024-09-21T10:11:26.523924+0000","last_fullsized":"2024-09-21T10:11:26.523924+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:55:05.695643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1259'8272","reported_seq":8327,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.695703+0000","last_change":"2024-09-21T10:08:48.536593+0000","last_active":"2024-09-21T10:11:27.695703+0000","last_peered":"2024-09-21T10:11:27.695703+0000","last_clean":"2024-09-21T10:11:27.695703+0000","last_became_active":"2024-09-21T10:08:48.536286+0000","last_became_peered":"2024-09-21T10:08:48.536286+0000","last_unstale":"2024-09-21T10:11:27.695703+0000","last_undegraded":"2024-09-21T10:11:27.695703+0000","last_fullsized":"2024-09-21T10:11:27.695703+0000","mapping_epoch":1238,"log_start":"1242'100","ondisk_log_start":"1242'100","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":8172,"log_dups_size":0,"ondisk_log_size":8172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:34.600055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215875584,"num_objects":3294,"num_object_clones":0,"num_object_copies":9882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8272,"num_write_kb":370112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"288'100","reported_seq":55,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501986+0000","last_change":"2024-09-21T10:09:25.437099+0000","last_active":"2024-09-21T10:11:26.501986+0000","last_peered":"2024-09-21T10:11:26.501986+0000","last_clean":"2024-09-21T10:11:26.501986+0000","last_became_active":"2024-09-21T10:09:25.436542+0000","last_became_peered":"2024-09-21T10:09:25.436542+0000","last_unstale":"2024-09-21T10:11:26.501986+0000","last_undegraded":"2024-09-21T10:11:26.501986+0000","last_fullsized":"2024-09-21T10:11:26.501986+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:03.581657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:03.581657+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:18.627778+0000","scrub_duration":0.0038308109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":886,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.513068+0000","last_change":"2024-09-21T10:10:13.678563+0000","last_active":"2024-09-21T10:11:27.513068+0000","last_peered":"2024-09-21T10:11:27.513068+0000","last_clean":"2024-09-21T10:11:27.513068+0000","last_became_active":"2024-09-21T10:10:13.676594+0000","last_became_peered":"2024-09-21T10:10:13.676594+0000","last_unstale":"2024-09-21T10:11:27.513068+0000","last_undegraded":"2024-09-21T10:11:27.513068+0000","last_fullsized":"2024-09-21T10:11:27.513068+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:34.275762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1969,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553475+0000","last_change":"2024-09-21T10:08:45.528426+0000","last_active":"2024-09-21T10:11:27.553475+0000","last_peered":"2024-09-21T10:11:27.553475+0000","last_clean":"2024-09-21T10:11:27.553475+0000","last_became_active":"2024-09-21T10:08:45.528049+0000","last_became_peered":"2024-09-21T10:08:45.528049+0000","last_unstale":"2024-09-21T10:11:27.553475+0000","last_undegraded":"2024-09-21T10:11:27.553475+0000","last_fullsized":"2024-09-21T10:11:27.553475+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:10:10.687040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"1259'8175","reported_seq":13288,"reported_epoch":1259,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:27.683375+0000","last_change":"2024-09-21T10:11:25.622396+0000","last_active":"2024-09-21T10:11:27.683375+0000","last_peered":"2024-09-21T10:11:27.683375+0000","last_clean":"2024-09-21T10:09:24.416711+0000","last_became_active":"2024-09-21T10:10:13.814696+0000","last_became_peered":"2024-09-21T10:10:13.814696+0000","last_unstale":"2024-09-21T10:11:27.683375+0000","last_undegraded":"2024-09-21T10:10:13.798377+0000","last_fullsized":"2024-09-21T10:10:13.622503+0000","mapping_epoch":1252,"log_start":"1243'3400","ondisk_log_start":"1243'3400","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4775,"log_dups_size":0,"ondisk_log_size":4775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218300416,"num_objects":3331,"num_object_clones":0,"num_object_copies":9993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1848,"num_objects_unfound":0,"num_objects_dirty":3331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8175,"num_write_kb":368192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2524,"num_bytes_recovered":165412864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,4],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"2,3,4","objects":892},{"shards":"3,4","objects":1482},{"shards":"3,6,7","objects":956}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":1221,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.509544+0000","last_change":"2024-09-21T10:09:25.602575+0000","last_active":"2024-09-21T10:11:27.509544+0000","last_peered":"2024-09-21T10:11:27.509544+0000","last_clean":"2024-09-21T10:11:27.509544+0000","last_became_active":"2024-09-21T10:09:25.451591+0000","last_became_peered":"2024-09-21T10:09:25.451591+0000","last_unstale":"2024-09-21T10:11:27.509544+0000","last_undegraded":"2024-09-21T10:11:27.509544+0000","last_fullsized":"2024-09-21T10:11:27.509544+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:25.602524+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:25.602524+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:11:24.338561+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1879,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.514385+0000","last_change":"2024-09-21T10:08:25.436371+0000","last_active":"2024-09-21T10:11:27.514385+0000","last_peered":"2024-09-21T10:11:27.514385+0000","last_clean":"2024-09-21T10:11:27.514385+0000","last_became_active":"2024-09-21T10:08:25.435921+0000","last_became_peered":"2024-09-21T10:08:25.435921+0000","last_unstale":"2024-09-21T10:11:27.514385+0000","last_undegraded":"2024-09-21T10:11:27.514385+0000","last_fullsized":"2024-09-21T10:11:27.514385+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:27.612386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1201,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.502160+0000","last_change":"2024-09-21T10:10:13.661328+0000","last_active":"2024-09-21T10:11:26.502160+0000","last_peered":"2024-09-21T10:11:26.502160+0000","last_clean":"2024-09-21T10:11:26.502160+0000","last_became_active":"2024-09-21T10:10:13.661028+0000","last_became_peered":"2024-09-21T10:10:13.661028+0000","last_unstale":"2024-09-21T10:11:26.502160+0000","last_undegraded":"2024-09-21T10:11:26.502160+0000","last_fullsized":"2024-09-21T10:11:26.502160+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:36.965214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1259'8336","reported_seq":8397,"reported_epoch":1259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:28.017032+0000","last_change":"2024-09-21T10:10:13.725956+0000","last_active":"2024-09-21T10:11:28.017032+0000","last_peered":"2024-09-21T10:11:28.017032+0000","last_clean":"2024-09-21T10:09:45.416832+0000","last_became_active":"2024-09-21T10:10:13.721727+0000","last_became_peered":"2024-09-21T10:10:13.721727+0000","last_unstale":"2024-09-21T10:11:28.017032+0000","last_undegraded":"2024-09-21T10:10:13.632913+0000","last_fullsized":"2024-09-21T10:10:13.613387+0000","mapping_epoch":1252,"log_start":"1246'3700","ondisk_log_start":"1246'3700","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4636,"log_dups_size":0,"ondisk_log_size":4636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217579520,"num_objects":3320,"num_object_clones":0,"num_object_copies":9960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":841,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8336,"num_write_kb":372992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":841},{"shards":"1,2","objects":2479}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":315,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.557916+0000","last_change":"2024-09-21T10:00:15.524812+0000","last_active":"2024-09-21T10:11:27.557916+0000","last_peered":"2024-09-21T10:11:27.557916+0000","last_clean":"2024-09-21T10:11:27.557916+0000","last_became_active":"2024-09-21T09:59:43.361751+0000","last_became_peered":"2024-09-21T09:59:43.361751+0000","last_unstale":"2024-09-21T10:11:27.557916+0000","last_undegraded":"2024-09-21T10:11:27.557916+0000","last_fullsized":"2024-09-21T10:11:27.557916+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1121,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:13.513881+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":16,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524155+0000","last_change":"2024-09-21T10:10:13.636008+0000","last_active":"2024-09-21T10:11:26.524155+0000","last_peered":"2024-09-21T10:11:26.524155+0000","last_clean":"2024-09-21T10:11:26.524155+0000","last_became_active":"2024-09-21T10:10:13.635632+0000","last_became_peered":"2024-09-21T10:10:13.635632+0000","last_unstale":"2024-09-21T10:11:26.524155+0000","last_undegraded":"2024-09-21T10:11:26.524155+0000","last_fullsized":"2024-09-21T10:11:26.524155+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:28.265954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1224,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.502478+0000","last_change":"2024-09-21T10:08:45.524238+0000","last_active":"2024-09-21T10:11:26.502478+0000","last_peered":"2024-09-21T10:11:26.502478+0000","last_clean":"2024-09-21T10:11:26.502478+0000","last_became_active":"2024-09-21T10:08:45.523517+0000","last_became_peered":"2024-09-21T10:08:45.523517+0000","last_unstale":"2024-09-21T10:11:26.502478+0000","last_undegraded":"2024-09-21T10:11:26.502478+0000","last_fullsized":"2024-09-21T10:11:26.502478+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:13.958459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1259'8349","reported_seq":8664,"reported_epoch":1259,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:27.703946+0000","last_change":"2024-09-21T10:11:25.602632+0000","last_active":"2024-09-21T10:11:27.703946+0000","last_peered":"2024-09-21T10:11:27.703946+0000","last_clean":"2024-09-21T10:09:24.411450+0000","last_became_active":"2024-09-21T10:09:26.644140+0000","last_became_peered":"2024-09-21T10:09:26.644140+0000","last_unstale":"2024-09-21T10:11:27.703946+0000","last_undegraded":"2024-09-21T10:10:13.672461+0000","last_fullsized":"2024-09-21T10:09:26.425499+0000","mapping_epoch":1245,"log_start":"1246'3600","ondisk_log_start":"1246'3600","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4749,"log_dups_size":0,"ondisk_log_size":4749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224985088,"num_objects":3433,"num_object_clones":0,"num_object_copies":10299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2400,"num_objects_misplaced":1008,"num_objects_unfound":0,"num_objects_dirty":3433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8349,"num_write_kb":377024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":131,"num_bytes_recovered":8585216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3","objects":25},{"shards":"1,3,7","objects":1008}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"288'100","reported_seq":57,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.504822+0000","last_change":"2024-09-21T10:08:45.524802+0000","last_active":"2024-09-21T10:11:27.504822+0000","last_peered":"2024-09-21T10:11:27.504822+0000","last_clean":"2024-09-21T10:11:27.504822+0000","last_became_active":"2024-09-21T10:08:45.524408+0000","last_became_peered":"2024-09-21T10:08:45.524408+0000","last_unstale":"2024-09-21T10:11:27.504822+0000","last_undegraded":"2024-09-21T10:11:27.504822+0000","last_fullsized":"2024-09-21T10:11:27.504822+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:36.488660+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:08:36.488660+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":23,"num_write":5,"num_write_kb":50,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1205,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501529+0000","last_change":"2024-09-21T10:09:25.449815+0000","last_active":"2024-09-21T10:11:26.501529+0000","last_peered":"2024-09-21T10:11:26.501529+0000","last_clean":"2024-09-21T10:11:26.501529+0000","last_became_active":"2024-09-21T10:09:25.449325+0000","last_became_peered":"2024-09-21T10:09:25.449325+0000","last_unstale":"2024-09-21T10:11:26.501529+0000","last_undegraded":"2024-09-21T10:11:26.501529+0000","last_fullsized":"2024-09-21T10:11:26.501529+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:12.890694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1247,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.502281+0000","last_change":"2024-09-21T10:10:13.641608+0000","last_active":"2024-09-21T10:11:26.502281+0000","last_peered":"2024-09-21T10:11:26.502281+0000","last_clean":"2024-09-21T10:11:26.502281+0000","last_became_active":"2024-09-21T10:10:13.640299+0000","last_became_peered":"2024-09-21T10:10:13.640299+0000","last_unstale":"2024-09-21T10:11:26.502281+0000","last_undegraded":"2024-09-21T10:11:26.502281+0000","last_fullsized":"2024-09-21T10:11:26.502281+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:46.978168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1259'8227","reported_seq":8282,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.680759+0000","last_change":"2024-09-21T10:08:48.535207+0000","last_active":"2024-09-21T10:11:27.680759+0000","last_peered":"2024-09-21T10:11:27.680759+0000","last_clean":"2024-09-21T10:11:27.680759+0000","last_became_active":"2024-09-21T10:08:48.534901+0000","last_became_peered":"2024-09-21T10:08:48.534901+0000","last_unstale":"2024-09-21T10:11:27.680759+0000","last_undegraded":"2024-09-21T10:11:27.680759+0000","last_fullsized":"2024-09-21T10:11:27.680759+0000","mapping_epoch":1238,"log_start":"1243'3500","ondisk_log_start":"1243'3500","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4727,"log_dups_size":0,"ondisk_log_size":4727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:15.810011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215678976,"num_objects":3291,"num_object_clones":0,"num_object_copies":9873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8227,"num_write_kb":368576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":327,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553026+0000","last_change":"2024-09-21T10:10:13.622553+0000","last_active":"2024-09-21T10:11:27.553026+0000","last_peered":"2024-09-21T10:11:27.553026+0000","last_clean":"2024-09-21T10:11:27.553026+0000","last_became_active":"2024-09-21T10:10:13.622268+0000","last_became_peered":"2024-09-21T10:10:13.622268+0000","last_unstale":"2024-09-21T10:11:27.553026+0000","last_undegraded":"2024-09-21T10:11:27.553026+0000","last_fullsized":"2024-09-21T10:11:27.553026+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:24.686892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:24.686892+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:55.474818+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1892,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.509399+0000","last_change":"2024-09-21T10:09:25.455306+0000","last_active":"2024-09-21T10:11:27.509399+0000","last_peered":"2024-09-21T10:11:27.509399+0000","last_clean":"2024-09-21T10:11:27.509399+0000","last_became_active":"2024-09-21T10:09:25.454568+0000","last_became_peered":"2024-09-21T10:09:25.454568+0000","last_unstale":"2024-09-21T10:11:27.509399+0000","last_undegraded":"2024-09-21T10:11:27.509399+0000","last_fullsized":"2024-09-21T10:11:27.509399+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:55:25.499158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":57,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.504421+0000","last_change":"2024-09-21T10:08:45.523818+0000","last_active":"2024-09-21T10:11:27.504421+0000","last_peered":"2024-09-21T10:11:27.504421+0000","last_clean":"2024-09-21T10:11:27.504421+0000","last_became_active":"2024-09-21T10:08:45.523507+0000","last_became_peered":"2024-09-21T10:08:45.523507+0000","last_unstale":"2024-09-21T10:11:27.504421+0000","last_undegraded":"2024-09-21T10:11:27.504421+0000","last_fullsized":"2024-09-21T10:11:27.504421+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:32.923210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1259'8448","reported_seq":8503,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.669069+0000","last_change":"2024-09-21T10:08:48.535314+0000","last_active":"2024-09-21T10:11:27.669069+0000","last_peered":"2024-09-21T10:11:27.669069+0000","last_clean":"2024-09-21T10:11:27.669069+0000","last_became_active":"2024-09-21T10:08:48.535036+0000","last_became_peered":"2024-09-21T10:08:48.535036+0000","last_unstale":"2024-09-21T10:11:27.669069+0000","last_undegraded":"2024-09-21T10:11:27.669069+0000","last_fullsized":"2024-09-21T10:11:27.669069+0000","mapping_epoch":1238,"log_start":"1246'3700","ondisk_log_start":"1246'3700","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4748,"log_dups_size":0,"ondisk_log_size":4748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:42.985277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216268800,"num_objects":3300,"num_object_clones":0,"num_object_copies":9900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8448,"num_write_kb":375936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":401,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.552964+0000","last_change":"2024-09-21T10:10:13.652304+0000","last_active":"2024-09-21T10:11:27.552964+0000","last_peered":"2024-09-21T10:11:27.552964+0000","last_clean":"2024-09-21T10:11:27.552964+0000","last_became_active":"2024-09-21T10:10:13.648459+0000","last_became_peered":"2024-09-21T10:10:13.648459+0000","last_unstale":"2024-09-21T10:11:27.552964+0000","last_undegraded":"2024-09-21T10:11:27.552964+0000","last_fullsized":"2024-09-21T10:11:27.552964+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.705004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.705004+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":33,"num_write":6,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26834,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":356,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553011+0000","last_change":"2024-09-21T10:09:25.471697+0000","last_active":"2024-09-21T10:11:27.553011+0000","last_peered":"2024-09-21T10:11:27.553011+0000","last_clean":"2024-09-21T10:11:27.553011+0000","last_became_active":"2024-09-21T10:09:25.458890+0000","last_became_peered":"2024-09-21T10:09:25.458890+0000","last_unstale":"2024-09-21T10:11:27.553011+0000","last_undegraded":"2024-09-21T10:11:27.553011+0000","last_fullsized":"2024-09-21T10:11:27.553011+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:07.347060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1870,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.510588+0000","last_change":"2024-09-21T10:10:13.664494+0000","last_active":"2024-09-21T10:11:27.510588+0000","last_peered":"2024-09-21T10:11:27.510588+0000","last_clean":"2024-09-21T10:11:27.510588+0000","last_became_active":"2024-09-21T10:10:13.647871+0000","last_became_peered":"2024-09-21T10:10:13.647871+0000","last_unstale":"2024-09-21T10:11:27.510588+0000","last_undegraded":"2024-09-21T10:11:27.510588+0000","last_fullsized":"2024-09-21T10:11:27.510588+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:13.390934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":16,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524158+0000","last_change":"2024-09-21T10:10:13.642176+0000","last_active":"2024-09-21T10:11:26.524158+0000","last_peered":"2024-09-21T10:11:26.524158+0000","last_clean":"2024-09-21T10:11:26.524158+0000","last_became_active":"2024-09-21T10:10:13.641997+0000","last_became_peered":"2024-09-21T10:10:13.641997+0000","last_unstale":"2024-09-21T10:11:26.524158+0000","last_undegraded":"2024-09-21T10:11:26.524158+0000","last_fullsized":"2024-09-21T10:11:26.524158+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:46.940248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1259'8519","reported_seq":8583,"reported_epoch":1259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:29.855832+0000","last_change":"2024-09-21T10:10:13.695829+0000","last_active":"2024-09-21T10:11:29.855832+0000","last_peered":"2024-09-21T10:11:29.855832+0000","last_clean":"2024-09-21T10:09:45.420818+0000","last_became_active":"2024-09-21T10:10:13.689557+0000","last_became_peered":"2024-09-21T10:10:13.689557+0000","last_unstale":"2024-09-21T10:11:29.855832+0000","last_undegraded":"2024-09-21T10:10:13.680588+0000","last_fullsized":"2024-09-21T10:10:13.628840+0000","mapping_epoch":1252,"log_start":"1243'3000","ondisk_log_start":"1243'3000","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210698240,"num_objects":3215,"num_object_clones":0,"num_object_copies":9645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":805,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8519,"num_write_kb":375488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,2,5","objects":805},{"shards":"0,5","objects":2410}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":71,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.504577+0000","last_change":"2024-09-21T10:10:13.637642+0000","last_active":"2024-09-21T10:11:27.504577+0000","last_peered":"2024-09-21T10:11:27.504577+0000","last_clean":"2024-09-21T10:11:27.504577+0000","last_became_active":"2024-09-21T10:10:13.637267+0000","last_became_peered":"2024-09-21T10:10:13.637267+0000","last_unstale":"2024-09-21T10:11:27.504577+0000","last_undegraded":"2024-09-21T10:11:27.504577+0000","last_fullsized":"2024-09-21T10:11:27.504577+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:04:35.993739+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":22,"num_write":4,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1242,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.505228+0000","last_change":"2024-09-21T10:09:25.465784+0000","last_active":"2024-09-21T10:11:27.505228+0000","last_peered":"2024-09-21T10:11:27.505228+0000","last_clean":"2024-09-21T10:11:27.505228+0000","last_became_active":"2024-09-21T10:09:25.465445+0000","last_became_peered":"2024-09-21T10:09:25.465445+0000","last_unstale":"2024-09-21T10:11:27.505228+0000","last_undegraded":"2024-09-21T10:11:27.505228+0000","last_fullsized":"2024-09-21T10:11:27.505228+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:42:02.016170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1954,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.554845+0000","last_change":"2024-09-21T10:08:25.429287+0000","last_active":"2024-09-21T10:11:27.554845+0000","last_peered":"2024-09-21T10:11:27.554845+0000","last_clean":"2024-09-21T10:11:27.554845+0000","last_became_active":"2024-09-21T10:08:25.428923+0000","last_became_peered":"2024-09-21T10:08:25.428923+0000","last_unstale":"2024-09-21T10:11:27.554845+0000","last_undegraded":"2024-09-21T10:11:27.554845+0000","last_fullsized":"2024-09-21T10:11:27.554845+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:09.255665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1970,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.558682+0000","last_change":"2024-09-21T10:08:45.532253+0000","last_active":"2024-09-21T10:11:27.558682+0000","last_peered":"2024-09-21T10:11:27.558682+0000","last_clean":"2024-09-21T10:11:27.558682+0000","last_became_active":"2024-09-21T10:08:45.531519+0000","last_became_peered":"2024-09-21T10:08:45.531519+0000","last_unstale":"2024-09-21T10:11:27.558682+0000","last_undegraded":"2024-09-21T10:11:27.558682+0000","last_fullsized":"2024-09-21T10:11:27.558682+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:25.378783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"1259'8441","reported_seq":10237,"reported_epoch":1259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:28.033844+0000","last_change":"2024-09-21T10:10:13.643523+0000","last_active":"2024-09-21T10:11:28.033844+0000","last_peered":"2024-09-21T10:11:28.033844+0000","last_clean":"2024-09-21T10:09:24.391963+0000","last_became_active":"2024-09-21T10:09:26.716693+0000","last_became_peered":"2024-09-21T10:09:26.716693+0000","last_unstale":"2024-09-21T10:11:28.033844+0000","last_undegraded":"2024-09-21T10:10:13.639837+0000","last_fullsized":"2024-09-21T10:09:26.423890+0000","mapping_epoch":1245,"log_start":"1246'3800","ondisk_log_start":"1246'3800","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4641,"log_dups_size":0,"ondisk_log_size":4641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210567168,"num_objects":3213,"num_object_clones":0,"num_object_copies":9639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2762,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8441,"num_write_kb":372928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":868,"num_bytes_recovered":56885248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5","objects":451},{"shards":"1,5,7","objects":1015}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":387,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553161+0000","last_change":"2024-09-21T10:09:25.450579+0000","last_active":"2024-09-21T10:11:27.553161+0000","last_peered":"2024-09-21T10:11:27.553161+0000","last_clean":"2024-09-21T10:11:27.553161+0000","last_became_active":"2024-09-21T10:09:25.445022+0000","last_became_peered":"2024-09-21T10:09:25.445022+0000","last_unstale":"2024-09-21T10:11:27.553161+0000","last_undegraded":"2024-09-21T10:11:27.553161+0000","last_fullsized":"2024-09-21T10:11:27.553161+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:33.068697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:08:33.068697+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":34,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1918,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553005+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:11:27.553005+0000","last_peered":"2024-09-21T10:11:27.553005+0000","last_clean":"2024-09-21T10:11:27.553005+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:11:27.553005+0000","last_undegraded":"2024-09-21T10:11:27.553005+0000","last_fullsized":"2024-09-21T10:11:27.553005+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":71,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.505336+0000","last_change":"2024-09-21T10:10:13.635570+0000","last_active":"2024-09-21T10:11:27.505336+0000","last_peered":"2024-09-21T10:11:27.505336+0000","last_clean":"2024-09-21T10:11:27.505336+0000","last_became_active":"2024-09-21T10:10:13.635099+0000","last_became_peered":"2024-09-21T10:10:13.635099+0000","last_unstale":"2024-09-21T10:11:27.505336+0000","last_undegraded":"2024-09-21T10:11:27.505336+0000","last_fullsized":"2024-09-21T10:11:27.505336+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:43.686232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":57,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.504350+0000","last_change":"2024-09-21T10:08:45.525625+0000","last_active":"2024-09-21T10:11:27.504350+0000","last_peered":"2024-09-21T10:11:27.504350+0000","last_clean":"2024-09-21T10:11:27.504350+0000","last_became_active":"2024-09-21T10:08:45.525299+0000","last_became_peered":"2024-09-21T10:08:45.525299+0000","last_unstale":"2024-09-21T10:11:27.504350+0000","last_undegraded":"2024-09-21T10:11:27.504350+0000","last_fullsized":"2024-09-21T10:11:27.504350+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:08.084485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"1259'8547","reported_seq":9621,"reported_epoch":1259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:29.883825+0000","last_change":"2024-09-21T10:09:46.486729+0000","last_active":"2024-09-21T10:11:29.883825+0000","last_peered":"2024-09-21T10:11:29.883825+0000","last_clean":"2024-09-21T10:09:24.421724+0000","last_became_active":"2024-09-21T10:09:26.681479+0000","last_became_peered":"2024-09-21T10:09:26.681479+0000","last_unstale":"2024-09-21T10:11:29.883825+0000","last_undegraded":"2024-09-21T10:09:46.484067+0000","last_fullsized":"2024-09-21T10:09:26.423258+0000","mapping_epoch":1245,"log_start":"1243'3000","ondisk_log_start":"1243'3000","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":5547,"log_dups_size":0,"ondisk_log_size":5547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212140032,"num_objects":3237,"num_object_clones":0,"num_object_copies":9711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3133,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8547,"num_write_kb":377088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":508,"num_bytes_recovered":33292288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":104},{"shards":"0,5,7","objects":986}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":666,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.554750+0000","last_change":"2024-09-21T10:06:19.901581+0000","last_active":"2024-09-21T10:11:27.554750+0000","last_peered":"2024-09-21T10:11:27.554750+0000","last_clean":"2024-09-21T10:11:27.554750+0000","last_became_active":"2024-09-21T10:06:13.443630+0000","last_became_peered":"2024-09-21T10:06:13.443630+0000","last_unstale":"2024-09-21T10:11:27.554750+0000","last_undegraded":"2024-09-21T10:11:27.554750+0000","last_fullsized":"2024-09-21T10:11:27.554750+0000","mapping_epoch":1120,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1213,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:19.901507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:19.901507+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:41.572099+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":72,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.505318+0000","last_change":"2024-09-21T10:10:13.636061+0000","last_active":"2024-09-21T10:11:27.505318+0000","last_peered":"2024-09-21T10:11:27.505318+0000","last_clean":"2024-09-21T10:11:27.505318+0000","last_became_active":"2024-09-21T10:10:13.635006+0000","last_became_peered":"2024-09-21T10:10:13.635006+0000","last_unstale":"2024-09-21T10:11:27.505318+0000","last_undegraded":"2024-09-21T10:11:27.505318+0000","last_fullsized":"2024-09-21T10:11:27.505318+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:24.272055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1202,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501907+0000","last_change":"2024-09-21T10:10:13.686243+0000","last_active":"2024-09-21T10:11:26.501907+0000","last_peered":"2024-09-21T10:11:26.501907+0000","last_clean":"2024-09-21T10:11:26.501907+0000","last_became_active":"2024-09-21T10:10:13.685894+0000","last_became_peered":"2024-09-21T10:10:13.685894+0000","last_unstale":"2024-09-21T10:11:26.501907+0000","last_undegraded":"2024-09-21T10:11:26.501907+0000","last_fullsized":"2024-09-21T10:11:26.501907+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:12.394732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":57,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.504203+0000","last_change":"2024-09-21T10:08:45.525546+0000","last_active":"2024-09-21T10:11:27.504203+0000","last_peered":"2024-09-21T10:11:27.504203+0000","last_clean":"2024-09-21T10:11:27.504203+0000","last_became_active":"2024-09-21T10:08:45.525116+0000","last_became_peered":"2024-09-21T10:08:45.525116+0000","last_unstale":"2024-09-21T10:11:27.504203+0000","last_undegraded":"2024-09-21T10:11:27.504203+0000","last_fullsized":"2024-09-21T10:11:27.504203+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:58.739744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"1259'8473","reported_seq":8528,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.937332+0000","last_change":"2024-09-21T10:08:48.537781+0000","last_active":"2024-09-21T10:11:27.937332+0000","last_peered":"2024-09-21T10:11:27.937332+0000","last_clean":"2024-09-21T10:11:27.937332+0000","last_became_active":"2024-09-21T10:08:48.537433+0000","last_became_peered":"2024-09-21T10:08:48.537433+0000","last_unstale":"2024-09-21T10:11:27.937332+0000","last_undegraded":"2024-09-21T10:11:27.937332+0000","last_fullsized":"2024-09-21T10:11:27.937332+0000","mapping_epoch":1238,"log_start":"1247'3900","ondisk_log_start":"1247'3900","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4573,"log_dups_size":0,"ondisk_log_size":4573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:12.113257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216072192,"num_objects":3297,"num_object_clones":0,"num_object_copies":9891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8473,"num_write_kb":376640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":37,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524387+0000","last_change":"2024-09-21T10:11:13.146483+0000","last_active":"2024-09-21T10:11:26.524387+0000","last_peered":"2024-09-21T10:11:26.524387+0000","last_clean":"2024-09-21T10:11:26.524387+0000","last_became_active":"2024-09-21T10:10:13.648610+0000","last_became_peered":"2024-09-21T10:10:13.648610+0000","last_unstale":"2024-09-21T10:11:26.524387+0000","last_undegraded":"2024-09-21T10:11:26.524387+0000","last_fullsized":"2024-09-21T10:11:26.524387+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.730973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.730973+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":34,"num_write":6,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":874,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.513962+0000","last_change":"2024-09-21T09:57:39.543492+0000","last_active":"2024-09-21T10:11:27.513962+0000","last_peered":"2024-09-21T10:11:27.513962+0000","last_clean":"2024-09-21T10:11:27.513962+0000","last_became_active":"2024-09-21T09:57:39.543196+0000","last_became_peered":"2024-09-21T09:57:39.543196+0000","last_unstale":"2024-09-21T10:11:27.513962+0000","last_undegraded":"2024-09-21T10:11:27.513962+0000","last_fullsized":"2024-09-21T10:11:27.513962+0000","mapping_epoch":1100,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:53.796415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":64,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.504017+0000","last_change":"2024-09-21T10:09:25.453087+0000","last_active":"2024-09-21T10:11:27.504017+0000","last_peered":"2024-09-21T10:11:27.504017+0000","last_clean":"2024-09-21T10:11:27.504017+0000","last_became_active":"2024-09-21T10:09:25.449388+0000","last_became_peered":"2024-09-21T10:09:25.449388+0000","last_unstale":"2024-09-21T10:11:27.504017+0000","last_undegraded":"2024-09-21T10:11:27.504017+0000","last_fullsized":"2024-09-21T10:11:27.504017+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:05.107434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":57,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.505149+0000","last_change":"2024-09-21T10:08:45.523135+0000","last_active":"2024-09-21T10:11:27.505149+0000","last_peered":"2024-09-21T10:11:27.505149+0000","last_clean":"2024-09-21T10:11:27.505149+0000","last_became_active":"2024-09-21T10:08:45.522774+0000","last_became_peered":"2024-09-21T10:08:45.522774+0000","last_unstale":"2024-09-21T10:11:27.505149+0000","last_undegraded":"2024-09-21T10:11:27.505149+0000","last_fullsized":"2024-09-21T10:11:27.505149+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:19:19.243019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"1259'8092","reported_seq":10476,"reported_epoch":1259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:28.466259+0000","last_change":"2024-09-21T10:09:46.459618+0000","last_active":"2024-09-21T10:11:28.466259+0000","last_peered":"2024-09-21T10:11:28.466259+0000","last_clean":"2024-09-21T10:09:24.391727+0000","last_became_active":"2024-09-21T10:09:26.567025+0000","last_became_peered":"2024-09-21T10:09:26.567025+0000","last_unstale":"2024-09-21T10:11:28.466259+0000","last_undegraded":"2024-09-21T10:09:46.458461+0000","last_fullsized":"2024-09-21T10:09:26.423498+0000","mapping_epoch":1245,"log_start":"1243'1900","ondisk_log_start":"1243'1900","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":6192,"log_dups_size":0,"ondisk_log_size":6192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:06:55.852645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208011264,"num_objects":3174,"num_object_clones":0,"num_object_copies":9522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2306,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8092,"num_write_kb":360512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1172,"num_bytes_recovered":76808192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4","objects":868},{"shards":"1,4,7","objects":1019}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":26,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524334+0000","last_change":"2024-09-21T10:10:37.256938+0000","last_active":"2024-09-21T10:11:26.524334+0000","last_peered":"2024-09-21T10:11:26.524334+0000","last_clean":"2024-09-21T10:11:26.524334+0000","last_became_active":"2024-09-21T10:10:13.648466+0000","last_became_peered":"2024-09-21T10:10:13.648466+0000","last_unstale":"2024-09-21T10:11:26.524334+0000","last_undegraded":"2024-09-21T10:11:26.524334+0000","last_fullsized":"2024-09-21T10:11:26.524334+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:10:37.256819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:10:37.256819+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:42.960519+0000","scrub_duration":0.0021766020000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":57,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.503924+0000","last_change":"2024-09-21T10:08:45.531369+0000","last_active":"2024-09-21T10:11:27.503924+0000","last_peered":"2024-09-21T10:11:27.503924+0000","last_clean":"2024-09-21T10:11:27.503924+0000","last_became_active":"2024-09-21T10:08:45.530951+0000","last_became_peered":"2024-09-21T10:08:45.530951+0000","last_unstale":"2024-09-21T10:11:27.503924+0000","last_undegraded":"2024-09-21T10:11:27.503924+0000","last_fullsized":"2024-09-21T10:11:27.503924+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:27:50.056329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1210,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.554205+0000","last_change":"2024-09-21T10:09:25.453642+0000","last_active":"2024-09-21T10:11:27.554205+0000","last_peered":"2024-09-21T10:11:27.554205+0000","last_clean":"2024-09-21T10:11:27.554205+0000","last_became_active":"2024-09-21T10:09:25.453318+0000","last_became_peered":"2024-09-21T10:09:25.453318+0000","last_unstale":"2024-09-21T10:11:27.554205+0000","last_undegraded":"2024-09-21T10:11:27.554205+0000","last_fullsized":"2024-09-21T10:11:27.554205+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:05.373220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1941,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553571+0000","last_change":"2024-09-21T10:09:25.452807+0000","last_active":"2024-09-21T10:11:27.553571+0000","last_peered":"2024-09-21T10:11:27.553571+0000","last_clean":"2024-09-21T10:11:27.553571+0000","last_became_active":"2024-09-21T10:09:25.451745+0000","last_became_peered":"2024-09-21T10:09:25.451745+0000","last_unstale":"2024-09-21T10:11:27.553571+0000","last_undegraded":"2024-09-21T10:11:27.553571+0000","last_fullsized":"2024-09-21T10:11:27.553571+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:33:16.553471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"1259'8320","reported_seq":15557,"reported_epoch":1259,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:11:28.045963+0000","last_change":"2024-09-21T10:11:27.551139+0000","last_active":"2024-09-21T10:11:28.045963+0000","last_peered":"2024-09-21T10:11:28.045963+0000","last_clean":"2024-09-21T10:09:24.420829+0000","last_became_active":"2024-09-21T10:11:27.520829+0000","last_became_peered":"2024-09-21T10:11:27.520829+0000","last_unstale":"2024-09-21T10:11:28.045963+0000","last_undegraded":"2024-09-21T10:11:28.045963+0000","last_fullsized":"2024-09-21T10:11:28.045963+0000","mapping_epoch":1258,"log_start":"1246'3700","ondisk_log_start":"1246'3700","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4620,"log_dups_size":0,"ondisk_log_size":4620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209125400,"num_objects":3192,"num_object_clones":0,"num_object_copies":9576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3192,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8320,"num_write_kb":368321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3591,"num_bytes_recovered":235274240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":["1","4","5","7"],"object_location_counts":[{"shards":"1,4,5","objects":3192}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":159,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.502113+0000","last_change":"2024-09-21T10:10:13.623475+0000","last_active":"2024-09-21T10:11:26.502113+0000","last_peered":"2024-09-21T10:11:26.502113+0000","last_clean":"2024-09-21T10:11:26.502113+0000","last_became_active":"2024-09-21T10:10:13.622580+0000","last_became_peered":"2024-09-21T10:10:13.622580+0000","last_unstale":"2024-09-21T10:11:26.502113+0000","last_undegraded":"2024-09-21T10:11:26.502113+0000","last_fullsized":"2024-09-21T10:11:26.502113+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:36.469530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:08:36.469530+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:47.757854+0000","scrub_duration":0.001636942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1880,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.512183+0000","last_change":"2024-09-21T10:09:25.477103+0000","last_active":"2024-09-21T10:11:27.512183+0000","last_peered":"2024-09-21T10:11:27.512183+0000","last_clean":"2024-09-21T10:11:27.512183+0000","last_became_active":"2024-09-21T10:09:25.439742+0000","last_became_peered":"2024-09-21T10:09:25.439742+0000","last_unstale":"2024-09-21T10:11:27.512183+0000","last_undegraded":"2024-09-21T10:11:27.512183+0000","last_fullsized":"2024-09-21T10:11:27.512183+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:59:19.911474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":71,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.504015+0000","last_change":"2024-09-21T10:10:13.637236+0000","last_active":"2024-09-21T10:11:27.504015+0000","last_peered":"2024-09-21T10:11:27.504015+0000","last_clean":"2024-09-21T10:11:27.504015+0000","last_became_active":"2024-09-21T10:10:13.636907+0000","last_became_peered":"2024-09-21T10:10:13.636907+0000","last_unstale":"2024-09-21T10:11:27.504015+0000","last_undegraded":"2024-09-21T10:11:27.504015+0000","last_fullsized":"2024-09-21T10:11:27.504015+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:33.507370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1874,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.509142+0000","last_change":"2024-09-21T10:09:25.455618+0000","last_active":"2024-09-21T10:11:27.509142+0000","last_peered":"2024-09-21T10:11:27.509142+0000","last_clean":"2024-09-21T10:11:27.509142+0000","last_became_active":"2024-09-21T10:09:25.454991+0000","last_became_peered":"2024-09-21T10:09:25.454991+0000","last_unstale":"2024-09-21T10:11:27.509142+0000","last_undegraded":"2024-09-21T10:11:27.509142+0000","last_fullsized":"2024-09-21T10:11:27.509142+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:03.365552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1259'8119","reported_seq":9668,"reported_epoch":1259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:11:27.691405+0000","last_change":"2024-09-21T10:10:13.677528+0000","last_active":"2024-09-21T10:11:27.691405+0000","last_peered":"2024-09-21T10:11:27.691405+0000","last_clean":"2024-09-21T10:09:24.399521+0000","last_became_active":"2024-09-21T10:09:26.604841+0000","last_became_peered":"2024-09-21T10:09:26.604841+0000","last_unstale":"2024-09-21T10:11:27.691405+0000","last_undegraded":"2024-09-21T10:09:26.493324+0000","last_fullsized":"2024-09-21T10:09:26.422905+0000","mapping_epoch":1245,"log_start":"1243'3400","ondisk_log_start":"1243'3400","created":1238,"last_epoch_clean":1239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:47.514402+0000","last_clean_scrub_stamp":"2024-09-21T10:08:47.514402+0000","objects_scrubbed":0,"log_size":4719,"log_dups_size":0,"ondisk_log_size":4719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211615744,"num_objects":3229,"num_object_clones":0,"num_object_copies":9687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2740,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8119,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":746,"num_bytes_recovered":48889856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4","objects":489},{"shards":"3,4,7","objects":1040}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":346,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.512652+0000","last_change":"2024-09-21T10:10:13.677133+0000","last_active":"2024-09-21T10:11:27.512652+0000","last_peered":"2024-09-21T10:11:27.512652+0000","last_clean":"2024-09-21T10:11:27.512652+0000","last_became_active":"2024-09-21T10:10:13.670149+0000","last_became_peered":"2024-09-21T10:10:13.670149+0000","last_unstale":"2024-09-21T10:11:27.512652+0000","last_undegraded":"2024-09-21T10:11:27.512652+0000","last_fullsized":"2024-09-21T10:11:27.512652+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.423664+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.423664+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:11:34.650236+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":1980,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.558964+0000","last_change":"2024-09-21T10:08:45.540447+0000","last_active":"2024-09-21T10:11:27.558964+0000","last_peered":"2024-09-21T10:11:27.558964+0000","last_clean":"2024-09-21T10:11:27.558964+0000","last_became_active":"2024-09-21T10:08:45.528704+0000","last_became_peered":"2024-09-21T10:08:45.528704+0000","last_unstale":"2024-09-21T10:11:27.558964+0000","last_undegraded":"2024-09-21T10:11:27.558964+0000","last_fullsized":"2024-09-21T10:11:27.558964+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:09.404941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524315+0000","last_change":"2024-09-21T10:10:13.648679+0000","last_active":"2024-09-21T10:11:26.524315+0000","last_peered":"2024-09-21T10:11:26.524315+0000","last_clean":"2024-09-21T10:11:26.524315+0000","last_became_active":"2024-09-21T10:10:13.648229+0000","last_became_peered":"2024-09-21T10:10:13.648229+0000","last_unstale":"2024-09-21T10:11:26.524315+0000","last_undegraded":"2024-09-21T10:11:26.524315+0000","last_fullsized":"2024-09-21T10:11:26.524315+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:08.922979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1184,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501584+0000","last_change":"2024-09-21T10:10:13.621002+0000","last_active":"2024-09-21T10:11:26.501584+0000","last_peered":"2024-09-21T10:11:26.501584+0000","last_clean":"2024-09-21T10:11:26.501584+0000","last_became_active":"2024-09-21T10:10:13.620575+0000","last_became_peered":"2024-09-21T10:10:13.620575+0000","last_unstale":"2024-09-21T10:11:26.501584+0000","last_undegraded":"2024-09-21T10:11:26.501584+0000","last_fullsized":"2024-09-21T10:11:26.501584+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:06:32.926448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1247,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501851+0000","last_change":"2024-09-21T10:10:13.640158+0000","last_active":"2024-09-21T10:11:26.501851+0000","last_peered":"2024-09-21T10:11:26.501851+0000","last_clean":"2024-09-21T10:11:26.501851+0000","last_became_active":"2024-09-21T10:10:13.639801+0000","last_became_peered":"2024-09-21T10:10:13.639801+0000","last_unstale":"2024-09-21T10:11:26.501851+0000","last_undegraded":"2024-09-21T10:11:26.501851+0000","last_fullsized":"2024-09-21T10:11:26.501851+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:25.153985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1267,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.502137+0000","last_change":"2024-09-21T10:10:13.631294+0000","last_active":"2024-09-21T10:11:26.502137+0000","last_peered":"2024-09-21T10:11:26.502137+0000","last_clean":"2024-09-21T10:11:26.502137+0000","last_became_active":"2024-09-21T10:10:13.622468+0000","last_became_peered":"2024-09-21T10:10:13.622468+0000","last_unstale":"2024-09-21T10:11:26.502137+0000","last_undegraded":"2024-09-21T10:11:26.502137+0000","last_fullsized":"2024-09-21T10:11:26.502137+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:53.404968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18976,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":71,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.505064+0000","last_change":"2024-09-21T10:10:13.635438+0000","last_active":"2024-09-21T10:11:27.505064+0000","last_peered":"2024-09-21T10:11:27.505064+0000","last_clean":"2024-09-21T10:11:27.505064+0000","last_became_active":"2024-09-21T10:10:13.634268+0000","last_became_peered":"2024-09-21T10:10:13.634268+0000","last_unstale":"2024-09-21T10:11:27.505064+0000","last_undegraded":"2024-09-21T10:11:27.505064+0000","last_fullsized":"2024-09-21T10:11:27.505064+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:00:07.541446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":57,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.510988+0000","last_change":"2024-09-21T10:09:25.473376+0000","last_active":"2024-09-21T10:11:27.510988+0000","last_peered":"2024-09-21T10:11:27.510988+0000","last_clean":"2024-09-21T10:11:27.510988+0000","last_became_active":"2024-09-21T10:09:25.441146+0000","last_became_peered":"2024-09-21T10:09:25.441146+0000","last_unstale":"2024-09-21T10:11:27.510988+0000","last_undegraded":"2024-09-21T10:11:27.510988+0000","last_fullsized":"2024-09-21T10:11:27.510988+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:50:08.490080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"288'100","reported_seq":70,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553310+0000","last_change":"2024-09-21T10:09:25.453332+0000","last_active":"2024-09-21T10:11:27.553310+0000","last_peered":"2024-09-21T10:11:27.553310+0000","last_clean":"2024-09-21T10:11:27.553310+0000","last_became_active":"2024-09-21T10:09:25.452984+0000","last_became_peered":"2024-09-21T10:09:25.452984+0000","last_unstale":"2024-09-21T10:11:27.553310+0000","last_undegraded":"2024-09-21T10:11:27.553310+0000","last_fullsized":"2024-09-21T10:11:27.553310+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.389965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.389965+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:24.624741+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":659,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.503927+0000","last_change":"2024-09-21T10:09:25.463949+0000","last_active":"2024-09-21T10:11:27.503927+0000","last_peered":"2024-09-21T10:11:27.503927+0000","last_clean":"2024-09-21T10:11:27.503927+0000","last_became_active":"2024-09-21T10:09:25.449976+0000","last_became_peered":"2024-09-21T10:09:25.449976+0000","last_unstale":"2024-09-21T10:11:27.503927+0000","last_undegraded":"2024-09-21T10:11:27.503927+0000","last_fullsized":"2024-09-21T10:11:27.503927+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:09.931834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1876,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.509654+0000","last_change":"2024-09-21T10:10:13.825052+0000","last_active":"2024-09-21T10:11:27.509654+0000","last_peered":"2024-09-21T10:11:27.509654+0000","last_clean":"2024-09-21T10:11:27.509654+0000","last_became_active":"2024-09-21T10:10:13.809124+0000","last_became_peered":"2024-09-21T10:10:13.809124+0000","last_unstale":"2024-09-21T10:11:27.509654+0000","last_undegraded":"2024-09-21T10:11:27.509654+0000","last_fullsized":"2024-09-21T10:11:27.509654+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:49:32.388321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1941,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553916+0000","last_change":"2024-09-21T09:46:47.116628+0000","last_active":"2024-09-21T10:11:27.553916+0000","last_peered":"2024-09-21T10:11:27.553916+0000","last_clean":"2024-09-21T10:11:27.553916+0000","last_became_active":"2024-09-21T09:46:47.116374+0000","last_became_peered":"2024-09-21T09:46:47.116374+0000","last_unstale":"2024-09-21T10:11:27.553916+0000","last_undegraded":"2024-09-21T10:11:27.553916+0000","last_fullsized":"2024-09-21T10:11:27.553916+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:29.490317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1911,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.509042+0000","last_change":"2024-09-21T10:10:13.656027+0000","last_active":"2024-09-21T10:11:27.509042+0000","last_peered":"2024-09-21T10:11:27.509042+0000","last_clean":"2024-09-21T10:11:27.509042+0000","last_became_active":"2024-09-21T10:10:13.652862+0000","last_became_peered":"2024-09-21T10:10:13.652862+0000","last_unstale":"2024-09-21T10:11:27.509042+0000","last_undegraded":"2024-09-21T10:11:27.509042+0000","last_fullsized":"2024-09-21T10:11:27.509042+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:54:16.108044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":16,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.524459+0000","last_change":"2024-09-21T10:10:13.634975+0000","last_active":"2024-09-21T10:11:26.524459+0000","last_peered":"2024-09-21T10:11:26.524459+0000","last_clean":"2024-09-21T10:11:26.524459+0000","last_became_active":"2024-09-21T10:10:13.634667+0000","last_became_peered":"2024-09-21T10:10:13.634667+0000","last_unstale":"2024-09-21T10:11:26.524459+0000","last_undegraded":"2024-09-21T10:11:26.524459+0000","last_fullsized":"2024-09-21T10:11:26.524459+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:52.296990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1890,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.509310+0000","last_change":"2024-09-21T10:08:45.520524+0000","last_active":"2024-09-21T10:11:27.509310+0000","last_peered":"2024-09-21T10:11:27.509310+0000","last_clean":"2024-09-21T10:11:27.509310+0000","last_became_active":"2024-09-21T10:08:45.520119+0000","last_became_peered":"2024-09-21T10:08:45.520119+0000","last_unstale":"2024-09-21T10:11:27.509310+0000","last_undegraded":"2024-09-21T10:11:27.509310+0000","last_fullsized":"2024-09-21T10:11:27.509310+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:55.559438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1953,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.558815+0000","last_change":"2024-09-21T10:09:25.457529+0000","last_active":"2024-09-21T10:11:27.558815+0000","last_peered":"2024-09-21T10:11:27.558815+0000","last_clean":"2024-09-21T10:11:27.558815+0000","last_became_active":"2024-09-21T10:09:25.457214+0000","last_became_peered":"2024-09-21T10:09:25.457214+0000","last_unstale":"2024-09-21T10:11:27.558815+0000","last_undegraded":"2024-09-21T10:11:27.558815+0000","last_fullsized":"2024-09-21T10:11:27.558815+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:37.912673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":57,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.503474+0000","last_change":"2024-09-21T10:08:45.523144+0000","last_active":"2024-09-21T10:11:27.503474+0000","last_peered":"2024-09-21T10:11:27.503474+0000","last_clean":"2024-09-21T10:11:27.503474+0000","last_became_active":"2024-09-21T10:08:45.522526+0000","last_became_peered":"2024-09-21T10:08:45.522526+0000","last_unstale":"2024-09-21T10:11:27.503474+0000","last_undegraded":"2024-09-21T10:11:27.503474+0000","last_fullsized":"2024-09-21T10:11:27.503474+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:45.926968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1882,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.510831+0000","last_change":"2024-09-21T10:08:45.515262+0000","last_active":"2024-09-21T10:11:27.510831+0000","last_peered":"2024-09-21T10:11:27.510831+0000","last_clean":"2024-09-21T10:11:27.510831+0000","last_became_active":"2024-09-21T10:08:45.514926+0000","last_became_peered":"2024-09-21T10:08:45.514926+0000","last_unstale":"2024-09-21T10:11:27.510831+0000","last_undegraded":"2024-09-21T10:11:27.510831+0000","last_fullsized":"2024-09-21T10:11:27.510831+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:49.954351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1226,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501800+0000","last_change":"2024-09-21T10:08:45.524864+0000","last_active":"2024-09-21T10:11:26.501800+0000","last_peered":"2024-09-21T10:11:26.501800+0000","last_clean":"2024-09-21T10:11:26.501800+0000","last_became_active":"2024-09-21T10:08:45.524533+0000","last_became_peered":"2024-09-21T10:08:45.524533+0000","last_unstale":"2024-09-21T10:11:26.501800+0000","last_undegraded":"2024-09-21T10:11:26.501800+0000","last_fullsized":"2024-09-21T10:11:26.501800+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:52.547302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1910,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553468+0000","last_change":"2024-09-21T10:08:45.525340+0000","last_active":"2024-09-21T10:11:27.553468+0000","last_peered":"2024-09-21T10:11:27.553468+0000","last_clean":"2024-09-21T10:11:27.553468+0000","last_became_active":"2024-09-21T10:08:45.525023+0000","last_became_peered":"2024-09-21T10:08:45.525023+0000","last_unstale":"2024-09-21T10:11:27.553468+0000","last_undegraded":"2024-09-21T10:11:27.553468+0000","last_fullsized":"2024-09-21T10:11:27.553468+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:37.674092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":1952,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.553669+0000","last_change":"2024-09-21T10:09:25.453096+0000","last_active":"2024-09-21T10:11:27.553669+0000","last_peered":"2024-09-21T10:11:27.553669+0000","last_clean":"2024-09-21T10:11:27.553669+0000","last_became_active":"2024-09-21T10:09:25.452401+0000","last_became_peered":"2024-09-21T10:09:25.452401+0000","last_unstale":"2024-09-21T10:11:27.553669+0000","last_undegraded":"2024-09-21T10:11:27.553669+0000","last_fullsized":"2024-09-21T10:11:27.553669+0000","mapping_epoch":1244,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.053333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1224,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501605+0000","last_change":"2024-09-21T10:08:45.516048+0000","last_active":"2024-09-21T10:11:26.501605+0000","last_peered":"2024-09-21T10:11:26.501605+0000","last_clean":"2024-09-21T10:11:26.501605+0000","last_became_active":"2024-09-21T10:08:45.515656+0000","last_became_peered":"2024-09-21T10:08:45.515656+0000","last_unstale":"2024-09-21T10:11:26.501605+0000","last_undegraded":"2024-09-21T10:11:26.501605+0000","last_fullsized":"2024-09-21T10:11:26.501605+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:05:22.491802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":196,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.502030+0000","last_change":"2024-09-21T10:08:45.535971+0000","last_active":"2024-09-21T10:11:26.502030+0000","last_peered":"2024-09-21T10:11:26.502030+0000","last_clean":"2024-09-21T10:11:26.502030+0000","last_became_active":"2024-09-21T10:08:45.530341+0000","last_became_peered":"2024-09-21T10:08:45.530341+0000","last_unstale":"2024-09-21T10:11:26.502030+0000","last_undegraded":"2024-09-21T10:11:26.502030+0000","last_fullsized":"2024-09-21T10:11:26.502030+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:58:34.894434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1204,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501557+0000","last_change":"2024-09-21T10:08:25.430667+0000","last_active":"2024-09-21T10:11:26.501557+0000","last_peered":"2024-09-21T10:11:26.501557+0000","last_clean":"2024-09-21T10:11:26.501557+0000","last_became_active":"2024-09-21T10:08:25.430357+0000","last_became_peered":"2024-09-21T10:08:25.430357+0000","last_unstale":"2024-09-21T10:11:26.501557+0000","last_undegraded":"2024-09-21T10:11:26.501557+0000","last_fullsized":"2024-09-21T10:11:26.501557+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:19.972020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":198,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501997+0000","last_change":"2024-09-21T10:10:13.639231+0000","last_active":"2024-09-21T10:11:26.501997+0000","last_peered":"2024-09-21T10:11:26.501997+0000","last_clean":"2024-09-21T10:11:26.501997+0000","last_became_active":"2024-09-21T10:10:13.622497+0000","last_became_peered":"2024-09-21T10:10:13.622497+0000","last_unstale":"2024-09-21T10:11:26.501997+0000","last_undegraded":"2024-09-21T10:11:26.501997+0000","last_fullsized":"2024-09-21T10:11:26.501997+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:31:46.263003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1868,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.510069+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:11:27.510069+0000","last_peered":"2024-09-21T10:11:27.510069+0000","last_clean":"2024-09-21T10:11:27.510069+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:11:27.510069+0000","last_undegraded":"2024-09-21T10:11:27.510069+0000","last_fullsized":"2024-09-21T10:11:27.510069+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1883,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.511686+0000","last_change":"2024-09-21T10:10:13.651599+0000","last_active":"2024-09-21T10:11:27.511686+0000","last_peered":"2024-09-21T10:11:27.511686+0000","last_clean":"2024-09-21T10:11:27.511686+0000","last_became_active":"2024-09-21T10:10:13.651250+0000","last_became_peered":"2024-09-21T10:10:13.651250+0000","last_unstale":"2024-09-21T10:11:27.511686+0000","last_undegraded":"2024-09-21T10:11:27.511686+0000","last_fullsized":"2024-09-21T10:11:27.511686+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:34.100056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1247,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-09-21T10:11:26.501506+0000","last_change":"2024-09-21T10:10:13.639970+0000","last_active":"2024-09-21T10:11:26.501506+0000","last_peered":"2024-09-21T10:11:26.501506+0000","last_clean":"2024-09-21T10:11:26.501506+0000","last_became_active":"2024-09-21T10:10:13.639671+0000","last_became_peered":"2024-09-21T10:10:13.639671+0000","last_unstale":"2024-09-21T10:11:26.501506+0000","last_undegraded":"2024-09-21T10:11:26.501506+0000","last_fullsized":"2024-09-21T10:11:26.501506+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:53.941605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":1950,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-09-21T10:11:27.554006+0000","last_change":"2024-09-21T09:46:13.273199+0000","last_active":"2024-09-21T10:11:27.554006+0000","last_peered":"2024-09-21T10:11:27.554006+0000","last_clean":"2024-09-21T10:11:27.554006+0000","last_became_active":"2024-09-21T09:46:13.272874+0000","last_became_peered":"2024-09-21T09:46:13.272874+0000","last_unstale":"2024-09-21T10:11:27.554006+0000","last_undegraded":"2024-09-21T10:11:27.554006+0000","last_fullsized":"2024-09-21T10:11:27.554006+0000","mapping_epoch":547,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:16.938057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3437232152,"num_objects":52449,"num_object_clones":0,"num_object_copies":157347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22238,"num_objects_misplaced":2856,"num_objects_unfound":0,"num_objects_dirty":52449,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133555,"num_write_kb":5952065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9542,"num_bytes_recovered":625278976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11779518464,"data_stored":11779506248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82455,"ondisk_log_size":82455,"up":48,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":19400,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34026,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":778212,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":94,"num_read_kb":238,"num_write":47,"num_write_kb":518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":515524,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":210453397948,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594176,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1062752,"kb_used_data":464536,"kb_used_omap":1,"kb_used_meta":598206,"kb_avail":92682400,"statfs":{"total":95995035648,"available":94906777600,"internally_reserved":0,"allocated":475684864,"data_stored":472774313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612563288},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790406,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2897820,"kb_used_data":1995348,"kb_used_omap":1,"kb_used_meta":902462,"kb_avail":90847332,"statfs":{"total":95995035648,"available":93027667968,"internally_reserved":0,"allocated":2043236352,"data_stored":2040326257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":924121432},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1224,"seq":5257039970342,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1324332,"kb_used_data":991964,"kb_used_omap":290,"kb_used_meta":332061,"kb_avail":92420820,"statfs":{"total":95995035648,"available":94638919680,"internally_reserved":0,"allocated":1015771136,"data_stored":1012856536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":297377,"internal_metadata":340031071},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999274,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2852296,"kb_used_data":1996540,"kb_used_omap":25,"kb_used_meta":855718,"kb_avail":90892856,"statfs":{"total":95995035648,"available":93074284544,"internally_reserved":0,"allocated":2044456960,"data_stored":2041562479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":876256199},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111217,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1637204,"kb_used_data":895112,"kb_used_omap":31,"kb_used_meta":742048,"kb_avail":92107948,"statfs":{"total":95995035648,"available":94318538752,"internally_reserved":0,"allocated":916594688,"data_stored":913693107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":759857384},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318223160,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2431428,"kb_used_data":2054524,"kb_used_omap":26,"kb_used_meta":376869,"kb_avail":91313724,"statfs":{"total":95995035648,"available":93505253376,"internally_reserved":0,"allocated":2103832576,"data_stored":2100937456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":385913868},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335103,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1821672,"kb_used_data":1180060,"kb_used_omap":31,"kb_used_meta":641568,"kb_avail":91923480,"statfs":{"total":95995035648,"available":94129643520,"internally_reserved":0,"allocated":1208381440,"data_stored":1205467406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":656965728},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1186922496,"data_stored":1186922496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2080837632,"data_stored":2080833560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":905576448,"data_stored":905576448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2031681536,"data_stored":2031681536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":999100416,"data_stored":999096344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2010255360,"data_stored":2010251288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":461963264,"data_stored":461963264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2103181312,"data_stored":2103181312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:11:39.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_4 pgp_num 16 2024-09-21T10:11:39.451 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.475 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:39.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:39.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:39.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:39.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:39.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:39 smithi017 ceph-mon[102913]: pgmap v1875: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 22238/138936 objects degraded (16.006%); 2106/138936 objects misplaced (1.516%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:11:39.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:39 smithi017 ceph-mon[105151]: pgmap v1875: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 22238/138936 objects degraded (16.006%); 2106/138936 objects misplaced (1.516%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:11:39.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:39.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:39.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:39.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:39.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:39.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:39 smithi134 ceph-mon[90085]: pgmap v1875: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 22238/138936 objects degraded (16.006%); 2106/138936 objects misplaced (1.516%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:11:40.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:40.067 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:40.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:40.169 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:40.193 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:40.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:40.295 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:40.319 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:40.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:40.420 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:40.443 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:40.472 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:40.472 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:11:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:11:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:11:40.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:40.589 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:11:40.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:40.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:40.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:40.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:40.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:40.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:40.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:40.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/788885645' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:40.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:40 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:40.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:40.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/788885645' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:40.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:40 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:40.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:40.985 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/788885645' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:40 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:41.008 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:41.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:41.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:41.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:41.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:41.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:41.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:41.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:41.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:41.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:41.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:41.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:41.561 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:41.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:41.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:41.706 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:41.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:41.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:41.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:41.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:41.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:41.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:41 smithi017 ceph-mon[102913]: pgmap v1876: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 22238/136293 objects degraded (16.316%); 2106/136293 objects misplaced (1.545%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T10:11:41.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:41 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:41.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:41 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:41.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:41 smithi017 ceph-mon[102913]: osdmap e1260: 8 total, 8 up, 7 in 2024-09-21T10:11:41.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:41 smithi017 ceph-mon[105151]: pgmap v1876: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 22238/136293 objects degraded (16.316%); 2106/136293 objects misplaced (1.545%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T10:11:41.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:41 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:41.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:41 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:41.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:41 smithi017 ceph-mon[105151]: osdmap e1260: 8 total, 8 up, 7 in 2024-09-21T10:11:41.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:41.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:41 smithi134 ceph-mon[90085]: pgmap v1876: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 22238/136293 objects degraded (16.316%); 2106/136293 objects misplaced (1.545%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T10:11:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:41 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:41 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:41 smithi134 ceph-mon[90085]: osdmap e1260: 8 total, 8 up, 7 in 2024-09-21T10:11:42.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:42.133 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:42.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:42.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:42.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:42.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:42.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:42.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:42.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:42.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:42.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:42.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:42.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:42.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:42.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:42.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:42.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:42.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:42.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:42.969 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:43.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:43.106 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:43.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:43.208 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:43.231 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:43.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:43.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:43.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:43.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:43.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:43.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:43.623 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:43.647 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:43.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:43 smithi017 ceph-mon[102913]: pgmap v1878: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 22238/136293 objects degraded (16.316%); 2106/136293 objects misplaced (1.545%); 4.7 MiB/s, 74 objects/s recovering 2024-09-21T10:11:43.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:43 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 22238/136293 objects degraded (16.316%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:43 smithi017 ceph-mon[105151]: pgmap v1878: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 22238/136293 objects degraded (16.316%); 2106/136293 objects misplaced (1.545%); 4.7 MiB/s, 74 objects/s recovering 2024-09-21T10:11:43.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:43 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 22238/136293 objects degraded (16.316%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:43.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:43.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:43.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:43.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:43.914 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:43 smithi134 ceph-mon[90085]: pgmap v1878: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 22238/136293 objects degraded (16.316%); 2106/136293 objects misplaced (1.545%); 4.7 MiB/s, 74 objects/s recovering 2024-09-21T10:11:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:43 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 22238/136293 objects degraded (16.316%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:44.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:44.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:44.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:11:44.157 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:44.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:44.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:44.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:44.335 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:44.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:44.437 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:44.460 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:44.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:44.561 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:44.584 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:44.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:44.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:44.721 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:44.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:44.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:44.840 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:11:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:11:44] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.51.0" 2024-09-21T10:11:44.875 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:44.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:44.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:45.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:45.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:45.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:45.168 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:45.190 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:45.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:45.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:45.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:45.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:45.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:45.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:45.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:45.567 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:45.576 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:11:45.576 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:45.586 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:45.608 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:45.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:45.626 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:11:45.626 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:45.659 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:11:45.659 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:45.695 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:11:45.695 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:45.709 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:45.727 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:11:45.727 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:45.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:45.759 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:11:45.759 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:45.759 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:45 smithi017 ceph-mon[102913]: pgmap v1879: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 22238/127785 objects degraded (17.403%); 1681/127785 objects misplaced (1.315%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:11:45.760 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:45 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/197176047' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:45.760 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:45 smithi017 ceph-mon[105151]: pgmap v1879: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 22238/127785 objects degraded (17.403%); 1681/127785 objects misplaced (1.315%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:11:45.760 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:45 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/197176047' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:45.789 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:11:45.789 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:45.818 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:11:45.818 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:45.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:45.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:45.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:45 smithi134 ceph-mon[90085]: pgmap v1879: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 22238/127785 objects degraded (17.403%); 1681/127785 objects misplaced (1.315%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:11:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:45 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/197176047' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:46.019 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:46.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:46.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:46.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:46.262 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:46.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.284 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:46.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:46.385 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:46.409 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:46.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:46.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:46.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:46.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:46.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:46.692 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:46.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:46.817 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:46.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:11:46.919 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:46.943 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:47.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:47.066 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:47.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:47.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:47.313 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:47.338 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:47.440 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:47.461 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:47.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:47.563 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:47.587 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:47.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:47.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:47 smithi017 ceph-mon[102913]: pgmap v1880: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 22238/125706 objects degraded (17.690%); 1681/125706 objects misplaced (1.337%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:11:47.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:47 smithi017 ceph-mon[105151]: pgmap v1880: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 22238/125706 objects degraded (17.690%); 1681/125706 objects misplaced (1.337%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:11:47.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:47.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:47.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:47.842 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:47.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:47.944 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:47.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.969 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:47.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:47 smithi134 ceph-mon[90085]: pgmap v1880: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 22238/125706 objects degraded (17.690%); 1681/125706 objects misplaced (1.337%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T10:11:48.071 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:48.093 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:48.194 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:48.216 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:48.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:48.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:48.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:48.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:48.450 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:48.473 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:48.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:48.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:48.598 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:48.700 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:48.725 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:48.826 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:48.850 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:48 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 22238/125706 objects degraded (17.690%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:48.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:48 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 22238/125706 objects degraded (17.690%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:48.952 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:48.975 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:48.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:48 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 22238/125706 objects degraded (17.690%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:49.077 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:49.103 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:49.205 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:49.229 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:49.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:49.331 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:49.355 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:49.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:49.480 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:49.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:49.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:49.730 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:49.752 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:49.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:49.854 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:49.877 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:49.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:49 smithi017 ceph-mon[102913]: pgmap v1881: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 22258/118194 objects degraded (18.832%); 1224/118194 objects misplaced (1.036%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:11:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:49 smithi017 ceph-mon[105151]: pgmap v1881: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 22258/118194 objects degraded (18.832%); 1224/118194 objects misplaced (1.036%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:11:49.978 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:49 smithi134 ceph-mon[90085]: pgmap v1881: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 22258/118194 objects degraded (18.832%); 1224/118194 objects misplaced (1.036%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:11:50.002 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:50.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:50.103 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:50.127 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:50.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:50.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:50.254 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:50.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:50.356 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:50.380 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:50.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:50.482 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:11:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:11:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:11:50.505 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:50.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:50.606 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:50.630 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:50.727 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:50.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:50.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:50.879 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:50.901 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:51.003 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:51.026 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:51.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:51.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:51.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:51.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:51.277 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:51.300 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:51.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:51.401 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:51.423 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:51.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:51.525 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:51.547 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:51.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:51.649 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:51.672 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:51.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:51.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:51.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.813 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:51.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:51.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:51 smithi017 ceph-mon[102913]: pgmap v1882: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 22258/115686 objects degraded (19.240%); 1224/115686 objects misplaced (1.058%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:11:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:51 smithi017 ceph-mon[105151]: pgmap v1882: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 22258/115686 objects degraded (19.240%); 1224/115686 objects misplaced (1.058%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:11:51.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:51.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:51.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:51 smithi134 ceph-mon[90085]: pgmap v1882: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 22258/115686 objects degraded (19.240%); 1224/115686 objects misplaced (1.058%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T10:11:52.055 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.080 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:52.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:52.182 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:52.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:52.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:52.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:52.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:52.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:52.451 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:52.475 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:52.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:52.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:52.622 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 9 pgp_num to 16 2024-09-21T10:11:52.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:11:52.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:52.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:52.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:52 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3665655025' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:11:52.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:52 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:11:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:52 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3665655025' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:11:52.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:52 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:11:52.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:52.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:52 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3665655025' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:11:52.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:52 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:11:53.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:53.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:53.106 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:53.127 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:53.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:53.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:53.252 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:53.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:53.354 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:53.377 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:53.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:53.479 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:53.502 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:53.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:53.603 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:53.616 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:53 smithi017 ceph-mon[105151]: pgmap v1883: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 22258/115686 objects degraded (19.240%); 1224/115686 objects misplaced (1.058%); 4.6 MiB/s, 72 objects/s recovering 2024-09-21T10:11:53.616 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:53 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-21T10:11:53.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.630 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:53.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:53.731 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:53.754 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:53.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:53.856 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:53.879 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:53.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:53 smithi017 ceph-mon[102913]: pgmap v1883: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 22258/115686 objects degraded (19.240%); 1224/115686 objects misplaced (1.058%); 4.6 MiB/s, 72 objects/s recovering 2024-09-21T10:11:53.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:53 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-21T10:11:53.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:53 smithi017 ceph-mon[102913]: osdmap e1261: 8 total, 8 up, 7 in 2024-09-21T10:11:53.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:53 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 22258/115686 objects degraded (19.240%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:53.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:53 smithi017 ceph-mon[105151]: osdmap e1261: 8 total, 8 up, 7 in 2024-09-21T10:11:53.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:53 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 22258/115686 objects degraded (19.240%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:53.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:53 smithi134 ceph-mon[90085]: pgmap v1883: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 22258/115686 objects degraded (19.240%); 1224/115686 objects misplaced (1.058%); 4.6 MiB/s, 72 objects/s recovering 2024-09-21T10:11:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:53 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-21T10:11:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:53 smithi134 ceph-mon[90085]: osdmap e1261: 8 total, 8 up, 7 in 2024-09-21T10:11:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:53 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 22258/115686 objects degraded (19.240%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:54.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:54.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:54.123 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:54.147 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:54.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:54.248 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:54.272 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:54.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:54.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:54.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:54.517 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:54.539 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:54.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:54.641 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:54.666 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:54.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:54.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:54.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.813 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:54.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:54.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:54.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:55.005 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:11:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:11:54] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.51.0" 2024-09-21T10:11:55.006 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:11:55.006 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:11:55.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:11:55.006 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:11:55.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.006 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263821 8.f 8:f02fefb5:::benchmark_data_smithi017_209235_object130915:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1223)", 2024-09-21T10:11:55.006 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.418315+0000", 2024-09-21T10:11:55.006 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 210.58343166899999, 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2255271790000002, 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263821 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418315+0000", 2024-09-21T10:11:55.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418315+0000", 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418317+0000", 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.468e-06 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418321+0000", 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7570000000000004e-06 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418333+0000", 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1931e-05 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418341+0000", 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5329999999999997e-06 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418739+0000", 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00039836800000000002 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418859+0000", 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119894 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:11:55.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.419410+0000", 2024-09-21T10:11:55.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00055092599999999998 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.423594+0000", 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041834790000000004 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.436988+0000", 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013394368 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.437000+0000", 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2503999999999999e-05 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.428163+0000", 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99116300300000004 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.428175+0000", 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1568e-05 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.643733+0000", 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2155582730000001 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.643842+0000", 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109107 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.016 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263824 8.f 8:fed575a8:::benchmark_data_smithi017_209235_object130918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1223)", 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.424585+0000", 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 210.57716232199999, 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2195379640000001, 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263824 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424585+0000", 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424585+0000", 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424587+0000", 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0449999999999999e-06 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424592+0000", 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4709999999999998e-06 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424613+0000", 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1331e-05 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424622+0000", 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3769999999999996e-06 2024-09-21T10:11:55.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424765+0000", 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014287600000000001 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424915+0000", 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015034299999999999 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.425037+0000", 2024-09-21T10:11:55.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012184399999999999 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.437031+0000", 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011994461999999999 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.437040+0000", 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7909999999999993e-06 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.428208+0000", 2024-09-21T10:11:55.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99116771800000003 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.428217+0000", 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1079999999999999e-06 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.644076+0000", 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.215859064 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.644123+0000", 2024-09-21T10:11:55.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6533999999999999e-05 2024-09-21T10:11:55.023 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.023 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.023 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.023 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263830 8.b 8:dc611968:::benchmark_data_smithi017_209235_object130924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.441220+0000", 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 210.56052679800001, 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1669228899999999, 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263830 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441220+0000", 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441220+0000", 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441222+0000", 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.762e-06 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441227+0000", 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1259999999999997e-06 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441241+0000", 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3661e-05 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441249+0000", 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.422e-06 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441313+0000", 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3777999999999998e-05 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441325+0000", 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2349e-05 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.426634+0000", 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98530858499999996 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.426652+0000", 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8334000000000001e-05 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.595823+0000", 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.169171245 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.595979+0000", 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015592700000000001 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.596130+0000", 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015037399999999999 2024-09-21T10:11:55.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.599707+0000", 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035773689999999999 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.603014+0000", 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033070360000000002 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.606794+0000", 2024-09-21T10:11:55.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003779691 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608021+0000", 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001227676 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608143+0000", 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121555 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263835 8.b 8:d7eca9d4:::benchmark_data_smithi017_209235_object130929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.454536+0000", 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 210.547210654, 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1538747279999999, 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263835 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.032 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454536+0000", 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454536+0000", 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454538+0000", 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0049999999999999e-06 2024-09-21T10:11:55.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454544+0000", 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4260000000000002e-06 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454558+0000", 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4073000000000001e-05 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454564+0000", 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2480000000000001e-06 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454623+0000", 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8721999999999998e-05 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454631+0000", 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2749999999999995e-06 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.427335+0000", 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97270446799999999 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.427346+0000", 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0023999999999999e-05 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.599745+0000", 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1723999119999999 2024-09-21T10:11:55.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.599872+0000", 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126871 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.600047+0000", 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017517100000000001 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608161+0000", 2024-09-21T10:11:55.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081138900000000003 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608210+0000", 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9044e-05 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608261+0000", 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0562000000000001e-05 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608317+0000", 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5630000000000001e-05 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608411+0000", 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4407000000000006e-05 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263838 8.b 8:d9aed5fa:::benchmark_data_smithi017_209235_object130932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.456576+0000", 2024-09-21T10:11:55.039 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 210.54517142, 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1539254689999998, 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263838 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456576+0000", 2024-09-21T10:11:55.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456576+0000", 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456577+0000", 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.251e-06 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456582+0000", 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7790000000000003e-06 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456594+0000", 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2867000000000001e-05 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456602+0000", 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9640000000000003e-06 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456677+0000", 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4278999999999998e-05 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456686+0000", 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4469999999999995e-06 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.427371+0000", 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97068533099999998 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.427380+0000", 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3790000000000006e-06 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.603501+0000", 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1761207929999999 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.603651+0000", 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015053700000000001 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.603792+0000", 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014081200000000001 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608224+0000", 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044316620000000003 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608466+0000", 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024204399999999999 2024-09-21T10:11:55.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.610334+0000", 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001868607 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.610399+0000", 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4605999999999998e-05 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.610501+0000", 2024-09-21T10:11:55.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102111 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57258 9.c 9:337f61f9:::benchmark_data_smithi017_237924_object57257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.412546+0000", 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 150.589200809, 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2051300349999998, 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.047 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57258 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412546+0000", 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412546+0000", 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412548+0000", 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.654e-06 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412567+0000", 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9585999999999998e-05 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412580+0000", 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2644e-05 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412588+0000", 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.746e-06 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.414696+0000", 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021078949999999998 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.414847+0000", 2024-09-21T10:11:55.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000151731 2024-09-21T10:11:55.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:11:55.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.414992+0000", 2024-09-21T10:11:55.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144349 2024-09-21T10:11:55.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.421028+0000", 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060366220000000002 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450408+0000", 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029379563000000001 2024-09-21T10:11:55.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450432+0000", 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.393e-05 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.422399+0000", 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.971966679 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.422410+0000", 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1615e-05 2024-09-21T10:11:55.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.617503+0000", 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1950931549999999 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.617676+0000", 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017286600000000001 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57263 9.c 9:35231ded:::benchmark_data_smithi017_237924_object57262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.415713+0000", 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 150.586034391, 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2043443279999999, 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57263 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.055 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415713+0000", 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415713+0000", 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415714+0000", 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3149999999999999e-06 2024-09-21T10:11:55.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415737+0000", 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3323999999999999e-05 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415748+0000", 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0557000000000001e-05 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415755+0000", 2024-09-21T10:11:55.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3209999999999999e-06 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.417937+0000", 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021822230000000001 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.418089+0000", 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015200700000000001 2024-09-21T10:11:55.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.418233+0000", 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014391100000000001 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.421035+0000", 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002802157 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450466+0000", 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029430464999999999 2024-09-21T10:11:55.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450474+0000", 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3310000000000004e-06 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.422444+0000", 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97196981699999996 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.422453+0000", 2024-09-21T10:11:55.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3000000000000007e-06 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.619820+0000", 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1973671800000001 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.620057+0000", 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023641999999999999 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57272 9.c 9:3bb2c359:::benchmark_data_smithi017_237924_object57271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.424002+0000", 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 150.577745142, 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1984504490000001, 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57272 2024-09-21T10:11:55.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424002+0000", 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424002+0000", 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424004+0000", 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9489999999999999e-06 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424030+0000", 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6444999999999998e-05 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424049+0000", 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8681999999999998e-05 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424058+0000", 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1819999999999995e-06 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424142+0000", 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3628999999999995e-05 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424420+0000", 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027829800000000001 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424592+0000", 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000171931 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450503+0000", 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025911338999999999 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450511+0000", 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9719999999999992e-06 2024-09-21T10:11:55.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.424670+0000", 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97415910100000003 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.424683+0000", 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2357e-05 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.620183+0000", 2024-09-21T10:11:55.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1954999120000001 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.622452+0000", 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002269652 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57278 9.c 9:32156efe:::benchmark_data_smithi017_237924_object57277:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.439362+0000", 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 150.56238453099999, 2024-09-21T10:11:55.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.195950984, 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57278 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439362+0000", 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439362+0000", 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439364+0000", 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2720000000000001e-06 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439395+0000", 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1324e-05 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439408+0000", 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2524000000000001e-05 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439416+0000", 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9730000000000003e-06 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450534+0000", 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011117392 2024-09-21T10:11:55.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450541+0000", 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5429999999999996e-06 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.424766+0000", 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97422412000000003 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.424774+0000", 2024-09-21T10:11:55.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9139999999999997e-06 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.622588+0000", 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1978139999999999 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.622840+0000", 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000251473 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.623023+0000", 2024-09-21T10:11:55.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018283899999999999 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.627435+0000", 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044125550000000003 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.631224+0000", 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037888930000000002 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.632444+0000", 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001219695 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.632578+0000", 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013418999999999999 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.635313+0000", 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027352769999999999 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.075 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57268 9.3 9:ce6a408f:::benchmark_data_smithi017_237924_object57267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.439429+0000", 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 150.562318221, 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2849673930000001, 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57268 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.076 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439429+0000", 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439429+0000", 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439429+0000", 2024-09-21T10:11:55.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.82e-07 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439727+0000", 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029737900000000001 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439741+0000", 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4508999999999999e-05 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439748+0000", 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2389999999999996e-06 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.440593+0000", 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00084440299999999997 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.440602+0000", 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4879999999999996e-06 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.421006+0000", 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98040421300000002 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.421015+0000", 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8769999999999999e-06 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.651013+0000", 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2299973500000001 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.651585+0000", 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00057230499999999995 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.724307+0000", 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.072722014000000001 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.724396+0000", 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9134000000000004e-05 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.081 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57280 9.7 9:e169e579:::benchmark_data_smithi017_237924_object57279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.439765+0000", 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 150.56198225099999, 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.353750952, 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.082 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57280 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439765+0000", 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439765+0000", 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439765+0000", 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4099999999999998e-07 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439789+0000", 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3578999999999999e-05 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439796+0000", 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5480000000000004e-06 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439802+0000", 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4709999999999998e-06 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.440744+0000", 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00094240600000000002 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.440755+0000", 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0886e-05 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.423951+0000", 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.983195504 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.423961+0000", 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9450000000000005e-06 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.768747+0000", 2024-09-21T10:11:55.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.344786491 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.770629+0000", 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001881364 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.772941+0000", 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023123430000000001 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.785246+0000", 2024-09-21T10:11:55.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012304864 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.788330+0000", 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030839629999999999 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.791240+0000", 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002909867 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.791286+0000", 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6421000000000002e-05 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.793516+0000", 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022296590000000002 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57290 9.7 9:eed5bd25:::benchmark_data_smithi017_237924_object57289:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.459346+0000", 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 150.54240089800001, 2024-09-21T10:11:55.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.339554245, 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57290 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459346+0000", 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459346+0000", 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459347+0000", 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.421e-06 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459376+0000", 2024-09-21T10:11:55.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8235e-05 2024-09-21T10:11:55.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459386+0000", 2024-09-21T10:11:55.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:55.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8360000000000004e-06 2024-09-21T10:11:55.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459392+0000", 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1260000000000003e-06 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459552+0000", 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016084000000000001 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459572+0000", 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9230999999999999e-05 2024-09-21T10:11:55.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.423986+0000", 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96441438400000001 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.423994+0000", 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2779999999999993e-06 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.777718+0000", 2024-09-21T10:11:55.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3537239160000001 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.777871+0000", 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000152242 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.778023+0000", 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000151989 2024-09-21T10:11:55.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.788345+0000", 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010322964 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.788384+0000", 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8717999999999998e-05 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.796197+0000", 2024-09-21T10:11:55.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0078130860000000003 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.796402+0000", 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020457 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.798900+0000", 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002498409 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57295 9.1 9:8fc19906:::benchmark_data_smithi017_237924_object57294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.464232+0000", 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 150.53751524399999, 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.230825754, 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57295 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.099 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464232+0000", 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464232+0000", 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464234+0000", 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8700000000000001e-06 2024-09-21T10:11:55.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464255+0000", 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1012000000000001e-05 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464272+0000", 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7889000000000001e-05 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464281+0000", 2024-09-21T10:11:55.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7649999999999999e-06 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464371+0000", 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9474000000000004e-05 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464384+0000", 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3433e-05 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.420648+0000", 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95626375299999999 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.420659+0000", 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1449e-05 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.685864+0000", 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.265204947 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.686055+0000", 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000190352 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.686232+0000", 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000177193 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.690565+0000", 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043328280000000004 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.690589+0000", 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4719000000000002e-05 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.692160+0000", 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001570335 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.692225+0000", 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5074999999999996e-05 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.695057+0000", 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028326599999999999 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:79090 9.c 9:361bee82:::benchmark_data_smithi017_237924_object79089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:02.332882+0000", 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 112.66886541700001, 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.441358203, 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.106 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 79090 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.332882+0000", 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.332882+0000", 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.332883+0000", 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.269e-06 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.332906+0000", 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3053000000000001e-05 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.332915+0000", 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5459999999999997e-06 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.332922+0000", 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.878e-06 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.334093+0000", 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011710220000000001 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.334257+0000", 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000164048 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.334443+0000", 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018556099999999999 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.346041+0000", 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011598249999999999 2024-09-21T10:11:55.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.346220+0000", 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000178887 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.771821+0000", 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.42560141899999998 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.771887+0000", 2024-09-21T10:11:55.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5184000000000005e-05 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.774240+0000", 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023530859999999999 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:79091 9.c 9:3c72d25c:::benchmark_data_smithi017_237924_object79090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:02.337619+0000", 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 112.66412744199999, 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.43902102799999998, 2024-09-21T10:11:55.112 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 79091 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.337619+0000", 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.337619+0000", 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.337621+0000", 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0020000000000001e-06 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.337827+0000", 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020592599999999999 2024-09-21T10:11:55.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.337843+0000", 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5752000000000001e-05 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.337853+0000", 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7350000000000008e-06 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.338685+0000", 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00083209199999999999 2024-09-21T10:11:55.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.338881+0000", 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000196138 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.339181+0000", 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030004400000000002 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.346274+0000", 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070932670000000003 2024-09-21T10:11:55.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.346347+0000", 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2263999999999998e-05 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.774344+0000", 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.42799749599999998 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.774404+0000", 2024-09-21T10:11:55.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9630999999999999e-05 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.776641+0000", 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022366809999999999 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:79092 9.c 9:3f68df63:::benchmark_data_smithi017_237924_object79091:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:02.338418+0000", 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 112.663328523, 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.440559859, 2024-09-21T10:11:55.118 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 79092 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.338418+0000", 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.338418+0000", 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.338420+0000", 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4249999999999999e-06 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.338449+0000", 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9289e-05 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.338461+0000", 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1792e-05 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.338469+0000", 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6009999999999997e-06 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.342815+0000", 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043467590000000004 2024-09-21T10:11:55.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.342984+0000", 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016830999999999999 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.343164+0000", 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000180192 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.346330+0000", 2024-09-21T10:11:55.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031661699999999998 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.346353+0000", 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2948000000000001e-05 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.776691+0000", 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.43033764800000002 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.776741+0000", 2024-09-21T10:11:55.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0049999999999997e-05 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:02.778978+0000", 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022376750000000002 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:98975 9.1 9:8a29331c:::benchmark_data_smithi017_237924_object5971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1254)", 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:35.765931+0000", 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 79.235816258, 2024-09-21T10:11:55.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.29428406699999998, 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 98975 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.765931+0000", 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.765931+0000", 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.765932+0000", 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4169999999999999e-06 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.765937+0000", 2024-09-21T10:11:55.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8590000000000004e-06 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.765947+0000", 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9299999999999998e-06 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.765954+0000", 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.255e-06 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.948910+0000", 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.182955492 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.949465+0000", 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00055577700000000003 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.969195+0000", 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019729211999999999 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.969321+0000", 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012613899999999999 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.969452+0000", 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013121399999999999 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.988624+0000", 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019171555 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.988671+0000", 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7681e-05 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.053680+0000", 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.065008791999999996 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.053746+0000", 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5807000000000002e-05 2024-09-21T10:11:55.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.060215+0000", 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064689370000000001 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:98995 9.1 9:824ebdc4:::benchmark_data_smithi017_237924_object5991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1254)", 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:35.800512+0000", 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 79.201234885999995, 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26422749200000001, 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.131 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 98995 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800512+0000", 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800512+0000", 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800513+0000", 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4079999999999999e-06 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800519+0000", 2024-09-21T10:11:55.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1290000000000004e-06 2024-09-21T10:11:55.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800531+0000", 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2651e-05 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800539+0000", 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.311e-06 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.949516+0000", 2024-09-21T10:11:55.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.148977785 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.950031+0000", 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00051417100000000001 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.985798+0000", 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035767357999999999 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.985904+0000", 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106198 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.986116+0000", 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000211468 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.038043+0000", 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.051927593000000001 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.060270+0000", 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022227064000000001 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.060322+0000", 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2167999999999999e-05 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.060423+0000", 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100259 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.064740+0000", 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043169289999999997 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:98996 9.1 9:8cb57ec8:::benchmark_data_smithi017_237924_object5992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1254)", 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:35.800562+0000", 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 79.201184686000005, 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26933248900000001, 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.139 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 98996 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800562+0000", 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800562+0000", 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800563+0000", 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6699999999999999e-07 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800565+0000", 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3010000000000001e-06 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800571+0000", 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2910000000000004e-06 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.800575+0000", 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.658e-06 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.950064+0000", 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.149489234 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.950417+0000", 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035320600000000001 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.996065+0000", 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.045647957000000003 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.996157+0000", 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2032000000000002e-05 2024-09-21T10:11:55.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.996306+0000", 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014835799999999999 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.042629+0000", 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.046323419999999997 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.064787+0000", 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022157909 2024-09-21T10:11:55.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.064823+0000", 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5500000000000002e-05 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.066048+0000", 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012258530000000001 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.069895+0000", 2024-09-21T10:11:55.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038463030000000001 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:98998 9.1 9:808cd8fd:::benchmark_data_smithi017_237924_object5994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1254)", 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:35.801989+0000", 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 79.199758150999997, 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27181077300000001, 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:11:55.146 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 98998 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.801989+0000", 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.801989+0000", 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:11:55.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.801990+0000", 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6700000000000001e-06 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.801995+0000", 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8439999999999997e-06 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.802007+0000", 2024-09-21T10:11:55.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1884e-05 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.802014+0000", 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.426e-06 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.950497+0000", 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14848315600000001 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:11:55.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:35.950848+0000", 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035074999999999999 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.010124+0000", 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.059276258999999998 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.010219+0000", 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5027999999999999e-05 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.010357+0000", 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013773799999999999 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.042637+0000", 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.032280548999999999 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.069952+0000", 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027314615 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.069997+0000", 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5354e-05 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.070055+0000", 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7528999999999998e-05 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:36.073800+0000", 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037449710000000002 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.152 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.153 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:11:55.153 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:11:55.153 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:11:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.153 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:55.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:55.254 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:55.277 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:55.379 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:55.403 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:55.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:55.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:55.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:55.649 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:55.673 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:55.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:55.774 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:55.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:55.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:55.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:55 smithi017 ceph-mon[102913]: pgmap v1885: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 22179/106974 objects degraded (20.733%); 740/106974 objects misplaced (0.692%); 5.8 MiB/s, 99 objects/s recovering 2024-09-21T10:11:55.862 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:55.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:55 smithi017 ceph-mon[105151]: pgmap v1885: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 22179/106974 objects degraded (20.733%); 740/106974 objects misplaced (0.692%); 5.8 MiB/s, 99 objects/s recovering 2024-09-21T10:11:55.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:55.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:55.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:55.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:55 smithi134 ceph-mon[90085]: pgmap v1885: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 22179/106974 objects degraded (20.733%); 740/106974 objects misplaced (0.692%); 5.8 MiB/s, 99 objects/s recovering 2024-09-21T10:11:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:56.023 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:56.047 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:56.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:56.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:56.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:56.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:56.231 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:56.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:56.314 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:56.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:56.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:56.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:56.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:56.562 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:56.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:56.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:56.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:56.711 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:56.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:56.722 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:11:56.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:56.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:56.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:56.939 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:56.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:57.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:57.090 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:57.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:57.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:57.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:57.267 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:57.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:57.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:57.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:57.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:57.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:57.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:57.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:57.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:57.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:57.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:57.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:57.839 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:57.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:57.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:57 smithi017 ceph-mon[102913]: pgmap v1886: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 22179/104751 objects degraded (21.173%); 740/104751 objects misplaced (0.706%); 5.8 MiB/s, 99 objects/s recovering 2024-09-21T10:11:57.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:57 smithi017 ceph-mon[105151]: pgmap v1886: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 22179/104751 objects degraded (21.173%); 740/104751 objects misplaced (0.706%); 5.8 MiB/s, 99 objects/s recovering 2024-09-21T10:11:57.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:57.963 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:57.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:57 smithi134 ceph-mon[90085]: pgmap v1886: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 22179/104751 objects degraded (21.173%); 740/104751 objects misplaced (0.706%); 5.8 MiB/s, 99 objects/s recovering 2024-09-21T10:11:58.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:58.106 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:58.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:58.208 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:58.232 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:58.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:58.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:58.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.389 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:58.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:58.490 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:58.518 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:58.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:58.619 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:58.643 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:11:58.646 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:58.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:58 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1520299312' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:11:58.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:58 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 22179/104751 objects degraded (21.173%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:58.749 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:58.773 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:58.875 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:58.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:58 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1520299312' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:11:58.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:58 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 22179/104751 objects degraded (21.173%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:58.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:58 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1520299312' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:11:58.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:58 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 22179/104751 objects degraded (21.173%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:11:58.899 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:58.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:59.002 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:59.027 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:59.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:59.129 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:59.153 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:59.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:59.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:155471 9.7 e1262/1245 9:e5c98e1a:::benchmark_data_smithi017_237924_object62467:head v 1262'9865, mlcod=0'0)", 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:11:59.177708+0000", 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.001817482, 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00184286, 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:11:59.257 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.177708+0000", 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.177708+0000", 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.177707+0000", 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:59.258 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.177713+0000", 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8549999999999998e-06 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.177724+0000", 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1296e-05 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.177730+0000", 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7919999999999998e-06 2024-09-21T10:11:59.259 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.177775+0000", 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4354000000000003e-05 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.177803+0000", 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.8124999999999999e-05 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.260 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:155477 9.b e1262/1252 9:d3f90699:::benchmark_data_smithi017_237924_object62473:head v 1262'9881, mlcod=1247'4555)", 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:11:59.179325+0000", 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.00020041500000000001, 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00028700399999999999, 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179325+0000", 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.261 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179325+0000", 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179324+0000", 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179338+0000", 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.358e-05 2024-09-21T10:11:59.262 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179352+0000", 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4487999999999999e-05 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179358+0000", 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2659999999999998e-06 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179415+0000", 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6363999999999998e-05 2024-09-21T10:11:59.263 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179437+0000", 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1951e-05 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:155472 9.5 e1262/1245 9:a4e6327c:::benchmark_data_smithi017_237924_object62468:head v 1262'9788, mlcod=1242'1)", 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:11:59.179375+0000", 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.00015064500000000001, 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000289122, 2024-09-21T10:11:59.264 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179375+0000", 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179375+0000", 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.265 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179374+0000", 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179381+0000", 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6760000000000001e-06 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179391+0000", 2024-09-21T10:11:59.266 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.9960000000000005e-06 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179395+0000", 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4059999999999998e-06 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179452+0000", 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6078000000000001e-05 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:11:59.267 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:59.179473+0000", 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1801e-05 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout: ], 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 3 2024-09-21T10:11:59.268 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:11:59.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:59.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:11:59.300 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:59.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:59.345 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:11:59.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:59.389 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:11:59.389 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:59.402 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:59.423 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:11:59.423 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:59.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:59.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:59.456 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:11:59.456 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:59.487 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:11:59.487 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:59.512 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:11:59.512 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:59.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:59.542 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:11:59.542 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:59.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:59.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:59.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[102913]: pgmap v1887: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 21868/96228 objects degraded (22.725%); 406/96228 objects misplaced (0.422%); 5.1 MiB/s, 120 objects/s recovering 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1520299312' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[102913]: osdmap e1262: 8 total, 8 up, 7 in 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/502163683' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[105151]: pgmap v1887: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 21868/96228 objects degraded (22.725%); 406/96228 objects misplaced (0.422%); 5.1 MiB/s, 120 objects/s recovering 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1520299312' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[105151]: osdmap e1262: 8 total, 8 up, 7 in 2024-09-21T10:11:59.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:11:59 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/502163683' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:59.719 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:59.766 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:11:59.766 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:11:59.766 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-21T10:11:59.766 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:11:59.766 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:11:59.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-21T10:11:59.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:59.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:11:59.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:59.962 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:59.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:11:59.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:59 smithi134 ceph-mon[90085]: pgmap v1887: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 21868/96228 objects degraded (22.725%); 406/96228 objects misplaced (0.422%); 5.1 MiB/s, 120 objects/s recovering 2024-09-21T10:11:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:59 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:59 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1520299312' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:11:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:59 smithi134 ceph-mon[90085]: osdmap e1262: 8 total, 8 up, 7 in 2024-09-21T10:11:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:11:59 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/502163683' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:00.023 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:12:00.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:00.130 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:00.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:00.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:00.259 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:00.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:00.383 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:00.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:00.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:12:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:12:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:12:00.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:00.528 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:00.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:00.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:00.653 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:00.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:00.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:00.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:00.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:00.907 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:00.933 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:01.035 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:01.060 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:01.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:01.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:01.179 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:01.185 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:01.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:01.310 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:01.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:01.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:01.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:01.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:01.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:01.601 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:01.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:01.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:01.724 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:01.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:01.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:01 smithi134 ceph-mon[90085]: pgmap v1889: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 923 op/s; 21868/93510 objects degraded (23.386%); 406/93510 objects misplaced (0.434%); 6.4 MiB/s, 150 objects/s recovering 2024-09-21T10:12:01.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:01.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:01.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:01.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:01 smithi017 ceph-mon[102913]: pgmap v1889: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 923 op/s; 21868/93510 objects degraded (23.386%); 406/93510 objects misplaced (0.434%); 6.4 MiB/s, 150 objects/s recovering 2024-09-21T10:12:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:01 smithi017 ceph-mon[105151]: pgmap v1889: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 923 op/s; 21868/93510 objects degraded (23.386%); 406/93510 objects misplaced (0.434%); 6.4 MiB/s, 150 objects/s recovering 2024-09-21T10:12:01.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:02.001 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:02.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:02.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:02.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:02.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:02.270 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:02.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:02.390 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:02.390 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:02.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:12:02.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:02.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:02.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:02.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.560 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:02.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:02.660 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:02.684 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:02.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:02.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:02.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:02.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:02.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:02.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:02.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:03.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:03.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:03.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:03.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:12:03.214 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:03.236 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:03.295 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:12:03.295 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "" 2024-09-21T10:12:03.295 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:12:03.317 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:03.335 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:12:03.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:03.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:03.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:03.494 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:03.518 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:03.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:03.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:03.624 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:03.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:03.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:03.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:03.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:03.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:03.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:03.917 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:03 smithi134 ceph-mon[90085]: pgmap v1890: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 21868/93510 objects degraded (23.386%); 406/93510 objects misplaced (0.434%); 5.2 MiB/s, 123 objects/s recovering 2024-09-21T10:12:03.917 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:03 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 21868/93510 objects degraded (23.386%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:03.917 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:03 smithi134 ceph-mon[90085]: osdmap e1263: 8 total, 8 up, 7 in 2024-09-21T10:12:03.935 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:03.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:03 smithi017 ceph-mon[102913]: pgmap v1890: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 21868/93510 objects degraded (23.386%); 406/93510 objects misplaced (0.434%); 5.2 MiB/s, 123 objects/s recovering 2024-09-21T10:12:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:03 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 21868/93510 objects degraded (23.386%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:03 smithi017 ceph-mon[102913]: osdmap e1263: 8 total, 8 up, 7 in 2024-09-21T10:12:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:03 smithi017 ceph-mon[105151]: pgmap v1890: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 21868/93510 objects degraded (23.386%); 406/93510 objects misplaced (0.434%); 5.2 MiB/s, 123 objects/s recovering 2024-09-21T10:12:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:03 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 21868/93510 objects degraded (23.386%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:03 smithi017 ceph-mon[105151]: osdmap e1263: 8 total, 8 up, 7 in 2024-09-21T10:12:04.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:04.060 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:04.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:04.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:04.183 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:04.285 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:04.305 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:04.407 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:04.428 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:04.530 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:04.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:04.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:04.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:04.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:04.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:04.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:04.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:04.896 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:12:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:12:04] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-09-21T10:12:04.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:05.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:05.104 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:05.107 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:05.132 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:05.234 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:05.256 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:05.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:05.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:05.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:05.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:05.506 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:05.531 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:05.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:05.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:05.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:05.691 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:05 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:12:05.692 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:05 smithi134 ceph-mon[90085]: pgmap v1892: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 21537/83892 objects degraded (25.672%); 5.8 MiB/s, 172 objects/s recovering 2024-09-21T10:12:05.692 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:05 smithi134 ceph-mon[90085]: osdmap e1264: 8 total, 8 up, 7 in 2024-09-21T10:12:05.697 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:05.697 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:05.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:05.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:05.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:05.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:05.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:05 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:12:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:05 smithi017 ceph-mon[102913]: pgmap v1892: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 21537/83892 objects degraded (25.672%); 5.8 MiB/s, 172 objects/s recovering 2024-09-21T10:12:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:05 smithi017 ceph-mon[102913]: osdmap e1264: 8 total, 8 up, 7 in 2024-09-21T10:12:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:05 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:12:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:05 smithi017 ceph-mon[105151]: pgmap v1892: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 868 op/s; 21537/83892 objects degraded (25.672%); 5.8 MiB/s, 172 objects/s recovering 2024-09-21T10:12:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:05 smithi017 ceph-mon[105151]: osdmap e1264: 8 total, 8 up, 7 in 2024-09-21T10:12:06.046 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:06.074 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:06.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:06.176 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:06.203 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:06.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:06.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:06.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:06.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:06.503 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:06.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:06.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:06.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.652 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:06.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:06 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:12:06.753 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:06.778 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:06.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:06.880 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:06.903 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:06 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:12:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:06 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:12:07.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:07.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:07.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:07.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:07.169 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:12:07.192 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:07.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:07.294 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:07.319 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:07.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:07.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:07.466 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:07.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:07.502 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:07.568 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:07.597 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:07.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:07.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:07.731 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:07.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:07.847 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:07 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:12:07.847 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:07 smithi134 ceph-mon[90085]: pgmap v1894: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 21537/82050 objects degraded (26.249%); 3.3 MiB/s, 94 objects/s recovering 2024-09-21T10:12:07.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:07.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:07 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:12:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:07 smithi017 ceph-mon[102913]: pgmap v1894: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 21537/82050 objects degraded (26.249%); 3.3 MiB/s, 94 objects/s recovering 2024-09-21T10:12:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:07 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:12:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:07 smithi017 ceph-mon[105151]: pgmap v1894: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 21537/82050 objects degraded (26.249%); 3.3 MiB/s, 94 objects/s recovering 2024-09-21T10:12:07.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:08.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:08.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:08.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:08.165 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:08.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:08.266 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:08.292 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:08.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:08.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:08.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:08.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:08.539 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:08.562 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:08.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:08.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:08.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:08.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:08 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 21537/82050 objects degraded (26.249%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:08.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:08.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:08.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:08 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 21537/82050 objects degraded (26.249%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:08.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:08 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 21537/82050 objects degraded (26.249%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:08.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:09.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:09.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:09.104 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:09.129 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:09.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:09.231 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:09.253 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:09.337 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:12:09.337 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:12:09.337 INFO:tasks.thrashosds.thrasher:Adding osd 7 2024-09-21T10:12:09.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd in 7 2024-09-21T10:12:09.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:09.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:09.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:09.435 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:09.435 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:09.530 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:09.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:09.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:09.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:09.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:09.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:09 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:12:09.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:09 smithi017 ceph-mon[102913]: pgmap v1895: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 21012/74739 objects degraded (28.114%); 3.2 MiB/s, 157 objects/s recovering 2024-09-21T10:12:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:09 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:12:09.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:09 smithi017 ceph-mon[105151]: pgmap v1895: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 21012/74739 objects degraded (28.114%); 3.2 MiB/s, 157 objects/s recovering 2024-09-21T10:12:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:09 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:12:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:09 smithi134 ceph-mon[90085]: pgmap v1895: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 781 op/s; 21012/74739 objects degraded (28.114%); 3.2 MiB/s, 157 objects/s recovering 2024-09-21T10:12:09.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:09.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:09.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:09.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:09.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:10.048 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:10.073 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:10.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:10.173 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:12:10.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:10.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:10.299 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:10.324 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:10.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:10.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:10.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:10.478 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:12:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:12:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:12:10.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:10.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:10.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:10.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:10.743 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:10.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:10.845 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:10.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.872 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:10.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:10.937 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:10.974 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:10.995 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:10.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:11.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:11.141 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:11.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:11.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:11.267 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:11.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:11.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:11.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:11.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:11.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:11.542 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:11.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:11.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:11.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:11.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:11.734 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:11 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:12:11.735 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:11 smithi134 ceph-mon[90085]: pgmap v1896: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 898 op/s; 21012/71937 objects degraded (29.209%); 3.2 MiB/s, 157 objects/s recovering 2024-09-21T10:12:11.789 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:11.815 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:11.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:11.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:11 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:12:11.851 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:11 smithi017 ceph-mon[102913]: pgmap v1896: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 898 op/s; 21012/71937 objects degraded (29.209%); 3.2 MiB/s, 157 objects/s recovering 2024-09-21T10:12:11.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:11 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:12:11.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:11 smithi017 ceph-mon[105151]: pgmap v1896: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 898 op/s; 21012/71937 objects degraded (29.209%); 3.2 MiB/s, 157 objects/s recovering 2024-09-21T10:12:11.917 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:11.941 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:11.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:12.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:12.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:12.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:12.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:12.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:12.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:12.322 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:12.348 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:12.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:12.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:12.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:12.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:12.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:12.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:12.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:12.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:12.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:12 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:12:12.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.783 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:12.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:12.884 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:12.909 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:12.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:12 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:12:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:12 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:12:13.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:13.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:13.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:13.154 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:13.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:13.254 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:13.255 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:13.281 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:13.304 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:13.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:13.406 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:13.429 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:13.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:13.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:13.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:13.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:13.672 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:13.696 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:13.798 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:13.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.822 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:13.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:13.895 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:13 smithi134 ceph-mon[90085]: pgmap v1897: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 21012/71937 objects degraded (29.209%); 0 B/s, 59 objects/s recovering 2024-09-21T10:12:13.896 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:13 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 21012/71937 objects degraded (29.209%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:13.923 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:13.947 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:13.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:13 smithi017 ceph-mon[102913]: pgmap v1897: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 21012/71937 objects degraded (29.209%); 0 B/s, 59 objects/s recovering 2024-09-21T10:12:13.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:13 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 21012/71937 objects degraded (29.209%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:13 smithi017 ceph-mon[105151]: pgmap v1897: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 21012/71937 objects degraded (29.209%); 0 B/s, 59 objects/s recovering 2024-09-21T10:12:13.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:13 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 21012/71937 objects degraded (29.209%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:14.049 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:14.074 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:14.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:14.175 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:14.202 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:14.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:14.238 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:12:14.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:14.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:14.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:14.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:14.472 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:14.574 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:14.580 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:14.603 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:14.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:14.705 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:14.729 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:14.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:14.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:14.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:14.868 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:14 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:12:14.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:14 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:12:14.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:12:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:12:14] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.51.0" 2024-09-21T10:12:14.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:14 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:12:14.963 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:14.989 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:15.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:15.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:15.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:15.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:15.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:15.383 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:15.408 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:15.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:15.510 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:15.534 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:15.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:15.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:15.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:15.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:15.773 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:15.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:15.849 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:15 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:12:15.849 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:15 smithi134 ceph-mon[90085]: pgmap v1898: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.3 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 20593/62793 objects degraded (32.795%); 0 B/s, 94 objects/s recovering 2024-09-21T10:12:15.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:15.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:15.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:15 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:12:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:15 smithi017 ceph-mon[102913]: pgmap v1898: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.3 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 20593/62793 objects degraded (32.795%); 0 B/s, 94 objects/s recovering 2024-09-21T10:12:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:15 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:12:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:15 smithi017 ceph-mon[105151]: pgmap v1898: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.3 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 20593/62793 objects degraded (32.795%); 0 B/s, 94 objects/s recovering 2024-09-21T10:12:16.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:16.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:16.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:16.183 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:16.211 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:16.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:16.335 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:16.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:16.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:16.486 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:16.486 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:16.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:16.608 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:16.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:16.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:16.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:16 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:12:16.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:16.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:16.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:16.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:16.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:16 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:12:16.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:16 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:12:16.994 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:17.019 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:17.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:17.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:17.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:17.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:17.256 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:12:17.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:17.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:17.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:17.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:17.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:17.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:17.550 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:17.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:17.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:17.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:17.706 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:17.731 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:17.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:17.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:17.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:17.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:17.885 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:17 smithi134 ceph-mon[90085]: pgmap v1899: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.2 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 20593/60831 objects degraded (33.853%); 0 B/s, 79 objects/s recovering 2024-09-21T10:12:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:17 smithi017 ceph-mon[102913]: pgmap v1899: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.2 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 20593/60831 objects degraded (33.853%); 0 B/s, 79 objects/s recovering 2024-09-21T10:12:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:17 smithi017 ceph-mon[105151]: pgmap v1899: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.2 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 20593/60831 objects degraded (33.853%); 0 B/s, 79 objects/s recovering 2024-09-21T10:12:17.980 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:18.004 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:18.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:18.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:18.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:18.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:18.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:18.316 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:18.418 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:18.443 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:18.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:18.544 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:18.567 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:18.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:18.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:18.693 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:18.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:18.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:18 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 20593/60831 objects degraded (33.853%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:18.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:18 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 20593/60831 objects degraded (33.853%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:18.781 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:18 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 20593/60831 objects degraded (33.853%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:18.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:18.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:18.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:18.981 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:12:18.982 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:19.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:19.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:19.028 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:12:19.029 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:19.065 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:12:19.065 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:19.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:19.114 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:12:19.115 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:19.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:19.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:19.148 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:12:19.148 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:19.193 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:12:19.193 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:19.220 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:19.223 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:12:19.223 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:19.248 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:19.254 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:12:19.254 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:19.270 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:19.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:19.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:19.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:19.518 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:19.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.543 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:19.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:19.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:19.665 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:12:19.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:19.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:19.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:19.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:19.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:19.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:19 smithi017 ceph-mon[102913]: pgmap v1900: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.1 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 20082/52536 objects degraded (38.225%); 0 B/s, 121 objects/s recovering 2024-09-21T10:12:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:19 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/919003556' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:19 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2241656296' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:19 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:19 smithi017 ceph-mon[105151]: pgmap v1900: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.1 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 20082/52536 objects degraded (38.225%); 0 B/s, 121 objects/s recovering 2024-09-21T10:12:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:19 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/919003556' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:19 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2241656296' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:19 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:19.958 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:19 smithi134 ceph-mon[90085]: pgmap v1900: 109 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.1 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 819 op/s; 20082/52536 objects degraded (38.225%); 0 B/s, 121 objects/s recovering 2024-09-21T10:12:19.958 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:19 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/919003556' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:19.958 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:19 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2241656296' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:19.958 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:19 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:19.962 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:12:19.962 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:12:19.962 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 0 2024-09-21T10:12:19.962 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:12:19.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:20.074 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:20.097 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:20.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:20.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:20.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:20.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:20.347 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:20.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:20.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:20.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:20.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:12:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:12:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:12:20.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:20.599 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:20.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:20.700 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:20.722 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:20.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:20.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:12:20.824 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:20.850 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:20.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:20.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:20 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:12:20.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:20 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:20.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:20 smithi017 ceph-mon[102913]: osdmap e1265: 8 total, 8 up, 7 in 2024-09-21T10:12:20.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:20 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:12:20.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:20 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:20.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:20 smithi017 ceph-mon[105151]: osdmap e1265: 8 total, 8 up, 7 in 2024-09-21T10:12:20.952 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:20.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:20.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:20 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:12:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:20 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:20 smithi134 ceph-mon[90085]: osdmap e1265: 8 total, 8 up, 7 in 2024-09-21T10:12:21.077 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:21.078 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:21.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:21.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:21.211 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:21.240 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:21.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:21.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:21.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:21.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:21.477 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:21.501 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:21.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:21.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:21.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:21.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:21.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:21.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:21.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:21.918 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:21.940 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:21.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[102913]: pgmap v1902: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 20069/49713 objects degraded (40.370%); 0 B/s, 94 objects/s recovering 2024-09-21T10:12:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[102913]: osdmap e1266: 8 total, 8 up, 7 in 2024-09-21T10:12:21.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:12:21.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:21.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[105151]: pgmap v1902: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 20069/49713 objects degraded (40.370%); 0 B/s, 94 objects/s recovering 2024-09-21T10:12:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[105151]: osdmap e1266: 8 total, 8 up, 7 in 2024-09-21T10:12:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:12:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:21.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:21 smithi134 ceph-mon[90085]: pgmap v1902: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 20069/49713 objects degraded (40.370%); 0 B/s, 94 objects/s recovering 2024-09-21T10:12:21.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:21 smithi134 ceph-mon[90085]: osdmap e1266: 8 total, 8 up, 7 in 2024-09-21T10:12:21.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:12:21.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:21.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:22.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:22.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:22.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:22.236 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:22.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:22.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:22.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:22.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:22.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:22.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:22.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:22.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:22.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:22.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:22.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:22.803 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:22.905 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:22.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:23.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:23.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:23.189 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:23.215 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:23.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:23.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:23.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:23.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:12:23.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:23.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:23.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:23.628 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:12:23.628 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:12:23.628 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:12:23.628 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:12:23.628 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:12:23.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:23.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:23 smithi017 ceph-mon[102913]: pgmap v1904: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.0 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 20069/49713 objects degraded (40.370%); 0 B/s, 65 objects/s recovering 2024-09-21T10:12:23.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:23 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 20069/49713 objects degraded (40.370%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:23 smithi017 ceph-mon[105151]: pgmap v1904: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.0 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 20069/49713 objects degraded (40.370%); 0 B/s, 65 objects/s recovering 2024-09-21T10:12:23.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:23 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 20069/49713 objects degraded (40.370%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:23.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:23.850 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:23.878 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:23.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:12:23.956 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:23 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:12:23.697+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1266 definitely_dead 0 2024-09-21T10:12:23.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:23 smithi134 ceph-mon[90085]: pgmap v1904: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 1.0 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 20069/49713 objects degraded (40.370%); 0 B/s, 65 objects/s recovering 2024-09-21T10:12:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:23 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 20069/49713 objects degraded (40.370%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:24.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:24.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:24.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:24.255 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:24.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:24.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:24.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:24.375 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:12:24.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:24.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:24.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:24.611 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:24.635 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:24.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:24.688 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.7. 2024-09-21T10:12:24.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:24.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:24.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:24.861 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:24.882 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:24.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2415861664' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:12:24.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:24 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:12:24.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2415861664' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:12:24.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:24 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:12:24.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:12:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:12:24] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.51.0" 2024-09-21T10:12:24.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2415861664' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:12:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:24 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:12:25.006 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:25.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:25.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:25.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:25.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:25.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:25.253 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:25.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:25.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:25.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:25.479 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:25.500 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:25.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:25.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:25.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[102913]: pgmap v1905: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 867 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 19504/41655 objects degraded (46.823%); 0 B/s, 136 objects/s recovering 2024-09-21T10:12:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-09-21T10:12:25.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[102913]: osdmap e1267: 8 total, 8 up, 8 in 2024-09-21T10:12:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:25.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[105151]: pgmap v1905: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 867 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 19504/41655 objects degraded (46.823%); 0 B/s, 136 objects/s recovering 2024-09-21T10:12:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-09-21T10:12:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[105151]: osdmap e1267: 8 total, 8 up, 8 in 2024-09-21T10:12:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:25.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:25.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:25.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:25.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:25.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:25.896 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:25.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:25 smithi134 ceph-mon[90085]: pgmap v1905: 109 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 867 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 19504/41655 objects degraded (46.823%); 0 B/s, 136 objects/s recovering 2024-09-21T10:12:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:25 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-09-21T10:12:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:25 smithi134 ceph-mon[90085]: osdmap e1267: 8 total, 8 up, 8 in 2024-09-21T10:12:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:25.997 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:26.018 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:26.120 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:26.141 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:26.242 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:26.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.265 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:26.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:26.314 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:26.366 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:26.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:26.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:26.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:26.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:26.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:26.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:26.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:26.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:26.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:26.897 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:26 smithi017 ceph-mon[102913]: osdmap e1268: 8 total, 8 up, 8 in 2024-09-21T10:12:26.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:26 smithi017 ceph-mon[105151]: osdmap e1268: 8 total, 8 up, 8 in 2024-09-21T10:12:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:26 smithi134 ceph-mon[90085]: osdmap e1268: 8 total, 8 up, 8 in 2024-09-21T10:12:26.998 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:27.019 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:27.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:27.120 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:27.147 INFO:tasks.thrashosds.thrasher:Added osd 7 2024-09-21T10:12:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.148 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:27.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:27.249 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:27.277 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:27.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:27.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:27.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:27.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:27.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:27.641 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:27.661 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:27.684 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:27.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:27.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:27.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:27.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:27.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: pgmap v1908: 109 pgs: 2 remapped+peering, 5 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 832 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 550 op/s; 12785/39990 objects degraded (31.970%); 13 KiB/s, 124 objects/s recovering 2024-09-21T10:12:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: 9.9 starting backfill to osd.7 from (0'0,1243'3641] MAX to 1266'10953 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: osdmap e1269: 8 total, 8 up, 8 in 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: 9.e starting backfill to osd.7 from (0'0,1243'3478] MAX to 1266'10529 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: 9.c starting backfill to osd.7 from (0'0,1243'3485] MAX to 1266'10583 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: 9.7 starting backfill to osd.7 from (0'0,1243'3651] MAX to 1266'10940 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: 9.1 starting backfill to osd.7 from (0'0,1243'3479] MIN to 1266'10698 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: 9.5 starting backfill to osd.7 from (0'0,1243'3600] MAX to 1266'10928 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: 9.3 starting backfill to osd.7 from (0'0,1243'3539] MIN to 1266'10963 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: 9.d starting backfill to osd.7 from (0'0,1243'3590] MAX to 1266'10750 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[102913]: 9.a starting backfill to osd.7 from (0'0,1243'3629] MAX to 1266'10848 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: pgmap v1908: 109 pgs: 2 remapped+peering, 5 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 832 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 550 op/s; 12785/39990 objects degraded (31.970%); 13 KiB/s, 124 objects/s recovering 2024-09-21T10:12:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: 9.9 starting backfill to osd.7 from (0'0,1243'3641] MAX to 1266'10953 2024-09-21T10:12:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: osdmap e1269: 8 total, 8 up, 8 in 2024-09-21T10:12:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: 9.e starting backfill to osd.7 from (0'0,1243'3478] MAX to 1266'10529 2024-09-21T10:12:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: 9.c starting backfill to osd.7 from (0'0,1243'3485] MAX to 1266'10583 2024-09-21T10:12:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: 9.7 starting backfill to osd.7 from (0'0,1243'3651] MAX to 1266'10940 2024-09-21T10:12:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: 9.1 starting backfill to osd.7 from (0'0,1243'3479] MIN to 1266'10698 2024-09-21T10:12:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: 9.5 starting backfill to osd.7 from (0'0,1243'3600] MAX to 1266'10928 2024-09-21T10:12:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: 9.3 starting backfill to osd.7 from (0'0,1243'3539] MIN to 1266'10963 2024-09-21T10:12:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: 9.d starting backfill to osd.7 from (0'0,1243'3590] MAX to 1266'10750 2024-09-21T10:12:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:27 smithi017 ceph-mon[105151]: 9.a starting backfill to osd.7 from (0'0,1243'3629] MAX to 1266'10848 2024-09-21T10:12:27.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:27.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:27.986 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: pgmap v1908: 109 pgs: 2 remapped+peering, 5 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 832 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 550 op/s; 12785/39990 objects degraded (31.970%); 13 KiB/s, 124 objects/s recovering 2024-09-21T10:12:27.986 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: 9.9 starting backfill to osd.7 from (0'0,1243'3641] MAX to 1266'10953 2024-09-21T10:12:27.986 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: osdmap e1269: 8 total, 8 up, 8 in 2024-09-21T10:12:27.986 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: 9.e starting backfill to osd.7 from (0'0,1243'3478] MAX to 1266'10529 2024-09-21T10:12:27.986 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: 9.c starting backfill to osd.7 from (0'0,1243'3485] MAX to 1266'10583 2024-09-21T10:12:27.986 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: 9.7 starting backfill to osd.7 from (0'0,1243'3651] MAX to 1266'10940 2024-09-21T10:12:27.986 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: 9.1 starting backfill to osd.7 from (0'0,1243'3479] MIN to 1266'10698 2024-09-21T10:12:27.987 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: 9.5 starting backfill to osd.7 from (0'0,1243'3600] MAX to 1266'10928 2024-09-21T10:12:27.987 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: 9.3 starting backfill to osd.7 from (0'0,1243'3539] MIN to 1266'10963 2024-09-21T10:12:27.987 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: 9.d starting backfill to osd.7 from (0'0,1243'3590] MAX to 1266'10750 2024-09-21T10:12:27.987 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:27 smithi134 ceph-mon[90085]: 9.a starting backfill to osd.7 from (0'0,1243'3629] MAX to 1266'10848 2024-09-21T10:12:28.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:28.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:28.215 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:28.264 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:28.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:28.366 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:28.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:28.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:28.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:28.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:28.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:28.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:28.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:28.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:28.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:28.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:28.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:28 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 12785/39990 objects degraded (31.970%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:28 smithi017 ceph-mon[102913]: osdmap e1270: 8 total, 8 up, 8 in 2024-09-21T10:12:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:28 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 12785/39990 objects degraded (31.970%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:28 smithi017 ceph-mon[105151]: osdmap e1270: 8 total, 8 up, 8 in 2024-09-21T10:12:28.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:28 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 12785/39990 objects degraded (31.970%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:28 smithi134 ceph-mon[90085]: osdmap e1270: 8 total, 8 up, 8 in 2024-09-21T10:12:29.001 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:29.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:29.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:29.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:29.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:29.230 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:29.254 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:29.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:29.354 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:12:29.354 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:42848 9.3 9:c5380b05:::benchmark_data_smithi017_237924_object42847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:15.381535+0000", 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 193.96951539, 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.106211818, 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:29.355 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 42848 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.381535+0000", 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.381535+0000", 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.356 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.381534+0000", 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.381591+0000", 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7092000000000002e-05 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.381603+0000", 2024-09-21T10:12:29.357 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2031000000000001e-05 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.381611+0000", 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5150000000000006e-06 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.381654+0000", 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2644e-05 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.358 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.381756+0000", 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000101881 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.381858+0000", 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000101773 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.383592+0000", 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0017345699999999999 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.359 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.384711+0000", 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0011191350000000001 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.487494+0000", 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.102783022 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.487640+0000", 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014536700000000001 2024-09-21T10:12:29.360 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.487747+0000", 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010730000000000001 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:42849 9.e 9:79bc4a2a:::benchmark_data_smithi017_237924_object42848:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:15.382004+0000", 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 193.96904682100001, 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.108702405, 2024-09-21T10:12:29.361 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 42849 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382004+0000", 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.362 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382004+0000", 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382003+0000", 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.363 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382030+0000", 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.6684999999999999e-05 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382046+0000", 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5936000000000001e-05 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382051+0000", 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.028e-06 2024-09-21T10:12:29.364 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382108+0000", 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7667999999999997e-05 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382223+0000", 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011503899999999999 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382362+0000", 2024-09-21T10:12:29.365 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000138603 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.385848+0000", 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0034859349999999999 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.385853+0000", 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8930000000000004e-06 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.366 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.490618+0000", 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.104764998 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.490671+0000", 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3603000000000002e-05 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.490706+0000", 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4814999999999998e-05 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.367 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:42850 9.d 9:b23d0213:::benchmark_data_smithi017_237924_object42849:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:15.382134+0000", 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 193.96891659100001, 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.108687583, 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:29.368 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:29.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:29.370 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 42850 2024-09-21T10:12:29.370 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.370 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.370 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.370 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.370 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382134+0000", 2024-09-21T10:12:29.370 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.370 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.370 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382134+0000", 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382133+0000", 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382197+0000", 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3464000000000001e-05 2024-09-21T10:12:29.371 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382206+0000", 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8999999999999995e-06 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382216+0000", 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0915e-05 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382254+0000", 2024-09-21T10:12:29.372 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7669999999999997e-05 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382338+0000", 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3877999999999999e-05 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.382426+0000", 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8269000000000005e-05 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:29.373 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.385452+0000", 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0030258630000000002 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.385749+0000", 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0002965 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.490768+0000", 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.105019217 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.374 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.490799+0000", 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0902999999999999e-05 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.490822+0000", 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2728999999999999e-05 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.375 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:42861 9.d 9:b61735b6:::benchmark_data_smithi017_237924_object42860:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:15.387692+0000", 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 193.96335901200001, 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.11350774199999999, 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 42861 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.387692+0000", 2024-09-21T10:12:29.376 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.387692+0000", 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.387691+0000", 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.377 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.387756+0000", 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4647999999999995e-05 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.387761+0000", 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.242e-06 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.387764+0000", 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.926e-06 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.378 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.389080+0000", 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001316661 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.389174+0000", 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.3938999999999996e-05 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.389262+0000", 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7878000000000001e-05 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.379 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.392676+0000", 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003413391 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.392750+0000", 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3994999999999999e-05 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.501100+0000", 2024-09-21T10:12:29.380 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.108350714 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.501157+0000", 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7012999999999998e-05 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:15.501199+0000", 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1912999999999997e-05 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.381 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:49785 9.3 9:c9de1927:::benchmark_data_smithi017_237924_object49784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:19.707349+0000", 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 189.64370181499999, 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.108428816, 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 49785 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.382 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.707349+0000", 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.707349+0000", 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.707348+0000", 2024-09-21T10:12:29.383 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.707402+0000", 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4795e-05 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.707414+0000", 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1158999999999999e-05 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.384 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.707419+0000", 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6609999999999997e-06 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.707474+0000", 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4800999999999999e-05 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.707650+0000", 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000175599 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.385 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.707777+0000", 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000127563 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.709456+0000", 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0016791550000000001 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.386 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.712304+0000", 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0028475729999999999 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.815685+0000", 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.103380915 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.815737+0000", 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1663999999999999e-05 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.387 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.815778+0000", 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0970999999999998e-05 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:49790 9.3 9:cbb8be35:::benchmark_data_smithi017_237924_object49789:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:19.709782+0000", 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 189.641268908, 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.109241952, 2024-09-21T10:12:29.388 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 49790 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.709782+0000", 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.389 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.709782+0000", 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.709781+0000", 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.709829+0000", 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8276000000000003e-05 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.390 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.709838+0000", 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.9309999999999997e-06 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.709842+0000", 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1640000000000003e-06 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.709886+0000", 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3693999999999999e-05 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.391 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.709991+0000", 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010501900000000001 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.710115+0000", 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012442200000000001 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.712034+0000", 2024-09-21T10:12:29.392 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001918459 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.712466+0000", 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00043250899999999998 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.818932+0000", 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.106465795 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.393 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.818987+0000", 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4959e-05 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.819024+0000", 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6483999999999998e-05 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.394 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:49802 9.7 9:e1bbb961:::benchmark_data_smithi017_237924_object49801:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:19.717424+0000", 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 189.63362642300001, 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.106185391, 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 49802 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.395 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.717424+0000", 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.717424+0000", 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.717423+0000", 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.396 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.717486+0000", 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3372000000000002e-05 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.717498+0000", 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1167e-05 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.717506+0000", 2024-09-21T10:12:29.397 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0770000000000008e-06 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.717541+0000", 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5098999999999997e-05 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.717619+0000", 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8418999999999996e-05 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.398 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.717713+0000", 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.3893000000000004e-05 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.720974+0000", 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003261328 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.721028+0000", 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4036000000000002e-05 2024-09-21T10:12:29.399 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.823532+0000", 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.102503706 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.823573+0000", 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1310000000000003e-05 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.400 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.823609+0000", 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6081999999999998e-05 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:49803 9.7 9:e3e991e7:::benchmark_data_smithi017_237924_object49802:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:19.718676+0000", 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 189.63237412500001, 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.105053165, 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:29.401 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 49803 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.718676+0000", 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.718676+0000", 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.402 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.718676+0000", 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.718707+0000", 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1195999999999997e-05 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.718716+0000", 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5009999999999993e-06 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.403 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.718722+0000", 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1930000000000001e-06 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.719425+0000", 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00070357500000000001 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.719544+0000", 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000118755 2024-09-21T10:12:29.404 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.719674+0000", 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012977400000000001 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.721102+0000", 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0014281400000000001 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.721495+0000", 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00039283699999999999 2024-09-21T10:12:29.405 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.823655+0000", 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10215999000000001 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.823699+0000", 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4286000000000003e-05 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:19.823730+0000", 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0709e-05 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:173491 9.3 9.22c85303 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:24.703930+0000", 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 4.6471201769999997, 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.98604968599999998, 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:29.407 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 173491 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703930+0000", 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703930+0000", 2024-09-21T10:12:29.408 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703929+0000", 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703935+0000", 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7830000000000004e-06 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.409 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703947+0000", 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2495e-05 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703955+0000", 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0749999999999998e-06 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.704021+0000", 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5544999999999997e-05 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.410 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.704032+0000", 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1714e-05 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689967+0000", 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.98593412800000002 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689980+0000", 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3451e-05 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.411 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:173537 9.9 e1269/1268 9:9abd2ba7:::benchmark_data_smithi017_237924_object80533:head v 1269'10956, mlcod=1254'6601)", 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:26.958423+0000", 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 2.3926276570000002, 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.021072695999999998, 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.958423+0000", 2024-09-21T10:12:29.412 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.958423+0000", 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.958422+0000", 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.958429+0000", 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4139999999999999e-06 2024-09-21T10:12:29.413 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.958437+0000", 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.507e-06 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.958441+0000", 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3279999999999999e-06 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.958478+0000", 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6199999999999999e-05 2024-09-21T10:12:29.414 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.958491+0000", 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3361e-05 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.979465+0000", 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.020974172999999999 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.979496+0000", 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0482999999999999e-05 2024-09-21T10:12:29.415 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:173644 9.5 e1269/1268 9:a391214c:::benchmark_data_smithi017_237924_object80640:head v 1269'10938, mlcod=1253'5848)", 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:27.225037+0000", 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 2.126013462, 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.015928320999999999, 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.225037+0000", 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.225037+0000", 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.225036+0000", 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.417 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.225049+0000", 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2955e-05 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.225082+0000", 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3352999999999997e-05 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.418 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.225090+0000", 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9659999999999996e-06 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.225189+0000", 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8072000000000006e-05 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.225218+0000", 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9278e-05 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.419 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.240925+0000", 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.015707069000000001 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.240965+0000", 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0506000000000001e-05 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.420 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:173646 9.d e1269/1268 9:b68338c4:::benchmark_data_smithi017_237924_object80642:head v 1269'10758, mlcod=1269'10756)", 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:27.226401+0000", 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 2.1246499710000002, 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014517366, 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.226401+0000", 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.421 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.226401+0000", 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.226400+0000", 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.226406+0000", 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5979999999999996e-06 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.422 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.226418+0000", 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1832e-05 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.226425+0000", 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0140000000000001e-06 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.228009+0000", 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0015841679999999999 2024-09-21T10:12:29.423 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.228089+0000", 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0266999999999996e-05 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.240883+0000", 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.01279364 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.240918+0000", 2024-09-21T10:12:29.424 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5224e-05 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:173652 9.5 e1269/1268 9:ab17b72b:::benchmark_data_smithi017_237924_object80648:head v 1269'10939, mlcod=1253'5848)", 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:27.227778+0000", 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 2.1232726820000001, 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013210530999999999, 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.425 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.426 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.426 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.426 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.426 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.227778+0000", 2024-09-21T10:12:29.426 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.426 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.227778+0000", 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.227777+0000", 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.227789+0000", 2024-09-21T10:12:29.427 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1623999999999999e-05 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.227807+0000", 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8105999999999999e-05 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.227814+0000", 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6370000000000002e-06 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.233514+0000", 2024-09-21T10:12:29.428 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0057008959999999996 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.233574+0000", 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9503000000000003e-05 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.240968+0000", 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0073937849999999999 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.429 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.240988+0000", 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0608999999999999e-05 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:173670 9.d e1269/1268 9:b9805015:::benchmark_data_smithi017_237924_object80666:head v 1269'10761, mlcod=1269'10757)", 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:27.267067+0000", 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 2.0839838199999998, 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013045070000000001, 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.430 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.267067+0000", 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.267067+0000", 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.267066+0000", 2024-09-21T10:12:29.431 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.267072+0000", 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9519999999999999e-06 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.267085+0000", 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2917e-05 2024-09-21T10:12:29.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.267091+0000", 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0220000000000001e-06 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.267121+0000", 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.0661999999999999e-05 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.433 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.267142+0000", 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1027e-05 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.279984+0000", 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.01284164 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.280112+0000", 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012775199999999999 2024-09-21T10:12:29.434 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:173673 9.d e1269/1268 9:bf91f921:::benchmark_data_smithi017_237924_object80669:head v 1269'10762, mlcod=1269'10757)", 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:27.269756+0000", 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 2.0812948310000001, 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010443973, 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.435 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.269756+0000", 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.269756+0000", 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.269755+0000", 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.436 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.269762+0000", 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.0690000000000004e-06 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.269778+0000", 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6390000000000001e-05 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.269785+0000", 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8419999999999999e-06 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.437 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.270887+0000", 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0011014410000000001 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.270907+0000", 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0284000000000001e-05 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.280117+0000", 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0092096950000000004 2024-09-21T10:12:29.438 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.280200+0000", 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3171999999999996e-05 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:174445 9.9 e1269/1268 9:9f8c662e:::benchmark_data_smithi017_237924_object81441:head v 1269'11010, mlcod=1254'6601)", 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:28.221291+0000", 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 1.1297599199999999, 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013250524, 2024-09-21T10:12:29.439 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.221291+0000", 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.221291+0000", 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.440 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.221290+0000", 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967296 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.221294+0000", 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5990000000000002e-06 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.221299+0000", 2024-09-21T10:12:29.441 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5099999999999998e-06 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.221303+0000", 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3560000000000001e-06 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.224087+0000", 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0027846120000000001 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.442 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.224109+0000", 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1767999999999999e-05 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.234403+0000", 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010293382 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.234541+0000", 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013853100000000001 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.443 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.444 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.444 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.444 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:174465 9.c e1269/1268 9:3a6a31fb:::benchmark_data_smithi017_237924_object81461:head v 1269'10646, mlcod=1254'6420)", 2024-09-21T10:12:29.444 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:28.242719+0000", 2024-09-21T10:12:29.445 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 1.1083311689999999, 2024-09-21T10:12:29.445 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013177517999999999, 2024-09-21T10:12:29.445 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.445 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.445 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.242719+0000", 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.242719+0000", 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.242719+0000", 2024-09-21T10:12:29.446 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.242726+0000", 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.029e-06 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.242737+0000", 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0818e-05 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.447 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.242743+0000", 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0560000000000001e-06 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.242783+0000", 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0122e-05 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.242802+0000", 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9703000000000001e-05 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.448 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.255844+0000", 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013042054000000001 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.255897+0000", 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2426999999999998e-05 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.449 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:174475 9.7 e1269/1268 9:e0035f67:::benchmark_data_smithi017_237924_object81471:head v 1269'10999, mlcod=0'0)", 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:28.245194+0000", 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 1.1058564360000001, 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010752559, 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245194+0000", 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245194+0000", 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245193+0000", 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245203+0000", 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.9080000000000007e-06 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245214+0000", 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.093e-05 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245219+0000", 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2349999999999999e-06 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245279+0000", 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9757999999999999e-05 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245302+0000", 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3555999999999999e-05 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.255909+0000", 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010606167999999999 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.255947+0000", 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8092000000000001e-05 2024-09-21T10:12:29.453 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:174474 9.d e1269/1268 9:b667798a:::benchmark_data_smithi017_237924_object81470:head v 1269'10812, mlcod=1269'10810)", 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:28.245840+0000", 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 1.105210413, 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011777595, 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.454 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245840+0000", 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245840+0000", 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245839+0000", 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:12:29.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245849+0000", 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8220000000000002e-06 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245861+0000", 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1388e-05 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.456 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245866+0000", 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4609999999999999e-06 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245929+0000", 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3200999999999993e-05 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.245948+0000", 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9170000000000001e-05 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.257579+0000", 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011630821 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.257618+0000", 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8408999999999998e-05 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.458 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:174468 9.e e1269/1268 9:7656e2b9:::benchmark_data_smithi017_237924_object81464:head v 1269'10605, mlcod=1256'7238)", 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:28.247815+0000", 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 1.103235306, 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0096976639999999999, 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.247815+0000", 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.459 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.247815+0000", 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.247814+0000", 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.247825+0000", 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1214e-05 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.247844+0000", 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8301999999999999e-05 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.247851+0000", 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4739999999999999e-06 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.247942+0000", 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1312999999999998e-05 2024-09-21T10:12:29.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.247964+0000", 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2048e-05 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.257472+0000", 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0095077519999999995 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:28.257513+0000", 2024-09-21T10:12:29.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0797000000000003e-05 2024-09-21T10:12:29.463 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.463 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.463 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.463 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:29.463 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:29.463 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:12:29.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.465 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:29.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:29.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:29.580 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:12:29.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:29.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:29.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:29.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:29.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:29.842 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:29.874 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:29.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:29.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[102913]: pgmap v1910: 109 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+undersized+remapped, 2 remapped+peering, 5 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 742 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 791 op/s; 4968/35640 objects degraded (13.939%); 2201/35640 objects misplaced (6.176%); 2.7 MiB/s, 265 objects/s recovering 2024-09-21T10:12:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/756475772' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:12:29.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[102913]: osdmap e1271: 8 total, 8 up, 8 in 2024-09-21T10:12:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[105151]: pgmap v1910: 109 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+undersized+remapped, 2 remapped+peering, 5 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 742 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 791 op/s; 4968/35640 objects degraded (13.939%); 2201/35640 objects misplaced (6.176%); 2.7 MiB/s, 265 objects/s recovering 2024-09-21T10:12:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/756475772' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:29.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:12:29.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:29 smithi017 ceph-mon[105151]: osdmap e1271: 8 total, 8 up, 8 in 2024-09-21T10:12:29.975 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:29 smithi134 ceph-mon[90085]: pgmap v1910: 109 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+undersized+remapped, 2 remapped+peering, 5 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 742 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 791 op/s; 4968/35640 objects degraded (13.939%); 2201/35640 objects misplaced (6.176%); 2.7 MiB/s, 265 objects/s recovering 2024-09-21T10:12:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:29 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/756475772' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:29 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:29 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:29 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:12:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:29 smithi134 ceph-mon[90085]: osdmap e1271: 8 total, 8 up, 8 in 2024-09-21T10:12:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:30.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:30.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:30.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:30.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:30.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:30.273 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:12:30.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:30.414 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.467 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:30.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:12:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:12:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:12:30.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:30.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:30.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:30.710 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:30.740 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:30.842 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:30.872 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:30.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:30.996 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:12:30.996 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:31.020 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:31.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:31.040 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:12:31.040 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:31.078 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:12:31.078 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:31.120 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:12:31.120 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:31.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:31.154 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:12:31.154 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:31.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:31.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:31.185 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:12:31.185 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:31.215 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:12:31.216 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:31.244 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:31.254 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:12:31.254 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:31.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:31.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:31.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:31.425 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:31.456 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:31.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:31.557 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:31.595 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:31.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:31.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:31.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:31.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:31.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:31.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:31.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:31 smithi017 ceph-mon[102913]: pgmap v1913: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 98 active+clean; 714 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 22 KiB/s rd, 47 KiB/s wr, 410 op/s; 4016/34323 objects degraded (11.701%); 2201/34323 objects misplaced (6.413%); 6.4 MiB/s, 281 objects/s recovering 2024-09-21T10:12:31.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:31 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4069119455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:31 smithi017 ceph-mon[105151]: pgmap v1913: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 98 active+clean; 714 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 22 KiB/s rd, 47 KiB/s wr, 410 op/s; 4016/34323 objects degraded (11.701%); 2201/34323 objects misplaced (6.413%); 6.4 MiB/s, 281 objects/s recovering 2024-09-21T10:12:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:31 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4069119455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:31.984 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:31 smithi134 ceph-mon[90085]: pgmap v1913: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 98 active+clean; 714 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 22 KiB/s rd, 47 KiB/s wr, 410 op/s; 4016/34323 objects degraded (11.701%); 2201/34323 objects misplaced (6.413%); 6.4 MiB/s, 281 objects/s recovering 2024-09-21T10:12:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:31 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4069119455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:32.012 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:32.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:32.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:32.147 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:12:32.147 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:12:32.147 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-21T10:12:32.147 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:12:32.148 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:12:32.148 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-21T10:12:32.150 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:32.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:32.252 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:32.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.293 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:32.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:32.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:32.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:32.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:32.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:32.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:32.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:32.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:32.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:32.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:32.866 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:32.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:32.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:33.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:33.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:33.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:33.141 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:33.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:33.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:33.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:33.385 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:33.413 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:33.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:33.515 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:33.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:33.642 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:33.672 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:33.774 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:33.802 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:33.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:33.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:33.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:33.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:33 smithi017 ceph-mon[102913]: pgmap v1914: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 98 active+clean; 714 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 4016/34323 objects degraded (11.701%); 2201/34323 objects misplaced (6.413%); 5.0 MiB/s, 223 objects/s recovering 2024-09-21T10:12:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 4016/34323 objects degraded (11.701%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:33 smithi017 ceph-mon[102913]: osdmap e1272: 8 total, 8 up, 8 in 2024-09-21T10:12:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:33 smithi017 ceph-mon[105151]: pgmap v1914: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 98 active+clean; 714 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 4016/34323 objects degraded (11.701%); 2201/34323 objects misplaced (6.413%); 5.0 MiB/s, 223 objects/s recovering 2024-09-21T10:12:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 4016/34323 objects degraded (11.701%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:33 smithi017 ceph-mon[105151]: osdmap e1272: 8 total, 8 up, 8 in 2024-09-21T10:12:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:33 smithi134 ceph-mon[90085]: pgmap v1914: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 98 active+clean; 714 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 4016/34323 objects degraded (11.701%); 2201/34323 objects misplaced (6.413%); 5.0 MiB/s, 223 objects/s recovering 2024-09-21T10:12:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 4016/34323 objects degraded (11.701%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:12:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:33 smithi134 ceph-mon[90085]: osdmap e1272: 8 total, 8 up, 8 in 2024-09-21T10:12:34.031 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:34.052 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:34.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:34.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:12:34.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:34.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:34.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:34.278 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:34.300 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:34.401 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:34.423 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:34.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:34.524 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:34.548 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:34.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:34.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:34.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:34.791 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:34.814 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:34.837 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:34.837 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:34.915 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:34.943 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:34.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:34.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:12:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:12:34] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-09-21T10:12:35.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:35.071 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:35.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:35.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:12:35.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:35.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:35.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:35.328 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:35.353 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:35.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:35.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:35.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:35.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:35.578 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:35.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:35.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:35.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:35.753 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:35.855 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:35.881 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:35.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:35 smithi017 ceph-mon[102913]: pgmap v1916: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 99 active+clean; 538 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 3377/25851 objects degraded (13.063%); 1520/25851 objects misplaced (5.880%); 2.3 MiB/s, 154 objects/s recovering 2024-09-21T10:12:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:35 smithi017 ceph-mon[102913]: osdmap e1273: 8 total, 8 up, 8 in 2024-09-21T10:12:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:35 smithi017 ceph-mon[105151]: pgmap v1916: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 99 active+clean; 538 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 3377/25851 objects degraded (13.063%); 1520/25851 objects misplaced (5.880%); 2.3 MiB/s, 154 objects/s recovering 2024-09-21T10:12:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:35 smithi017 ceph-mon[105151]: osdmap e1273: 8 total, 8 up, 8 in 2024-09-21T10:12:35.983 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:35 smithi134 ceph-mon[90085]: pgmap v1916: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 99 active+clean; 538 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 3377/25851 objects degraded (13.063%); 1520/25851 objects misplaced (5.880%); 2.3 MiB/s, 154 objects/s recovering 2024-09-21T10:12:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:35 smithi134 ceph-mon[90085]: osdmap e1273: 8 total, 8 up, 8 in 2024-09-21T10:12:36.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:36.068 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:12:36.109 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:36.143 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:36.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:36.159 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:36.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:36.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.282 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:36.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:36.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:36.410 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:36.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:36.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:36.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:36.544 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:36.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.585 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:36.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:36.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:36.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.710 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:36.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:36.811 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:36.841 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:36.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:36.942 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:36.973 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:37.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:37.117 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:37.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:37.219 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:37.247 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:37.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:37.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:37.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.386 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:37.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:37.435 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:37.487 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:37.514 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:37.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:37.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:37.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.665 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:37.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:37.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:37.812 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:37.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:37.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[102913]: pgmap v1918: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 497 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 2487/23877 objects degraded (10.416%); 1520/23877 objects misplaced (6.366%); 2.0 MiB/s, 180 objects/s recovering 2024-09-21T10:12:37.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.848 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[105151]: pgmap v1918: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 497 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 2487/23877 objects degraded (10.416%); 1520/23877 objects misplaced (6.366%); 2.0 MiB/s, 180 objects/s recovering 2024-09-21T10:12:37.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.849 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.849 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.860 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:37 smithi134 ceph-mon[90085]: pgmap v1918: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 497 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 2487/23877 objects degraded (10.416%); 1520/23877 objects misplaced (6.366%); 2.0 MiB/s, 180 objects/s recovering 2024-09-21T10:12:37.860 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.860 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.860 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.860 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:37.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:37.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:38.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:38.106 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:38.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:38.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:38.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:38.356 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:38.389 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:38.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:38.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:38.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:38.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:38.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:38.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:38.754 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:38.790 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:38.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:38.840 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 2487/23877 objects degraded (10.416%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:12:38.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 2487/23877 objects degraded (10.416%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:12:38.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 2487/23877 objects degraded (10.416%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:12:38.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:38.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:38.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:39.091 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:39.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:39.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:39.251 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:39.353 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:39.383 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:39.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:39.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:39.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:39.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:39.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:39.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:39 smithi134 ceph-mon[90085]: pgmap v1919: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 342 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 1855/16446 objects degraded (11.279%); 921/16446 objects misplaced (5.600%); 0 B/s, 109 objects/s recovering 2024-09-21T10:12:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:12:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:12:39.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:39.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:39.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:39.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:39.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:39 smithi017 ceph-mon[102913]: pgmap v1919: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 342 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 1855/16446 objects degraded (11.279%); 921/16446 objects misplaced (5.600%); 0 B/s, 109 objects/s recovering 2024-09-21T10:12:39.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:12:39.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:12:39.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:39.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:39 smithi017 ceph-mon[105151]: pgmap v1919: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 342 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 1855/16446 objects degraded (11.279%); 921/16446 objects misplaced (5.600%); 0 B/s, 109 objects/s recovering 2024-09-21T10:12:39.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:12:39.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:12:39.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:39.924 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:39.951 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:39.982 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:12:39.982 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:12:39.982 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:12:40.053 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:40.078 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:40.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:40.180 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:40.208 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:40.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:40.310 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.338 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:40.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:40.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:40.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:12:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:12:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:12:40.583 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:40.611 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:40.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:40.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:40.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:40.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:40.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:40.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:40.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:40.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:41.015 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:41.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:41.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:41.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:41.255 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:41.285 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:41.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:41.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:41.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:41.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:41 smithi134 ceph-mon[90085]: pgmap v1920: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 285 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 1349/13725 objects degraded (9.829%); 921/13725 objects misplaced (6.710%); 0 B/s, 173 objects/s recovering 2024-09-21T10:12:41.534 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:41.565 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:41.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:41.667 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:41.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:41.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:41.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:41 smithi017 ceph-mon[102913]: pgmap v1920: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 285 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 1349/13725 objects degraded (9.829%); 921/13725 objects misplaced (6.710%); 0 B/s, 173 objects/s recovering 2024-09-21T10:12:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:12:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:41.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:41 smithi017 ceph-mon[105151]: pgmap v1920: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 285 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 1349/13725 objects degraded (9.829%); 921/13725 objects misplaced (6.710%); 0 B/s, 173 objects/s recovering 2024-09-21T10:12:41.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:41.821 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:41.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:41.853 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:41.854 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:41.923 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:41.952 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:41.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:42.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:42.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:42.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:42.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:42.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:42.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:42.338 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:42.367 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:42.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:42.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:42.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:42.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:42.610 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:42.636 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:42.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:42.738 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:42.763 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:42.865 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:42.891 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:42.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:42.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:43.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:43.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:43.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:43.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:43.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:43.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:43.301 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:43.403 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:43.426 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:43.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:43.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:43.554 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:43.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:43.656 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:43.680 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:43.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:43.747 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:12:43.747 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:43.748 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:43.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:43 smithi017 ceph-mon[102913]: pgmap v1921: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 285 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 1349/13725 objects degraded (9.829%); 921/13725 objects misplaced (6.710%); 0 B/s, 107 objects/s recovering 2024-09-21T10:12:43.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:43 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/994159816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:43.782 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:43 smithi017 ceph-mon[105151]: pgmap v1921: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 285 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 1349/13725 objects degraded (9.829%); 921/13725 objects misplaced (6.710%); 0 B/s, 107 objects/s recovering 2024-09-21T10:12:43.782 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:43 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/994159816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:43.782 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:12:43.782 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:43.782 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:43.808 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:43.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:43.817 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:12:43.817 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:43.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:43.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:43.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:43 smithi134 ceph-mon[90085]: pgmap v1921: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 285 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 1349/13725 objects degraded (9.829%); 921/13725 objects misplaced (6.710%); 0 B/s, 107 objects/s recovering 2024-09-21T10:12:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:43 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/994159816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:44.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:44.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:44.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:44.211 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:44.239 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:44.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:44.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:44.384 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:44.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:44.432 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 93001 objects 2024-09-21T10:12:44.432 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :138.093 2024-09-21T10:12:44.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:44.491 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:12:44.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:44.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:44.514 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:12:44.613 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:44.635 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:44.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:44.737 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:44 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 1349/13725 objects degraded (9.829%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:12:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1433524090' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:44 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:44.753 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:44.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:44.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:44.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:44.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:44.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:44 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 1349/13725 objects degraded (9.829%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:12:44.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1433524090' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:44.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:44 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:44.926 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:12:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:12:44] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-09-21T10:12:44.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:44 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 1349/13725 objects degraded (9.829%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:12:44.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1433524090' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:44.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:44 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:44.987 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:45.009 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:45.110 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:45.131 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:45.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:45.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:45.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:45.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:45.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:45.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:45.483 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:45.505 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:45.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:45.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:45.633 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:45.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:45.694 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:12:45.694 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:12:45.694 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.29000048511979326 2024-09-21T10:12:45.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 5 0.29000048511979326 2024-09-21T10:12:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:45 smithi017 ceph-mon[102913]: pgmap v1922: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 85 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 594/4131 objects degraded (14.379%); 171/4131 objects misplaced (4.139%); 0 B/s, 135 objects/s recovering 2024-09-21T10:12:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:45 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:45 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:45 smithi017 ceph-mon[102913]: osdmap e1274: 8 total, 8 up, 8 in 2024-09-21T10:12:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:45 smithi017 ceph-mon[105151]: pgmap v1922: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 85 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 594/4131 objects degraded (14.379%); 171/4131 objects misplaced (4.139%); 0 B/s, 135 objects/s recovering 2024-09-21T10:12:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:45 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:45 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:45 smithi017 ceph-mon[105151]: osdmap e1274: 8 total, 8 up, 8 in 2024-09-21T10:12:45.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:45.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:45.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:45.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:45.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:45.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:45.950 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:45 smithi134 ceph-mon[90085]: pgmap v1922: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 5 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 85 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 594/4131 objects degraded (14.379%); 171/4131 objects misplaced (4.139%); 0 B/s, 135 objects/s recovering 2024-09-21T10:12:45.950 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:45 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:45.950 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:45 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:45.950 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:45 smithi134 ceph-mon[90085]: osdmap e1274: 8 total, 8 up, 8 in 2024-09-21T10:12:45.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:45.992 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:46.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:46.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:46.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:46.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:46.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:46.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:46.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:46.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:46.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:46.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:46.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:46.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:46.505 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:12:46.505 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:12:46.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263811 8.4 8.5ca8d1b4 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.443394+0000", 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 262.058461303, 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97607429599999995, 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:12:46.506 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263811 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443394+0000", 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443394+0000", 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443393+0000", 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443399+0000", 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9609999999999999e-06 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443411+0000", 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1708e-05 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443418+0000", 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1439999999999997e-06 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443601+0000", 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00018249299999999999 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443606+0000", 2024-09-21T10:12:46.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3630000000000002e-06 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.419459+0000", 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.975852371 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.419468+0000", 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.7180000000000008e-06 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263812 8.4 8.c9d94834 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.443432+0000", 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 262.05842293699999, 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97606504900000002, 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:12:46.511 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263812 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443432+0000", 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443432+0000", 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443432+0000", 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443436+0000", 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4700000000000004e-06 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443443+0000", 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.063e-06 2024-09-21T10:12:46.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443448+0000", 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5460000000000002e-06 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443625+0000", 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017739599999999999 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443630+0000", 2024-09-21T10:12:46.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.583e-06 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.419489+0000", 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97585938500000002 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.419497+0000", 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1659999999999994e-06 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263816 8.2 8.2cca6682 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.443453+0000", 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 262.05840195899998, 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97134293500000002, 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263816 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443453+0000", 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443453+0000", 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443453+0000", 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.517 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443455+0000", 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4449999999999999e-06 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443461+0000", 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7130000000000002e-06 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443465+0000", 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0980000000000004e-06 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443533+0000", 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7779000000000003e-05 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443543+0000", 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0209e-05 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.414784+0000", 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97124098699999994 2024-09-21T10:12:46.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.414796+0000", 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2169e-05 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263823 8.2 8.34cbd772 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.443476+0000", 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 262.05837968700001, 2024-09-21T10:12:46.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97135318000000004, 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263823 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443476+0000", 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443476+0000", 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443475+0000", 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:12:46.522 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443478+0000", 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.6970000000000002e-06 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443484+0000", 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.314e-06 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443488+0000", 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3100000000000002e-06 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443565+0000", 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6998999999999999e-05 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.443572+0000", 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4609999999999996e-06 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.414820+0000", 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97124847199999997 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.414829+0000", 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.3599999999999996e-06 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.525 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45620.0:263843 8.2 8.659c36a2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:08:24.462475+0000", 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 262.03938076499998, 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95498026199999997, 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45620", 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 263843 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462475+0000", 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462475+0000", 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462472+0000", 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999971 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462482+0000", 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0180000000000001e-05 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462501+0000", 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8695999999999999e-05 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462509+0000", 2024-09-21T10:12:46.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9389999999999997e-06 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462560+0000", 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0914000000000001e-05 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:24.462570+0000", 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0215e-05 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.417442+0000", 2024-09-21T10:12:46.529 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95487207600000001 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:08:25.417455+0000", 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2867999999999999e-05 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:57282 9.e 9:7d14f317:::benchmark_data_smithi017_237924_object57281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:24.450175+0000", 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 202.051679885, 2024-09-21T10:12:46.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1387745909999998, 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 57282 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450175+0000", 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450175+0000", 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450173+0000", 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450368+0000", 2024-09-21T10:12:46.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019504900000000001 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450404+0000", 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5596000000000003e-05 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450416+0000", 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2003e-05 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450508+0000", 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1644999999999994e-05 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450524+0000", 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6843000000000001e-05 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.534 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:25.429753+0000", 2024-09-21T10:12:46.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97922897900000005 2024-09-21T10:12:46.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:46.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:25.429766+0000", 2024-09-21T10:12:46.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2617e-05 2024-09-21T10:12:46.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.570352+0000", 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.140585728 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.571070+0000", 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00071844699999999999 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.571211+0000", 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000140648 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.573315+0000", 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002104201 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.580792+0000", 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0074773510000000001 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.587425+0000", 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0066328389999999997 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.587470+0000", 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4362e-05 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.588950+0000", 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001480386 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:75080 9.e 9:705ed5a1:::benchmark_data_smithi017_237924_object75079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:12:46.539 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:54.667325+0000", 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 171.83452997699999, 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.17556476400000001, 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 75080 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.667325+0000", 2024-09-21T10:12:46.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.667325+0000", 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.667324+0000", 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.541 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.667379+0000", 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4941000000000001e-05 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.667390+0000", 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1507e-05 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.667395+0000", 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5499999999999996e-06 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.667431+0000", 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5722999999999997e-05 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.667540+0000", 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010915499999999999 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.667654+0000", 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011461 2024-09-21T10:12:46.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.669785+0000", 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021301029999999999 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.678710+0000", 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0089258389999999996 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.841644+0000", 2024-09-21T10:12:46.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.162933673 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.841684+0000", 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0008e-05 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.842890+0000", 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00120597 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:75085 9.e 9:7cca31fe:::benchmark_data_smithi017_237924_object75084:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:54.672011+0000", 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 171.829843869, 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.171715697, 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 75085 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.672011+0000", 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.672011+0000", 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.672010+0000", 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:46.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.672078+0000", 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7781999999999996e-05 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.672092+0000", 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3455000000000001e-05 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.672097+0000", 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9250000000000003e-06 2024-09-21T10:12:46.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.672130+0000", 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2991000000000003e-05 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.672224+0000", 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.4485000000000001e-05 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.672327+0000", 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010273899999999999 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.673752+0000", 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0014254070000000001 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.678894+0000", 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0051417590000000001 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.842966+0000", 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.16407212199999999 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.843022+0000", 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5726000000000001e-05 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:54.843727+0000", 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00070528799999999999 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.551 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:75598 9.e 9:73fe0178:::benchmark_data_smithi017_237924_object75597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:55.777188+0000", 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 170.72466748400001, 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.13456630999999999, 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.552 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 75598 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.777188+0000", 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.777188+0000", 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.777187+0000", 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.777292+0000", 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010524899999999999 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.777300+0000", 2024-09-21T10:12:46.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1620000000000008e-06 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.777304+0000", 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6160000000000002e-06 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.777339+0000", 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5528999999999998e-05 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.777495+0000", 2024-09-21T10:12:46.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000156167 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.777630+0000", 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000134831 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.779350+0000", 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0017196609999999999 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.556 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.787759+0000", 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0084091240000000005 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.910610+0000", 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.122850757 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.910651+0000", 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0791999999999998e-05 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:55.911754+0000", 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0011035660000000001 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:79087 9.e 9:793e427c:::benchmark_data_smithi017_237924_object79086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:10:02.331927+0000", 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 164.16992828299999, 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.41695789700000002, 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.558 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 79087 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.331927+0000", 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.331927+0000", 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.331926+0000", 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.331958+0000", 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2879000000000001e-05 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.331972+0000", 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.378e-05 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.331977+0000", 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.045e-06 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.332026+0000", 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8508999999999999e-05 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.332178+0000", 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015234600000000001 2024-09-21T10:12:46.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.332366+0000", 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00018824799999999999 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.334401+0000", 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020343309999999999 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.346410+0000", 2024-09-21T10:12:46.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.012008960000000001 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.747516+0000", 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.40110595399999999 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.747575+0000", 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8909999999999997e-05 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.563 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.748885+0000", 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0013103360000000001 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:79094 9.e 9:757239c9:::benchmark_data_smithi017_237924_object79093:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:10:02.340712+0000", 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 164.16114315999999, 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.41794367199999999, 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.564 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 79094 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.340712+0000", 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.340712+0000", 2024-09-21T10:12:46.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.340711+0000", 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.340743+0000", 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2224000000000002e-05 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.340756+0000", 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2490000000000001e-05 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.340760+0000", 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5449999999999997e-06 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.340801+0000", 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0528000000000002e-05 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.340969+0000", 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016827000000000001 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.341157+0000", 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000187925 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.343624+0000", 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00246693 2024-09-21T10:12:46.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.346801+0000", 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0031772480000000001 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.757494+0000", 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.41069340199999999 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.757539+0000", 2024-09-21T10:12:46.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4257e-05 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:02.758656+0000", 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001117053 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:79840 9.e 9:792fa858:::benchmark_data_smithi017_237924_object79839:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:10:03.940067+0000", 2024-09-21T10:12:46.570 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 162.56178827799999, 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.20645629099999999, 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 79840 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.940067+0000", 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.571 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.940067+0000", 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.940065+0000", 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.940110+0000", 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4826000000000001e-05 2024-09-21T10:12:46.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.940121+0000", 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0726e-05 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.940127+0000", 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3149999999999997e-06 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.940176+0000", 2024-09-21T10:12:46.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8872999999999998e-05 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.940335+0000", 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000158536 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.940515+0000", 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00018011500000000001 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.574 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.942527+0000", 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020118610000000002 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:03.946886+0000", 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.004359262 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:04.145325+0000", 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.198438848 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.575 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:04.145366+0000", 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0893999999999998e-05 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:04.146523+0000", 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001157709 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:103222 9.e 9:716172c3:::benchmark_data_smithi017_237924_object10218:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1254)", 2024-09-21T10:12:46.576 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:10:42.358206+0000", 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 124.143648852, 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.107625553, 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 103222 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.358206+0000", 2024-09-21T10:12:46.577 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.358206+0000", 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.358205+0000", 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.358211+0000", 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4690000000000002e-06 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.358224+0000", 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2463000000000001e-05 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.358229+0000", 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9150000000000004e-06 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.579 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.358267+0000", 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8513999999999998e-05 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.358371+0000", 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000103977 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.358771+0000", 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00039973999999999999 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.361209+0000", 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002437809 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.363731+0000", 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0025227029999999998 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.464708+0000", 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.100976336 2024-09-21T10:12:46.581 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.464748+0000", 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0204999999999999e-05 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:42.465832+0000", 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0010840279999999999 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:122657 9.e 9:79145779:::benchmark_data_smithi017_237924_object29653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:11:11.495089+0000", 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 95.006766370999998, 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.183346706, 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122657 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.583 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.495089+0000", 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.495089+0000", 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.495087+0000", 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.584 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.495094+0000", 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9770000000000003e-06 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.495109+0000", 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4224e-05 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.495114+0000", 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6749999999999999e-06 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.495159+0000", 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.477e-05 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.495275+0000", 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000115596 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.495387+0000", 2024-09-21T10:12:46.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0001128 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.497876+0000", 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0024885559999999998 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.499326+0000", 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0014498639999999999 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.587 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.677255+0000", 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.177928952 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.677294+0000", 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8903999999999999e-05 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.678436+0000", 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0011419609999999999 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.588 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:122665 9.e 9:7906daa3:::benchmark_data_smithi017_237924_object29661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:11:11.507773+0000", 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 94.994082399999996, 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.20008250899999999, 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122665 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.507773+0000", 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.507773+0000", 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.507772+0000", 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.507780+0000", 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.4619999999999996e-06 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.507792+0000", 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2170999999999999e-05 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.507799+0000", 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4110000000000001e-06 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.507841+0000", 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2577999999999998e-05 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.507960+0000", 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000118496 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.592 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.508072+0000", 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000112471 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.512305+0000", 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0042326990000000004 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.515233+0000", 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029282140000000002 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.706829+0000", 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.19159591000000001 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.706902+0000", 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3139999999999994e-05 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.707855+0000", 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00095331300000000001 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.594 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:122667 9.e 9:7821907b:::benchmark_data_smithi017_237924_object29663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:11:11.510110+0000", 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 94.991745495999993, 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.19833347600000001, 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122667 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.595 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.510110+0000", 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.510110+0000", 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.510109+0000", 2024-09-21T10:12:46.596 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.510114+0000", 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1399999999999999e-06 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.510124+0000", 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0186999999999999e-05 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.510129+0000", 2024-09-21T10:12:46.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.374e-06 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.510290+0000", 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016137500000000001 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.510389+0000", 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8785999999999999e-05 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.510508+0000", 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000119164 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.512445+0000", 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0019367099999999999 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.515276+0000", 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0028313560000000002 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.707890+0000", 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.19261431200000001 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.707923+0000", 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2162999999999997e-05 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.708443+0000", 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00052076300000000004 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.600 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:122672 9.e 9:74df9389:::benchmark_data_smithi017_237924_object29668:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:11:11.520846+0000", 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 94.981008989000003, 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.18819591099999999, 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 122672 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.601 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.520846+0000", 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.520846+0000", 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.520843+0000", 2024-09-21T10:12:46.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999971 2024-09-21T10:12:46.603 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.603 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.603 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.603 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.520859+0000", 2024-09-21T10:12:46.603 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5483999999999999e-05 2024-09-21T10:12:46.603 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.603 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.603 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:46.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:46.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:12:46.604 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.520916+0000", 2024-09-21T10:12:46.604 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7717999999999999e-05 2024-09-21T10:12:46.604 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.604 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.520935+0000", 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8243000000000001e-05 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.521060+0000", 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000125081 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.521200+0000", 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014023400000000001 2024-09-21T10:12:46.605 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.521303+0000", 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000103154 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.523569+0000", 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002265665 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.524583+0000", 2024-09-21T10:12:46.606 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001014339 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.708480+0000", 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.183896543 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.708512+0000", 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2254e-05 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.607 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:11.709042+0000", 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00053021299999999997 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:125636 9.e 9:7f04e03c:::benchmark_data_smithi017_237924_object32632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:11:16.203347+0000", 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 90.298507864000001, 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.189468635, 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.608 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 125636 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.203347+0000", 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.203347+0000", 2024-09-21T10:12:46.609 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.203346+0000", 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.203353+0000", 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6359999999999994e-06 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.610 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.203368+0000", 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4552000000000001e-05 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.203374+0000", 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1809999999999998e-06 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.203421+0000", 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7034999999999999e-05 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.611 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.203589+0000", 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016805999999999999 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.203765+0000", 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017617300000000001 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.206194+0000", 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0024286049999999999 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.612 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.207361+0000", 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0011669510000000001 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.391375+0000", 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.18401373900000001 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.391441+0000", 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.6775999999999999e-05 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.613 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:11:16.392816+0000", 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001374592 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:173513 9.e 9:71c17054:::benchmark_data_smithi017_237924_object80509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:24.724062+0000", 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 21.777793245000002, 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4376691149999998, 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.614 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 173513 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724062+0000", 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.615 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724062+0000", 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724060+0000", 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724071+0000", 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1567e-05 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.616 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724093+0000", 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1767999999999999e-05 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724101+0000", 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.4549999999999995e-06 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724163+0000", 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1303000000000006e-05 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.617 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724176+0000", 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2779000000000001e-05 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689519+0000", 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96534338099999994 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689531+0000", 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1839e-05 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.618 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.123518+0000", 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4339874020000001 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.123669+0000", 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015062300000000001 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1,7", 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.123813+0000", 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000144487 2024-09-21T10:12:46.619 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.134005+0000", 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010191817000000001 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.134077+0000", 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1769999999999999e-05 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.147373+0000", 2024-09-21T10:12:46.620 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013295968999999999 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161450+0000", 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014077265 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161485+0000", 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5074000000000003e-05 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161731+0000", 2024-09-21T10:12:46.621 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000246016 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:173524 9.e 9:757b1ab8:::benchmark_data_smithi017_237924_object80520:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:26.932658+0000", 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 19.569196868999999, 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.22930012899999999, 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:12:46.622 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 173524 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.932658+0000", 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.932658+0000", 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.623 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.932657+0000", 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:26.932665+0000", 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8189999999999998e-06 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.091025+0000", 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.15835991799999999 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.624 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.091046+0000", 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1067999999999999e-05 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.127439+0000", 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.036392597999999998 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.127587+0000", 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000148503 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.625 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1,7", 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.127729+0000", 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014148399999999999 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.134067+0000", 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0063377700000000004 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.134083+0000", 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6450999999999999e-05 2024-09-21T10:12:46.626 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.147488+0000", 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013404717 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161782+0000", 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014293685 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161814+0000", 2024-09-21T10:12:46.627 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2777999999999997e-05 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161959+0000", 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00014421799999999999 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:12:46.628 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:12:46.705 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:46.727 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:46.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:46.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:46.956 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:46.977 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:46.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:47.078 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:47.100 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:47.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:47.202 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:47.223 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:47.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:47.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:47.349 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:47.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:47.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:12:47.450 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:47.472 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:47.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:47.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:12:47.574 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:47.596 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:47.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:47.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:47 smithi017 ceph-mon[102913]: pgmap v1924: 109 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+remapped, 3 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 43 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 151/2118 objects degraded (7.129%); 171/2118 objects misplaced (8.074%); 0 B/s, 91 objects/s recovering 2024-09-21T10:12:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:47 smithi017 ceph-mon[105151]: pgmap v1924: 109 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+remapped, 3 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 43 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 151/2118 objects degraded (7.129%); 171/2118 objects misplaced (8.074%); 0 B/s, 91 objects/s recovering 2024-09-21T10:12:47.720 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:47.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:47.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:47 smithi134 ceph-mon[90085]: pgmap v1924: 109 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+remapped, 3 active+recovery_wait+degraded+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 43 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 151/2118 objects degraded (7.129%); 171/2118 objects misplaced (8.074%); 0 B/s, 91 objects/s recovering 2024-09-21T10:12:47.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:47.845 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:47.947 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:47.961 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:47.968 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:47.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:47.971 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:48.070 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:48.091 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:48.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:48.097 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-09-21T10:12:48.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-09-21T10:12:48.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:48.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:48.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:48.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:48.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:48.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:48.445 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:48.467 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:48.568 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:48.589 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:48.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:48.691 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:48.712 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:48.794 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:48 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 151/2118 objects degraded (7.129%), 3 pgs degraded) 2024-09-21T10:12:48.795 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:48 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 151/2118 objects degraded (7.129%), 3 pgs degraded) 2024-09-21T10:12:48.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:48.834 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:48.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:48.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:48.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:48 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 151/2118 objects degraded (7.129%), 3 pgs degraded) 2024-09-21T10:12:49.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:49.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:49.187 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:49.210 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:49.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:49.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:49.334 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:49.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:49.436 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:49.458 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:49.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:49.560 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:49.562 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 0.29 (4a3d) 2024-09-21T10:12:49.591 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:49.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:49.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:49 smithi017 ceph-mon[102913]: pgmap v1925: 109 pgs: 2 active+recovering+remapped, 5 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 4.2 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 0 B/s, 137 objects/s recovering 2024-09-21T10:12:49.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:49 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2529156390' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 0.29000048511979326} : dispatch 2024-09-21T10:12:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:49 smithi017 ceph-mon[105151]: pgmap v1925: 109 pgs: 2 active+recovering+remapped, 5 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 4.2 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 0 B/s, 137 objects/s recovering 2024-09-21T10:12:49.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:49 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2529156390' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 0.29000048511979326} : dispatch 2024-09-21T10:12:49.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:49.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:49.819 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:49.840 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:49.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:49.942 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:49.963 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:49.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:49 smithi134 ceph-mon[90085]: pgmap v1925: 109 pgs: 2 active+recovering+remapped, 5 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 100 active+clean; 4.2 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 0 B/s, 137 objects/s recovering 2024-09-21T10:12:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:49 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2529156390' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 0.29000048511979326} : dispatch 2024-09-21T10:12:50.065 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:50.086 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:50.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:50.188 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:50.205 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:50.209 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:50.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:50.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:50.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:50.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:50.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:50.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:12:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:12:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:12:50.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:50.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:50.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:50 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2529156390' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.29000048511979326}]': finished 2024-09-21T10:12:50.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:50 smithi017 ceph-mon[102913]: osdmap e1275: 8 total, 8 up, 8 in 2024-09-21T10:12:50.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:50 smithi017 ceph-mon[102913]: osdmap e1276: 8 total, 8 up, 8 in 2024-09-21T10:12:50.615 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:50 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2529156390' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.29000048511979326}]': finished 2024-09-21T10:12:50.615 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:50 smithi017 ceph-mon[105151]: osdmap e1275: 8 total, 8 up, 8 in 2024-09-21T10:12:50.616 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:50 smithi017 ceph-mon[105151]: osdmap e1276: 8 total, 8 up, 8 in 2024-09-21T10:12:50.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:50.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:50.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:50.828 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:50.850 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:50.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:50.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:50.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:50.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:50 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2529156390' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.29000048511979326}]': finished 2024-09-21T10:12:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:50 smithi134 ceph-mon[90085]: osdmap e1275: 8 total, 8 up, 8 in 2024-09-21T10:12:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:50 smithi134 ceph-mon[90085]: osdmap e1276: 8 total, 8 up, 8 in 2024-09-21T10:12:51.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:51.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:51.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:51.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:51.225 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:51.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:51.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:51.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:51.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:51.476 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:51.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:51.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:51.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:51.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:51.721 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:51.743 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:51.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:51.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:51 smithi017 ceph-mon[102913]: pgmap v1927: 109 pgs: 2 remapped+peering, 5 peering, 1 active+recovering+remapped, 3 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 0 B/s, 171 objects/s recovering 2024-09-21T10:12:51.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:51 smithi017 ceph-mon[102913]: osdmap e1277: 8 total, 8 up, 8 in 2024-09-21T10:12:51.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:51 smithi017 ceph-mon[105151]: pgmap v1927: 109 pgs: 2 remapped+peering, 5 peering, 1 active+recovering+remapped, 3 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 0 B/s, 171 objects/s recovering 2024-09-21T10:12:51.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:51 smithi017 ceph-mon[105151]: osdmap e1277: 8 total, 8 up, 8 in 2024-09-21T10:12:51.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:51.872 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:51.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:51.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:51.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:51 smithi134 ceph-mon[90085]: pgmap v1927: 109 pgs: 2 remapped+peering, 5 peering, 1 active+recovering+remapped, 3 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 0 B/s, 171 objects/s recovering 2024-09-21T10:12:51.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:51 smithi134 ceph-mon[90085]: osdmap e1277: 8 total, 8 up, 8 in 2024-09-21T10:12:51.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:51.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:52.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:52.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:52.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:52.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:52.247 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:52.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:52.349 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:52.370 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:52.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:52.394 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:12:52.394 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:12:52.395 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-21T10:12:52.395 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:12:52.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:52.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:52.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:52.598 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:52.620 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:52.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:52.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:52.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:52.847 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:52.869 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:52.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[102913]: 9.9 continuing backfill to osd.7 from (1253'5400,1273'11784] MIN to 1273'11784 2024-09-21T10:12:52.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[102913]: 9.5 continuing backfill to osd.7 from (1253'5400,1273'11694] MIN to 1273'11694 2024-09-21T10:12:52.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[102913]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1273'11748 2024-09-21T10:12:52.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[102913]: 9.7 continuing backfill to osd.7 from (1254'6200,1273'11716] MIN to 1273'11716 2024-09-21T10:12:52.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[102913]: 9.a continuing backfill to osd.7 from (1254'6800,1273'11654] MIN to 1273'11654 2024-09-21T10:12:52.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[102913]: 9.d continuing backfill to osd.7 from (1254'6600,1273'11512] MIN to 1273'11512 2024-09-21T10:12:52.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[102913]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1273'11566 2024-09-21T10:12:52.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[105151]: 9.9 continuing backfill to osd.7 from (1253'5400,1273'11784] MIN to 1273'11784 2024-09-21T10:12:52.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[105151]: 9.5 continuing backfill to osd.7 from (1253'5400,1273'11694] MIN to 1273'11694 2024-09-21T10:12:52.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[105151]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1273'11748 2024-09-21T10:12:52.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[105151]: 9.7 continuing backfill to osd.7 from (1254'6200,1273'11716] MIN to 1273'11716 2024-09-21T10:12:52.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[105151]: 9.a continuing backfill to osd.7 from (1254'6800,1273'11654] MIN to 1273'11654 2024-09-21T10:12:52.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[105151]: 9.d continuing backfill to osd.7 from (1254'6600,1273'11512] MIN to 1273'11512 2024-09-21T10:12:52.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:52 smithi017 ceph-mon[105151]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1273'11566 2024-09-21T10:12:52.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:52.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:52 smithi134 ceph-mon[90085]: 9.9 continuing backfill to osd.7 from (1253'5400,1273'11784] MIN to 1273'11784 2024-09-21T10:12:52.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:52 smithi134 ceph-mon[90085]: 9.5 continuing backfill to osd.7 from (1253'5400,1273'11694] MIN to 1273'11694 2024-09-21T10:12:52.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:52 smithi134 ceph-mon[90085]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1273'11748 2024-09-21T10:12:52.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:52 smithi134 ceph-mon[90085]: 9.7 continuing backfill to osd.7 from (1254'6200,1273'11716] MIN to 1273'11716 2024-09-21T10:12:52.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:52 smithi134 ceph-mon[90085]: 9.a continuing backfill to osd.7 from (1254'6800,1273'11654] MIN to 1273'11654 2024-09-21T10:12:52.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:52 smithi134 ceph-mon[90085]: 9.d continuing backfill to osd.7 from (1254'6600,1273'11512] MIN to 1273'11512 2024-09-21T10:12:52.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:52 smithi134 ceph-mon[90085]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1273'11566 2024-09-21T10:12:52.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:52.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:53.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:53.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:53.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:53.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:53.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:53.349 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:53.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:53.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:53.500 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:53.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:53.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:53.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:53.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:53.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:12:53.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:53.751 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:53.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:53.852 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:53.874 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:53.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:53 smithi017 ceph-mon[102913]: pgmap v1930: 109 pgs: 2 remapped+peering, 5 peering, 1 active+recovering+remapped, 3 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 11 B/s, 163 objects/s recovering 2024-09-21T10:12:53.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:53 smithi017 ceph-mon[105151]: pgmap v1930: 109 pgs: 2 remapped+peering, 5 peering, 1 active+recovering+remapped, 3 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 11 B/s, 163 objects/s recovering 2024-09-21T10:12:53.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:53 smithi134 ceph-mon[90085]: pgmap v1930: 109 pgs: 2 remapped+peering, 5 peering, 1 active+recovering+remapped, 3 active+recovery_wait+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 11 B/s, 163 objects/s recovering 2024-09-21T10:12:53.998 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:53.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:54.100 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:54.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:54.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:54.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:54.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:54.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:54.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:54.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:54.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:54.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:54.499 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:54.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:54.601 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:54.625 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:54.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:54.737 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:54.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:54.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:54.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:54.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:54.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:54 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 2/81 objects degraded (2.469%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:12:54.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:54 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 2/81 objects degraded (2.469%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:12:54.905 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:12:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:12:54] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-09-21T10:12:54.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:54 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 2/81 objects degraded (2.469%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:12:55.014 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:55.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:55.116 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:55.142 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:55.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:55.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:55.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:55.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:55.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:55.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:55.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:55.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:55.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:55.623 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:55.649 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:55.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:55.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:55 smithi017 ceph-mon[102913]: pgmap v1931: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 2 remapped+peering, 5 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 6 active+remapped+backfill_wait, 91 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 2/81 objects degraded (2.469%); 5701/81 objects misplaced (7038.272%); 11 B/s, 296 objects/s recovering 2024-09-21T10:12:55.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:55.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:55 smithi017 ceph-mon[105151]: pgmap v1931: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 2 remapped+peering, 5 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 6 active+remapped+backfill_wait, 91 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 2/81 objects degraded (2.469%); 5701/81 objects misplaced (7038.272%); 11 B/s, 296 objects/s recovering 2024-09-21T10:12:55.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:55.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:55.776 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:55.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:55.831 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:12:55.831 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:55.852 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:12:55.853 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:55.871 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:12:55.872 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:55.878 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:55.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:12:55.889 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:55.904 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:55.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:55.907 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:12:55.907 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:55.914 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:55.924 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:12:55.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:55.942 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:12:55.942 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:55.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:12:55.960 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:55 smithi134 ceph-mon[90085]: pgmap v1931: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 2 remapped+peering, 5 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 6 active+remapped+backfill_wait, 91 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 2/81 objects degraded (2.469%); 5701/81 objects misplaced (7038.272%); 11 B/s, 296 objects/s recovering 2024-09-21T10:12:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:56.007 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:56.032 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:56.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:56.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:56.160 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:56.262 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:56.288 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:56.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:56.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:56.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:56.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:56.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:56.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:56.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:56.639 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:56.657 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:12:56.665 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:56.767 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:56.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:56.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:56.894 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:56.919 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:56 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2287221399' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:56 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3532034982' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:56 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2287221399' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:56 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3532034982' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:56.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:56 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2287221399' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:56.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:56 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3532034982' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:57.020 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:57.045 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:57.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:57.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:57.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:57.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:57.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:57.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:57.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:57.521 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:57.547 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:57.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:57.649 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:57.675 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:57.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:57.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:57.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:57.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:57.903 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:57.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:57 smithi017 ceph-mon[102913]: pgmap v1932: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 3 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 6 active+remapped+backfill_wait, 96 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2/81 objects degraded (2.469%); 12232/81 objects misplaced (15101.235%); 92 B/s, 183 objects/s recovering 2024-09-21T10:12:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:57 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3532034982' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:12:57.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:57 smithi017 ceph-mon[102913]: osdmap e1278: 8 total, 8 up, 8 in 2024-09-21T10:12:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:57 smithi017 ceph-mon[105151]: pgmap v1932: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 3 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 6 active+remapped+backfill_wait, 96 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2/81 objects degraded (2.469%); 12232/81 objects misplaced (15101.235%); 92 B/s, 183 objects/s recovering 2024-09-21T10:12:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:57 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3532034982' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:12:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:57 smithi017 ceph-mon[105151]: osdmap e1278: 8 total, 8 up, 8 in 2024-09-21T10:12:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:57 smithi134 ceph-mon[90085]: pgmap v1932: 109 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+remapped, 3 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 6 active+remapped+backfill_wait, 96 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2/81 objects degraded (2.469%); 12232/81 objects misplaced (15101.235%); 92 B/s, 183 objects/s recovering 2024-09-21T10:12:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:57 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3532034982' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:12:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:57 smithi134 ceph-mon[90085]: osdmap e1278: 8 total, 8 up, 8 in 2024-09-21T10:12:58.030 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:58.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:58.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:58.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:58.182 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:58.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:58.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:58.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:58.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:58.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:58.435 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:58.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:58.536 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:58.561 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:58.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:58.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:58.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:58.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:58.813 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:58.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:58.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:59.040 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:59.066 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:59.167 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:59.193 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:59.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:59.202 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:12:59.202 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:12:59.203 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-21T10:12:59.203 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:12:59.203 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:12:59.203 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-21T10:12:59.294 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.320 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:59.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:59.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:59.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:59.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:59.489 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:59.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:59.569 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:59.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:59.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:59.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:12:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:59.797 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:59.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:59.880 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:12:59 smithi134 ceph-mon[90085]: pgmap v1934: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 81 B/s, 438 objects/s recovering 2024-09-21T10:12:59.920 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:59.942 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:12:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:12:59 smithi017 ceph-mon[102913]: pgmap v1934: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 81 B/s, 438 objects/s recovering 2024-09-21T10:12:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:12:59 smithi017 ceph-mon[105151]: pgmap v1934: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 81 B/s, 438 objects/s recovering 2024-09-21T10:13:00.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:00.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:00.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:00.169 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:00.190 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:00.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:00.292 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:00.313 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:00.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:00.415 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:00.436 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:13:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:13:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:13:00.538 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:00.558 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:00.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:00.660 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.682 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:00.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:00.783 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:00.805 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:00.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:00.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:00.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:00.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:00 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:13:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:00 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:13:00.952 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:00 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:13:01.030 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:01.053 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:01.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:01.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:01.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:01.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:01.203 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:13:01.203 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "" 2024-09-21T10:13:01.204 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:13:01.278 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:01.300 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:01.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:01.402 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:01.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:01.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:01.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:01.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.550 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:01.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:01.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:01.676 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:01.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:01.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:01 smithi017 ceph-mon[102913]: pgmap v1935: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 64 B/s, 388 objects/s recovering 2024-09-21T10:13:01.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:01 smithi017 ceph-mon[105151]: pgmap v1935: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 64 B/s, 388 objects/s recovering 2024-09-21T10:13:01.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:01.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:01.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:01.891 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:01 smithi134 ceph-mon[90085]: pgmap v1935: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 64 B/s, 388 objects/s recovering 2024-09-21T10:13:01.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:01.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:01.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:02.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:02.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:02.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:02.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:02.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:02.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:02.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:02.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:02.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:02.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:02.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:02.524 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:02.546 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:02.647 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:02.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:02.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:02.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:02.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:02.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:02.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:02.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:03.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:03.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:03.144 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:03.169 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:03.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:03.270 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:03.295 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:03.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:03.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:03.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:03.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:03.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:03.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:03.645 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:03.671 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:03.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:03.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:03.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:03.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:03.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:03 smithi017 ceph-mon[102913]: pgmap v1936: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 56 B/s, 344 objects/s recovering 2024-09-21T10:13:03.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:03 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 1/81 objects degraded (1.235%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:13:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:03 smithi017 ceph-mon[105151]: pgmap v1936: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 56 B/s, 344 objects/s recovering 2024-09-21T10:13:03.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:03 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 1/81 objects degraded (1.235%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:13:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:03 smithi134 ceph-mon[90085]: pgmap v1936: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 56 B/s, 344 objects/s recovering 2024-09-21T10:13:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:03 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 1/81 objects degraded (1.235%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:13:04.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:04.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:04.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:04.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:04.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:04.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:04.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:04.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:04.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:04.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.422 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:04.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:04.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:04.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:04.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:04.647 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:04.671 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:04.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:04.773 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:04.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:04.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:04.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:04.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:04.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:05.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:05.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:05.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:05.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:05.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:05.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:13:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:13:04] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.51.0" 2024-09-21T10:13:05.270 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:05.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:05.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:05.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:05.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:05.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:05.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:05.548 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:05.650 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:05.675 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:05.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:05.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:05.801 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:05.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:05.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:05.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:05.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:05 smithi017 ceph-mon[102913]: pgmap v1937: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 56 B/s, 357 objects/s recovering 2024-09-21T10:13:05.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:05 smithi017 ceph-mon[105151]: pgmap v1937: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 56 B/s, 357 objects/s recovering 2024-09-21T10:13:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:05 smithi134 ceph-mon[90085]: pgmap v1937: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 56 B/s, 357 objects/s recovering 2024-09-21T10:13:06.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:06.050 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:06.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:06.151 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:06.173 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:06.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:06.275 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:06.295 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:06.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:06.397 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:06.419 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:06.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:06.520 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:06.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:06.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:06.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:06.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:06.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:06.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:06.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:06.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:06.914 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:06.915 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:13:06.915 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:13:06.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:07.016 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:07.038 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:07.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:07.140 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:07.161 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:07.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:07.263 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:07.283 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:07.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:07.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:07.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:07.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:07.511 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:07.532 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:07.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:07.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:07.655 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:07.757 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:07.778 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:07.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:07.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:07.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:07.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:07 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:13:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:07 smithi017 ceph-mon[102913]: pgmap v1938: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 0 B/s, 356 objects/s recovering 2024-09-21T10:13:07.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:07 smithi017 ceph-mon[102913]: osdmap e1279: 8 total, 8 up, 8 in 2024-09-21T10:13:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:07 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:13:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:07 smithi017 ceph-mon[105151]: pgmap v1938: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 0 B/s, 356 objects/s recovering 2024-09-21T10:13:07.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:07 smithi017 ceph-mon[105151]: osdmap e1279: 8 total, 8 up, 8 in 2024-09-21T10:13:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:07 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:13:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:07 smithi134 ceph-mon[90085]: pgmap v1938: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 9293/81 objects misplaced (11472.840%); 0 B/s, 356 objects/s recovering 2024-09-21T10:13:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:07 smithi134 ceph-mon[90085]: osdmap e1279: 8 total, 8 up, 8 in 2024-09-21T10:13:08.006 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:08.028 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:08.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:08.129 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:08.150 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:08.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:08.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:08.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:08.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:08.377 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:08.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.400 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:08.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:08.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.523 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:08.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:08.623 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:08.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.646 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:08.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:08.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:08.770 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:08.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:08.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:08.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:08.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:08 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:13:08.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:08 smithi017 ceph-mon[102913]: osdmap e1280: 8 total, 8 up, 8 in 2024-09-21T10:13:08.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:08 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:13:08.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:08 smithi017 ceph-mon[105151]: osdmap e1280: 8 total, 8 up, 8 in 2024-09-21T10:13:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:08 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:13:08.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:08 smithi134 ceph-mon[90085]: osdmap e1280: 8 total, 8 up, 8 in 2024-09-21T10:13:08.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:09.026 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:09.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:09.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:09.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:09.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:09.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:09.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:09.505 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:09.531 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:09.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:09.657 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:09.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:09.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:09.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:09.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:09.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:09.910 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:09.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:09 smithi017 ceph-mon[102913]: 9.7 continuing backfill to osd.7 from (1254'6200,1273'11716] MIN to 1273'11716 2024-09-21T10:13:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:09 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:13:09.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:09 smithi017 ceph-mon[102913]: pgmap v1941: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 404 objects/s recovering 2024-09-21T10:13:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:09 smithi017 ceph-mon[105151]: 9.7 continuing backfill to osd.7 from (1254'6200,1273'11716] MIN to 1273'11716 2024-09-21T10:13:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:09 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:13:09.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:09 smithi017 ceph-mon[105151]: pgmap v1941: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 404 objects/s recovering 2024-09-21T10:13:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:09 smithi134 ceph-mon[90085]: 9.7 continuing backfill to osd.7 from (1254'6200,1273'11716] MIN to 1273'11716 2024-09-21T10:13:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:09 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:13:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:09 smithi134 ceph-mon[90085]: pgmap v1941: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 404 objects/s recovering 2024-09-21T10:13:10.013 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:10.036 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:10.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:10.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:10.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:10.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:10.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:10.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:10.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:10.390 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:10.416 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:13:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:13:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:13:10.518 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:10.543 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:10.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:10.645 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:10.670 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:10.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:10.772 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:10.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:10.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:10.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:10.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:10 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:13:10.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:10.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:10 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:13:10.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:10 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:13:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:11.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:11.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:11.147 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:11.171 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:11.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:11.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:11.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:11.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:11.398 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:11.423 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:11.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:11.525 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:11.550 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:11.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:11.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:11.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:11.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:11.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:11.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.804 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:11.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:11.905 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:11.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:11.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:11 smithi017 ceph-mon[102913]: pgmap v1942: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 404 objects/s recovering 2024-09-21T10:13:11.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:11 smithi017 ceph-mon[105151]: pgmap v1942: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 404 objects/s recovering 2024-09-21T10:13:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:11 smithi134 ceph-mon[90085]: pgmap v1942: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 404 objects/s recovering 2024-09-21T10:13:12.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:12.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:12.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:12.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:12.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:12.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:12.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:12.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:12.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:12.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:12.431 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:12.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:12.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:12.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:12.657 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:12.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:12.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:12.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:12.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.810 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:12.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:12.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:12.936 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:12.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:13.038 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:13.062 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:13.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:13.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:13.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:13.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:13.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:13.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:13.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:13.415 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:13.441 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:13.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:13.542 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:13.567 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:13.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:13.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:13.693 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:13.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:13.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:13.818 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:13.920 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:13.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:13 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:13:13.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:13 smithi017 ceph-mon[102913]: pgmap v1943: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 231 objects/s recovering 2024-09-21T10:13:13.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:13 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:13:13.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:13 smithi017 ceph-mon[105151]: pgmap v1943: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 231 objects/s recovering 2024-09-21T10:13:13.943 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:13 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:13:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:13 smithi134 ceph-mon[90085]: pgmap v1943: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 231 objects/s recovering 2024-09-21T10:13:14.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:14.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:14.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:14.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:14.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:14.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:14.296 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:14.321 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:14.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:14.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:14.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:14.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:14.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:14.671 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:14.697 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:14.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:14.704 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_4' removed 2024-09-21T10:13:14.799 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:14.821 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:14.923 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:14.944 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:14.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:14 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:13:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3232886077' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:13:15.044 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:14 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:13:15.044 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3232886077' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:13:15.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:14 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:13:15.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3232886077' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:13:15.045 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:13:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:13:14] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.51.0" 2024-09-21T10:13:15.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:15.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:15.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:15.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:15.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:15.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:15.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:15.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:15.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:15.422 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:15.447 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:15.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:13:15.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:15.573 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:15.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:15.675 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:15.698 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:15.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:15.706 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:15 smithi017 ceph-mon[102913]: pgmap v1944: 109 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 482 objects/s recovering 2024-09-21T10:13:15.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:15.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:15.925 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:15.950 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:15.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:15 smithi134 ceph-mon[90085]: pgmap v1944: 109 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 482 objects/s recovering 2024-09-21T10:13:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:15 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3232886077' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:13:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:15 smithi134 ceph-mon[90085]: osdmap e1281: 8 total, 8 up, 8 in 2024-09-21T10:13:16.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:16.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:16.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:16.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:15 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3232886077' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:13:16.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:15 smithi017 ceph-mon[102913]: osdmap e1281: 8 total, 8 up, 8 in 2024-09-21T10:13:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:15 smithi017 ceph-mon[105151]: pgmap v1944: 109 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 9 active+remapped+backfill_wait, 97 active+clean; 844 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%); 6531/81 objects misplaced (8062.963%); 0 B/s, 482 objects/s recovering 2024-09-21T10:13:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:15 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3232886077' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:13:16.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:15 smithi017 ceph-mon[105151]: osdmap e1281: 8 total, 8 up, 8 in 2024-09-21T10:13:16.203 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:16.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:16.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:16.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:16.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:16.432 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:16.457 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:16.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:16.559 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:16.582 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:16.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:16.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:16.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:16.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:16.838 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:16.940 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:16.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:16.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:16 smithi134 ceph-mon[90085]: 1.9 scrub starts 2024-09-21T10:13:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:16 smithi134 ceph-mon[90085]: 1.9 scrub ok 2024-09-21T10:13:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:16 smithi134 ceph-mon[90085]: 1.13 scrub starts 2024-09-21T10:13:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:16 smithi134 ceph-mon[90085]: 1.13 scrub ok 2024-09-21T10:13:17.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:17.089 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:17.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:17.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:16 smithi017 ceph-mon[102913]: 1.9 scrub starts 2024-09-21T10:13:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:16 smithi017 ceph-mon[102913]: 1.9 scrub ok 2024-09-21T10:13:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:16 smithi017 ceph-mon[102913]: 1.13 scrub starts 2024-09-21T10:13:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:16 smithi017 ceph-mon[102913]: 1.13 scrub ok 2024-09-21T10:13:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:16 smithi017 ceph-mon[105151]: 1.9 scrub starts 2024-09-21T10:13:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:16 smithi017 ceph-mon[105151]: 1.9 scrub ok 2024-09-21T10:13:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:16 smithi017 ceph-mon[105151]: 1.13 scrub starts 2024-09-21T10:13:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:16 smithi017 ceph-mon[105151]: 1.13 scrub ok 2024-09-21T10:13:17.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:17.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:17.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:17.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:17.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:17.450 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:17.473 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:17.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:13:17.575 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:17.598 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:17.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:17.699 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:17.722 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:17.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:17.823 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:17.845 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:17.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:17.947 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:17 smithi017 ceph-mon[102913]: pgmap v1946: 93 pgs: 1 active+recovery_wait+degraded, 92 active+clean; 792 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%) 2024-09-21T10:13:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:17 smithi017 ceph-mon[102913]: 1.d scrub starts 2024-09-21T10:13:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:17 smithi017 ceph-mon[102913]: 1.d scrub ok 2024-09-21T10:13:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:17 smithi017 ceph-mon[105151]: pgmap v1946: 93 pgs: 1 active+recovery_wait+degraded, 92 active+clean; 792 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%) 2024-09-21T10:13:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:17 smithi017 ceph-mon[105151]: 1.d scrub starts 2024-09-21T10:13:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:17 smithi017 ceph-mon[105151]: 1.d scrub ok 2024-09-21T10:13:17.968 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:17.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:17 smithi134 ceph-mon[90085]: pgmap v1946: 93 pgs: 1 active+recovery_wait+degraded, 92 active+clean; 792 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 1/81 objects degraded (1.235%) 2024-09-21T10:13:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:17 smithi134 ceph-mon[90085]: 1.d scrub starts 2024-09-21T10:13:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:17 smithi134 ceph-mon[90085]: 1.d scrub ok 2024-09-21T10:13:18.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:18.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:18.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:18.199 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:18.222 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:18.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:18.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:18.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:18.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:18.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:18.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:18.581 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:18.606 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:18.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:18.708 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:18.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.734 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:18.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:18.835 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.862 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:18.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:18.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:18.968 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[102913]: 1.7 scrub ok 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[102913]: 1.2 scrub starts 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[102913]: 1.2 scrub ok 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[102913]: 1.3 scrub starts 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[102913]: 1.3 scrub ok 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[105151]: 1.7 scrub ok 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[105151]: 1.2 scrub starts 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[105151]: 1.2 scrub ok 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[105151]: 1.3 scrub starts 2024-09-21T10:13:18.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:18 smithi017 ceph-mon[105151]: 1.3 scrub ok 2024-09-21T10:13:18.970 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:13:18.971 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:13:18.971 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-09-21T10:13:18.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_5 16 2024-09-21T10:13:18.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:18.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:18 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:13:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:18 smithi134 ceph-mon[90085]: 1.7 scrub ok 2024-09-21T10:13:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:18 smithi134 ceph-mon[90085]: 1.2 scrub starts 2024-09-21T10:13:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:18 smithi134 ceph-mon[90085]: 1.2 scrub ok 2024-09-21T10:13:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:18 smithi134 ceph-mon[90085]: 1.3 scrub starts 2024-09-21T10:13:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:18 smithi134 ceph-mon[90085]: 1.3 scrub ok 2024-09-21T10:13:19.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:19.112 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:19.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:19.239 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:19.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:19.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:19.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.365 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:19.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:19.458 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:13:19.458 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:13:19.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:19.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:19.590 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:19.616 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:19.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:19.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:19.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:19.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:19.867 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:19.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:19.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:19.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:19.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:19 smithi134 ceph-mon[90085]: pgmap v1947: 93 pgs: 93 active+clean; 757 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:19 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/81 objects degraded (1.235%), 1 pg degraded) 2024-09-21T10:13:20.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:20.118 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:19 smithi017 ceph-mon[102913]: pgmap v1947: 93 pgs: 93 active+clean; 757 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:20.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:19 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/81 objects degraded (1.235%), 1 pg degraded) 2024-09-21T10:13:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:19 smithi017 ceph-mon[105151]: pgmap v1947: 93 pgs: 93 active+clean; 757 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:19 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/81 objects degraded (1.235%), 1 pg degraded) 2024-09-21T10:13:20.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:20.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:20.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:20.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:20.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:20.474 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:13:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:13:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:13:20.500 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:20.602 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:20.628 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:20.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:20.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:20.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:20.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:20.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:20.987 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:21.013 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:21.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:21.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:21.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:21.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:21.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:13:21.240 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:21.241 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:21.242 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:21.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:21.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:21.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:21.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:21.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:21.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:21.510 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.537 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:21.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:21.638 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:21.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.665 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:21.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:21.767 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:21.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:21.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:21.835 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:21 smithi017 ceph-mon[102913]: pgmap v1948: 93 pgs: 93 active+clean; 757 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:21.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:21 smithi017 ceph-mon[102913]: 1.0 scrub starts 2024-09-21T10:13:21.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:21 smithi017 ceph-mon[102913]: 1.0 scrub ok 2024-09-21T10:13:21.836 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:21 smithi017 ceph-mon[105151]: pgmap v1948: 93 pgs: 93 active+clean; 757 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:21.836 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:21 smithi017 ceph-mon[105151]: 1.0 scrub starts 2024-09-21T10:13:21.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:21 smithi017 ceph-mon[105151]: 1.0 scrub ok 2024-09-21T10:13:21.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.920 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:21.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:21 smithi134 ceph-mon[90085]: pgmap v1948: 93 pgs: 93 active+clean; 757 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:21 smithi134 ceph-mon[90085]: 1.0 scrub starts 2024-09-21T10:13:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:21 smithi134 ceph-mon[90085]: 1.0 scrub ok 2024-09-21T10:13:22.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:22.044 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:22.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:22.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:22.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:22.271 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:22.297 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:22.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:22.398 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:22.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:22.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:22.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.553 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:22.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:22.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:22.679 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:22.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:22.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:22.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:22.905 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:22.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:22.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:23.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:23.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:23.158 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:23.184 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:23.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:23.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:23.311 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:23.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:23.413 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:23.438 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:23.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:23.565 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:23.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:23.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:23.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:23.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:23.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:23.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:23.942 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:23.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:23 smithi017 ceph-mon[102913]: pgmap v1949: 93 pgs: 93 active+clean; 652 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:23 smithi017 ceph-mon[102913]: 1.10 scrub starts 2024-09-21T10:13:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:23 smithi017 ceph-mon[102913]: 1.10 scrub ok 2024-09-21T10:13:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:23 smithi017 ceph-mon[105151]: pgmap v1949: 93 pgs: 93 active+clean; 652 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:23 smithi017 ceph-mon[105151]: 1.10 scrub starts 2024-09-21T10:13:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:23 smithi017 ceph-mon[105151]: 1.10 scrub ok 2024-09-21T10:13:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:23 smithi134 ceph-mon[90085]: pgmap v1949: 93 pgs: 93 active+clean; 652 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:23 smithi134 ceph-mon[90085]: 1.10 scrub starts 2024-09-21T10:13:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:23 smithi134 ceph-mon[90085]: 1.10 scrub ok 2024-09-21T10:13:24.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:24.071 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:24.172 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:24.198 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:24.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:24.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:24.416 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:24.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:24.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:24.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:24.575 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:24.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:24.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:24.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:24.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:24.829 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:24.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:24.854 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:13:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:13:24] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.51.0" 2024-09-21T10:13:24.931 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:24.956 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:24.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:25.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:25.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:25.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:25.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:25.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:25.306 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:25.332 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:25.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:25.434 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:25.460 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:25.562 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:25.588 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:25.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:25.690 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:25.713 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:25.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:25.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:25.838 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:25.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:25 smithi017 ceph-mon[102913]: pgmap v1950: 93 pgs: 93 active+clean; 652 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:25.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:25.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:25.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:25.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:25 smithi017 ceph-mon[105151]: pgmap v1950: 93 pgs: 93 active+clean; 652 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:25.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:25.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:25.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:25.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:25.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:25.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:25 smithi134 ceph-mon[90085]: pgmap v1950: 93 pgs: 93 active+clean; 652 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:26.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:26.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:26.188 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:26.214 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:26.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:26.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:26.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:26.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:26.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:26.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:26.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:26.590 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:26.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:26.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:26.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:26.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:26.733 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:13:26.818 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:26.844 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:26.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:26.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:26.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:26.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:26 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2046311706' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:26 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:27.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:26 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2046311706' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:27.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:26 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:27.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:26 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2046311706' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:27.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:26 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:27.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:27.095 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:27.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:27.140 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:13:27.140 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:13:27.157 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:13:27.157 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:13:27.173 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:13:27.173 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:13:27.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:13:27.190 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:13:27.197 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:27.206 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:13:27.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:13:27.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.223 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:27.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:27.224 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:13:27.224 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:13:27.240 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:13:27.240 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:13:27.257 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:13:27.257 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:13:27.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:27.349 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:27.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:27.451 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:27.474 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:27.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:27.576 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:27.600 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:27.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:27.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:27.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:27.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:27.734 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_5' created 2024-09-21T10:13:27.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:27.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:27.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[102913]: pgmap v1951: 93 pgs: 93 active+clean; 626 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3.2 KiB/s, 0 objects/s recovering 2024-09-21T10:13:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[102913]: osdmap e1282: 8 total, 8 up, 8 in 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4241939605' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/671321758' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[105151]: pgmap v1951: 93 pgs: 93 active+clean; 626 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3.2 KiB/s, 0 objects/s recovering 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[105151]: osdmap e1282: 8 total, 8 up, 8 in 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4241939605' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:27 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/671321758' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:13:27.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:27.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:27.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:27 smithi134 ceph-mon[90085]: pgmap v1951: 93 pgs: 93 active+clean; 626 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3.2 KiB/s, 0 objects/s recovering 2024-09-21T10:13:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:27 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:13:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:27 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:13:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:27 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:27 smithi134 ceph-mon[90085]: osdmap e1282: 8 total, 8 up, 8 in 2024-09-21T10:13:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:27 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4241939605' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:27 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/671321758' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:13:28.077 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:28.102 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:28.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:28.204 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:28.230 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:28.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:28.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:28.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:28.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:28.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-09-21T10:13:28.458 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:28.482 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:28.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:28.584 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:28.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:28.708 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:28.731 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:28.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:28.833 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:28.855 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:28.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:28.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:28.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:28.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:28 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/671321758' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-21T10:13:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:28 smithi134 ceph-mon[90085]: osdmap e1283: 8 total, 8 up, 8 in 2024-09-21T10:13:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:28 smithi134 ceph-mon[90085]: 1.12 scrub starts 2024-09-21T10:13:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:28 smithi134 ceph-mon[90085]: 1.12 scrub ok 2024-09-21T10:13:29.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:29.104 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:29.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:29.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:28 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/671321758' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-21T10:13:29.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:28 smithi017 ceph-mon[102913]: osdmap e1283: 8 total, 8 up, 8 in 2024-09-21T10:13:29.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:28 smithi017 ceph-mon[102913]: 1.12 scrub starts 2024-09-21T10:13:29.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:28 smithi017 ceph-mon[102913]: 1.12 scrub ok 2024-09-21T10:13:29.113 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:28 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/671321758' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-21T10:13:29.113 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:28 smithi017 ceph-mon[105151]: osdmap e1283: 8 total, 8 up, 8 in 2024-09-21T10:13:29.113 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:28 smithi017 ceph-mon[105151]: 1.12 scrub starts 2024-09-21T10:13:29.113 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:28 smithi017 ceph-mon[105151]: 1.12 scrub ok 2024-09-21T10:13:29.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:29.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:29.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:29.330 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:29.352 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:29.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:29.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:29.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:29.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:29.600 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:29.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:29.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:29.724 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:29.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:29.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:29 smithi017 ceph-mon[102913]: pgmap v1954: 109 pgs: 4 creating+peering, 12 unknown, 93 active+clean; 599 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:29.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:29 smithi017 ceph-mon[102913]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:13:29.781 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:29 smithi017 ceph-mon[102913]: osdmap e1284: 8 total, 8 up, 8 in 2024-09-21T10:13:29.781 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:29 smithi017 ceph-mon[105151]: pgmap v1954: 109 pgs: 4 creating+peering, 12 unknown, 93 active+clean; 599 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:29.781 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:29 smithi017 ceph-mon[105151]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:13:29.781 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:29 smithi017 ceph-mon[105151]: osdmap e1284: 8 total, 8 up, 8 in 2024-09-21T10:13:29.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:29.848 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:29.950 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:29.973 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:30.048 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:29 smithi134 ceph-mon[90085]: pgmap v1954: 109 pgs: 4 creating+peering, 12 unknown, 93 active+clean; 599 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:30.048 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:29 smithi134 ceph-mon[90085]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:13:30.048 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:29 smithi134 ceph-mon[90085]: osdmap e1284: 8 total, 8 up, 8 in 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263842 8.e 8:74c05c70:::benchmark_data_smithi017_209235_object130936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.460994+0000", 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 305.590094685, 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1964085500000001, 2024-09-21T10:13:30.063 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263842 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.460994+0000", 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.460994+0000", 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.460995+0000", 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9300000000000006e-07 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.460999+0000", 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9559999999999999e-06 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.461009+0000", 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6919999999999997e-06 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.461014+0000", 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.938e-06 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.461072+0000", 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7902999999999998e-05 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.461083+0000", 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0606e-05 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.420582+0000", 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95949984399999999 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.420593+0000", 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0464e-05 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.642934+0000", 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.222341165 2024-09-21T10:13:30.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.643068+0000", 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013448799999999999 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.643209+0000", 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014089300000000001 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.646941+0000", 2024-09-21T10:13:30.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037320299999999999 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.653755+0000", 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068134429999999998 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657243+0000", 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034884680000000002 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657293+0000", 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9672999999999997e-05 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657403+0000", 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109994 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.071 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263850 8.e 8:73dca25d:::benchmark_data_smithi017_209235_object130944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.477238+0000", 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 305.57385069200001, 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1804316350000001, 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263850 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477238+0000", 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477238+0000", 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477240+0000", 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.604e-06 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477246+0000", 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4470000000000003e-06 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477262+0000", 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5455999999999999e-05 2024-09-21T10:13:30.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477269+0000", 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.311e-06 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477333+0000", 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3355999999999994e-05 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477343+0000", 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0161e-05 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421884+0000", 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94454095100000002 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421895+0000", 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1535999999999999e-05 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.646983+0000", 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.225087781 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.647090+0000", 2024-09-21T10:13:30.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000107135 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.647161+0000", 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1082999999999997e-05 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.653697+0000", 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00653588 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657450+0000", 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037531679999999999 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657463+0000", 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2928999999999999e-05 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657549+0000", 2024-09-21T10:13:30.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6212999999999996e-05 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657670+0000", 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120624 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.080 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263816 8.2 8:41665334:::benchmark_data_smithi017_209235_object130910:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1225)", 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:25.417025+0000", 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.63406400600002, 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1940257160000001, 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263816 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417025+0000", 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417025+0000", 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417027+0000", 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9709999999999998e-06 2024-09-21T10:13:30.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417032+0000", 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3190000000000002e-06 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417045+0000", 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2417e-05 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417053+0000", 2024-09-21T10:13:30.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2640000000000008e-06 2024-09-21T10:13:30.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.418942+0000", 2024-09-21T10:13:30.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001888881 2024-09-21T10:13:30.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.418954+0000", 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.243e-05 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.610963+0000", 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.192009044 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.611051+0000", 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7390000000000002e-05 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263823 8.2 8:4eebd32c:::benchmark_data_smithi017_209235_object130917:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1225)", 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:25.417069+0000", 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.63402014299999, 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.204893024, 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:13:30.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263823 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417069+0000", 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417069+0000", 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417069+0000", 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0800000000000005e-07 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417072+0000", 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4660000000000002e-06 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417079+0000", 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5810000000000002e-06 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417084+0000", 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3000000000000001e-06 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.418981+0000", 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001897328 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.418990+0000", 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5369999999999997e-06 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.611241+0000", 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1922517100000001 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.611362+0000", 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120532 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.621839+0000", 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010477405 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.621962+0000", 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122657 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263843 8.2 8:456c39a6:::benchmark_data_smithi017_209235_object130937:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1225)", 2024-09-21T10:13:30.091 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:25.417091+0000", 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.63399855799997, 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.204318803, 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263843 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417091+0000", 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417091+0000", 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417091+0000", 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8200000000000001e-07 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417093+0000", 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.869e-06 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417099+0000", 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0510000000000002e-06 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417104+0000", 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0830000000000003e-06 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.419019+0000", 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001915536 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.419029+0000", 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0440000000000001e-06 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.611391+0000", 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1923624049999999 2024-09-21T10:13:30.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.611509+0000", 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118362 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.611660+0000", 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015029999999999999 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.619525+0000", 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0078652389999999996 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.619624+0000", 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9499000000000004e-05 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.621143+0000", 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015186310000000001 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.621306+0000", 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016311599999999999 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.621409+0000", 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103286 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57281 9.9 9:92fb5fa7:::benchmark_data_smithi017_237924_object57280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:13:30.098 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.448907+0000", 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 245.602182293, 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2562489019999998, 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57281 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448907+0000", 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448907+0000", 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448909+0000", 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7600000000000001e-06 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448954+0000", 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5025999999999999e-05 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448968+0000", 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4243000000000001e-05 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448996+0000", 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8161e-05 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.449193+0000", 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019727500000000001 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.449205+0000", 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1917e-05 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.458659+0000", 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0094541319999999 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.458672+0000", 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2559e-05 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.687854+0000", 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.229181708 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.687998+0000", 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014425500000000001 2024-09-21T10:13:30.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.688153+0000", 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015539199999999999 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.693095+0000", 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049422550000000004 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.693218+0000", 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122077 2024-09-21T10:13:30.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.702384+0000", 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0091666160000000007 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.702457+0000", 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2898999999999994e-05 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.705156+0000", 2024-09-21T10:13:30.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026986269999999999 2024-09-21T10:13:30.106 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.106 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.106 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.106 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57298 9.5 9:a7783708:::benchmark_data_smithi017_237924_object57297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:13:30.106 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.469242+0000", 2024-09-21T10:13:30.106 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 245.58184704799999, 2024-09-21T10:13:30.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.293953723, 2024-09-21T10:13:30.107 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.107 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57298 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469242+0000", 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469242+0000", 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469243+0000", 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.344e-06 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469266+0000", 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2154000000000002e-05 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469281+0000", 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5262000000000001e-05 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469290+0000", 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7800000000000006e-06 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469364+0000", 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4004999999999994e-05 2024-09-21T10:13:30.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469375+0000", 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.17e-05 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.434823+0000", 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96544806999999999 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.434837+0000", 2024-09-21T10:13:30.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3692e-05 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.741938+0000", 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.307101399 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.742085+0000", 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000146911 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.742239+0000", 2024-09-21T10:13:30.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015382899999999999 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.747492+0000", 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052529810000000003 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.752487+0000", 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049951450000000003 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.756963+0000", 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044756479999999996 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.757079+0000", 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116134 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.763196+0000", 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061166689999999999 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.114 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:147301 9.9 9:936e4338:::benchmark_data_smithi017_237924_object54297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1260)", 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:11:48.131412+0000", 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 101.91967744, 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16806140999999999, 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147301 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.115 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.131412+0000", 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.131412+0000", 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.131413+0000", 2024-09-21T10:13:30.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.824e-06 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.131420+0000", 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3570000000000003e-06 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.131435+0000", 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5149e-05 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.131443+0000", 2024-09-21T10:13:30.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1100000000000003e-06 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.133445+0000", 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020023929999999999 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.133595+0000", 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014960099999999999 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:13:30.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.133761+0000", 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016601499999999999 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.137085+0000", 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033233970000000001 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.140525+0000", 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034405479999999999 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.152373+0000", 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011847768 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.152437+0000", 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3691999999999999e-05 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.299473+0000", 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14703655600000001 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.120 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:147306 9.9 9:96175ea5:::benchmark_data_smithi017_237924_object54302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1260)", 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:11:48.136355+0000", 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 101.914734528, 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16471823599999999, 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147306 2024-09-21T10:13:30.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.136355+0000", 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.136355+0000", 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.136356+0000", 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.61e-06 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.136360+0000", 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9040000000000002e-06 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.136373+0000", 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3104e-05 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.136380+0000", 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2989999999999999e-06 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.137231+0000", 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00085069200000000001 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.137380+0000", 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014879199999999999 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.137530+0000", 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014955799999999999 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.140668+0000", 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031381989999999999 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.141558+0000", 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00089026100000000005 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.299532+0000", 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15797440900000001 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.299585+0000", 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2581999999999999e-05 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.301073+0000", 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001487826 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.126 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:147325 9.9 9:921b73e1:::benchmark_data_smithi017_237924_object54321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1260)", 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:11:48.214128+0000", 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 101.836961249, 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.154925642, 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.127 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147325 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.214128+0000", 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.214128+0000", 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.214130+0000", 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.491e-06 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.214139+0000", 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3000000000000002e-06 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.299382+0000", 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.085243125000000003 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.299402+0000", 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0035e-05 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.301118+0000", 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017158449999999999 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.301261+0000", 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014367699999999999 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.301421+0000", 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016009600000000001 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.314824+0000", 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013402251 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.314838+0000", 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4653e-05 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.331357+0000", 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016518201 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.331404+0000", 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7448999999999999e-05 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:11:48.369053+0000", 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.037649518999999999 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.132 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:171056 9.b 9:dbe40d8e:::benchmark_data_smithi017_237924_object78052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:21.281199+0000", 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 68.769890419999996, 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.085546774000000006, 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.133 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 171056 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.281199+0000", 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.281199+0000", 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.281200+0000", 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2500000000000004e-07 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.281204+0000", 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0740000000000003e-06 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.281213+0000", 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8060000000000006e-06 2024-09-21T10:13:30.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.281220+0000", 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5459999999999997e-06 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.289682+0000", 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0084622770000000007 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.290108+0000", 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042547199999999998 2024-09-21T10:13:30.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.290534+0000", 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042661400000000002 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.295531+0000", 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049969799999999998 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.302356+0000", 2024-09-21T10:13:30.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068247769999999998 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.366007+0000", 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.063651442000000003 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.366067+0000", 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9938000000000001e-05 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.366745+0000", 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00067802300000000004 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:171065 9.5 9:ab459791:::benchmark_data_smithi017_237924_object78061:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:21.291088+0000", 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 68.760001482000007, 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077851437999999995, 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 171065 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.291088+0000", 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.291088+0000", 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.291089+0000", 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4389999999999999e-06 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.291095+0000", 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6890000000000001e-06 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.291117+0000", 2024-09-21T10:13:30.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1784e-05 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.291126+0000", 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0159999999999997e-06 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.295181+0000", 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040556489999999997 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.295305+0000", 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012420000000000001 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.297573+0000", 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002267261 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.308269+0000", 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010696746 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.309188+0000", 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00091860599999999998 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.367381+0000", 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.058192630000000002 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.367430+0000", 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9713999999999999e-05 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:21.368939+0000", 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001508704 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:172994 9.5 9:adbb7b66:::benchmark_data_smithi017_237924_object79990:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:23.902093+0000", 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 66.148996056000001, 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.073517015000000005, 2024-09-21T10:13:30.145 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 172994 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.902093+0000", 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.902093+0000", 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.902093+0000", 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1699999999999999e-07 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.902096+0000", 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2520000000000002e-06 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.902102+0000", 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.246e-06 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.902107+0000", 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7119999999999999e-06 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.905802+0000", 2024-09-21T10:13:30.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036952899999999999 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.906015+0000", 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021288500000000001 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.906178+0000", 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016328800000000001 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.909528+0000", 2024-09-21T10:13:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033498260000000002 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.914170+0000", 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046422700000000004 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.974049+0000", 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.05987845 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.974106+0000", 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7788000000000003e-05 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:23.975610+0000", 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015035910000000001 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173500 9.5 9:adc0c03e:::benchmark_data_smithi017_237924_object80496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.683064+0000", 2024-09-21T10:13:30.151 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 65.368025391000003, 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2520501240000002, 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173500 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683064+0000", 2024-09-21T10:13:30.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683064+0000", 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683066+0000", 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8190000000000001e-06 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683072+0000", 2024-09-21T10:13:30.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9789999999999998e-06 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683083+0000", 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1467e-05 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683093+0000", 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.594e-06 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686032+0000", 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029398359999999999 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686156+0000", 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012317 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686284+0000", 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012887799999999999 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.695220+0000", 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0089355019999999997 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.700524+0000", 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053036070000000001 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.719426+0000", 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018902077 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.719439+0000", 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3179999999999999e-05 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.693124+0000", 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.973685315 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.693142+0000", 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8020999999999999e-05 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917018+0000", 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2238759670000001 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917129+0000", 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111148 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.935025+0000", 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017895409000000001 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.935114+0000", 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9154999999999996e-05 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173510 9.5 9:aed0fd9d:::benchmark_data_smithi017_237924_object80506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.718247+0000", 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 65.332842459000005, 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2162618580000002, 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.159 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173510 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718247+0000", 2024-09-21T10:13:30.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718247+0000", 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718249+0000", 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2330000000000001e-06 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718255+0000", 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1990000000000003e-06 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718269+0000", 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.375e-05 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718279+0000", 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.679e-06 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.721856+0000", 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035772970000000001 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.721865+0000", 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1989999999999995e-06 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.693179+0000", 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97131427199999998 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.693188+0000", 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2259999999999994e-06 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917704+0000", 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2245152340000001 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917832+0000", 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128565 2024-09-21T10:13:30.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5,7", 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917978+0000", 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014551200000000001 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.920920+0000", 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029422060000000002 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.921118+0000", 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000197648 2024-09-21T10:13:30.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.921345+0000", 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022725100000000001 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.934019+0000", 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012674505000000001 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.934102+0000", 2024-09-21T10:13:30.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2096999999999997e-05 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.934508+0000", 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040698500000000001 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173512 9.9 9:99bb9c6d:::benchmark_data_smithi017_237924_object80508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.718318+0000", 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 65.332771557000001, 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.211803164, 2024-09-21T10:13:30.167 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173512 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718318+0000", 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718318+0000", 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718318+0000", 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2800000000000003e-07 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718320+0000", 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6929999999999999e-06 2024-09-21T10:13:30.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718325+0000", 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4310000000000002e-06 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718330+0000", 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.515e-06 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.724088+0000", 2024-09-21T10:13:30.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057589590000000001 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.724097+0000", 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5499999999999995e-06 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.696969+0000", 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97287164800000003 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.696980+0000", 2024-09-21T10:13:30.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1403000000000001e-05 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.887650+0000", 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1906697939999999 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.887743+0000", 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3257000000000005e-05 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.887820+0000", 2024-09-21T10:13:30.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6514000000000006e-05 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894542+0000", 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067218820000000002 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894590+0000", 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8013000000000003e-05 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894646+0000", 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6104999999999999e-05 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929714+0000", 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035068662 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929780+0000", 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5324e-05 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930121+0000", 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034108599999999999 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173518 9.9 9:935b61bf:::benchmark_data_smithi017_237924_object80514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.729828+0000", 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 65.321261254000007, 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2030523099999999, 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.175 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173518 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729828+0000", 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729828+0000", 2024-09-21T10:13:30.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729830+0000", 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.756e-06 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729836+0000", 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3099999999999997e-06 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729853+0000", 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7350000000000002e-05 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729862+0000", 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9190000000000005e-06 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729929+0000", 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7291000000000003e-05 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729937+0000", 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3529999999999997e-06 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.698207+0000", 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96827046900000002 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.698217+0000", 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9399999999999997e-06 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.891189+0000", 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.192971875 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.891337+0000", 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000147396 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.891474+0000", 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013725 2024-09-21T10:13:30.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894704+0000", 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032297950000000001 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894749+0000", 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5677000000000003e-05 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894763+0000", 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4248e-05 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.932394+0000", 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.037630080000000003 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.932448+0000", 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4033999999999998e-05 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.932880+0000", 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00043256700000000002 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173523 9.b 9:da84e5ec:::benchmark_data_smithi017_237924_object80519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:26.930242+0000", 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 63.120847046999998, 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17958227399999999, 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.183 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173523 2024-09-21T10:13:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930242+0000", 2024-09-21T10:13:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930242+0000", 2024-09-21T10:13:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930243+0000", 2024-09-21T10:13:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.423e-06 2024-09-21T10:13:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930249+0000", 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6099999999999997e-06 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930263+0000", 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4301000000000001e-05 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930271+0000", 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3769999999999998e-06 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930345+0000", 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4672999999999995e-05 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930467+0000", 2024-09-21T10:13:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012116999999999999 2024-09-21T10:13:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:13:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930645+0000", 2024-09-21T10:13:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017858199999999999 2024-09-21T10:13:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933362+0000", 2024-09-21T10:13:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027165800000000001 2024-09-21T10:13:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933459+0000", 2024-09-21T10:13:30.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7238000000000007e-05 2024-09-21T10:13:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.191 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:30.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.108981+0000", 2024-09-21T10:13:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.175522122 2024-09-21T10:13:30.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.109109+0000", 2024-09-21T10:13:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012760899999999999 2024-09-21T10:13:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.109824+0000", 2024-09-21T10:13:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00071558899999999998 2024-09-21T10:13:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.195 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173533 9.b 9:d9d1f2a4:::benchmark_data_smithi017_237924_object80529:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-09-21T10:13:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:26.947935+0000", 2024-09-21T10:13:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 63.103153665999997, 2024-09-21T10:13:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.162530853, 2024-09-21T10:13:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.196 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173533 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947935+0000", 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947935+0000", 2024-09-21T10:13:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947937+0000", 2024-09-21T10:13:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.539e-06 2024-09-21T10:13:30.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947942+0000", 2024-09-21T10:13:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5530000000000003e-06 2024-09-21T10:13:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947958+0000", 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6538999999999999e-05 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947966+0000", 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9379999999999992e-06 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.948027+0000", 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1144999999999998e-05 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.948113+0000", 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5628000000000001e-05 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.948227+0000", 2024-09-21T10:13:30.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113869 2024-09-21T10:13:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.951310+0000", 2024-09-21T10:13:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003083222 2024-09-21T10:13:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.951399+0000", 2024-09-21T10:13:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8776000000000005e-05 2024-09-21T10:13:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.109879+0000", 2024-09-21T10:13:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15848061599999999 2024-09-21T10:13:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.109980+0000", 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100931 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.110466+0000", 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000486097 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173542 9.b 9:d75940a4:::benchmark_data_smithi017_237924_object80538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:26.993838+0000", 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 63.057251587000003, 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.117099365, 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:13:30.205 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:13:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:13:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173542 2024-09-21T10:13:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:13:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:13:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.993838+0000", 2024-09-21T10:13:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.993838+0000", 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.993839+0000", 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2309999999999999e-06 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.993844+0000", 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8520000000000003e-06 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.993855+0000", 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1144999999999999e-05 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.993860+0000", 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4099999999999999e-06 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.993920+0000", 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9660000000000001e-05 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.994039+0000", 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011871600000000001 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.994134+0000", 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5698000000000004e-05 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.996407+0000", 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022732049999999999 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.996636+0000", 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022901200000000001 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.110514+0000", 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.113877433 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:13:30.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.110570+0000", 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5988e-05 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.110937+0000", 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000367015 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:13:30.212 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:13:30.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:30.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:30.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:30.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:30.362 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:30.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:30.464 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:30.491 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:30.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:13:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:13:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:13:30.592 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:30.616 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:30.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:30.739 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:30.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:13:30.842 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:30.866 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:30.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:30.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:13:30.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:30.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:30.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:31.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:31.116 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:30 smithi017 ceph-mon[102913]: osdmap e1285: 8 total, 8 up, 8 in 2024-09-21T10:13:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:30 smithi017 ceph-mon[102913]: 1.b scrub starts 2024-09-21T10:13:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:30 smithi017 ceph-mon[102913]: 1.b scrub ok 2024-09-21T10:13:31.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:30 smithi017 ceph-mon[105151]: osdmap e1285: 8 total, 8 up, 8 in 2024-09-21T10:13:31.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:30 smithi017 ceph-mon[105151]: 1.b scrub starts 2024-09-21T10:13:31.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:30 smithi017 ceph-mon[105151]: 1.b scrub ok 2024-09-21T10:13:31.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:31.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:31.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:30 smithi134 ceph-mon[90085]: osdmap e1285: 8 total, 8 up, 8 in 2024-09-21T10:13:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:30 smithi134 ceph-mon[90085]: 1.b scrub starts 2024-09-21T10:13:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:30 smithi134 ceph-mon[90085]: 1.b scrub ok 2024-09-21T10:13:31.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:31.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:31.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:31.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:31.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:31.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:31.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:13:31.591 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:31.617 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:31.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:31.719 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:31.744 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:31.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:31.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:31.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:31.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:31 smithi017 ceph-mon[102913]: pgmap v1957: 109 pgs: 6 creating+peering, 7 unknown, 96 active+clean; 599 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:31 smithi017 ceph-mon[105151]: pgmap v1957: 109 pgs: 6 creating+peering, 7 unknown, 96 active+clean; 599 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:31.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:31.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:31.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:32.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:32.119 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:32.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:32.220 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:32.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:31 smithi134 ceph-mon[90085]: pgmap v1957: 109 pgs: 6 creating+peering, 7 unknown, 96 active+clean; 599 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:32.246 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:32.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:32.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:32.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:32.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:32.473 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:32.499 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:32.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:32.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:32.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:32.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:32.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:32.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:32.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:32.848 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:32.873 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:32.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:32.945 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:32.966 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:32.974 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:32.990 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:32.999 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:32.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:33.100 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:33.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:33.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:33.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:33.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:33.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:33.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:33.372 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:33.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:33.473 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:33.499 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:33.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:33.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:33.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:33.725 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:33.749 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:33.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:33.753 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-09-21T10:13:33.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:33.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:33.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:33.974 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:34.000 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:34.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:34.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:33 smithi017 ceph-mon[102913]: pgmap v1958: 109 pgs: 6 creating+peering, 103 active+clean; 599 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:34.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:33 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2618137065' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:13:34.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:33 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:13:34.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:33 smithi017 ceph-mon[105151]: pgmap v1958: 109 pgs: 6 creating+peering, 103 active+clean; 599 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:34.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:33 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2618137065' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:13:34.080 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:33 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:13:34.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:34.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:34.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:33 smithi134 ceph-mon[90085]: pgmap v1958: 109 pgs: 6 creating+peering, 103 active+clean; 599 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:33 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2618137065' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:13:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:33 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:13:34.251 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:34.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:34.375 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:34.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:34.477 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:34.502 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:34.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:34.604 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:34.629 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:34.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:34.730 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:34.755 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:34.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:34.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:34.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:34.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:35.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:35.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:35.018 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:34 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:13:35.018 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:34 smithi017 ceph-mon[102913]: osdmap e1286: 8 total, 8 up, 8 in 2024-09-21T10:13:35.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:34 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:13:35.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:34 smithi017 ceph-mon[105151]: osdmap e1286: 8 total, 8 up, 8 in 2024-09-21T10:13:35.019 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:13:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:13:34] "GET /metrics HTTP/1.1" 200 34236 "" "Prometheus/2.51.0" 2024-09-21T10:13:35.109 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:35.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:35.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:34 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:13:35.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:34 smithi134 ceph-mon[90085]: osdmap e1286: 8 total, 8 up, 8 in 2024-09-21T10:13:35.237 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:35.263 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:35.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:35.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:35.390 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:35.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:35.492 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:35.514 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:35.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:35.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:35.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:35.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:35.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:35.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:35.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:35.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:35.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:36.005 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:36.026 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:36.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:36.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:36.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:35 smithi017 ceph-mon[105151]: pgmap v1960: 109 pgs: 109 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:35 smithi017 ceph-mon[105151]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:13:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:35 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:13:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:35 smithi017 ceph-mon[102913]: pgmap v1960: 109 pgs: 109 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:35 smithi017 ceph-mon[102913]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:13:36.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:35 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:13:36.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:35 smithi134 ceph-mon[90085]: pgmap v1960: 109 pgs: 109 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:36.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:35 smithi134 ceph-mon[90085]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:13:36.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:35 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:13:36.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:36.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:36.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:36.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:36.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:36.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:36.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:36.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:36.637 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:36.659 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:36.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:36.760 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:36.781 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:36.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:36.883 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:36.904 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:36.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:37.005 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:37.028 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:37.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:37.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:37.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:37.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:37.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:37.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:37.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:37.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:37.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:37.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:37.505 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:37.520 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:13:37.520 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:13:37.520 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-21T10:13:37.520 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:13:37.528 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:37.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:37.653 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:37.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:37.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:37.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:37.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:37.901 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:37.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:38.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:38.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:38.051 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:37 smithi017 ceph-mon[102913]: pgmap v1961: 109 pgs: 109 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:38.051 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:37 smithi017 ceph-mon[105151]: pgmap v1961: 109 pgs: 109 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:38.126 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:38.151 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:38.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:38.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:37 smithi134 ceph-mon[90085]: pgmap v1961: 109 pgs: 109 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:38.253 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:38.277 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:38.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:38.401 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:38.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:38.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:38.529 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:38.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:38.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:38.655 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:38.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:38.756 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:38.767 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:13:38.783 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:38.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:38.884 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:38.909 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:38.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:39.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:39.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:39.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:39.132 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:38 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3298415665' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:13:39.133 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:38 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3298415665' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:13:39.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:39.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:39.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:38 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3298415665' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:13:39.259 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:39.285 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:39.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:39.386 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:39.410 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:39.442 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-09-21T10:13:39.442 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2024-09-21T10:13:39.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:13:39.444 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:13:39.444 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:13:39.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:39.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:39.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:39.551 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-21T10:13:39.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:13:39.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_268344 2024-09-21T10:13:39.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:13:39.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:13:39.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:39.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:39.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:39.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:39.834 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:39.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:39.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:13:39.936 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:39.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.972 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:39.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:40.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:40.088 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[102913]: pgmap v1962: 109 pgs: 109 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:40.088 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:40.088 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3298415665' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:13:40.088 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[102913]: osdmap e1287: 8 total, 8 up, 8 in 2024-09-21T10:13:40.088 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:13:40.088 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:40.088 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:40.089 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[105151]: pgmap v1962: 109 pgs: 109 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:40.089 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:40.089 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3298415665' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:13:40.090 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[105151]: osdmap e1287: 8 total, 8 up, 8 in 2024-09-21T10:13:40.090 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:13:40.090 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:40.090 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:40.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:40.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:40.167 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:40.183 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:13:40.183 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:13:40.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:39 smithi134 ceph-mon[90085]: pgmap v1962: 109 pgs: 109 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:13:40.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:39 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:40.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:39 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3298415665' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:13:40.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:39 smithi134 ceph-mon[90085]: osdmap e1287: 8 total, 8 up, 8 in 2024-09-21T10:13:40.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:13:40.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:40.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:40.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:13:40.222 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:13:40.230 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:40.259 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:40.266 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:13:40.266 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:13:40.301 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:13:40.301 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:13:40.336 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:13:40.336 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:13:40.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:40.370 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:13:40.371 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:13:40.391 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:40.408 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:13:40.408 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:13:40.440 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:13:40.440 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:13:40.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:13:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:13:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:13:40.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:40.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:40.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3079 3063 191.423 191.438 0.00292084 0.00520478 2024-09-21T10:13:40.640 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:40.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:40.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:40.810 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:40.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:40.950 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:41.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:41.077 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:41.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:41.179 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-21T10:13:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]} : dispatch 2024-09-21T10:13:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]} : dispatch 2024-09-21T10:13:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]} : dispatch 2024-09-21T10:13:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3087570193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:41.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-21T10:13:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]} : dispatch 2024-09-21T10:13:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]} : dispatch 2024-09-21T10:13:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]} : dispatch 2024-09-21T10:13:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3087570193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:41.207 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:41.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-21T10:13:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]} : dispatch 2024-09-21T10:13:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]} : dispatch 2024-09-21T10:13:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]} : dispatch 2024-09-21T10:13:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3087570193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:41.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:41.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:41.309 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:41.347 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:41.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:41.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:13:41.449 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:41.476 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:41.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 6486 6470 202.171 212.938 0.00616061 0.00493612 2024-09-21T10:13:41.578 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:41.610 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:41.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:41.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:41.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:41.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:41.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:41.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:41.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:41 smithi134 ceph-mon[90085]: pgmap v1964: 109 pgs: 109 active+clean; 19 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2024-09-21T10:13:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-09-21T10:13:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]}]': finished 2024-09-21T10:13:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-21T10:13:41.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-09-21T10:13:41.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:41 smithi134 ceph-mon[90085]: osdmap e1288: 8 total, 8 up, 8 in 2024-09-21T10:13:41.996 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:42.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.030 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:42.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:42.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[102913]: pgmap v1964: 109 pgs: 109 active+clean; 19 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2024-09-21T10:13:42.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-09-21T10:13:42.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]}]': finished 2024-09-21T10:13:42.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-21T10:13:42.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-09-21T10:13:42.077 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[102913]: osdmap e1288: 8 total, 8 up, 8 in 2024-09-21T10:13:42.077 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[105151]: pgmap v1964: 109 pgs: 109 active+clean; 19 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2024-09-21T10:13:42.077 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-09-21T10:13:42.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.3", "id": [1, 2]}]': finished 2024-09-21T10:13:42.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-21T10:13:42.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-09-21T10:13:42.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:41 smithi017 ceph-mon[105151]: osdmap e1288: 8 total, 8 up, 8 in 2024-09-21T10:13:42.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:42.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:42.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:42.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:42.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:42.436 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:42.472 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:42.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:42.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 9324 9308 193.898 177.375 0.00166101 0.00506965 2024-09-21T10:13:42.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:42.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:42.710 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:42.730 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:42.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:42.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:42 smithi017 ceph-mon[102913]: osdmap e1289: 8 total, 8 up, 8 in 2024-09-21T10:13:42.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:42 smithi017 ceph-mon[105151]: osdmap e1289: 8 total, 8 up, 8 in 2024-09-21T10:13:42.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:42.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:42.957 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:42.984 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:43.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:43.116 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:43.218 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:42 smithi134 ceph-mon[90085]: osdmap e1289: 8 total, 8 up, 8 in 2024-09-21T10:13:43.248 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:43.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:43.350 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:43.378 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:43.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:43.480 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:43.513 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:43.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:43.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 11396 11380 177.796 129.5 0.00968177 0.00561449 2024-09-21T10:13:43.615 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:43.645 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:43.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:43.662 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:43.747 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:43.776 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:43.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:43.878 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:43.920 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:43.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:44.022 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:44.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:43 smithi017 ceph-mon[102913]: pgmap v1967: 109 pgs: 109 active+clean; 131 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 22 MiB/s wr, 347 op/s 2024-09-21T10:13:44.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:43 smithi017 ceph-mon[105151]: pgmap v1967: 109 pgs: 109 active+clean; 131 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 22 MiB/s wr, 347 op/s 2024-09-21T10:13:44.053 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:44.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:44.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:44.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:44.200 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:43 smithi134 ceph-mon[90085]: pgmap v1967: 109 pgs: 109 active+clean; 131 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 22 MiB/s wr, 347 op/s 2024-09-21T10:13:44.297 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:44.318 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:44.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:44.420 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:44.450 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:44.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:44.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:44.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 13640 13624 170.284 140.25 0.00387929 0.00586406 2024-09-21T10:13:44.587 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:44.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:13:44.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:44.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:44.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:44.815 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:44.844 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:44.946 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:44.949 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:13:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:13:44] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-09-21T10:13:44.967 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:44.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:45.068 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:45.102 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:45.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:45.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:45.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:45.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:45.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:45.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:45.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:45.462 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:45.496 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:45.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:45.552 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 15124 15108 157.36 92.75 0.00285667 0.0063242 2024-09-21T10:13:45.598 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:45.622 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:45.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:45.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:45.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:45.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:45.863 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:45.896 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:45.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:45.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:46.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:46.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:46.080 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:13:46.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:45 smithi017 ceph-mon[105151]: pgmap v1968: 109 pgs: 109 active+clean; 372 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2024-09-21T10:13:46.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:45 smithi017 ceph-mon[105151]: from='client.36237 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:13:46.082 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:45 smithi017 ceph-mon[102913]: pgmap v1968: 109 pgs: 109 active+clean; 372 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2024-09-21T10:13:46.082 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:45 smithi017 ceph-mon[102913]: from='client.36237 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:13:46.083 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:13:46.089 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:46.138 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:46.167 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:45 smithi134 ceph-mon[90085]: pgmap v1968: 109 pgs: 109 active+clean; 372 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2024-09-21T10:13:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:45 smithi134 ceph-mon[90085]: from='client.36237 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:13:46.269 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:46.293 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:46.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:46.395 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:46.429 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:46.530 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:46.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 15 17210 17195 153.511 130.438 0.00525263 0.00650748 2024-09-21T10:13:46.565 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:46.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:46.667 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:46.709 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:46.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:46.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:46.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:46.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:46.949 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:46.988 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:47.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:47.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:47.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:47.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:47.270 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:47.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:47.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:47.403 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:47.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:47.505 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:47.536 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:47.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:47.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 19953 19937 155.742 171.375 0.00504356 0.00641438 2024-09-21T10:13:47.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:47.683 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:47.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:47.821 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:47.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:47.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:47.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:48.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:48.097 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:48.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:48.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:47 smithi017 ceph-mon[102913]: pgmap v1969: 109 pgs: 109 active+clean; 834 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:48.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:48.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:48.120 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:47 smithi017 ceph-mon[105151]: pgmap v1969: 109 pgs: 109 active+clean; 834 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:48.120 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:48.120 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:48.198 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:48.232 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:48.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:47 smithi134 ceph-mon[90085]: pgmap v1969: 109 pgs: 109 active+clean; 834 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:48.334 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:48.367 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:48.468 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:48.508 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:48.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:48.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 22400 22384 155.429 152.938 0.00466965 0.00642725 2024-09-21T10:13:48.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:48.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:48.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:48.753 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:48.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:48.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:48.871 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1968,"stamp":"2024-09-21T10:13:44.455301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":389945293,"num_objects":5967,"num_object_clones":0,"num_object_copies":17901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6004,"num_write_kb":380255,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":615122,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":8099,"ondisk_log_size":8099,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":333,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8175448,"kb_used_data":2656632,"kb_used_omap":411,"kb_used_meta":5518308,"kb_avail":741785768,"statfs":{"total":767960285184,"available":759588626432,"internally_reserved":0,"allocated":2720391168,"data_stored":2695727143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":420990,"internal_metadata":5650748290},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,4,6],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":389349376,"num_objects":5941,"num_object_clones":0,"num_object_copies":17823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5941,"num_write_kb":380224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001372"},"pg_stats":[{"pgid":"1.11","version":"288'100","reported_seq":736,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.556735+0000","last_change":"2024-09-21T10:12:50.589581+0000","last_active":"2024-09-21T10:13:39.556735+0000","last_peered":"2024-09-21T10:13:39.556735+0000","last_clean":"2024-09-21T10:13:39.556735+0000","last_became_active":"2024-09-21T10:12:50.589222+0000","last_became_peered":"2024-09-21T10:12:50.589222+0000","last_unstale":"2024-09-21T10:13:39.556735+0000","last_undegraded":"2024-09-21T10:13:39.556735+0000","last_fullsized":"2024-09-21T10:13:39.556735+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.888350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.888350+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:37:10.080751+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1947,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.780067+0000","last_change":"2024-09-21T10:12:25.715091+0000","last_active":"2024-09-21T10:13:41.780067+0000","last_peered":"2024-09-21T10:13:41.780067+0000","last_clean":"2024-09-21T10:13:41.780067+0000","last_became_active":"2024-09-21T10:12:25.711946+0000","last_became_peered":"2024-09-21T10:12:25.711946+0000","last_unstale":"2024-09-21T10:13:41.780067+0000","last_undegraded":"2024-09-21T10:13:41.780067+0000","last_fullsized":"2024-09-21T10:13:41.780067+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:08:07.743522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":36,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555268+0000","last_change":"2024-09-21T10:12:25.720808+0000","last_active":"2024-09-21T10:13:39.555268+0000","last_peered":"2024-09-21T10:13:39.555268+0000","last_clean":"2024-09-21T10:13:39.555268+0000","last_became_active":"2024-09-21T10:12:25.706699+0000","last_became_peered":"2024-09-21T10:12:25.706699+0000","last_unstale":"2024-09-21T10:13:39.555268+0000","last_undegraded":"2024-09-21T10:13:39.555268+0000","last_fullsized":"2024-09-21T10:13:39.555268+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:29.044701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"288'100","reported_seq":50,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554915+0000","last_change":"2024-09-21T10:13:22.620513+0000","last_active":"2024-09-21T10:13:39.554915+0000","last_peered":"2024-09-21T10:13:39.554915+0000","last_clean":"2024-09-21T10:13:39.554915+0000","last_became_active":"2024-09-21T10:12:50.581542+0000","last_became_peered":"2024-09-21T10:12:50.581542+0000","last_unstale":"2024-09-21T10:13:39.554915+0000","last_undegraded":"2024-09-21T10:13:39.554915+0000","last_fullsized":"2024-09-21T10:13:39.554915+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:22.620446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:22.620446+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:13.111755+0000","scrub_duration":0.0014078199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":40,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555522+0000","last_change":"2024-09-21T10:12:50.584513+0000","last_active":"2024-09-21T10:13:39.555522+0000","last_peered":"2024-09-21T10:13:39.555522+0000","last_clean":"2024-09-21T10:13:39.555522+0000","last_became_active":"2024-09-21T10:12:50.584107+0000","last_became_peered":"2024-09-21T10:12:50.584107+0000","last_unstale":"2024-09-21T10:13:39.555522+0000","last_undegraded":"2024-09-21T10:13:39.555522+0000","last_fullsized":"2024-09-21T10:13:39.555522+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:23.047403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2013,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554127+0000","last_change":"2024-09-21T10:12:25.700249+0000","last_active":"2024-09-21T10:13:39.554127+0000","last_peered":"2024-09-21T10:13:39.554127+0000","last_clean":"2024-09-21T10:13:39.554127+0000","last_became_active":"2024-09-21T10:12:25.699912+0000","last_became_peered":"2024-09-21T10:12:25.699912+0000","last_unstale":"2024-09-21T10:13:39.554127+0000","last_undegraded":"2024-09-21T10:13:39.554127+0000","last_fullsized":"2024-09-21T10:13:39.554127+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:47:59.054215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"288'100","reported_seq":126,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789948+0000","last_change":"2024-09-21T10:13:16.405658+0000","last_active":"2024-09-21T10:13:41.789948+0000","last_peered":"2024-09-21T10:13:41.789948+0000","last_clean":"2024-09-21T10:13:41.789948+0000","last_became_active":"2024-09-21T10:08:45.521658+0000","last_became_peered":"2024-09-21T10:08:45.521658+0000","last_unstale":"2024-09-21T10:13:41.789948+0000","last_undegraded":"2024-09-21T10:13:41.789948+0000","last_fullsized":"2024-09-21T10:13:41.789948+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:16.405582+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:16.405582+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:32.832948+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2014,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554291+0000","last_change":"2024-09-21T10:12:25.703395+0000","last_active":"2024-09-21T10:13:39.554291+0000","last_peered":"2024-09-21T10:13:39.554291+0000","last_clean":"2024-09-21T10:13:39.554291+0000","last_became_active":"2024-09-21T10:12:25.701460+0000","last_became_peered":"2024-09-21T10:12:25.701460+0000","last_unstale":"2024-09-21T10:13:39.554291+0000","last_undegraded":"2024-09-21T10:13:39.554291+0000","last_fullsized":"2024-09-21T10:13:39.554291+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:27:12.806968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":30,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554820+0000","last_change":"2024-09-21T10:12:25.700080+0000","last_active":"2024-09-21T10:13:39.554820+0000","last_peered":"2024-09-21T10:13:39.554820+0000","last_clean":"2024-09-21T10:13:39.554820+0000","last_became_active":"2024-09-21T10:12:25.699762+0000","last_became_peered":"2024-09-21T10:12:25.699762+0000","last_unstale":"2024-09-21T10:13:39.554820+0000","last_undegraded":"2024-09-21T10:13:39.554820+0000","last_fullsized":"2024-09-21T10:13:39.554820+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:41.509175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"288'100","reported_seq":147,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789429+0000","last_change":"2024-09-21T10:13:28.346056+0000","last_active":"2024-09-21T10:13:41.789429+0000","last_peered":"2024-09-21T10:13:41.789429+0000","last_clean":"2024-09-21T10:13:41.789429+0000","last_became_active":"2024-09-21T10:12:50.582710+0000","last_became_peered":"2024-09-21T10:12:50.582710+0000","last_unstale":"2024-09-21T10:13:41.789429+0000","last_undegraded":"2024-09-21T10:13:41.789429+0000","last_fullsized":"2024-09-21T10:13:41.789429+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:28.346003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:28.346003+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.707712+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1947,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.781813+0000","last_change":"2024-09-21T10:12:25.707969+0000","last_active":"2024-09-21T10:13:41.781813+0000","last_peered":"2024-09-21T10:13:41.781813+0000","last_clean":"2024-09-21T10:13:41.781813+0000","last_became_active":"2024-09-21T10:12:25.707531+0000","last_became_peered":"2024-09-21T10:12:25.707531+0000","last_unstale":"2024-09-21T10:13:41.781813+0000","last_undegraded":"2024-09-21T10:13:41.781813+0000","last_fullsized":"2024-09-21T10:13:41.781813+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:08:47.289964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":75,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:38.768386+0000","last_change":"2024-09-21T10:12:50.620444+0000","last_active":"2024-09-21T10:13:38.768386+0000","last_peered":"2024-09-21T10:13:38.768386+0000","last_clean":"2024-09-21T10:13:38.768386+0000","last_became_active":"2024-09-21T10:12:50.589267+0000","last_became_peered":"2024-09-21T10:12:50.589267+0000","last_unstale":"2024-09-21T10:13:38.768386+0000","last_undegraded":"2024-09-21T10:13:38.768386+0000","last_fullsized":"2024-09-21T10:13:38.768386+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:42:11.055834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"1287'97","reported_seq":116,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.033145+0000","last_change":"2024-09-21T10:13:28.746733+0000","last_active":"2024-09-21T10:13:40.033145+0000","last_peered":"2024-09-21T10:13:40.033145+0000","last_clean":"2024-09-21T10:13:40.033145+0000","last_became_active":"2024-09-21T10:13:28.746116+0000","last_became_peered":"2024-09-21T10:13:28.746116+0000","last_unstale":"2024-09-21T10:13:40.033145+0000","last_undegraded":"2024-09-21T10:13:40.033145+0000","last_fullsized":"2024-09-21T10:13:40.033145+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:01.477994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6356992,"num_objects":97,"num_object_clones":0,"num_object_copies":291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":97,"num_write_kb":6208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"288'100","reported_seq":40,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555400+0000","last_change":"2024-09-21T10:13:16.697889+0000","last_active":"2024-09-21T10:13:39.555400+0000","last_peered":"2024-09-21T10:13:39.555400+0000","last_clean":"2024-09-21T10:13:39.555400+0000","last_became_active":"2024-09-21T10:12:25.699431+0000","last_became_peered":"2024-09-21T10:12:25.699431+0000","last_unstale":"2024-09-21T10:13:39.555400+0000","last_undegraded":"2024-09-21T10:13:39.555400+0000","last_fullsized":"2024-09-21T10:13:39.555400+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:16.697812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:16.697812+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:18.395903+0000","scrub_duration":0.001732562,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":34,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555742+0000","last_change":"2024-09-21T10:12:25.764496+0000","last_active":"2024-09-21T10:13:39.555742+0000","last_peered":"2024-09-21T10:13:39.555742+0000","last_clean":"2024-09-21T10:13:39.555742+0000","last_became_active":"2024-09-21T10:12:25.707263+0000","last_became_peered":"2024-09-21T10:12:25.707263+0000","last_unstale":"2024-09-21T10:13:39.555742+0000","last_undegraded":"2024-09-21T10:13:39.555742+0000","last_fullsized":"2024-09-21T10:13:39.555742+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:45.728577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":40,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554910+0000","last_change":"2024-09-21T10:12:50.573976+0000","last_active":"2024-09-21T10:13:39.554910+0000","last_peered":"2024-09-21T10:13:39.554910+0000","last_clean":"2024-09-21T10:13:39.554910+0000","last_became_active":"2024-09-21T10:12:50.573657+0000","last_became_peered":"2024-09-21T10:12:50.573657+0000","last_unstale":"2024-09-21T10:13:39.554910+0000","last_undegraded":"2024-09-21T10:13:39.554910+0000","last_fullsized":"2024-09-21T10:13:39.554910+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:44:44.245943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"1288'254","reported_seq":266,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.810392+0000","last_change":"2024-09-21T10:13:28.789587+0000","last_active":"2024-09-21T10:13:40.810392+0000","last_peered":"2024-09-21T10:13:40.810392+0000","last_clean":"2024-09-21T10:13:40.810392+0000","last_became_active":"2024-09-21T10:13:28.789374+0000","last_became_peered":"2024-09-21T10:13:28.789374+0000","last_unstale":"2024-09-21T10:13:40.810392+0000","last_undegraded":"2024-09-21T10:13:40.810392+0000","last_fullsized":"2024-09-21T10:13:40.810392+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":254,"log_dups_size":0,"ondisk_log_size":254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:46:42.415657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16646144,"num_objects":254,"num_object_clones":0,"num_object_copies":762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":254,"num_write_kb":16256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"288'100","reported_seq":409,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.779912+0000","last_change":"2024-09-21T10:10:13.678895+0000","last_active":"2024-09-21T10:13:41.779912+0000","last_peered":"2024-09-21T10:13:41.779912+0000","last_clean":"2024-09-21T10:13:41.779912+0000","last_became_active":"2024-09-21T10:10:13.678219+0000","last_became_peered":"2024-09-21T10:10:13.678219+0000","last_unstale":"2024-09-21T10:13:41.779912+0000","last_undegraded":"2024-09-21T10:13:41.779912+0000","last_fullsized":"2024-09-21T10:13:41.779912+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:27.395700+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:27.395700+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:26.476978+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":34,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555465+0000","last_change":"2024-09-21T10:12:25.711574+0000","last_active":"2024-09-21T10:13:39.555465+0000","last_peered":"2024-09-21T10:13:39.555465+0000","last_clean":"2024-09-21T10:13:39.555465+0000","last_became_active":"2024-09-21T10:12:25.699847+0000","last_became_peered":"2024-09-21T10:12:25.699847+0000","last_unstale":"2024-09-21T10:13:39.555465+0000","last_undegraded":"2024-09-21T10:13:39.555465+0000","last_fullsized":"2024-09-21T10:13:39.555465+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:09:56.556064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1254,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.776664+0000","last_change":"2024-09-21T10:08:45.515858+0000","last_active":"2024-09-21T10:13:40.776664+0000","last_peered":"2024-09-21T10:13:40.776664+0000","last_clean":"2024-09-21T10:13:40.776664+0000","last_became_active":"2024-09-21T10:08:45.515543+0000","last_became_peered":"2024-09-21T10:08:45.515543+0000","last_unstale":"2024-09-21T10:13:40.776664+0000","last_undegraded":"2024-09-21T10:13:40.776664+0000","last_fullsized":"2024-09-21T10:13:40.776664+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:29.565740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"1287'85","reported_seq":104,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.021585+0000","last_change":"2024-09-21T10:13:28.789306+0000","last_active":"2024-09-21T10:13:40.021585+0000","last_peered":"2024-09-21T10:13:40.021585+0000","last_clean":"2024-09-21T10:13:40.021585+0000","last_became_active":"2024-09-21T10:13:28.789015+0000","last_became_peered":"2024-09-21T10:13:28.789015+0000","last_unstale":"2024-09-21T10:13:40.021585+0000","last_undegraded":"2024-09-21T10:13:40.021585+0000","last_fullsized":"2024-09-21T10:13:40.021585+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":85,"log_dups_size":0,"ondisk_log_size":85,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:03.609333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5570560,"num_objects":85,"num_object_clones":0,"num_object_copies":255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":85,"num_write_kb":5440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"288'100","reported_seq":45,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:38.770364+0000","last_change":"2024-09-21T10:10:13.624076+0000","last_active":"2024-09-21T10:13:38.770364+0000","last_peered":"2024-09-21T10:13:38.770364+0000","last_clean":"2024-09-21T10:13:38.770364+0000","last_became_active":"2024-09-21T10:10:13.623709+0000","last_became_peered":"2024-09-21T10:10:13.623709+0000","last_unstale":"2024-09-21T10:13:38.770364+0000","last_undegraded":"2024-09-21T10:13:38.770364+0000","last_fullsized":"2024-09-21T10:13:38.770364+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:23.746751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:23.746751+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:22.068163+0000","scrub_duration":0.0021986229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":48,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555238+0000","last_change":"2024-09-21T10:12:50.623002+0000","last_active":"2024-09-21T10:13:39.555238+0000","last_peered":"2024-09-21T10:13:39.555238+0000","last_clean":"2024-09-21T10:13:39.555238+0000","last_became_active":"2024-09-21T10:12:50.588355+0000","last_became_peered":"2024-09-21T10:12:50.588355+0000","last_unstale":"2024-09-21T10:13:39.555238+0000","last_undegraded":"2024-09-21T10:13:39.555238+0000","last_fullsized":"2024-09-21T10:13:39.555238+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:22.564459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":55,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:38.770018+0000","last_change":"2024-09-21T10:12:25.707043+0000","last_active":"2024-09-21T10:13:38.770018+0000","last_peered":"2024-09-21T10:13:38.770018+0000","last_clean":"2024-09-21T10:13:38.770018+0000","last_became_active":"2024-09-21T10:12:25.706636+0000","last_became_peered":"2024-09-21T10:12:25.706636+0000","last_unstale":"2024-09-21T10:13:38.770018+0000","last_undegraded":"2024-09-21T10:13:38.770018+0000","last_fullsized":"2024-09-21T10:13:38.770018+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:38.994917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"1289'583","reported_seq":606,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:42.660918+0000","last_change":"2024-09-21T10:13:28.789163+0000","last_active":"2024-09-21T10:13:42.660918+0000","last_peered":"2024-09-21T10:13:42.660918+0000","last_clean":"2024-09-21T10:13:42.660918+0000","last_became_active":"2024-09-21T10:13:28.788814+0000","last_became_peered":"2024-09-21T10:13:28.788814+0000","last_unstale":"2024-09-21T10:13:42.660918+0000","last_undegraded":"2024-09-21T10:13:42.660918+0000","last_fullsized":"2024-09-21T10:13:42.660918+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":583,"log_dups_size":0,"ondisk_log_size":583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:28:48.487989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38207488,"num_objects":583,"num_object_clones":0,"num_object_copies":1749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":583,"num_write_kb":37312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"288'100","reported_seq":95,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.776160+0000","last_change":"2024-09-21T10:12:25.697192+0000","last_active":"2024-09-21T10:13:40.776160+0000","last_peered":"2024-09-21T10:13:40.776160+0000","last_clean":"2024-09-21T10:13:40.776160+0000","last_became_active":"2024-09-21T10:12:25.696789+0000","last_became_peered":"2024-09-21T10:12:25.696789+0000","last_unstale":"2024-09-21T10:13:40.776160+0000","last_undegraded":"2024-09-21T10:13:40.776160+0000","last_fullsized":"2024-09-21T10:13:40.776160+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:03.581657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:03.581657+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:24.147270+0000","scrub_duration":0.0038308109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":947,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.780188+0000","last_change":"2024-09-21T10:10:13.678563+0000","last_active":"2024-09-21T10:13:41.780188+0000","last_peered":"2024-09-21T10:13:41.780188+0000","last_clean":"2024-09-21T10:13:41.780188+0000","last_became_active":"2024-09-21T10:10:13.676594+0000","last_became_peered":"2024-09-21T10:10:13.676594+0000","last_unstale":"2024-09-21T10:13:41.780188+0000","last_undegraded":"2024-09-21T10:13:41.780188+0000","last_fullsized":"2024-09-21T10:13:41.780188+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:34.275762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2026,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554177+0000","last_change":"2024-09-21T10:08:45.528426+0000","last_active":"2024-09-21T10:13:39.554177+0000","last_peered":"2024-09-21T10:13:39.554177+0000","last_clean":"2024-09-21T10:13:39.554177+0000","last_became_active":"2024-09-21T10:08:45.528049+0000","last_became_peered":"2024-09-21T10:08:45.528049+0000","last_unstale":"2024-09-21T10:13:39.554177+0000","last_undegraded":"2024-09-21T10:13:39.554177+0000","last_fullsized":"2024-09-21T10:13:39.554177+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:10:10.687040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"1289'709","reported_seq":732,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:43.428219+0000","last_change":"2024-09-21T10:13:28.747034+0000","last_active":"2024-09-21T10:13:43.428219+0000","last_peered":"2024-09-21T10:13:43.428219+0000","last_clean":"2024-09-21T10:13:43.428219+0000","last_became_active":"2024-09-21T10:13:28.746743+0000","last_became_peered":"2024-09-21T10:13:28.746743+0000","last_unstale":"2024-09-21T10:13:43.428219+0000","last_undegraded":"2024-09-21T10:13:43.428219+0000","last_fullsized":"2024-09-21T10:13:43.428219+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":709,"log_dups_size":0,"ondisk_log_size":709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:09.249849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46465024,"num_objects":709,"num_object_clones":0,"num_object_copies":2127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":709,"num_write_kb":45376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":40,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554982+0000","last_change":"2024-09-21T10:13:15.663291+0000","last_active":"2024-09-21T10:13:39.554982+0000","last_peered":"2024-09-21T10:13:39.554982+0000","last_clean":"2024-09-21T10:13:39.554982+0000","last_became_active":"2024-09-21T10:12:25.712199+0000","last_became_peered":"2024-09-21T10:12:25.712199+0000","last_unstale":"2024-09-21T10:13:39.554982+0000","last_undegraded":"2024-09-21T10:13:39.554982+0000","last_fullsized":"2024-09-21T10:13:39.554982+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:15.663228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:15.663228+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:44.919423+0000","scrub_duration":0.0014833489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1940,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.779681+0000","last_change":"2024-09-21T10:08:25.436371+0000","last_active":"2024-09-21T10:13:41.779681+0000","last_peered":"2024-09-21T10:13:41.779681+0000","last_clean":"2024-09-21T10:13:41.779681+0000","last_became_active":"2024-09-21T10:08:25.435921+0000","last_became_peered":"2024-09-21T10:08:25.435921+0000","last_unstale":"2024-09-21T10:13:41.779681+0000","last_undegraded":"2024-09-21T10:13:41.779681+0000","last_fullsized":"2024-09-21T10:13:41.779681+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:27.612386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1231,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.776124+0000","last_change":"2024-09-21T10:10:13.661328+0000","last_active":"2024-09-21T10:13:40.776124+0000","last_peered":"2024-09-21T10:13:40.776124+0000","last_clean":"2024-09-21T10:13:40.776124+0000","last_became_active":"2024-09-21T10:10:13.661028+0000","last_became_peered":"2024-09-21T10:10:13.661028+0000","last_unstale":"2024-09-21T10:13:40.776124+0000","last_undegraded":"2024-09-21T10:13:40.776124+0000","last_fullsized":"2024-09-21T10:13:40.776124+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:36.965214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"1289'611","reported_seq":634,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:42.636147+0000","last_change":"2024-09-21T10:13:28.789187+0000","last_active":"2024-09-21T10:13:42.636147+0000","last_peered":"2024-09-21T10:13:42.636147+0000","last_clean":"2024-09-21T10:13:42.636147+0000","last_became_active":"2024-09-21T10:13:28.788851+0000","last_became_peered":"2024-09-21T10:13:28.788851+0000","last_unstale":"2024-09-21T10:13:42.636147+0000","last_undegraded":"2024-09-21T10:13:42.636147+0000","last_fullsized":"2024-09-21T10:13:42.636147+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":611,"log_dups_size":0,"ondisk_log_size":611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:22.482804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40042496,"num_objects":611,"num_object_clones":0,"num_object_copies":1833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":611,"num_write_kb":39104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":387,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.783805+0000","last_change":"2024-09-21T10:13:15.622804+0000","last_active":"2024-09-21T10:13:41.783805+0000","last_peered":"2024-09-21T10:13:41.783805+0000","last_clean":"2024-09-21T10:13:41.783805+0000","last_became_active":"2024-09-21T10:12:50.586357+0000","last_became_peered":"2024-09-21T10:12:50.586357+0000","last_unstale":"2024-09-21T10:13:41.783805+0000","last_undegraded":"2024-09-21T10:13:41.783805+0000","last_fullsized":"2024-09-21T10:13:41.783805+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1281,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:28.894448+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":45,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:38.769783+0000","last_change":"2024-09-21T10:10:13.636008+0000","last_active":"2024-09-21T10:13:38.769783+0000","last_peered":"2024-09-21T10:13:38.769783+0000","last_clean":"2024-09-21T10:13:38.769783+0000","last_became_active":"2024-09-21T10:10:13.635632+0000","last_became_peered":"2024-09-21T10:10:13.635632+0000","last_unstale":"2024-09-21T10:13:38.769783+0000","last_undegraded":"2024-09-21T10:13:38.769783+0000","last_fullsized":"2024-09-21T10:13:38.769783+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:28.265954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1254,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.777113+0000","last_change":"2024-09-21T10:08:45.524238+0000","last_active":"2024-09-21T10:13:40.777113+0000","last_peered":"2024-09-21T10:13:40.777113+0000","last_clean":"2024-09-21T10:13:40.777113+0000","last_became_active":"2024-09-21T10:08:45.523517+0000","last_became_peered":"2024-09-21T10:08:45.523517+0000","last_unstale":"2024-09-21T10:13:40.777113+0000","last_undegraded":"2024-09-21T10:13:40.777113+0000","last_fullsized":"2024-09-21T10:13:40.777113+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:13.958459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"1288'250","reported_seq":263,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.809322+0000","last_change":"2024-09-21T10:13:28.744030+0000","last_active":"2024-09-21T10:13:40.809322+0000","last_peered":"2024-09-21T10:13:40.809322+0000","last_clean":"2024-09-21T10:13:40.809322+0000","last_became_active":"2024-09-21T10:13:28.743756+0000","last_became_peered":"2024-09-21T10:13:28.743756+0000","last_unstale":"2024-09-21T10:13:40.809322+0000","last_undegraded":"2024-09-21T10:13:40.809322+0000","last_fullsized":"2024-09-21T10:13:40.809322+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":250,"log_dups_size":0,"ondisk_log_size":250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:07.403842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16384000,"num_objects":250,"num_object_clones":0,"num_object_copies":750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":250,"num_write_kb":16000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"288'100","reported_seq":133,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789715+0000","last_change":"2024-09-21T10:13:30.346091+0000","last_active":"2024-09-21T10:13:41.789715+0000","last_peered":"2024-09-21T10:13:41.789715+0000","last_clean":"2024-09-21T10:13:41.789715+0000","last_became_active":"2024-09-21T10:12:50.586593+0000","last_became_peered":"2024-09-21T10:12:50.586593+0000","last_unstale":"2024-09-21T10:13:41.789715+0000","last_undegraded":"2024-09-21T10:13:41.789715+0000","last_fullsized":"2024-09-21T10:13:41.789715+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:30.346035+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:30.346035+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:01.387926+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1245,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.775693+0000","last_change":"2024-09-21T10:12:25.697464+0000","last_active":"2024-09-21T10:13:40.775693+0000","last_peered":"2024-09-21T10:13:40.775693+0000","last_clean":"2024-09-21T10:13:40.775693+0000","last_became_active":"2024-09-21T10:12:25.697050+0000","last_became_peered":"2024-09-21T10:12:25.697050+0000","last_unstale":"2024-09-21T10:13:40.775693+0000","last_undegraded":"2024-09-21T10:13:40.775693+0000","last_fullsized":"2024-09-21T10:13:40.775693+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:50.687008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1277,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.776425+0000","last_change":"2024-09-21T10:10:13.641608+0000","last_active":"2024-09-21T10:13:40.776425+0000","last_peered":"2024-09-21T10:13:40.776425+0000","last_clean":"2024-09-21T10:13:40.776425+0000","last_became_active":"2024-09-21T10:10:13.640299+0000","last_became_peered":"2024-09-21T10:10:13.640299+0000","last_unstale":"2024-09-21T10:13:40.776425+0000","last_undegraded":"2024-09-21T10:13:40.776425+0000","last_fullsized":"2024-09-21T10:13:40.776425+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:46.978168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"1289'699","reported_seq":713,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:43.479330+0000","last_change":"2024-09-21T10:13:28.745999+0000","last_active":"2024-09-21T10:13:43.479330+0000","last_peered":"2024-09-21T10:13:43.479330+0000","last_clean":"2024-09-21T10:13:43.479330+0000","last_became_active":"2024-09-21T10:13:28.745702+0000","last_became_peered":"2024-09-21T10:13:28.745702+0000","last_unstale":"2024-09-21T10:13:43.479330+0000","last_undegraded":"2024-09-21T10:13:43.479330+0000","last_fullsized":"2024-09-21T10:13:43.479330+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":699,"log_dups_size":0,"ondisk_log_size":699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:00.932542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45809664,"num_objects":699,"num_object_clones":0,"num_object_copies":2097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":699,"num_write_kb":44736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":388,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.783210+0000","last_change":"2024-09-21T10:10:13.622553+0000","last_active":"2024-09-21T10:13:41.783210+0000","last_peered":"2024-09-21T10:13:41.783210+0000","last_clean":"2024-09-21T10:13:41.783210+0000","last_became_active":"2024-09-21T10:10:13.622268+0000","last_became_peered":"2024-09-21T10:10:13.622268+0000","last_unstale":"2024-09-21T10:13:41.783210+0000","last_undegraded":"2024-09-21T10:13:41.783210+0000","last_fullsized":"2024-09-21T10:13:41.783210+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:24.686892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:24.686892+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:55.474818+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1960,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.782042+0000","last_change":"2024-09-21T10:12:25.709463+0000","last_active":"2024-09-21T10:13:41.782042+0000","last_peered":"2024-09-21T10:13:41.782042+0000","last_clean":"2024-09-21T10:13:41.782042+0000","last_became_active":"2024-09-21T10:12:25.706354+0000","last_became_peered":"2024-09-21T10:12:25.706354+0000","last_unstale":"2024-09-21T10:13:41.782042+0000","last_undegraded":"2024-09-21T10:13:41.782042+0000","last_fullsized":"2024-09-21T10:13:41.782042+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:37:56.890445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":118,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789718+0000","last_change":"2024-09-21T10:08:45.523818+0000","last_active":"2024-09-21T10:13:41.789718+0000","last_peered":"2024-09-21T10:13:41.789718+0000","last_clean":"2024-09-21T10:13:41.789718+0000","last_became_active":"2024-09-21T10:08:45.523507+0000","last_became_peered":"2024-09-21T10:08:45.523507+0000","last_unstale":"2024-09-21T10:13:41.789718+0000","last_undegraded":"2024-09-21T10:13:41.789718+0000","last_fullsized":"2024-09-21T10:13:41.789718+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:32.923210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"1288'272","reported_seq":285,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.806945+0000","last_change":"2024-09-21T10:13:28.745891+0000","last_active":"2024-09-21T10:13:40.806945+0000","last_peered":"2024-09-21T10:13:40.806945+0000","last_clean":"2024-09-21T10:13:40.806945+0000","last_became_active":"2024-09-21T10:13:28.745565+0000","last_became_peered":"2024-09-21T10:13:28.745565+0000","last_unstale":"2024-09-21T10:13:40.806945+0000","last_undegraded":"2024-09-21T10:13:40.806945+0000","last_fullsized":"2024-09-21T10:13:40.806945+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":272,"log_dups_size":0,"ondisk_log_size":272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:18.910062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17825792,"num_objects":272,"num_object_clones":0,"num_object_copies":816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":272,"num_write_kb":17408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":40,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554733+0000","last_change":"2024-09-21T10:13:20.647735+0000","last_active":"2024-09-21T10:13:39.554733+0000","last_peered":"2024-09-21T10:13:39.554733+0000","last_clean":"2024-09-21T10:13:39.554733+0000","last_became_active":"2024-09-21T10:12:25.708657+0000","last_became_peered":"2024-09-21T10:12:25.708657+0000","last_unstale":"2024-09-21T10:13:39.554733+0000","last_undegraded":"2024-09-21T10:13:39.554733+0000","last_fullsized":"2024-09-21T10:13:39.554733+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:20.647659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:20.647659+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:09.770331+0000","scrub_duration":0.0016005629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":439,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.784863+0000","last_change":"2024-09-21T10:12:56.081277+0000","last_active":"2024-09-21T10:13:41.784863+0000","last_peered":"2024-09-21T10:13:41.784863+0000","last_clean":"2024-09-21T10:13:41.784863+0000","last_became_active":"2024-09-21T10:12:50.586317+0000","last_became_peered":"2024-09-21T10:12:50.586317+0000","last_unstale":"2024-09-21T10:13:41.784863+0000","last_undegraded":"2024-09-21T10:13:41.784863+0000","last_fullsized":"2024-09-21T10:13:41.784863+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:57.776335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1938,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.778995+0000","last_change":"2024-09-21T10:12:50.577072+0000","last_active":"2024-09-21T10:13:41.778995+0000","last_peered":"2024-09-21T10:13:41.778995+0000","last_clean":"2024-09-21T10:13:41.778995+0000","last_became_active":"2024-09-21T10:12:50.576731+0000","last_became_peered":"2024-09-21T10:12:50.576731+0000","last_unstale":"2024-09-21T10:13:41.778995+0000","last_undegraded":"2024-09-21T10:13:41.778995+0000","last_fullsized":"2024-09-21T10:13:41.778995+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:56:06.937599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":55,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:38.770202+0000","last_change":"2024-09-21T10:12:25.703686+0000","last_active":"2024-09-21T10:13:38.770202+0000","last_peered":"2024-09-21T10:13:38.770202+0000","last_clean":"2024-09-21T10:13:38.770202+0000","last_became_active":"2024-09-21T10:12:25.703387+0000","last_became_peered":"2024-09-21T10:12:25.703387+0000","last_unstale":"2024-09-21T10:13:38.770202+0000","last_undegraded":"2024-09-21T10:13:38.770202+0000","last_fullsized":"2024-09-21T10:13:38.770202+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:04:12.166405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"1287'105","reported_seq":124,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.034903+0000","last_change":"2024-09-21T10:13:28.744415+0000","last_active":"2024-09-21T10:13:40.034903+0000","last_peered":"2024-09-21T10:13:40.034903+0000","last_clean":"2024-09-21T10:13:40.034903+0000","last_became_active":"2024-09-21T10:13:28.743851+0000","last_became_peered":"2024-09-21T10:13:28.743851+0000","last_unstale":"2024-09-21T10:13:40.034903+0000","last_undegraded":"2024-09-21T10:13:40.034903+0000","last_fullsized":"2024-09-21T10:13:40.034903+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:41:11.574935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6881280,"num_objects":105,"num_object_clones":0,"num_object_copies":315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":105,"num_write_kb":6720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":140,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789550+0000","last_change":"2024-09-21T10:13:18.313624+0000","last_active":"2024-09-21T10:13:41.789550+0000","last_peered":"2024-09-21T10:13:41.789550+0000","last_clean":"2024-09-21T10:13:41.789550+0000","last_became_active":"2024-09-21T10:10:13.637267+0000","last_became_peered":"2024-09-21T10:10:13.637267+0000","last_unstale":"2024-09-21T10:13:41.789550+0000","last_undegraded":"2024-09-21T10:13:41.789550+0000","last_fullsized":"2024-09-21T10:13:41.789550+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:18.313538+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:18.313538+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:11.379319+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":30,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554810+0000","last_change":"2024-09-21T10:12:25.708338+0000","last_active":"2024-09-21T10:13:39.554810+0000","last_peered":"2024-09-21T10:13:39.554810+0000","last_clean":"2024-09-21T10:13:39.554810+0000","last_became_active":"2024-09-21T10:12:25.708041+0000","last_became_peered":"2024-09-21T10:12:25.708041+0000","last_unstale":"2024-09-21T10:13:39.554810+0000","last_undegraded":"2024-09-21T10:13:39.554810+0000","last_fullsized":"2024-09-21T10:13:39.554810+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:07:48.291125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2011,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.556640+0000","last_change":"2024-09-21T10:08:25.429287+0000","last_active":"2024-09-21T10:13:39.556640+0000","last_peered":"2024-09-21T10:13:39.556640+0000","last_clean":"2024-09-21T10:13:39.556640+0000","last_became_active":"2024-09-21T10:08:25.428923+0000","last_became_peered":"2024-09-21T10:08:25.428923+0000","last_unstale":"2024-09-21T10:13:39.556640+0000","last_undegraded":"2024-09-21T10:13:39.556640+0000","last_fullsized":"2024-09-21T10:13:39.556640+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:09.255665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2027,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554100+0000","last_change":"2024-09-21T10:08:45.532253+0000","last_active":"2024-09-21T10:13:39.554100+0000","last_peered":"2024-09-21T10:13:39.554100+0000","last_clean":"2024-09-21T10:13:39.554100+0000","last_became_active":"2024-09-21T10:08:45.531519+0000","last_became_peered":"2024-09-21T10:08:45.531519+0000","last_unstale":"2024-09-21T10:13:39.554100+0000","last_undegraded":"2024-09-21T10:13:39.554100+0000","last_fullsized":"2024-09-21T10:13:39.554100+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:25.378783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"1289'590","reported_seq":613,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:42.647699+0000","last_change":"2024-09-21T10:13:28.743957+0000","last_active":"2024-09-21T10:13:42.647699+0000","last_peered":"2024-09-21T10:13:42.647699+0000","last_clean":"2024-09-21T10:13:42.647699+0000","last_became_active":"2024-09-21T10:13:28.743560+0000","last_became_peered":"2024-09-21T10:13:28.743560+0000","last_unstale":"2024-09-21T10:13:42.647699+0000","last_undegraded":"2024-09-21T10:13:42.647699+0000","last_fullsized":"2024-09-21T10:13:42.647699+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":590,"log_dups_size":0,"ondisk_log_size":590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:01:41.195242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38666240,"num_objects":590,"num_object_clones":0,"num_object_copies":1770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":590,"num_write_kb":37760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":40,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555212+0000","last_change":"2024-09-21T10:13:17.734264+0000","last_active":"2024-09-21T10:13:39.555212+0000","last_peered":"2024-09-21T10:13:39.555212+0000","last_clean":"2024-09-21T10:13:39.555212+0000","last_became_active":"2024-09-21T10:12:25.707372+0000","last_became_peered":"2024-09-21T10:12:25.707372+0000","last_unstale":"2024-09-21T10:13:39.555212+0000","last_undegraded":"2024-09-21T10:13:39.555212+0000","last_fullsized":"2024-09-21T10:13:39.555212+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:17.734169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:17.734169+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:59.937743+0000","scrub_duration":0.0017947760000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1979,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.783668+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:13:41.783668+0000","last_peered":"2024-09-21T10:13:41.783668+0000","last_clean":"2024-09-21T10:13:41.783668+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:13:41.783668+0000","last_undegraded":"2024-09-21T10:13:41.783668+0000","last_fullsized":"2024-09-21T10:13:41.783668+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":132,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789357+0000","last_change":"2024-09-21T10:10:13.635570+0000","last_active":"2024-09-21T10:13:41.789357+0000","last_peered":"2024-09-21T10:13:41.789357+0000","last_clean":"2024-09-21T10:13:41.789357+0000","last_became_active":"2024-09-21T10:10:13.635099+0000","last_became_peered":"2024-09-21T10:10:13.635099+0000","last_unstale":"2024-09-21T10:13:41.789357+0000","last_undegraded":"2024-09-21T10:13:41.789357+0000","last_fullsized":"2024-09-21T10:13:41.789357+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:43.686232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":125,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789624+0000","last_change":"2024-09-21T10:12:50.581491+0000","last_active":"2024-09-21T10:13:41.789624+0000","last_peered":"2024-09-21T10:13:41.789624+0000","last_clean":"2024-09-21T10:13:41.789624+0000","last_became_active":"2024-09-21T10:12:50.580970+0000","last_became_peered":"2024-09-21T10:12:50.580970+0000","last_unstale":"2024-09-21T10:13:41.789624+0000","last_undegraded":"2024-09-21T10:13:41.789624+0000","last_fullsized":"2024-09-21T10:13:41.789624+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:17.961335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"1288'253","reported_seq":266,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.805949+0000","last_change":"2024-09-21T10:13:28.746797+0000","last_active":"2024-09-21T10:13:40.805949+0000","last_peered":"2024-09-21T10:13:40.805949+0000","last_clean":"2024-09-21T10:13:40.805949+0000","last_became_active":"2024-09-21T10:13:28.746510+0000","last_became_peered":"2024-09-21T10:13:28.746510+0000","last_unstale":"2024-09-21T10:13:40.805949+0000","last_undegraded":"2024-09-21T10:13:40.805949+0000","last_fullsized":"2024-09-21T10:13:40.805949+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":253,"log_dups_size":0,"ondisk_log_size":253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:12:58.123721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16580608,"num_objects":253,"num_object_clones":0,"num_object_copies":759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":253,"num_write_kb":16192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":730,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.556534+0000","last_change":"2024-09-21T10:12:50.578738+0000","last_active":"2024-09-21T10:13:39.556534+0000","last_peered":"2024-09-21T10:13:39.556534+0000","last_clean":"2024-09-21T10:13:39.556534+0000","last_became_active":"2024-09-21T10:12:50.578363+0000","last_became_peered":"2024-09-21T10:12:50.578363+0000","last_unstale":"2024-09-21T10:13:39.556534+0000","last_undegraded":"2024-09-21T10:13:39.556534+0000","last_fullsized":"2024-09-21T10:13:39.556534+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:19.901507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:19.901507+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:08:56.749215+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":133,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789543+0000","last_change":"2024-09-21T10:10:13.636061+0000","last_active":"2024-09-21T10:13:41.789543+0000","last_peered":"2024-09-21T10:13:41.789543+0000","last_clean":"2024-09-21T10:13:41.789543+0000","last_became_active":"2024-09-21T10:10:13.635006+0000","last_became_peered":"2024-09-21T10:10:13.635006+0000","last_unstale":"2024-09-21T10:13:41.789543+0000","last_undegraded":"2024-09-21T10:13:41.789543+0000","last_fullsized":"2024-09-21T10:13:41.789543+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:24.272055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1232,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.776093+0000","last_change":"2024-09-21T10:10:13.686243+0000","last_active":"2024-09-21T10:13:40.776093+0000","last_peered":"2024-09-21T10:13:40.776093+0000","last_clean":"2024-09-21T10:13:40.776093+0000","last_became_active":"2024-09-21T10:10:13.685894+0000","last_became_peered":"2024-09-21T10:10:13.685894+0000","last_unstale":"2024-09-21T10:13:40.776093+0000","last_undegraded":"2024-09-21T10:13:40.776093+0000","last_fullsized":"2024-09-21T10:13:40.776093+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:12.394732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":118,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789383+0000","last_change":"2024-09-21T10:08:45.525546+0000","last_active":"2024-09-21T10:13:41.789383+0000","last_peered":"2024-09-21T10:13:41.789383+0000","last_clean":"2024-09-21T10:13:41.789383+0000","last_became_active":"2024-09-21T10:08:45.525116+0000","last_became_peered":"2024-09-21T10:08:45.525116+0000","last_unstale":"2024-09-21T10:13:41.789383+0000","last_undegraded":"2024-09-21T10:13:41.789383+0000","last_fullsized":"2024-09-21T10:13:41.789383+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:58.739744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"1288'257","reported_seq":270,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.804551+0000","last_change":"2024-09-21T10:13:28.747073+0000","last_active":"2024-09-21T10:13:40.804551+0000","last_peered":"2024-09-21T10:13:40.804551+0000","last_clean":"2024-09-21T10:13:40.804551+0000","last_became_active":"2024-09-21T10:13:28.746745+0000","last_became_peered":"2024-09-21T10:13:28.746745+0000","last_unstale":"2024-09-21T10:13:40.804551+0000","last_undegraded":"2024-09-21T10:13:40.804551+0000","last_fullsized":"2024-09-21T10:13:40.804551+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":257,"log_dups_size":0,"ondisk_log_size":257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:09:26.352129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16842752,"num_objects":257,"num_object_clones":0,"num_object_copies":771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":257,"num_write_kb":16448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":119,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:38.770178+0000","last_change":"2024-09-21T10:13:17.223079+0000","last_active":"2024-09-21T10:13:38.770178+0000","last_peered":"2024-09-21T10:13:38.770178+0000","last_clean":"2024-09-21T10:13:38.770178+0000","last_became_active":"2024-09-21T10:12:25.703148+0000","last_became_peered":"2024-09-21T10:12:25.703148+0000","last_unstale":"2024-09-21T10:13:38.770178+0000","last_undegraded":"2024-09-21T10:13:38.770178+0000","last_fullsized":"2024-09-21T10:13:38.770178+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:17.223016+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:17.223016+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:42.128679+0000","scrub_duration":0.001854694,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":942,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.780622+0000","last_change":"2024-09-21T10:12:50.579273+0000","last_active":"2024-09-21T10:13:41.780622+0000","last_peered":"2024-09-21T10:13:41.780622+0000","last_clean":"2024-09-21T10:13:41.780622+0000","last_became_active":"2024-09-21T10:12:50.578806+0000","last_became_peered":"2024-09-21T10:12:50.578806+0000","last_unstale":"2024-09-21T10:13:41.780622+0000","last_undegraded":"2024-09-21T10:13:41.780622+0000","last_fullsized":"2024-09-21T10:13:41.780622+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:04:53.664017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":132,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789824+0000","last_change":"2024-09-21T10:12:25.705453+0000","last_active":"2024-09-21T10:13:41.789824+0000","last_peered":"2024-09-21T10:13:41.789824+0000","last_clean":"2024-09-21T10:13:41.789824+0000","last_became_active":"2024-09-21T10:12:25.705105+0000","last_became_peered":"2024-09-21T10:12:25.705105+0000","last_unstale":"2024-09-21T10:13:41.789824+0000","last_undegraded":"2024-09-21T10:13:41.789824+0000","last_fullsized":"2024-09-21T10:13:41.789824+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:07:02.419776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":125,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789339+0000","last_change":"2024-09-21T10:12:50.585127+0000","last_active":"2024-09-21T10:13:41.789339+0000","last_peered":"2024-09-21T10:13:41.789339+0000","last_clean":"2024-09-21T10:13:41.789339+0000","last_became_active":"2024-09-21T10:12:50.584761+0000","last_became_peered":"2024-09-21T10:12:50.584761+0000","last_unstale":"2024-09-21T10:13:41.789339+0000","last_undegraded":"2024-09-21T10:13:41.789339+0000","last_fullsized":"2024-09-21T10:13:41.789339+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:15.425510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"1288'264","reported_seq":277,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.802344+0000","last_change":"2024-09-21T10:13:28.745515+0000","last_active":"2024-09-21T10:13:40.802344+0000","last_peered":"2024-09-21T10:13:40.802344+0000","last_clean":"2024-09-21T10:13:40.802344+0000","last_became_active":"2024-09-21T10:13:28.745221+0000","last_became_peered":"2024-09-21T10:13:28.745221+0000","last_unstale":"2024-09-21T10:13:40.802344+0000","last_undegraded":"2024-09-21T10:13:40.802344+0000","last_fullsized":"2024-09-21T10:13:40.802344+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":264,"log_dups_size":0,"ondisk_log_size":264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:40.235606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17301504,"num_objects":264,"num_object_clones":0,"num_object_copies":792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":264,"num_write_kb":16896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":55,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:38.769729+0000","last_change":"2024-09-21T10:10:37.256938+0000","last_active":"2024-09-21T10:13:38.769729+0000","last_peered":"2024-09-21T10:13:38.769729+0000","last_clean":"2024-09-21T10:13:38.769729+0000","last_became_active":"2024-09-21T10:10:13.648466+0000","last_became_peered":"2024-09-21T10:10:13.648466+0000","last_unstale":"2024-09-21T10:13:38.769729+0000","last_undegraded":"2024-09-21T10:13:38.769729+0000","last_fullsized":"2024-09-21T10:13:38.769729+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:10:37.256819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:10:37.256819+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:42.960519+0000","scrub_duration":0.0021766020000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":125,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789223+0000","last_change":"2024-09-21T10:12:50.585333+0000","last_active":"2024-09-21T10:13:41.789223+0000","last_peered":"2024-09-21T10:13:41.789223+0000","last_clean":"2024-09-21T10:13:41.789223+0000","last_became_active":"2024-09-21T10:12:50.584978+0000","last_became_peered":"2024-09-21T10:12:50.584978+0000","last_unstale":"2024-09-21T10:13:41.789223+0000","last_undegraded":"2024-09-21T10:13:41.789223+0000","last_fullsized":"2024-09-21T10:13:41.789223+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:16:10.033667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":30,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555314+0000","last_change":"2024-09-21T10:12:25.709819+0000","last_active":"2024-09-21T10:13:39.555314+0000","last_peered":"2024-09-21T10:13:39.555314+0000","last_clean":"2024-09-21T10:13:39.555314+0000","last_became_active":"2024-09-21T10:12:25.709548+0000","last_became_peered":"2024-09-21T10:12:25.709548+0000","last_unstale":"2024-09-21T10:13:39.555314+0000","last_undegraded":"2024-09-21T10:13:39.555314+0000","last_fullsized":"2024-09-21T10:13:39.555314+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:35.986614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2012,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554275+0000","last_change":"2024-09-21T10:12:50.582031+0000","last_active":"2024-09-21T10:13:39.554275+0000","last_peered":"2024-09-21T10:13:39.554275+0000","last_clean":"2024-09-21T10:13:39.554275+0000","last_became_active":"2024-09-21T10:12:50.581678+0000","last_became_peered":"2024-09-21T10:12:50.581678+0000","last_unstale":"2024-09-21T10:13:39.554275+0000","last_undegraded":"2024-09-21T10:13:39.554275+0000","last_fullsized":"2024-09-21T10:13:39.554275+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:40.292137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"1289'664","reported_seq":678,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:43.483060+0000","last_change":"2024-09-21T10:13:28.743256+0000","last_active":"2024-09-21T10:13:43.483060+0000","last_peered":"2024-09-21T10:13:43.483060+0000","last_clean":"2024-09-21T10:13:43.483060+0000","last_became_active":"2024-09-21T10:13:28.742998+0000","last_became_peered":"2024-09-21T10:13:28.742998+0000","last_unstale":"2024-09-21T10:13:43.483060+0000","last_undegraded":"2024-09-21T10:13:43.483060+0000","last_fullsized":"2024-09-21T10:13:43.483060+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":664,"log_dups_size":0,"ondisk_log_size":664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:13:15.751354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43515904,"num_objects":664,"num_object_clones":0,"num_object_copies":1992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":664,"num_write_kb":42496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":189,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.776044+0000","last_change":"2024-09-21T10:10:13.623475+0000","last_active":"2024-09-21T10:13:40.776044+0000","last_peered":"2024-09-21T10:13:40.776044+0000","last_clean":"2024-09-21T10:13:40.776044+0000","last_became_active":"2024-09-21T10:10:13.622580+0000","last_became_peered":"2024-09-21T10:10:13.622580+0000","last_unstale":"2024-09-21T10:13:40.776044+0000","last_undegraded":"2024-09-21T10:13:40.776044+0000","last_fullsized":"2024-09-21T10:13:40.776044+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:36.469530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:08:36.469530+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:47.757854+0000","scrub_duration":0.001636942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1952,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.781203+0000","last_change":"2024-09-21T10:12:25.718713+0000","last_active":"2024-09-21T10:13:41.781203+0000","last_peered":"2024-09-21T10:13:41.781203+0000","last_clean":"2024-09-21T10:13:41.781203+0000","last_became_active":"2024-09-21T10:12:25.713007+0000","last_became_peered":"2024-09-21T10:12:25.713007+0000","last_unstale":"2024-09-21T10:13:41.781203+0000","last_undegraded":"2024-09-21T10:13:41.781203+0000","last_fullsized":"2024-09-21T10:13:41.781203+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:09.753443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":132,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789354+0000","last_change":"2024-09-21T10:10:13.637236+0000","last_active":"2024-09-21T10:13:41.789354+0000","last_peered":"2024-09-21T10:13:41.789354+0000","last_clean":"2024-09-21T10:13:41.789354+0000","last_became_active":"2024-09-21T10:10:13.636907+0000","last_became_peered":"2024-09-21T10:10:13.636907+0000","last_unstale":"2024-09-21T10:13:41.789354+0000","last_undegraded":"2024-09-21T10:13:41.789354+0000","last_fullsized":"2024-09-21T10:13:41.789354+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:33.507370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1942,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.781856+0000","last_change":"2024-09-21T10:12:25.709467+0000","last_active":"2024-09-21T10:13:41.781856+0000","last_peered":"2024-09-21T10:13:41.781856+0000","last_clean":"2024-09-21T10:13:41.781856+0000","last_became_active":"2024-09-21T10:12:25.706170+0000","last_became_peered":"2024-09-21T10:12:25.706170+0000","last_unstale":"2024-09-21T10:13:41.781856+0000","last_undegraded":"2024-09-21T10:13:41.781856+0000","last_fullsized":"2024-09-21T10:13:41.781856+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:30.335358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"1288'248","reported_seq":260,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.797160+0000","last_change":"2024-09-21T10:13:28.789659+0000","last_active":"2024-09-21T10:13:40.797160+0000","last_peered":"2024-09-21T10:13:40.797160+0000","last_clean":"2024-09-21T10:13:40.797160+0000","last_became_active":"2024-09-21T10:13:28.789382+0000","last_became_peered":"2024-09-21T10:13:28.789382+0000","last_unstale":"2024-09-21T10:13:40.797160+0000","last_undegraded":"2024-09-21T10:13:40.797160+0000","last_fullsized":"2024-09-21T10:13:40.797160+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":248,"log_dups_size":0,"ondisk_log_size":248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:51.179733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16252928,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":248,"num_write_kb":15872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":421,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.780943+0000","last_change":"2024-09-21T10:12:50.578581+0000","last_active":"2024-09-21T10:13:41.780943+0000","last_peered":"2024-09-21T10:13:41.780943+0000","last_clean":"2024-09-21T10:13:41.780943+0000","last_became_active":"2024-09-21T10:12:50.578246+0000","last_became_peered":"2024-09-21T10:12:50.578246+0000","last_unstale":"2024-09-21T10:13:41.780943+0000","last_undegraded":"2024-09-21T10:13:41.780943+0000","last_fullsized":"2024-09-21T10:13:41.780943+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.423664+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.423664+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:38.323326+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":2037,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554386+0000","last_change":"2024-09-21T10:08:45.540447+0000","last_active":"2024-09-21T10:13:39.554386+0000","last_peered":"2024-09-21T10:13:39.554386+0000","last_clean":"2024-09-21T10:13:39.554386+0000","last_became_active":"2024-09-21T10:08:45.528704+0000","last_became_peered":"2024-09-21T10:08:45.528704+0000","last_unstale":"2024-09-21T10:13:39.554386+0000","last_undegraded":"2024-09-21T10:13:39.554386+0000","last_fullsized":"2024-09-21T10:13:39.554386+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:58:09.404941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":55,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:38.770101+0000","last_change":"2024-09-21T10:12:25.704090+0000","last_active":"2024-09-21T10:13:38.770101+0000","last_peered":"2024-09-21T10:13:38.770101+0000","last_clean":"2024-09-21T10:13:38.770101+0000","last_became_active":"2024-09-21T10:12:25.702945+0000","last_became_peered":"2024-09-21T10:12:25.702945+0000","last_unstale":"2024-09-21T10:13:38.770101+0000","last_undegraded":"2024-09-21T10:13:38.770101+0000","last_fullsized":"2024-09-21T10:13:38.770101+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:29.609613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1224,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.775768+0000","last_change":"2024-09-21T10:12:50.571672+0000","last_active":"2024-09-21T10:13:40.775768+0000","last_peered":"2024-09-21T10:13:40.775768+0000","last_clean":"2024-09-21T10:13:40.775768+0000","last_became_active":"2024-09-21T10:12:50.571358+0000","last_became_peered":"2024-09-21T10:12:50.571358+0000","last_unstale":"2024-09-21T10:13:40.775768+0000","last_undegraded":"2024-09-21T10:13:40.775768+0000","last_fullsized":"2024-09-21T10:13:40.775768+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:38.440163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1277,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.775855+0000","last_change":"2024-09-21T10:10:13.640158+0000","last_active":"2024-09-21T10:13:40.775855+0000","last_peered":"2024-09-21T10:13:40.775855+0000","last_clean":"2024-09-21T10:13:40.775855+0000","last_became_active":"2024-09-21T10:10:13.639801+0000","last_became_peered":"2024-09-21T10:10:13.639801+0000","last_unstale":"2024-09-21T10:13:40.775855+0000","last_undegraded":"2024-09-21T10:13:40.775855+0000","last_fullsized":"2024-09-21T10:13:40.775855+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:25.153985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1317,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.776046+0000","last_change":"2024-09-21T10:12:25.713745+0000","last_active":"2024-09-21T10:13:40.776046+0000","last_peered":"2024-09-21T10:13:40.776046+0000","last_clean":"2024-09-21T10:13:40.776046+0000","last_became_active":"2024-09-21T10:12:25.698011+0000","last_became_peered":"2024-09-21T10:12:25.698011+0000","last_unstale":"2024-09-21T10:13:40.776046+0000","last_undegraded":"2024-09-21T10:13:40.776046+0000","last_fullsized":"2024-09-21T10:13:40.776046+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:45.276913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23720,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":132,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789256+0000","last_change":"2024-09-21T10:10:13.635438+0000","last_active":"2024-09-21T10:13:41.789256+0000","last_peered":"2024-09-21T10:13:41.789256+0000","last_clean":"2024-09-21T10:13:41.789256+0000","last_became_active":"2024-09-21T10:10:13.634268+0000","last_became_peered":"2024-09-21T10:10:13.634268+0000","last_unstale":"2024-09-21T10:13:41.789256+0000","last_undegraded":"2024-09-21T10:13:41.789256+0000","last_fullsized":"2024-09-21T10:13:41.789256+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:00:07.541446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":36,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555304+0000","last_change":"2024-09-21T10:12:25.717602+0000","last_active":"2024-09-21T10:13:39.555304+0000","last_peered":"2024-09-21T10:13:39.555304+0000","last_clean":"2024-09-21T10:13:39.555304+0000","last_became_active":"2024-09-21T10:12:25.705211+0000","last_became_peered":"2024-09-21T10:12:25.705211+0000","last_unstale":"2024-09-21T10:13:39.555304+0000","last_undegraded":"2024-09-21T10:13:39.555304+0000","last_fullsized":"2024-09-21T10:13:39.555304+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:30.822884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"288'100","reported_seq":138,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.784244+0000","last_change":"2024-09-21T10:12:25.698905+0000","last_active":"2024-09-21T10:13:41.784244+0000","last_peered":"2024-09-21T10:13:41.784244+0000","last_clean":"2024-09-21T10:13:41.784244+0000","last_became_active":"2024-09-21T10:12:25.698552+0000","last_became_peered":"2024-09-21T10:12:25.698552+0000","last_unstale":"2024-09-21T10:13:41.784244+0000","last_undegraded":"2024-09-21T10:13:41.784244+0000","last_fullsized":"2024-09-21T10:13:41.784244+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.389965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.389965+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:36.307701+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":40,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.555588+0000","last_change":"2024-09-21T10:12:50.588976+0000","last_active":"2024-09-21T10:13:39.555588+0000","last_peered":"2024-09-21T10:13:39.555588+0000","last_clean":"2024-09-21T10:13:39.555588+0000","last_became_active":"2024-09-21T10:12:50.588232+0000","last_became_peered":"2024-09-21T10:12:50.588232+0000","last_unstale":"2024-09-21T10:13:39.555588+0000","last_undegraded":"2024-09-21T10:13:39.555588+0000","last_fullsized":"2024-09-21T10:13:39.555588+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:44:25.302039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1944,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.779179+0000","last_change":"2024-09-21T10:12:25.707393+0000","last_active":"2024-09-21T10:13:41.779179+0000","last_peered":"2024-09-21T10:13:41.779179+0000","last_clean":"2024-09-21T10:13:41.779179+0000","last_became_active":"2024-09-21T10:12:25.706457+0000","last_became_peered":"2024-09-21T10:12:25.706457+0000","last_unstale":"2024-09-21T10:13:41.779179+0000","last_undegraded":"2024-09-21T10:13:41.779179+0000","last_fullsized":"2024-09-21T10:13:41.779179+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:56.520799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2005,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554712+0000","last_change":"2024-09-21T10:12:50.591554+0000","last_active":"2024-09-21T10:13:39.554712+0000","last_peered":"2024-09-21T10:13:39.554712+0000","last_clean":"2024-09-21T10:13:39.554712+0000","last_became_active":"2024-09-21T10:12:50.591171+0000","last_became_peered":"2024-09-21T10:12:50.591171+0000","last_unstale":"2024-09-21T10:13:39.554712+0000","last_undegraded":"2024-09-21T10:13:39.554712+0000","last_fullsized":"2024-09-21T10:13:39.554712+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:13:22.080704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1979,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.788124+0000","last_change":"2024-09-21T10:13:41.788034+0000","last_active":"2024-09-21T10:13:41.788124+0000","last_peered":"2024-09-21T10:13:41.788124+0000","last_clean":"2024-09-21T10:13:41.788124+0000","last_became_active":"2024-09-21T10:13:41.787698+0000","last_became_peered":"2024-09-21T10:13:41.787698+0000","last_unstale":"2024-09-21T10:13:41.788124+0000","last_undegraded":"2024-09-21T10:13:41.788124+0000","last_fullsized":"2024-09-21T10:13:41.788124+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:08.268799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":45,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:38.770279+0000","last_change":"2024-09-21T10:10:13.634975+0000","last_active":"2024-09-21T10:13:38.770279+0000","last_peered":"2024-09-21T10:13:38.770279+0000","last_clean":"2024-09-21T10:13:38.770279+0000","last_became_active":"2024-09-21T10:10:13.634667+0000","last_became_peered":"2024-09-21T10:10:13.634667+0000","last_unstale":"2024-09-21T10:13:38.770279+0000","last_undegraded":"2024-09-21T10:13:38.770279+0000","last_fullsized":"2024-09-21T10:13:38.770279+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:52.296990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1951,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.781951+0000","last_change":"2024-09-21T10:08:45.520524+0000","last_active":"2024-09-21T10:13:41.781951+0000","last_peered":"2024-09-21T10:13:41.781951+0000","last_clean":"2024-09-21T10:13:41.781951+0000","last_became_active":"2024-09-21T10:08:45.520119+0000","last_became_peered":"2024-09-21T10:08:45.520119+0000","last_unstale":"2024-09-21T10:13:41.781951+0000","last_undegraded":"2024-09-21T10:13:41.781951+0000","last_fullsized":"2024-09-21T10:13:41.781951+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:55.559438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2017,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554196+0000","last_change":"2024-09-21T10:12:25.703790+0000","last_active":"2024-09-21T10:13:39.554196+0000","last_peered":"2024-09-21T10:13:39.554196+0000","last_clean":"2024-09-21T10:13:39.554196+0000","last_became_active":"2024-09-21T10:12:25.702989+0000","last_became_peered":"2024-09-21T10:12:25.702989+0000","last_unstale":"2024-09-21T10:13:39.554196+0000","last_undegraded":"2024-09-21T10:13:39.554196+0000","last_fullsized":"2024-09-21T10:13:39.554196+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:15.338067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":118,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789601+0000","last_change":"2024-09-21T10:08:45.523144+0000","last_active":"2024-09-21T10:13:41.789601+0000","last_peered":"2024-09-21T10:13:41.789601+0000","last_clean":"2024-09-21T10:13:41.789601+0000","last_became_active":"2024-09-21T10:08:45.522526+0000","last_became_peered":"2024-09-21T10:08:45.522526+0000","last_unstale":"2024-09-21T10:13:41.789601+0000","last_undegraded":"2024-09-21T10:13:41.789601+0000","last_fullsized":"2024-09-21T10:13:41.789601+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:45.926968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1943,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.780381+0000","last_change":"2024-09-21T10:08:45.515262+0000","last_active":"2024-09-21T10:13:41.780381+0000","last_peered":"2024-09-21T10:13:41.780381+0000","last_clean":"2024-09-21T10:13:41.780381+0000","last_became_active":"2024-09-21T10:08:45.514926+0000","last_became_peered":"2024-09-21T10:08:45.514926+0000","last_unstale":"2024-09-21T10:13:41.780381+0000","last_undegraded":"2024-09-21T10:13:41.780381+0000","last_fullsized":"2024-09-21T10:13:41.780381+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:49.954351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1256,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.775478+0000","last_change":"2024-09-21T10:08:45.524864+0000","last_active":"2024-09-21T10:13:40.775478+0000","last_peered":"2024-09-21T10:13:40.775478+0000","last_clean":"2024-09-21T10:13:40.775478+0000","last_became_active":"2024-09-21T10:08:45.524533+0000","last_became_peered":"2024-09-21T10:08:45.524533+0000","last_unstale":"2024-09-21T10:13:40.775478+0000","last_undegraded":"2024-09-21T10:13:40.775478+0000","last_fullsized":"2024-09-21T10:13:40.775478+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:52.547302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1971,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.784585+0000","last_change":"2024-09-21T10:08:45.525340+0000","last_active":"2024-09-21T10:13:41.784585+0000","last_peered":"2024-09-21T10:13:41.784585+0000","last_clean":"2024-09-21T10:13:41.784585+0000","last_became_active":"2024-09-21T10:08:45.525023+0000","last_became_peered":"2024-09-21T10:08:45.525023+0000","last_unstale":"2024-09-21T10:13:41.784585+0000","last_undegraded":"2024-09-21T10:13:41.784585+0000","last_fullsized":"2024-09-21T10:13:41.784585+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:37.674092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":2016,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554362+0000","last_change":"2024-09-21T10:12:25.700804+0000","last_active":"2024-09-21T10:13:39.554362+0000","last_peered":"2024-09-21T10:13:39.554362+0000","last_clean":"2024-09-21T10:13:39.554362+0000","last_became_active":"2024-09-21T10:12:25.700390+0000","last_became_peered":"2024-09-21T10:12:25.700390+0000","last_unstale":"2024-09-21T10:13:39.554362+0000","last_undegraded":"2024-09-21T10:13:39.554362+0000","last_fullsized":"2024-09-21T10:13:39.554362+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:07:23.705541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1264,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.789883+0000","last_change":"2024-09-21T10:13:41.789883+0000","last_active":"2024-09-21T10:13:41.789883+0000","last_peered":"2024-09-21T10:13:41.789883+0000","last_clean":"2024-09-21T10:13:41.789883+0000","last_became_active":"2024-09-21T10:13:41.789620+0000","last_became_peered":"2024-09-21T10:13:41.789620+0000","last_unstale":"2024-09-21T10:13:41.789883+0000","last_undegraded":"2024-09-21T10:13:41.789883+0000","last_fullsized":"2024-09-21T10:13:41.789883+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:36.869540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":226,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.776009+0000","last_change":"2024-09-21T10:08:45.535971+0000","last_active":"2024-09-21T10:13:40.776009+0000","last_peered":"2024-09-21T10:13:40.776009+0000","last_clean":"2024-09-21T10:13:40.776009+0000","last_became_active":"2024-09-21T10:08:45.530341+0000","last_became_peered":"2024-09-21T10:08:45.530341+0000","last_unstale":"2024-09-21T10:13:40.776009+0000","last_undegraded":"2024-09-21T10:13:40.776009+0000","last_fullsized":"2024-09-21T10:13:40.776009+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:58:34.894434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1234,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.775281+0000","last_change":"2024-09-21T10:08:25.430667+0000","last_active":"2024-09-21T10:13:40.775281+0000","last_peered":"2024-09-21T10:13:40.775281+0000","last_clean":"2024-09-21T10:13:40.775281+0000","last_became_active":"2024-09-21T10:08:25.430357+0000","last_became_peered":"2024-09-21T10:08:25.430357+0000","last_unstale":"2024-09-21T10:13:40.775281+0000","last_undegraded":"2024-09-21T10:13:40.775281+0000","last_fullsized":"2024-09-21T10:13:40.775281+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:19.972020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":228,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.775950+0000","last_change":"2024-09-21T10:10:13.639231+0000","last_active":"2024-09-21T10:13:40.775950+0000","last_peered":"2024-09-21T10:13:40.775950+0000","last_clean":"2024-09-21T10:13:40.775950+0000","last_became_active":"2024-09-21T10:10:13.622497+0000","last_became_peered":"2024-09-21T10:10:13.622497+0000","last_unstale":"2024-09-21T10:13:40.775950+0000","last_undegraded":"2024-09-21T10:13:40.775950+0000","last_fullsized":"2024-09-21T10:13:40.775950+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:31:46.263003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1929,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.779558+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:13:41.779558+0000","last_peered":"2024-09-21T10:13:41.779558+0000","last_clean":"2024-09-21T10:13:41.779558+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:13:41.779558+0000","last_undegraded":"2024-09-21T10:13:41.779558+0000","last_fullsized":"2024-09-21T10:13:41.779558+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1944,"reported_epoch":1289,"state":"active+clean","last_fresh":"2024-09-21T10:13:41.780791+0000","last_change":"2024-09-21T10:10:13.651599+0000","last_active":"2024-09-21T10:13:41.780791+0000","last_peered":"2024-09-21T10:13:41.780791+0000","last_clean":"2024-09-21T10:13:41.780791+0000","last_became_active":"2024-09-21T10:10:13.651250+0000","last_became_peered":"2024-09-21T10:10:13.651250+0000","last_unstale":"2024-09-21T10:13:41.780791+0000","last_undegraded":"2024-09-21T10:13:41.780791+0000","last_fullsized":"2024-09-21T10:13:41.780791+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:34.100056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1277,"reported_epoch":1288,"state":"active+clean","last_fresh":"2024-09-21T10:13:40.775188+0000","last_change":"2024-09-21T10:10:13.639970+0000","last_active":"2024-09-21T10:13:40.775188+0000","last_peered":"2024-09-21T10:13:40.775188+0000","last_clean":"2024-09-21T10:13:40.775188+0000","last_became_active":"2024-09-21T10:10:13.639671+0000","last_became_peered":"2024-09-21T10:10:13.639671+0000","last_unstale":"2024-09-21T10:13:40.775188+0000","last_undegraded":"2024-09-21T10:13:40.775188+0000","last_fullsized":"2024-09-21T10:13:40.775188+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:53.941605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":2020,"reported_epoch":1287,"state":"active+clean","last_fresh":"2024-09-21T10:13:39.554819+0000","last_change":"2024-09-21T10:12:50.594933+0000","last_active":"2024-09-21T10:13:39.554819+0000","last_peered":"2024-09-21T10:13:39.554819+0000","last_clean":"2024-09-21T10:13:39.554819+0000","last_became_active":"2024-09-21T10:12:50.578183+0000","last_became_peered":"2024-09-21T10:12:50.578183+0000","last_unstale":"2024-09-21T10:13:39.554819+0000","last_undegraded":"2024-09-21T10:13:39.554819+0000","last_fullsized":"2024-09-21T10:13:39.554819+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:17.508182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":389349376,"num_objects":5941,"num_object_clones":0,"num_object_copies":17823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5941,"num_write_kb":380224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1358823424,"data_stored":1358823424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5941,"ondisk_log_size":5941,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":24716,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34026,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397974,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2375612,"kb_used_data":1483440,"kb_used_omap":1,"kb_used_meta":892158,"kb_avail":91369540,"statfs":{"total":95995035648,"available":93562408960,"internally_reserved":0,"allocated":1519042560,"data_stored":1515952865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913570136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594203,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":662332,"kb_used_data":72432,"kb_used_omap":1,"kb_used_meta":589886,"kb_avail":93082820,"statfs":{"total":95995035648,"available":95316807680,"internally_reserved":0,"allocated":74170368,"data_stored":71081644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":604043608},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790432,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925280,"kb_used_data":35288,"kb_used_omap":1,"kb_used_meta":889982,"kb_avail":92819872,"statfs":{"total":95995035648,"available":95047548928,"internally_reserved":0,"allocated":36134912,"data_stored":33064836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":911341912},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1224,"seq":5257039970369,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":691412,"kb_used_data":354884,"kb_used_omap":290,"kb_used_meta":336221,"kb_avail":93053740,"statfs":{"total":95995035648,"available":95287029760,"internally_reserved":0,"allocated":363401216,"data_stored":360307649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":297377,"internal_metadata":344290911},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999301,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1106692,"kb_used_data":196792,"kb_used_omap":25,"kb_used_meta":909862,"kb_avail":92638460,"statfs":{"total":95995035648,"available":94861783040,"internally_reserved":0,"allocated":201515008,"data_stored":198429563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":931699655},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111244,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1078724,"kb_used_data":270008,"kb_used_omap":31,"kb_used_meta":808672,"kb_avail":92666428,"statfs":{"total":95995035648,"available":94890422272,"internally_reserved":0,"allocated":276488192,"data_stored":273403191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":828080360},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318223187,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":598204,"kb_used_data":181040,"kb_used_omap":26,"kb_used_meta":417125,"kb_avail":93146948,"statfs":{"total":95995035648,"available":95382474752,"internally_reserved":0,"allocated":185384960,"data_stored":182307022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":427136012},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335129,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":737192,"kb_used_data":62748,"kb_used_omap":31,"kb_used_meta":674400,"kb_avail":93007960,"statfs":{"total":95995035648,"available":95240151040,"internally_reserved":0,"allocated":64253952,"data_stored":61180373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":690585696},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":67764224,"data_stored":67764224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":181796864,"data_stored":181796864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":272433152,"data_stored":272433152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":210305024,"data_stored":210305024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":410320896,"data_stored":410320896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":33751040,"data_stored":33751040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":115998720,"data_stored":115998720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":66453504,"data_stored":66453504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:13:48.876 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:13:48.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_5 pg_num 26 2024-09-21T10:13:48.894 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:48.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:48.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:49.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:49.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:49.173 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:49.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.208 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:49.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:49.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:49.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:49.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:49.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:49.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:49.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:49.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 24328 24312 151.935 120.5 0.00643524 0.00657639 2024-09-21T10:13:49.586 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:49.614 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:49.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:49.716 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:49.746 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:49.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:49.848 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:49.879 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:49.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:49 smithi017 ceph-mon[102913]: pgmap v1970: 109 pgs: 109 active+clean; 1006 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:49 smithi017 ceph-mon[105151]: pgmap v1970: 109 pgs: 109 active+clean; 1006 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:49.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:49 smithi134 ceph-mon[90085]: pgmap v1970: 109 pgs: 109 active+clean; 1006 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:50.020 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:50.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:50.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:50.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:50.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:50.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:50.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:50.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:50.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:50.430 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:13:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:13:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:13:50.433 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:50.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:50.535 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:50.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 26363 26347 149.684 127.188 0.00387383 0.00667553 2024-09-21T10:13:50.566 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:50.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:50.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:50.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:50.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:50.844 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:50.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:50.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:50.978 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:51.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:51.116 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:51.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:51.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:51.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:51.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:51.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:51.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:51.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:51.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 28465 28449 148.157 131.375 0.00170729 0.00673349 2024-09-21T10:13:51.643 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:51.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.678 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:51.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:51.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:51.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:51.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:51.841 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:51 smithi134 ceph-mon[90085]: pgmap v1971: 109 pgs: 109 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:51.917 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:51.945 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:51 smithi017 ceph-mon[102913]: pgmap v1971: 109 pgs: 109 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:51.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:51 smithi017 ceph-mon[105151]: pgmap v1971: 109 pgs: 109 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:52.047 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:52.084 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:52.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:52.186 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:52.222 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:52.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:52.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:52.470 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:52.508 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:52.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:52.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 30604 30588 147.043 133.688 0.00456921 0.00679607 2024-09-21T10:13:52.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:52.652 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:52.754 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:52.790 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:52.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:52.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:52.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:53.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:53.066 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:53.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:53.144 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:13:53.144 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:13:53.168 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:53.200 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:53.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:53.332 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:53.434 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:53.466 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:53.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:53.553 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 32599 32583 145.445 124.688 0.00561968 0.00686978 2024-09-21T10:13:53.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:53.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:53.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:53.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:53.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:53.835 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:53.856 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:53.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:53.869 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:53 smithi017 ceph-mon[102913]: pgmap v1972: 109 pgs: 109 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:53.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:53 smithi017 ceph-mon[105151]: pgmap v1972: 109 pgs: 109 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:53.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:53 smithi134 ceph-mon[90085]: pgmap v1972: 109 pgs: 109 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:53.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:53.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:54.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:54.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:54.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:13:54.191 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:54.241 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:54.268 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:54.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:54.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:54.398 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:54.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:54.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:54.524 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:54.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:54.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 34184 34168 142.351 99.0625 0.00396598 0.00701679 2024-09-21T10:13:54.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:54.629 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:13:54.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:54.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:54.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:54.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:54.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:54.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:54.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:54 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3362292962' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:54.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:54 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:54 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3362292962' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:54 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:54.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:13:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:13:54] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-09-21T10:13:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:54 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3362292962' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:54 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:55.051 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:55.085 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:55.187 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:55.220 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:55.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:55.321 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:55.344 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:55.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:55.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:55.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:55.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 35799 35783 139.763 100.938 0.00822051 0.00714959 2024-09-21T10:13:55.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:55.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:55.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:55.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:55.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:55.854 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:55.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[102913]: pgmap v1973: 109 pgs: 109 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:55.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:55.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:55.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[102913]: osdmap e1290: 8 total, 8 up, 8 in 2024-09-21T10:13:55.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:55.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[105151]: pgmap v1973: 109 pgs: 109 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:55.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:55.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:55.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[105151]: osdmap e1290: 8 total, 8 up, 8 in 2024-09-21T10:13:55.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:55.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:55.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:55.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:55 smithi134 ceph-mon[90085]: pgmap v1973: 109 pgs: 109 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:13:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:55 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:55 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:55 smithi134 ceph-mon[90085]: osdmap e1290: 8 total, 8 up, 8 in 2024-09-21T10:13:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:56.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:56.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:56.130 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:56.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:56.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:56.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:56.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:56.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:56.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:56.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:56.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:56.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:56.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 37823 37807 138.982 126.5 0.00499361 0.00718999 2024-09-21T10:13:56.583 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:56.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:56.685 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:56.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:56.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:56.817 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:56.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:56.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:56.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:56.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:57.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:57.123 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:57.144 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:57.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:57.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:57.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.278 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:57.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:57.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:57.404 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:57.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:57.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:57 smithi017 ceph-mon[102913]: pgmap v1975: 109 pgs: 109 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-21T10:13:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:57.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:57 smithi017 ceph-mon[105151]: pgmap v1975: 109 pgs: 109 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-21T10:13:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:13:57.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:57 smithi134 ceph-mon[90085]: pgmap v1975: 109 pgs: 109 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-21T10:13:57.506 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:57.540 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:57.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 39646 39630 137.59 113.938 0.00805767 0.00726248 2024-09-21T10:13:57.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:57.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:57.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:57.782 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:57.817 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:57.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:57.918 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:57.947 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:57.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:58.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:58.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:58.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:58.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:58.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:58.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:58.339 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:58.371 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:58.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:58.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:58.508 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:58.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:58.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 41592 41576 136.749 121.625 0.00384025 0.00730783 2024-09-21T10:13:58.610 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:58.641 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:58.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:58.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:58.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:58.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:58.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:13:58.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:58.940 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:58.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:59.042 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:59.072 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:59.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:59.174 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:59.207 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:59.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:59.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:59.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:59.449 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:59.483 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:13:59.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:59.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:13:59.553217+0000 min lat: 0.00108589 max lat: 0.228925 avg lat: 0.00736687 2024-09-21T10:13:59.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:13:59.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 43430 43414 135.655 114.875 0.00790162 0.00736687 2024-09-21T10:13:59.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.615 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:59.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:59.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:59.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:59.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:59.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:59.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:13:59.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:59.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:13:59 smithi017 ceph-mon[102913]: pgmap v1976: 109 pgs: 109 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-21T10:13:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:13:59 smithi017 ceph-mon[105151]: pgmap v1976: 109 pgs: 109 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-21T10:13:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:13:59 smithi134 ceph-mon[90085]: pgmap v1976: 109 pgs: 109 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-21T10:14:00.004 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:00.025 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:00.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:00.127 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:00.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:00.260 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:00.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:00.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:00.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:00.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:14:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:14:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:14:00.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:00.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 45161 45145 134.346 108.188 0.0114147 0.0074385 2024-09-21T10:14:00.566 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:00.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:00.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:00.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:00.852 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:00.881 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:00.983 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:01.018 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:01.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:01.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:01.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:01.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:01.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:01.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:01.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:01.442 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:01.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:01.544 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:01.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 46947 46931 133.313 111.625 0.00679634 0.0074963 2024-09-21T10:14:01.580 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:01.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:01.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:01.828 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:01.862 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:01.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:01.879 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:01 smithi017 ceph-mon[102913]: pgmap v1977: 109 pgs: 109 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-21T10:14:01.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:01 smithi017 ceph-mon[105151]: pgmap v1977: 109 pgs: 109 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-21T10:14:01.964 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:01 smithi134 ceph-mon[90085]: pgmap v1977: 109 pgs: 109 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-21T10:14:01.996 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:01.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:02.098 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:02.128 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:02.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:02.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:02.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:02.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:02.376 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:02.413 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:02.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:02.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:02.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 48700 48684 132.28 109.562 0.0118453 0.00755423 2024-09-21T10:14:02.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:02.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:02.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:02.702 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:02.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:02.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:02.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:03.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.002 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:03.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:03.103 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:03.125 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:03.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:03.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:03.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:03.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:03.368 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:14:03.368 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:03.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:03.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:03.403 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:14:03.403 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:03.442 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:14:03.442 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:03.476 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:14:03.477 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:03.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:03.508 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:14:03.509 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:03.529 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:03.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:03.539 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 10 pg_num to 26 2024-09-21T10:14:03.541 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:14:03.541 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:03.554 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 50069 50053 130.333 85.5625 0.0134564 0.00766651 2024-09-21T10:14:03.571 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:14:03.571 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:03.602 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:14:03.603 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:03.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:03.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:03.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:03.738 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:03 smithi017 ceph-mon[102913]: pgmap v1978: 109 pgs: 109 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2024-09-21T10:14:03.738 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:03 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3948054666' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:14:03.738 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:03 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:14:03.738 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:03 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1185722046' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:03.738 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:03 smithi017 ceph-mon[105151]: pgmap v1978: 109 pgs: 109 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2024-09-21T10:14:03.739 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:03 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3948054666' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:14:03.739 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:03 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:14:03.739 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:03 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1185722046' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:03.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:03.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:03.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:03.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:03.945 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:03.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:03 smithi134 ceph-mon[90085]: pgmap v1978: 109 pgs: 109 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2024-09-21T10:14:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:03 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3948054666' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:14:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:03 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:14:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:03 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1185722046' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:04.047 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:04.077 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:04.179 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:04.206 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:04.308 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:04.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.338 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:04.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:04.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:04.468 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:04.555 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 51604 51588 128.957 95.9375 0.00744876 0.00774921 2024-09-21T10:14:04.570 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:04.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.634 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:04.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:04.735 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:04.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.758 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:04.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:04.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:04.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:04.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:14:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:14:04] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.51.0" 2024-09-21T10:14:04.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:04 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:14:04.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:04 smithi017 ceph-mon[102913]: osdmap e1291: 8 total, 8 up, 8 in 2024-09-21T10:14:04.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:04 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:14:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:04 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:14:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:04 smithi017 ceph-mon[105151]: osdmap e1291: 8 total, 8 up, 8 in 2024-09-21T10:14:04.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:04 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:14:04.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:04 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:14:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:04 smithi134 ceph-mon[90085]: osdmap e1291: 8 total, 8 up, 8 in 2024-09-21T10:14:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:04 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:14:05.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:05.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:05.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:05.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:05.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:05.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:05.255 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:05.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:05.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:05.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:05.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:05.482 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:05.506 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:05.555 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 51648 51632 124.102 2.75 0.0016579 0.0077687 2024-09-21T10:14:05.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:05.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:05.737 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:05.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:05.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:05.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:05.895 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:05 smithi134 ceph-mon[90085]: pgmap v1980: 109 pgs: 109 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-09-21T10:14:05.895 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:14:05.895 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:05 smithi134 ceph-mon[90085]: osdmap e1292: 8 total, 8 up, 8 in 2024-09-21T10:14:05.915 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:05.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:05 smithi017 ceph-mon[102913]: pgmap v1980: 109 pgs: 109 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-09-21T10:14:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:14:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:05 smithi017 ceph-mon[102913]: osdmap e1292: 8 total, 8 up, 8 in 2024-09-21T10:14:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:05 smithi017 ceph-mon[105151]: pgmap v1980: 109 pgs: 109 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-09-21T10:14:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:14:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:05 smithi017 ceph-mon[105151]: osdmap e1292: 8 total, 8 up, 8 in 2024-09-21T10:14:06.016 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:06.053 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:06.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:06.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:06.186 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:06.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:06.287 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:06.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.324 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:06.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:06.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:06.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:06.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:06.555 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 53407 53391 123.578 109.938 0.01072 0.00808762 2024-09-21T10:14:06.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:06.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:06.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:06.738 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:06.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:06.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:06 smithi134 ceph-mon[90085]: osdmap e1293: 8 total, 8 up, 8 in 2024-09-21T10:14:06.839 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.876 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:06.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:06.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:06 smithi017 ceph-mon[102913]: osdmap e1293: 8 total, 8 up, 8 in 2024-09-21T10:14:06.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:06 smithi017 ceph-mon[105151]: osdmap e1293: 8 total, 8 up, 8 in 2024-09-21T10:14:06.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:07.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:07.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:07.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:07.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:07.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:07.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:07.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.291 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:07.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:07.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:07.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:07.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:07.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:07.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 55338 55322 123.469 120.688 0.00728333 0.00809399 2024-09-21T10:14:07.570 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:07.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:07.672 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:07.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:07.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:07.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:07.847 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:07.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:07.933 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:07 smithi134 ceph-mon[90085]: pgmap v1983: 119 pgs: 3 peering, 8 unknown, 108 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-21T10:14:07.949 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:07 smithi017 ceph-mon[102913]: pgmap v1983: 119 pgs: 3 peering, 8 unknown, 108 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-21T10:14:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:07 smithi017 ceph-mon[105151]: pgmap v1983: 119 pgs: 3 peering, 8 unknown, 108 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-09-21T10:14:07.982 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:08.084 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:08.116 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:08.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:08.218 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:08.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:08.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:08.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:08.392 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:08.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:08.466 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:08.467 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:08.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:08.523 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:08.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:08.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 57357 57341 123.563 126.188 0.00745676 0.00808858 2024-09-21T10:14:08.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:08.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:08.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:08.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:08.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:08.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:08.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:08.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:08.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:09.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:09.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:09.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:09.200 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:09.239 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:09.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:09.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:09.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:09.482 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:09.512 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:09.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 59294 59278 123.479 121.062 0.00616649 0.00809387 2024-09-21T10:14:09.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:09.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:09.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:09.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:09.773 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:09 smithi017 ceph-mon[102913]: pgmap v1984: 119 pgs: 3 peering, 4 unknown, 112 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-21T10:14:09.773 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:09 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:09.773 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:09 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:09.773 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:09 smithi017 ceph-mon[105151]: pgmap v1984: 119 pgs: 3 peering, 4 unknown, 112 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-21T10:14:09.774 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:09 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:09.774 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:09 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:09.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:09.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:09.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:09.979 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:09.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:09 smithi134 ceph-mon[90085]: pgmap v1984: 119 pgs: 3 peering, 4 unknown, 112 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-21T10:14:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:09 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:09 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:10.080 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:10.109 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:10.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:10.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:10.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:10.353 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:10.385 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:10.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:14:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:14:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:14:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:10.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:10.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 61117 61101 123.171 113.938 0.00380159 0.00811402 2024-09-21T10:14:10.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:14:10.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:14:10.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:10.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:10.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:14:10.600 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:14:10.600 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:14:10.600 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:10.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:10.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:14:10.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:10.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:10.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:10.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:10.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:10.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:10.909 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:10.943 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:10.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:14:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:14:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:14:11.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:11.080 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:11.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:11.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:11.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:11.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:11.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:11.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:11.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:11.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:11.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:11.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:14:11.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 62487 62471 121.997 85.625 0.00254996 0.00808689 2024-09-21T10:14:11.566 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:11.587 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:11.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:11.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:11.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:11.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:11.834 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:11.936 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:11.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:11 smithi017 ceph-mon[102913]: pgmap v1985: 119 pgs: 119 active+clean; 3.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 28 KiB/s, 0 objects/s recovering 2024-09-21T10:14:11.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:14:11.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:11 smithi017 ceph-mon[102913]: osdmap e1294: 8 total, 8 up, 8 in 2024-09-21T10:14:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:11 smithi017 ceph-mon[105151]: pgmap v1985: 119 pgs: 119 active+clean; 3.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 28 KiB/s, 0 objects/s recovering 2024-09-21T10:14:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:14:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:11 smithi017 ceph-mon[105151]: osdmap e1294: 8 total, 8 up, 8 in 2024-09-21T10:14:11.958 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:11 smithi134 ceph-mon[90085]: pgmap v1985: 119 pgs: 119 active+clean; 3.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 28 KiB/s, 0 objects/s recovering 2024-09-21T10:14:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:14:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:11 smithi134 ceph-mon[90085]: osdmap e1294: 8 total, 8 up, 8 in 2024-09-21T10:14:12.059 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:12.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:12.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:12.182 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:12.203 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:12.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:12.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:12.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:12.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:12.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:12.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:12.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:12.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 62487 62471 118.301 0 - 0.00808689 2024-09-21T10:14:12.577 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:12.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:12.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:12.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:12.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:12.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:12.827 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:12.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:12.929 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:12.953 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:12.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:13.006 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:14:13.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:13.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:13.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:13.101 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:14:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:14:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:14:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:14:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.101 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57270 9.a 9:5e5a1cc8:::benchmark_data_smithi017_237924_object57269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.423992+0000", 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 288.67210079500001, 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2984869460000001, 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57270 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.423992+0000", 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.423992+0000", 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.423993+0000", 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4000000000000001e-07 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424017+0000", 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4017000000000001e-05 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424026+0000", 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7399999999999993e-06 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424032+0000", 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6229999999999999e-06 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424137+0000", 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010459 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424290+0000", 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153287 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424441+0000", 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015061100000000001 2024-09-21T10:14:13.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.446330+0000", 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021888744000000002 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.446341+0000", 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1245e-05 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.437328+0000", 2024-09-21T10:14:13.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99098753299999998 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.437346+0000", 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7173e-05 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.722057+0000", 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.284711774 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.722479+0000", 2024-09-21T10:14:13.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042196900000000001 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57273 9.a 9:52bf49fb:::benchmark_data_smithi017_237924_object57272:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.428944+0000", 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 288.66714866799998, 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3377784579999998, 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57273 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428944+0000", 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428944+0000", 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428946+0000", 2024-09-21T10:14:13.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.556e-06 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428977+0000", 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0793999999999997e-05 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428990+0000", 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2798e-05 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428999+0000", 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2830000000000007e-06 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.431002+0000", 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020034300000000001 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.431137+0000", 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134222 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-21T10:14:13.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.431270+0000", 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013351199999999999 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.446375+0000", 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015105159999999999 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.446384+0000", 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8429999999999999e-06 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.448028+0000", 2024-09-21T10:14:13.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0016439619999999 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.448042+0000", 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4419999999999999e-05 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.725409+0000", 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2773661299999999 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.729090+0000", 2024-09-21T10:14:13.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003681171 2024-09-21T10:14:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.766612+0000", 2024-09-21T10:14:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.037522487 2024-09-21T10:14:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.766723+0000", 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011069 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57274 9.d 9:b4817d95:::benchmark_data_smithi017_237924_object57273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.442406+0000", 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 288.653687553, 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2971451329999999, 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.115 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57274 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442406+0000", 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442406+0000", 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442407+0000", 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4789999999999999e-06 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442436+0000", 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9147999999999999e-05 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442447+0000", 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0674e-05 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442455+0000", 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0949999999999996e-06 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.445511+0000", 2024-09-21T10:14:13.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030560850000000001 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.445522+0000", 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1126e-05 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.437369+0000", 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.991847005 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.437380+0000", 2024-09-21T10:14:13.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1082e-05 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.738280+0000", 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.300899509 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.739551+0000", 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00127093 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57285 9.a 9:5a7c97ce:::benchmark_data_smithi017_237924_object57284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.453285+0000", 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 288.64280778599999, 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3177990209999999, 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57285 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453285+0000", 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453285+0000", 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453286+0000", 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0729999999999999e-06 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453316+0000", 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.959e-05 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453334+0000", 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7645000000000001e-05 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453342+0000", 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7350000000000002e-06 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.455020+0000", 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016776269999999999 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.455032+0000", 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1943000000000001e-05 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.448075+0000", 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99304325000000004 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.448081+0000", 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2790000000000004e-06 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.729139+0000", 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2810577990000001 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.729308+0000", 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000168338 2024-09-21T10:14:13.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.729489+0000", 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018096599999999999 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.740163+0000", 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010674865 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.742987+0000", 2024-09-21T10:14:13.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028231110000000001 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.766774+0000", 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023787557000000001 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.766827+0000", 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3258999999999999e-05 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.771084+0000", 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042569840000000001 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57301 9.a 9:548d621b:::benchmark_data_smithi017_237924_object57300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.475566+0000", 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 288.62052674900002, 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2977826700000001, 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.128 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57301 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475566+0000", 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475566+0000", 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475568+0000", 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1939999999999999e-06 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475591+0000", 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3132000000000001e-05 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475613+0000", 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2254000000000001e-05 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475623+0000", 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8500000000000006e-06 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475691+0000", 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8668000000000001e-05 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475704+0000", 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2285e-05 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.461721+0000", 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.986017276 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.461736+0000", 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.482e-05 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.734589+0000", 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.272852927 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.734767+0000", 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017805100000000001 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.734950+0000", 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018336700000000001 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.740209+0000", 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052588000000000001 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.742993+0000", 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027842850000000001 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.771175+0000", 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.02818128 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.771231+0000", 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6097000000000002e-05 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.773349+0000", 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021183840000000001 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:71121 9.0 9:089477fe:::benchmark_data_smithi017_237924_object71120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1248)", 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:45.644059+0000", 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 267.45203414399998, 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0834903279999999, 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.135 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 71121 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644059+0000", 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644059+0000", 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644061+0000", 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2460000000000002e-06 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644124+0000", 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2621999999999998e-05 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644145+0000", 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1228000000000001e-05 2024-09-21T10:14:13.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644157+0000", 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1630999999999999e-05 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644222+0000", 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5735999999999995e-05 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644230+0000", 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0260000000000007e-06 2024-09-21T10:14:13.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:46.454119+0000", 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.80988898399999998 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:46.454127+0000", 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7500000000000003e-06 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.668601+0000", 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.214473871 2024-09-21T10:14:13.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.668746+0000", 2024-09-21T10:14:13.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144979 2024-09-21T10:14:13.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:14:13.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.668897+0000", 2024-09-21T10:14:13.140 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[102913]: osdmap e1295: 8 total, 8 up, 8 in 2024-09-21T10:14:13.140 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[102913]: pgmap v1988: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s; 28 KiB/s, 0 objects/s recovering 2024-09-21T10:14:13.140 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:14:13.140 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2872757752' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:14:13.140 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:14:13.141 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[105151]: osdmap e1295: 8 total, 8 up, 8 in 2024-09-21T10:14:13.141 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[105151]: pgmap v1988: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s; 28 KiB/s, 0 objects/s recovering 2024-09-21T10:14:13.141 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:14:13.141 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2872757752' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:14:13.141 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:12 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:14:13.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000150808 2024-09-21T10:14:13.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.684975+0000", 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016078080000000002 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.685064+0000", 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9204999999999997e-05 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.724053+0000", 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.038988876999999998 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.724115+0000", 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1860999999999996e-05 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.727549+0000", 2024-09-21T10:14:13.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034344240000000002 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173494 9.a 9:50c8d2e5:::benchmark_data_smithi017_237924_object80490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.672366+0000", 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 108.423726952, 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2677540870000001, 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173494 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672366+0000", 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672366+0000", 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672367+0000", 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.296e-06 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672373+0000", 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3009999999999997e-06 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672384+0000", 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1387e-05 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672392+0000", 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7950000000000008e-06 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.673331+0000", 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00093947300000000002 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.673451+0000", 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119657 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.673880+0000", 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042885500000000001 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.681603+0000", 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077231490000000003 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686212+0000", 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046092870000000001 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.711576+0000", 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025363765999999999 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.711589+0000", 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3047999999999999e-05 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.691955+0000", 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98036599499999999 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.691968+0000", 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2887999999999999e-05 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.927698+0000", 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.235729887 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.927823+0000", 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124857 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.940061+0000", 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012238657999999999 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.940120+0000", 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8788e-05 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173497 9.a 9:5b90c862:::benchmark_data_smithi017_237924_object80493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.682723+0000", 2024-09-21T10:14:13.151 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 108.413369793, 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.265087597, 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173497 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682723+0000", 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682723+0000", 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682725+0000", 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5430000000000001e-06 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682731+0000", 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4389999999999997e-06 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682748+0000", 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7028e-05 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682756+0000", 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7300000000000005e-06 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683107+0000", 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035098600000000002 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683251+0000", 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014426200000000001 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683395+0000", 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014337700000000001 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686218+0000", 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028229240000000001 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712051+0000", 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025833394999999999 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712062+0000", 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0796e-05 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692006+0000", 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97994381699999999 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692014+0000", 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7839999999999992e-06 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928245+0000", 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.236230741 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928362+0000", 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117108 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947740+0000", 2024-09-21T10:14:13.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019377426999999999 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947811+0000", 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1240000000000002e-05 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173499 9.a 9:51167347:::benchmark_data_smithi017_237924_object80495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.682799+0000", 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 108.413294444, 2024-09-21T10:14:13.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2655015469999999, 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173499 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682799+0000", 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682799+0000", 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682799+0000", 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2100000000000002e-07 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682802+0000", 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.407e-06 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682808+0000", 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.404e-06 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682813+0000", 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3109999999999996e-06 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686247+0000", 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003433859 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686362+0000", 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114931 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686478+0000", 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116415 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.689225+0000", 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027468509999999998 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712092+0000", 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022866424 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712101+0000", 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9260000000000006e-06 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692045+0000", 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97994389900000001 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692055+0000", 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0183999999999999e-05 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928407+0000", 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2363518550000001 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928493+0000", 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6203000000000001e-05 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.948233+0000", 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019740119 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.948300+0000", 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7337999999999998e-05 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173506 9.a 9:5e300410:::benchmark_data_smithi017_237924_object80502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.691017+0000", 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 108.405075732, 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2555493759999998, 2024-09-21T10:14:13.165 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173506 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691017+0000", 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691017+0000", 2024-09-21T10:14:13.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691019+0000", 2024-09-21T10:14:13.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.68e-06 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691024+0000", 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8729999999999998e-06 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691036+0000", 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1798e-05 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691043+0000", 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6480000000000003e-06 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712526+0000", 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021482858000000001 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712537+0000", 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0359e-05 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692079+0000", 2024-09-21T10:14:13.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97954288099999998 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692089+0000", 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8290000000000003e-06 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928542+0000", 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2364527670000001 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928669+0000", 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127415 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928804+0000", 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134895 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933737+0000", 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049323420000000001 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933793+0000", 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6239999999999997e-05 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936412+0000", 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026192839999999999 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946186+0000", 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0097739379999999994 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946225+0000", 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8436999999999998e-05 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946567+0000", 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034213199999999998 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173517 9.a 9:533fe112:::benchmark_data_smithi017_237924_object80513:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.728151+0000", 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 108.367941702, 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.218795536, 2024-09-21T10:14:13.173 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173517 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728151+0000", 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728151+0000", 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728154+0000", 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2800000000000002e-06 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728160+0000", 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4169999999999997e-06 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728178+0000", 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8159999999999999e-05 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728189+0000", 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0264e-05 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728271+0000", 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2862999999999996e-05 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728284+0000", 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2395000000000001e-05 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692113+0000", 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96382926999999996 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692122+0000", 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6899999999999998e-06 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.931313+0000", 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2391907739999999 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.931439+0000", 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012628199999999999 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.931510+0000", 2024-09-21T10:14:13.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0813000000000001e-05 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936463+0000", 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049534410000000003 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936523+0000", 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9979000000000003e-05 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936592+0000", 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8757999999999995e-05 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946624+0000", 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01003248 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946662+0000", 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7984000000000001e-05 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946947+0000", 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028468599999999998 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173519 9.a 9:5cd44cac:::benchmark_data_smithi017_237924_object80515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.733984+0000", 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 108.362108726, 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2132821030000001, 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.181 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173519 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.733984+0000", 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.733984+0000", 2024-09-21T10:14:13.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.733986+0000", 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.291e-06 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.733990+0000", 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1500000000000001e-06 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.734003+0000", 2024-09-21T10:14:13.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2782999999999999e-05 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.734009+0000", 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2840000000000003e-06 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.734077+0000", 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8062e-05 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.734084+0000", 2024-09-21T10:14:13.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4170000000000003e-06 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692148+0000", 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95806323500000001 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692156+0000", 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8869999999999998e-06 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933829+0000", 2024-09-21T10:14:13.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2416723649999999 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933906+0000", 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6699999999999994e-05 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.934023+0000", 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117877 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936557+0000", 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002533712 2024-09-21T10:14:13.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936585+0000", 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7634999999999999e-05 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936595+0000", 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8619999999999998e-06 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947020+0000", 2024-09-21T10:14:13.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010425462 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947080+0000", 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9521000000000002e-05 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947266+0000", 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018686000000000001 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173491 9.3 9:c0ca1344:::benchmark_data_smithi017_237924_object80487:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1268)", 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:25.694632+0000", 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 107.401461595, 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2226775640000001, 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173491 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694632+0000", 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694632+0000", 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694633+0000", 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.516e-06 2024-09-21T10:14:13.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694638+0000", 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5619999999999997e-06 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694649+0000", 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1382e-05 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694658+0000", 2024-09-21T10:14:13.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4210000000000001e-06 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694720+0000", 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1409999999999996e-05 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694729+0000", 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3190000000000001e-06 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917233+0000", 2024-09-21T10:14:13.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2225039360000001 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917309+0000", 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6018000000000003e-05 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:174750 9.0 9:0193018d:::benchmark_data_smithi017_237924_object81746:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1270)", 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:28.597666+0000", 2024-09-21T10:14:13.193 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 104.498426688, 2024-09-21T10:14:13.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:13.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0207220859999999, 2024-09-21T10:14:13.194 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.194 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 174750 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597666+0000", 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597666+0000", 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597668+0000", 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5149999999999999e-06 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597673+0000", 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2650000000000004e-06 2024-09-21T10:14:13.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597686+0000", 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2313999999999999e-05 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597711+0000", 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5914999999999998e-05 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.598218+0000", 2024-09-21T10:14:13.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00050644500000000001 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.598227+0000", 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1719999999999996e-06 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.609945+0000", 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.011718382 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.610028+0000", 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2486999999999998e-05 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.610100+0000", 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2378000000000004e-05 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.612698+0000", 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025973670000000002 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.615409+0000", 2024-09-21T10:14:13.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027111029999999999 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618273+0000", 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028642310000000001 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618340+0000", 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6830999999999998e-05 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618389+0000", 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8680999999999997e-05 2024-09-21T10:14:13.200 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:174754 9.0 9:05c8886c:::benchmark_data_smithi017_237924_object81750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1270)", 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:28.602933+0000", 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 104.49316032, 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0185616390000001, 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 174754 2024-09-21T10:14:13.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602933+0000", 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602933+0000", 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602935+0000", 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3190000000000002e-06 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602940+0000", 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2179999999999998e-06 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602958+0000", 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7509000000000001e-05 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602967+0000", 2024-09-21T10:14:13.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9840000000000007e-06 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.603550+0000", 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00058335699999999995 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.603573+0000", 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2855999999999999e-05 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.612748+0000", 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.009174802 2024-09-21T10:14:13.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.612874+0000", 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012605099999999999 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.612996+0000", 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012250600000000001 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618448+0000", 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054516620000000003 2024-09-21T10:14:13.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618600+0000", 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000152304 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621393+0000", 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002792905 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621447+0000", 2024-09-21T10:14:13.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3891000000000003e-05 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621494+0000", 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7274999999999997e-05 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:174764 9.0 9:02d0e704:::benchmark_data_smithi017_237924_object81760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1270)", 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:28.619871+0000", 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 104.47622176500001, 2024-09-21T10:14:13.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.001702117, 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 174764 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619871+0000", 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619871+0000", 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619873+0000", 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.55e-06 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619877+0000", 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.031e-06 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619886+0000", 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2410000000000001e-06 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619892+0000", 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3219999999999998e-06 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619946+0000", 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3049e-05 2024-09-21T10:14:13.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619956+0000", 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0458999999999999e-05 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.615467+0000", 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99551071499999999 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.615602+0000", 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013529899999999999 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.615735+0000", 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000132652 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618655+0000", 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029202709999999999 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618709+0000", 2024-09-21T10:14:13.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4308999999999998e-05 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621505+0000", 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027956669999999999 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621541+0000", 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6576999999999999e-05 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621573+0000", 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1974999999999998e-05 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51624 10.12 10:4d635f7b:::benchmark_data_smithi017_268344_object51623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.574502+0000", 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.5215911569999996, 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0964674430000001, 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:13.214 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51624 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574502+0000", 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574502+0000", 2024-09-21T10:14:13.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574505+0000", 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8320000000000002e-06 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574574+0000", 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8708999999999996e-05 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574592+0000", 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8173e-05 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574601+0000", 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7899999999999994e-06 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652357+0000", 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077755938999999996 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652365+0000", 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9119999999999998e-06 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.670836+0000", 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0184705590000001 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.670969+0000", 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013352899999999999 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.218 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51630 10.12 10:4a1f2cd2:::benchmark_data_smithi017_268344_object51629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.574639+0000", 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.5214540030000006, 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1024330849999999, 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51630 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.219 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574639+0000", 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574639+0000", 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574640+0000", 2024-09-21T10:14:13.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4e-07 2024-09-21T10:14:13.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574688+0000", 2024-09-21T10:14:13.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8414000000000001e-05 2024-09-21T10:14:13.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574698+0000", 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5109999999999992e-06 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574705+0000", 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9750000000000001e-06 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652386+0000", 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077681106 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652392+0000", 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8440000000000003e-06 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.671339+0000", 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.018947681 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:14:13.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.672973+0000", 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001633633 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.676980+0000", 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040064209999999996 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.677072+0000", 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2659999999999997e-05 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.224 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51632 10.12 10:48e3c976:::benchmark_data_smithi017_268344_object51631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.575319+0000", 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.5207743980000004, 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1098050749999999, 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:13.225 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51632 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575319+0000", 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575319+0000", 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575320+0000", 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.305e-06 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575368+0000", 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7852999999999997e-05 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575383+0000", 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5378000000000001e-05 2024-09-21T10:14:13.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575390+0000", 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1919999999999999e-06 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652413+0000", 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077022304999999999 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652419+0000", 2024-09-21T10:14:13.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0480000000000004e-06 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673028+0000", 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.020609163 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673203+0000", 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175508 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673384+0000", 2024-09-21T10:14:13.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000181003 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.677117+0000", 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037329780000000001 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.679278+0000", 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021605589999999998 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.684683+0000", 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054047239999999996 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.684769+0000", 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6170999999999999e-05 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.685124+0000", 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035488799999999998 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51634 10.2 10:464e25bd:::benchmark_data_smithi017_268344_object51633:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.611011+0000", 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 8.4850818669999999, 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.053044901, 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51634 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:13.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611011+0000", 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611011+0000", 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611014+0000", 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7949999999999999e-06 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611049+0000", 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4715999999999998e-05 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611075+0000", 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6681000000000001e-05 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611089+0000", 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3224000000000001e-05 2024-09-21T10:14:13.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652442+0000", 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.041353194000000003 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652448+0000", 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3400000000000003e-06 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.653378+0000", 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0009295229999999 2024-09-21T10:14:13.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.653597+0000", 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000219378 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.654044+0000", 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000446792 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.662105+0000", 2024-09-21T10:14:13.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080612300000000008 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.663041+0000", 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00093635699999999997 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.663961+0000", 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00091951100000000003 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:13.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.664014+0000", 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2849999999999997e-05 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.664056+0000", 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.231e-05 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:13.238 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:14:13.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:13.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:13.363 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:13.390 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:13.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:13.492 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:13.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:12 smithi134 ceph-mon[90085]: osdmap e1295: 8 total, 8 up, 8 in 2024-09-21T10:14:13.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:12 smithi134 ceph-mon[90085]: pgmap v1988: 119 pgs: 119 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s; 28 KiB/s, 0 objects/s recovering 2024-09-21T10:14:13.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:12 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:14:13.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:12 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2872757752' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:14:13.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:12 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:14:13.514 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:13.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:13.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 62882 62866 115.547 12.3438 0.00260148 0.00858072 2024-09-21T10:14:13.616 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:13.637 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:13.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:13.738 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:13.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:13.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:13.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:13.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:13.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:13.985 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:14.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:14.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:14.104 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:14:14.104 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:14:14.104 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-21T10:14:14.104 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:14:14.105 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:14:14.105 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-21T10:14:14.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:14.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:14.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:14.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:14.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:14:14.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:14:14.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[102913]: osdmap e1296: 8 total, 8 up, 8 in 2024-09-21T10:14:14.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[102913]: 10.18 scrub starts 2024-09-21T10:14:14.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[102913]: 10.18 scrub ok 2024-09-21T10:14:14.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[102913]: osdmap e1297: 8 total, 8 up, 8 in 2024-09-21T10:14:14.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:14.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:14:14.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:14:14.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[105151]: osdmap e1296: 8 total, 8 up, 8 in 2024-09-21T10:14:14.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:13 smithi017 ceph-mon[105151]: 10.18 scrub starts 2024-09-21T10:14:14.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:14 smithi017 ceph-mon[105151]: 10.18 scrub ok 2024-09-21T10:14:14.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:14 smithi017 ceph-mon[105151]: osdmap e1297: 8 total, 8 up, 8 in 2024-09-21T10:14:14.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:14.254 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:14.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:14.356 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:14.378 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:14.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:14.385 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:14:14.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:14.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:14:14.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:14 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:14.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:14 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:14:14.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:14 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:14:14.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:14 smithi134 ceph-mon[90085]: osdmap e1296: 8 total, 8 up, 8 in 2024-09-21T10:14:14.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:14 smithi134 ceph-mon[90085]: 10.18 scrub starts 2024-09-21T10:14:14.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:14 smithi134 ceph-mon[90085]: 10.18 scrub ok 2024-09-21T10:14:14.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:14 smithi134 ceph-mon[90085]: osdmap e1297: 8 total, 8 up, 8 in 2024-09-21T10:14:14.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:14.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:14.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 62882 62866 112.246 0 - 0.00858072 2024-09-21T10:14:14.646 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:14.674 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:14.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:14.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:14.810 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:14.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:14.944 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:15.007 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:14:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:14:14] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.51.0" 2024-09-21T10:14:15.023 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:15.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:15.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:15.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:15.177 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:15.199 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:15.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:15 smithi134 ceph-mon[90085]: pgmap v1991: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 1855/185121 objects misplaced (1.002%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:14:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:15 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:14:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:15 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:14:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:15 smithi134 ceph-mon[90085]: osdmap e1298: 8 total, 8 up, 8 in 2024-09-21T10:14:15.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:15 smithi017 ceph-mon[102913]: pgmap v1991: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 1855/185121 objects misplaced (1.002%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:14:15.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:15 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:14:15.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:15 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:14:15.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:15 smithi017 ceph-mon[102913]: osdmap e1298: 8 total, 8 up, 8 in 2024-09-21T10:14:15.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:15 smithi017 ceph-mon[105151]: pgmap v1991: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 3.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 1855/185121 objects misplaced (1.002%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:14:15.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:15 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:14:15.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:15 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:14:15.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:15 smithi017 ceph-mon[105151]: osdmap e1298: 8 total, 8 up, 8 in 2024-09-21T10:14:15.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:15.322 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:15.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:15.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:15.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:15.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:15.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:15.556 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 63422 63406 110.066 16.875 0.002573 0.00890941 2024-09-21T10:14:15.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:15.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:15.673 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:15.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:15.796 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:15.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:15.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:15.907 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:15.920 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:15.941 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:15.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:16.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:16.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:16.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:16.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:16.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:16.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:16.294 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:16.318 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:16.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:16.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:16.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:16.543 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:16.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 63422 63406 107.091 0 - 0.00890941 2024-09-21T10:14:16.565 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:16.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:16.586 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:14:16.586 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "" 2024-09-21T10:14:16.586 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:14:16.667 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:16.691 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:16.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:16.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:16.821 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:16.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:16.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:16 smithi017 ceph-mon[105151]: osdmap e1299: 8 total, 8 up, 8 in 2024-09-21T10:14:16.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:16 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:16.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:16 smithi017 ceph-mon[102913]: osdmap e1299: 8 total, 8 up, 8 in 2024-09-21T10:14:16.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:16 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:16.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:16.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:16.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:16 smithi134 ceph-mon[90085]: osdmap e1299: 8 total, 8 up, 8 in 2024-09-21T10:14:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:16 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:17.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:14:17.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:17.133 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:17.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:17.235 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:17.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.284 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:17.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:17.385 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:17.406 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:17.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:17.507 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:17.547 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:17.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 64542 64526 106.115 35 0.0186865 0.00941882 2024-09-21T10:14:17.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:17.685 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:17.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:17.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:17.797 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:17 smithi017 ceph-mon[102913]: pgmap v1994: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25 MiB/s wr, 399 op/s; 1851/189456 objects degraded (0.977%); 1855/189456 objects misplaced (0.979%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:14:17.797 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:17 smithi017 ceph-mon[102913]: osdmap e1300: 8 total, 8 up, 8 in 2024-09-21T10:14:17.797 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:17 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 1851/189456 objects degraded (0.977%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:17.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:17 smithi017 ceph-mon[105151]: pgmap v1994: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25 MiB/s wr, 399 op/s; 1851/189456 objects degraded (0.977%); 1855/189456 objects misplaced (0.979%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:14:17.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:17 smithi017 ceph-mon[105151]: osdmap e1300: 8 total, 8 up, 8 in 2024-09-21T10:14:17.798 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:17 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 1851/189456 objects degraded (0.977%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:17.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:17.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:17.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:17.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:17.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:17 smithi134 ceph-mon[90085]: pgmap v1994: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25 MiB/s wr, 399 op/s; 1851/189456 objects degraded (0.977%); 1855/189456 objects misplaced (0.979%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:14:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:17 smithi134 ceph-mon[90085]: osdmap e1300: 8 total, 8 up, 8 in 2024-09-21T10:14:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:17 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 1851/189456 objects degraded (0.977%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:18.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:18.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:18.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:18.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:18.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:18.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:18.368 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:18.396 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:18.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:18.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:18.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:18.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:18.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 66113 66097 105.911 98.1875 0.00234282 0.00943342 2024-09-21T10:14:18.586 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:18.606 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:14:18.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:18.630 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:14:18.630 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:18.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:18 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1554809916' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:18.631 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:18 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1554809916' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:18.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:18.666 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:14:18.666 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:18.690 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:18.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:18.699 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:14:18.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:18.738 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:14:18.738 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:18.771 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:14:18.771 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:18.791 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:18.806 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:14:18.807 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:18.830 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:18.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:18.842 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:14:18.843 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:18.871 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:18.871 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:18.933 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:18.969 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:18.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:18 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1554809916' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:19.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:19.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:19.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:19.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:19.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:19.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:19.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:19.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:19.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:19.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:19.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:19.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:14:19.556150+0000 min lat: 0.00108589 max lat: 2.06285 avg lat: 0.00946813 2024-09-21T10:14:19.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:14:19.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 67584 67568 105.562 91.9375 0.0116422 0.00946813 2024-09-21T10:14:19.628 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:19.650 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:19.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:19.751 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:19.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.785 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:19.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:19.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:19.918 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:19.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:19.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:19 smithi017 ceph-mon[105151]: pgmap v1996: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 729 KiB/s wr, 11 op/s; 1851/191130 objects degraded (0.968%); 1855/191130 objects misplaced (0.971%) 2024-09-21T10:14:19.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:19 smithi017 ceph-mon[102913]: pgmap v1996: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 729 KiB/s wr, 11 op/s; 1851/191130 objects degraded (0.968%); 1855/191130 objects misplaced (0.971%) 2024-09-21T10:14:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:19 smithi134 ceph-mon[90085]: pgmap v1996: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 729 KiB/s wr, 11 op/s; 1851/191130 objects degraded (0.968%); 1855/191130 objects misplaced (0.971%) 2024-09-21T10:14:20.020 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:20.052 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:20.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:20.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:20.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:20.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:20.294 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:20.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:20.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:20.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:20.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:14:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:14:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:14:20.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 68864 68848 104.938 80 0.0109645 0.00952473 2024-09-21T10:14:20.563 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:20.596 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:20.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:20.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:20.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:20.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:20.838 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:20.875 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:20.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:20.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:21.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.020 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:21.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:21.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:21.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:21.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:21.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:21.299 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:21.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:21.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:21.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:21.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:21.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:14:21.545 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:21.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 70463 70447 104.819 99.9375 0.00738126 0.00953509 2024-09-21T10:14:21.580 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:21.682 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:21.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:21.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:21.808 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:21 smithi017 ceph-mon[102913]: pgmap v1997: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 116 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 288 op/s; 5871/196164 objects degraded (2.993%); 2917/196164 objects misplaced (1.487%); 9.9 MiB/s, 157 objects/s recovering 2024-09-21T10:14:21.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:21 smithi017 ceph-mon[105151]: pgmap v1997: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 116 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 288 op/s; 5871/196164 objects degraded (2.993%); 2917/196164 objects misplaced (1.487%); 9.9 MiB/s, 157 objects/s recovering 2024-09-21T10:14:21.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:21.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:21.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:21.974 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:21 smithi134 ceph-mon[90085]: pgmap v1997: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 116 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 18 MiB/s wr, 288 op/s; 5871/196164 objects degraded (2.993%); 2917/196164 objects misplaced (1.487%); 9.9 MiB/s, 157 objects/s recovering 2024-09-21T10:14:22.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:22.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:22.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:22.148 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:22.165 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:22.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:22.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:22.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:22.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:22.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:22.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:14:22.533 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:22.557 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 71877 71861 104.436 88.375 0.00785332 0.00957034 2024-09-21T10:14:22.570 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:22.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:22.642 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:14:22.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:22.717 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:22.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:22.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:22.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:22.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:22 smithi017 ceph-mon[102913]: 10.13 scrub starts 2024-09-21T10:14:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:22 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1405217511' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:22 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:22 smithi017 ceph-mon[105151]: 10.13 scrub starts 2024-09-21T10:14:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:22 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1405217511' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:22 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:22.978 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:22 smithi134 ceph-mon[90085]: 10.13 scrub starts 2024-09-21T10:14:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:22 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1405217511' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:22 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:23.015 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:23.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:23.117 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:23.154 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:23.256 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:23.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:23.390 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:23.423 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:23.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:23.525 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:23.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 15 73476 73461 104.335 100 0.0081143 0.00957936 2024-09-21T10:14:23.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.565 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:23.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:23.666 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:23.698 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:23.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:23.799 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:23.841 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:23.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:23.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[102913]: 10.13 scrub ok 2024-09-21T10:14:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[102913]: pgmap v1998: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 116 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 5871/203304 objects degraded (2.888%); 2917/203304 objects misplaced (1.435%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:14:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5871/203304 objects degraded (2.888%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:14:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:14:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[102913]: osdmap e1301: 8 total, 8 up, 8 in 2024-09-21T10:14:23.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[105151]: 10.13 scrub ok 2024-09-21T10:14:23.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[105151]: pgmap v1998: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 116 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 5871/203304 objects degraded (2.888%); 2917/203304 objects misplaced (1.435%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:14:23.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5871/203304 objects degraded (2.888%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:14:23.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:14:23.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:23.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:23 smithi017 ceph-mon[105151]: osdmap e1301: 8 total, 8 up, 8 in 2024-09-21T10:14:23.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:23.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:23 smithi134 ceph-mon[90085]: 10.13 scrub ok 2024-09-21T10:14:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:23 smithi134 ceph-mon[90085]: pgmap v1998: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 116 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 5871/203304 objects degraded (2.888%); 2917/203304 objects misplaced (1.435%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:14:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:23 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5871/203304 objects degraded (2.888%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:14:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:23 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:14:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:23 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:23 smithi134 ceph-mon[90085]: osdmap e1301: 8 total, 8 up, 8 in 2024-09-21T10:14:24.088 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:24.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:24.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:24.226 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:24.255 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:24.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:24.356 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:24.390 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:24.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:24.492 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:24.526 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:24.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:24.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 75123 75107 104.302 102.875 0.00826699 0.0095831 2024-09-21T10:14:24.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:24.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.662 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:24.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:24.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:24.800 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:24.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:24.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:24.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:24.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:24.980 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:14:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:14:24] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.51.0" 2024-09-21T10:14:25.032 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:25.051 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:25.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:25.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:25.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:25.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:25.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:25.281 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:25.302 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:25.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:25.403 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:25.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:25.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:25.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:25.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:25.558 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 75768 75752 102.911 40.3125 0.00274028 0.00956176 2024-09-21T10:14:25.650 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:25.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:25.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:25.818 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:25.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:25.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:25 smithi017 ceph-mon[102913]: pgmap v2000: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 608 op/s; 5871/210936 objects degraded (2.783%); 2054/210936 objects misplaced (0.974%); 14 MiB/s, 226 objects/s recovering 2024-09-21T10:14:25.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:25 smithi017 ceph-mon[102913]: osdmap e1302: 8 total, 8 up, 8 in 2024-09-21T10:14:25.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:25.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:25.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:25 smithi017 ceph-mon[105151]: pgmap v2000: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 608 op/s; 5871/210936 objects degraded (2.783%); 2054/210936 objects misplaced (0.974%); 14 MiB/s, 226 objects/s recovering 2024-09-21T10:14:25.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:25 smithi017 ceph-mon[105151]: osdmap e1302: 8 total, 8 up, 8 in 2024-09-21T10:14:25.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:25.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:25.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:25 smithi134 ceph-mon[90085]: pgmap v2000: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 608 op/s; 5871/210936 objects degraded (2.783%); 2054/210936 objects misplaced (0.974%); 14 MiB/s, 226 objects/s recovering 2024-09-21T10:14:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:25 smithi134 ceph-mon[90085]: osdmap e1302: 8 total, 8 up, 8 in 2024-09-21T10:14:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:26.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:26.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:26.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:26.205 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:26.206 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:26.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:26.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:26.355 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:26.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:26.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:14:26.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:26.536 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:26.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:26.559 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 77089 77073 102.476 82.5625 0.00681678 0.00975344 2024-09-21T10:14:26.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:26.648 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:14:26.648 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:14:26.648 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-21T10:14:26.648 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:14:26.648 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:14:26.649 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-21T10:14:26.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:26.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:26.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:26.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:26.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:26.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:26 smithi017 ceph-mon[102913]: osdmap e1303: 8 total, 8 up, 8 in 2024-09-21T10:14:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:26 smithi017 ceph-mon[105151]: osdmap e1303: 8 total, 8 up, 8 in 2024-09-21T10:14:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:26.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:26.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:26.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:26 smithi134 ceph-mon[90085]: osdmap e1303: 8 total, 8 up, 8 in 2024-09-21T10:14:26.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:27.047 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:14:27.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:27.116 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:27.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:27.258 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:27.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:27.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:27.406 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:27.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:27.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:27.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:27.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:27.560 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 78514 78498 102.193 89.0625 0.00411254 0.00977981 2024-09-21T10:14:27.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:27.686 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:27.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:27.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.830 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:27.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:27.931 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:27.961 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:27.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:27 smithi017 ceph-mon[102913]: pgmap v2003: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 4020/224541 objects degraded (1.790%); 3781/224541 objects misplaced (1.684%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:14:27.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:27 smithi017 ceph-mon[105151]: pgmap v2003: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 4020/224541 objects degraded (1.790%); 3781/224541 objects misplaced (1.684%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:14:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:27 smithi134 ceph-mon[90085]: pgmap v2003: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 116 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 4020/224541 objects degraded (1.790%); 3781/224541 objects misplaced (1.684%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:14:28.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:14:28.063 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:28.095 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:28.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:28.196 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:28.230 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:28.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:28.332 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:28.364 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:28.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:28.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:28.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.527 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:28.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:28.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 79871 79855 101.838 84.8125 0.0109793 0.00981457 2024-09-21T10:14:28.625 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:28.628 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:28.655 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:28.686 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:14:28.686 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:28.721 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:14:28.721 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:28.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:28.758 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:14:28.758 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:28.795 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:14:28.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:28.797 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:28.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:28.826 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:14:28.827 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:28.860 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:14:28.861 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:28.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:28 smithi017 ceph-mon[102913]: 10.3 scrub starts 2024-09-21T10:14:28.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:28 smithi017 ceph-mon[102913]: 10.3 scrub ok 2024-09-21T10:14:28.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:28 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 4020/224541 objects degraded (1.790%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:28.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:28 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2910519115' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:28.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:28 smithi017 ceph-mon[105151]: 10.3 scrub starts 2024-09-21T10:14:28.861 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:28 smithi017 ceph-mon[105151]: 10.3 scrub ok 2024-09-21T10:14:28.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:28 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 4020/224541 objects degraded (1.790%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:28.862 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:28 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2910519115' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:28.896 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:14:28.896 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:28.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:28.927 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:14:28.928 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:28.935 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:28.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:28.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:28 smithi134 ceph-mon[90085]: 10.3 scrub starts 2024-09-21T10:14:28.984 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:28 smithi134 ceph-mon[90085]: 10.3 scrub ok 2024-09-21T10:14:28.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:28 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 4020/224541 objects degraded (1.790%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:28.985 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:28 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2910519115' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:29.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:29.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:29.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:29.128 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:14:29.128 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "" 2024-09-21T10:14:29.128 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:14:29.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:29.253 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:29.355 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:29.397 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:29.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:29.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:29.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:29.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 81198 81182 101.46 82.9375 0.00999445 0.00985151 2024-09-21T10:14:29.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:29.645 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:29.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:29 smithi134 ceph-mon[90085]: pgmap v2004: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 115 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 4020/228918 objects degraded (1.756%); 3781/228918 objects misplaced (1.652%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:14:29.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:29.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:29.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:29.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:29.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:29.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:30.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:30.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:30.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:30.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:30.185 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:29 smithi017 ceph-mon[102913]: pgmap v2004: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 115 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 4020/228918 objects degraded (1.756%); 3781/228918 objects misplaced (1.652%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:14:30.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:29 smithi017 ceph-mon[105151]: pgmap v2004: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 115 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 4020/228918 objects degraded (1.756%); 3781/228918 objects misplaced (1.652%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:14:30.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:30.328 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:30.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:30.430 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:30.465 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:30.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:14:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:14:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:14:30.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 82327 82311 100.854 70.5625 0.0113241 0.0099102 2024-09-21T10:14:30.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:30.603 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:30.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:30.704 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:30.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.737 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:30.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:30.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:30.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:30.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:30.961 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:30.993 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:30.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:31.095 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:31.126 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:31.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:31.257 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:31.359 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:31.393 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:31.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:31.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:31.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.528 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:31.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:31.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 83535 83519 100.366 75.5 0.0100263 0.00995872 2024-09-21T10:14:31.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:31.663 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:31.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:31.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:31.807 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:31.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:31.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:31.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:31 smithi017 ceph-mon[102913]: 10.15 scrub starts 2024-09-21T10:14:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:31 smithi017 ceph-mon[102913]: pgmap v2005: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 4020/235806 objects degraded (1.705%); 3239/235806 objects misplaced (1.374%); 12 MiB/s, 195 objects/s recovering 2024-09-21T10:14:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:31 smithi017 ceph-mon[105151]: 10.15 scrub starts 2024-09-21T10:14:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:31 smithi017 ceph-mon[105151]: pgmap v2005: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 4020/235806 objects degraded (1.705%); 3239/235806 objects misplaced (1.374%); 12 MiB/s, 195 objects/s recovering 2024-09-21T10:14:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:31 smithi134 ceph-mon[90085]: 10.15 scrub starts 2024-09-21T10:14:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:31 smithi134 ceph-mon[90085]: pgmap v2005: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 4020/235806 objects degraded (1.705%); 3239/235806 objects misplaced (1.374%); 12 MiB/s, 195 objects/s recovering 2024-09-21T10:14:32.047 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:32.078 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:32.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:32.180 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:32.203 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:32.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:32.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:32.336 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:32.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:32.476 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:32.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:32.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 84864 84848 100.04 83.0625 0.0120405 0.00999091 2024-09-21T10:14:32.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:32.617 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:32.719 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:32.752 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:32.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:32.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:32.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:32 smithi134 ceph-mon[90085]: 10.15 scrub ok 2024-09-21T10:14:32.999 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:33.032 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:33.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:33.134 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:33.165 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:33.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:33.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:32 smithi017 ceph-mon[102913]: 10.15 scrub ok 2024-09-21T10:14:33.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:32 smithi017 ceph-mon[105151]: 10.15 scrub ok 2024-09-21T10:14:33.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:33.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:33.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:33.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:33.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:33.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:33.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:33.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 86187 86171 99.7182 82.6875 0.0119819 0.0100234 2024-09-21T10:14:33.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:33.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:33.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:33.724 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:33.826 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:33.863 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:33.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:33.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:34.005 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:34.107 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:34.132 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:34.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:34.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:33 smithi017 ceph-mon[102913]: pgmap v2006: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 4020/243918 objects degraded (1.648%); 3239/243918 objects misplaced (1.328%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:14:34.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:33 smithi017 ceph-mon[102913]: 10.5 scrub starts 2024-09-21T10:14:34.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 4020/243918 objects degraded (1.648%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:33 smithi017 ceph-mon[105151]: pgmap v2006: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 4020/243918 objects degraded (1.648%); 3239/243918 objects misplaced (1.328%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:14:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:33 smithi017 ceph-mon[105151]: 10.5 scrub starts 2024-09-21T10:14:34.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 4020/243918 objects degraded (1.648%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:34.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:33 smithi134 ceph-mon[90085]: pgmap v2006: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 4020/243918 objects degraded (1.648%); 3239/243918 objects misplaced (1.328%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:14:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:33 smithi134 ceph-mon[90085]: 10.5 scrub starts 2024-09-21T10:14:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 4020/243918 objects degraded (1.648%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:34.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:34.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:34.375 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:34.400 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:34.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:34.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:34.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:34.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 87567 87551 99.4732 86.25 0.00276289 0.0100439 2024-09-21T10:14:34.640 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:34.674 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:34.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:34.814 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:14:34.814 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:14:34.815 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-21T10:14:34.815 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.621030 2024-09-21T10:14:34.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 6 0.6210299007075906 2024-09-21T10:14:34.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:34.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:34.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:34.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:35.048 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:35.080 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:35.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:34 smithi134 ceph-mon[90085]: 10.5 scrub ok 2024-09-21T10:14:35.145 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:34 smithi134 ceph-mon[90085]: 10.1 deep-scrub starts 2024-09-21T10:14:35.181 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:35.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:34 smithi017 ceph-mon[102913]: 10.5 scrub ok 2024-09-21T10:14:35.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:34 smithi017 ceph-mon[102913]: 10.1 deep-scrub starts 2024-09-21T10:14:35.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:14:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:14:34] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.51.0" 2024-09-21T10:14:35.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:34 smithi017 ceph-mon[105151]: 10.5 scrub ok 2024-09-21T10:14:35.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:34 smithi017 ceph-mon[105151]: 10.1 deep-scrub starts 2024-09-21T10:14:35.219 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:35.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:35.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:35.353 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:35.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:35.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:35.490 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:35.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:35.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 88761 88745 99.0294 74.625 0.00967489 0.0100931 2024-09-21T10:14:35.592 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:35.616 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:35.718 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:35.751 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:35.852 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:35.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.893 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:35.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:35.905 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:35 smithi134 ceph-mon[90085]: pgmap v2007: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s; 4020/250503 objects degraded (1.605%); 3239/250503 objects misplaced (1.293%); 4.2 MiB/s, 67 objects/s recovering 2024-09-21T10:14:35.905 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:35 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:35.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:36.027 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:36.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:36.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:35 smithi017 ceph-mon[102913]: pgmap v2007: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s; 4020/250503 objects degraded (1.605%); 3239/250503 objects misplaced (1.293%); 4.2 MiB/s, 67 objects/s recovering 2024-09-21T10:14:36.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:35 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:36.095 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:35 smithi017 ceph-mon[105151]: pgmap v2007: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s; 4020/250503 objects degraded (1.605%); 3239/250503 objects misplaced (1.293%); 4.2 MiB/s, 67 objects/s recovering 2024-09-21T10:14:36.095 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:35 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:36.129 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.170 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:36.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:36.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:36.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:36.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:36.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:36.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:36.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:36.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 90142 90126 98.8062 86.3125 0.00990324 0.010116 2024-09-21T10:14:36.596 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:36.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.737 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:36.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:36.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:36.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:36.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:36.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:37.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:37.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:37.107 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:36 smithi134 ceph-mon[90085]: 10.10 scrub starts 2024-09-21T10:14:37.116 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:37.150 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:36 smithi017 ceph-mon[102913]: 10.10 scrub starts 2024-09-21T10:14:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:36 smithi017 ceph-mon[105151]: 10.10 scrub starts 2024-09-21T10:14:37.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:37.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:37.392 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:37.426 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:37.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:37.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:37.561 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:37.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 91446 91430 98.5077 81.5 0.00955295 0.0101461 2024-09-21T10:14:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:37.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:37.703 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:37.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:37.805 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:37.833 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:37.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:37.973 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:37.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:38.075 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:38.107 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:37 smithi017 ceph-mon[102913]: pgmap v2008: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s; 4020/263154 objects degraded (1.528%); 2893/263154 objects misplaced (1.099%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:14:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:37 smithi017 ceph-mon[105151]: pgmap v2008: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s; 4020/263154 objects degraded (1.528%); 2893/263154 objects misplaced (1.099%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:14:38.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:38.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:38.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:38.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:37 smithi134 ceph-mon[90085]: pgmap v2008: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 116 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s; 4020/263154 objects degraded (1.528%); 2893/263154 objects misplaced (1.099%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:14:38.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:38.378 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:38.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:38.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:38.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:38.561 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 92608 92592 98.0689 72.625 0.0149744 0.0101924 2024-09-21T10:14:38.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:38.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:38.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:38.763 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:38.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:38.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:38.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:38.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:38.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:39.029 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:39.050 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:39.151 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:39.174 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 4020/263154 objects degraded (1.528%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 4020/263154 objects degraded (1.528%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 4020/263154 objects degraded (1.528%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:39.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.305 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:39.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:39.406 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:39.442 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:39.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:39.544 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:39.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:14:39.560750+0000 min lat: 0.00108589 max lat: 2.06285 avg lat: 0.0102578 2024-09-21T10:14:39.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:14:39.562 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 93576 93560 97.4427 60.5 0.0125359 0.0102578 2024-09-21T10:14:39.579 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:39.681 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:39.711 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:39.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:39.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:39.848 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:39.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:39.972 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:40.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:40.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:40.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:39 smithi017 ceph-mon[102913]: pgmap v2009: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 34 MiB/s wr, 551 op/s; 4020/267495 objects degraded (1.503%); 2893/267495 objects misplaced (1.082%); 4.6 MiB/s, 73 objects/s recovering 2024-09-21T10:14:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:39 smithi017 ceph-mon[102913]: 10.1 scrub starts 2024-09-21T10:14:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:39 smithi017 ceph-mon[105151]: pgmap v2009: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 34 MiB/s wr, 551 op/s; 4020/267495 objects degraded (1.503%); 2893/267495 objects misplaced (1.082%); 4.6 MiB/s, 73 objects/s recovering 2024-09-21T10:14:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:39 smithi017 ceph-mon[105151]: 10.1 scrub starts 2024-09-21T10:14:40.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.218 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:39 smithi134 ceph-mon[90085]: pgmap v2009: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 34 MiB/s wr, 551 op/s; 4020/267495 objects degraded (1.503%); 2893/267495 objects misplaced (1.082%); 4.6 MiB/s, 73 objects/s recovering 2024-09-21T10:14:40.218 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:39 smithi134 ceph-mon[90085]: 10.1 scrub starts 2024-09-21T10:14:40.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:40.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:40.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:40.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:40.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:14:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:14:40.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:14:40.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:40.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:40.564 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 94936 94920 97.2348 85 0.0138048 0.0102793 2024-09-21T10:14:40.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:40.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:40.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:40.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:40.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:41.079 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:41.091 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:41.091 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:41.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:41.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:41.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:41.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:41.353 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:41.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:41.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:41.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:41.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:41.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 96315 96299 97.0565 86.1875 0.0259101 0.0102985 2024-09-21T10:14:41.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:41.668 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:41.770 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:41.796 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:41.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:41.897 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:41.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:41.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:41.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:41 smithi017 ceph-mon[102913]: pgmap v2010: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 624 op/s; 4020/274470 objects degraded (1.465%); 2387/274470 objects misplaced (0.870%); 7.3 MiB/s, 116 objects/s recovering 2024-09-21T10:14:41.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:41 smithi017 ceph-mon[102913]: 10.10 scrub starts 2024-09-21T10:14:41.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:41 smithi017 ceph-mon[105151]: pgmap v2010: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 624 op/s; 4020/274470 objects degraded (1.465%); 2387/274470 objects misplaced (0.870%); 7.3 MiB/s, 116 objects/s recovering 2024-09-21T10:14:41.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:41 smithi017 ceph-mon[105151]: 10.10 scrub starts 2024-09-21T10:14:42.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:42.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:42.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:42.166 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:42.196 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:41 smithi134 ceph-mon[90085]: pgmap v2010: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 624 op/s; 4020/274470 objects degraded (1.465%); 2387/274470 objects misplaced (0.870%); 7.3 MiB/s, 116 objects/s recovering 2024-09-21T10:14:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:41 smithi134 ceph-mon[90085]: 10.10 scrub starts 2024-09-21T10:14:42.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:42.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:42.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:42.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:42.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:42.564 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 97609 97593 96.7996 80.875 0.00483175 0.0103256 2024-09-21T10:14:42.584 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:42.614 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:42.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:42.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:42.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:42.839 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:42.857 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:42.882 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:42.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:42 smithi017 ceph-mon[102913]: 10.1 scrub starts 2024-09-21T10:14:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:42 smithi017 ceph-mon[105151]: 10.1 scrub starts 2024-09-21T10:14:42.984 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:43.015 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:43.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:43.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:14:43.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:43.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:42 smithi134 ceph-mon[90085]: 10.1 scrub starts 2024-09-21T10:14:43.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:43.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:43.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:43.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:43.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.450 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:43.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:43.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:43.564 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 98678 98662 96.3308 66.8125 0.00563973 0.010376 2024-09-21T10:14:43.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:43.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:43.605 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:14:43.693 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:43.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:43.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:43.821 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:43.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.842 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:43.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:43.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[102913]: pgmap v2011: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 40 MiB/s wr, 646 op/s; 4020/282165 objects degraded (1.425%); 2387/282165 objects misplaced (0.846%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T10:14:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3694781643' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 4020/282165 objects degraded (1.425%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3694781643' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:14:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[102913]: osdmap e1304: 8 total, 8 up, 8 in 2024-09-21T10:14:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[105151]: pgmap v2011: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 40 MiB/s wr, 646 op/s; 4020/282165 objects degraded (1.425%); 2387/282165 objects misplaced (0.846%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T10:14:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3694781643' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 4020/282165 objects degraded (1.425%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3694781643' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:14:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:43 smithi017 ceph-mon[105151]: osdmap e1304: 8 total, 8 up, 8 in 2024-09-21T10:14:43.978 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:43.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:44.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:44.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:44.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:44.210 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:44.231 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:44.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:43 smithi134 ceph-mon[90085]: pgmap v2011: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 40 MiB/s wr, 646 op/s; 4020/282165 objects degraded (1.425%); 2387/282165 objects misplaced (0.846%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T10:14:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:43 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3694781643' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:43 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 4020/282165 objects degraded (1.425%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:43 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:43 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3694781643' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:14:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:43 smithi134 ceph-mon[90085]: osdmap e1304: 8 total, 8 up, 8 in 2024-09-21T10:14:44.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:44.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:44.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:44.482 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:44.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:44.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 99169 99153 95.3208 30.6875 0.00277632 0.0103826 2024-09-21T10:14:44.584 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:44.606 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:44.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:44.754 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:44.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:44.810 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:14:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:14:44] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45620.0:102478 8.3 e1210/1200 8:c8ad84b4:::benchmark_data_smithi017_209235_object102477:head v 1210'6349, mlcod=1210'6347)", 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:06:06.338847+0000", 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 518.50671167400003, 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18747862500000001, 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.850 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338847+0000", 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338847+0000", 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338847+0000", 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3e-07 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338876+0000", 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8527999999999999e-05 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338887+0000", 2024-09-21T10:14:44.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1355999999999999e-05 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.338894+0000", 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4659999999999996e-06 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.339766+0000", 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00087272700000000001 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.339786+0000", 2024-09-21T10:14:44.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9544000000000001e-05 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.526275+0000", 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.186489404 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:06:06.526325+0000", 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9870000000000002e-05 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263811 8.4 8:2d8b153a:::benchmark_data_smithi017_209235_object130905:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1225)", 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:25.416982+0000", 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 379.42857583799997, 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.216450821, 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263811 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.855 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.416982+0000", 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.416982+0000", 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.416985+0000", 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2989999999999999e-06 2024-09-21T10:14:44.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.416994+0000", 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.867e-06 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417020+0000", 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5885999999999999e-05 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417032+0000", 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2258e-05 2024-09-21T10:14:44.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417137+0000", 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000104911 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417145+0000", 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.653e-06 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421071+0000", 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039252009999999997 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421080+0000", 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8889999999999997e-06 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.633364+0000", 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2122831169999999 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.633433+0000", 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9740000000000007e-05 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263812 8.4 8:2c129b93:::benchmark_data_smithi017_209235_object130906:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1225)", 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:25.417057+0000", 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 379.42850108599998, 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.216718411, 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.860 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263812 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417057+0000", 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417057+0000", 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417058+0000", 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8500000000000001e-07 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417061+0000", 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4429999999999999e-06 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417070+0000", 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7260000000000008e-06 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417077+0000", 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6719999999999998e-06 2024-09-21T10:14:44.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417205+0000", 2024-09-21T10:14:44.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012882900000000001 2024-09-21T10:14:44.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:44.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:14:44.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.417212+0000", 2024-09-21T10:14:44.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3929999999999996e-06 2024-09-21T10:14:44.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421109+0000", 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038974330000000001 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421117+0000", 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0020000000000006e-06 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.633668+0000", 2024-09-21T10:14:44.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.212550883 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.633776+0000", 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000107445 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:75073 9.6 9:6aa94b22:::benchmark_data_smithi017_237924_object75072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:14:44.867 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:54.659681+0000", 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 290.185877581, 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18124369600000001, 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 75073 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659681+0000", 2024-09-21T10:14:44.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659681+0000", 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659682+0000", 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4589999999999999e-06 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659731+0000", 2024-09-21T10:14:44.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8841999999999998e-05 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659742+0000", 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1379000000000001e-05 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659749+0000", 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0729999999999999e-06 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659872+0000", 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122776 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.660030+0000", 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000157269 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.660168+0000", 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013878799999999999 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.667039+0000", 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068708140000000003 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.668555+0000", 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001515567 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.840475+0000", 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.171919828 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.840758+0000", 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028381400000000002 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.840924+0000", 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016608700000000001 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:75077 9.6 9:6d1e6353:::benchmark_data_smithi017_237924_object75076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:54.659954+0000", 2024-09-21T10:14:44.873 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 290.18560459000003, 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.181617579, 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 75077 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659954+0000", 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659954+0000", 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659955+0000", 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9400000000000004e-07 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.659999+0000", 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3992999999999997e-05 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.660009+0000", 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0696000000000001e-05 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.660016+0000", 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0500000000000003e-06 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.663210+0000", 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031933560000000001 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.663385+0000", 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017536799999999999 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.663528+0000", 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014314800000000001 2024-09-21T10:14:44.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.668589+0000", 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050608199999999997 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.668615+0000", 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6037000000000002e-05 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.841152+0000", 2024-09-21T10:14:44.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.172536825 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.841415+0000", 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000263007 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:54.841571+0000", 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015648499999999999 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80760 9.b e1250/1249 9:d9fd3bc4:::benchmark_data_smithi017_237924_object80759:head v 1250'5132, mlcod=0'0)", 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.820464+0000", 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 279.02509397699998, 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.208616776, 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820464+0000", 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820464+0000", 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820466+0000", 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7799999999999999e-06 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820499+0000", 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2573000000000001e-05 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820516+0000", 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7314999999999999e-05 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820526+0000", 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0151e-05 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820634+0000", 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010801 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.820662+0000", 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7682999999999999e-05 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.023957+0000", 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20329536100000001 2024-09-21T10:14:44.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029081+0000", 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005123903 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80770 9.7 e1250/1245 9:eee90d88:::benchmark_data_smithi017_237924_object80769:head v 1250'5097, mlcod=0'0)", 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.829175+0000", 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 279.01638289300001, 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19592934400000001, 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.884 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829175+0000", 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829175+0000", 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829177+0000", 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2219999999999999e-06 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829211+0000", 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3881000000000003e-05 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829227+0000", 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6685999999999999e-05 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829237+0000", 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0144e-05 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.830053+0000", 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00081578499999999997 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.830080+0000", 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6486999999999999e-05 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.025043+0000", 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19496342 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.025105+0000", 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1718999999999996e-05 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80775 9.7 e1250/1245 9:e0ae36b0:::benchmark_data_smithi017_237924_object80774:head v 1250'5098, mlcod=0'0)", 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.829263+0000", 2024-09-21T10:14:44.888 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 279.01629568800001, 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20849586000000001, 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829263+0000", 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829263+0000", 2024-09-21T10:14:44.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829263+0000", 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1899999999999998e-07 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829328+0000", 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4927999999999999e-05 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829337+0000", 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0310000000000004e-06 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829343+0000", 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8669999999999999e-06 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.011152+0000", 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18180923600000001 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.011181+0000", 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9369e-05 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.037655+0000", 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.026473577000000002 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.037758+0000", 2024-09-21T10:14:44.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103433 2024-09-21T10:14:44.893 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.893 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.893 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.893 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80769 9.b e1250/1249 9:df051d48:::benchmark_data_smithi017_237924_object80768:head v 1250'5133, mlcod=0'0)", 2024-09-21T10:14:44.893 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.829522+0000", 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 279.01603635100003, 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.199615032, 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829522+0000", 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829522+0000", 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829523+0000", 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6099999999999999e-07 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829546+0000", 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3050000000000001e-05 2024-09-21T10:14:44.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829557+0000", 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1068e-05 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829563+0000", 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4910000000000002e-06 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829648+0000", 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4617000000000007e-05 2024-09-21T10:14:44.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.829672+0000", 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3853e-05 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029090+0000", 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19941814999999999 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029137+0000", 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6941999999999999e-05 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80777 9.1 e1250/1249 9:812de87f:::benchmark_data_smithi017_237924_object80776:head v 1250'4961, mlcod=0'0)", 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.833168+0000", 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 279.01239042700001, 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.193889176, 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.898 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.833168+0000", 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.833168+0000", 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.833169+0000", 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.215e-06 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.833193+0000", 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3935e-05 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.012562+0000", 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17936929200000001 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.012572+0000", 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5880000000000004e-06 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.013907+0000", 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013347509999999999 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.013974+0000", 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7248000000000004e-05 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.027004+0000", 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013029924999999999 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.027057+0000", 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3222e-05 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45851.0:80781 9.b e1250/1249 9:d7a6da48:::benchmark_data_smithi017_237924_object80780:head v 1250'5134, mlcod=0'0)", 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:10:05.835569+0000", 2024-09-21T10:14:44.902 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 279.00998898699999, 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19390075200000001, 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.835569+0000", 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.835569+0000", 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.835571+0000", 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3009999999999999e-06 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:05.835599+0000", 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8442999999999999e-05 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.011337+0000", 2024-09-21T10:14:44.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.175737857 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.011351+0000", 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.36e-05 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.014656+0000", 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033057830000000001 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.014681+0000", 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.461e-05 2024-09-21T10:14:44.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029141+0000", 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014459830999999999 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:10:06.029470+0000", 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032932700000000003 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.906 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51618 10.19 10:9ba5b59b:::benchmark_data_smithi017_268344_object51617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.575097+0000", 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 40.270461578999999, 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.12423435, 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51618 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575097+0000", 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575097+0000", 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575099+0000", 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9230000000000001e-06 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575123+0000", 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4786e-05 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575137+0000", 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4002000000000001e-05 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575146+0000", 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8279999999999992e-06 2024-09-21T10:14:44.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.743905+0000", 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16875853499999999 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.743912+0000", 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1520000000000003e-06 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.699201+0000", 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95528904199999998 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.699331+0000", 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013008200000000001 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51620 10.19 10:9a036bcb:::benchmark_data_smithi017_268344_object51619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:44.911 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.575160+0000", 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 40.270398528000001, 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1246337529999999, 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51620 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575160+0000", 2024-09-21T10:14:44.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575160+0000", 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575160+0000", 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4199999999999995e-07 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575184+0000", 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3688000000000001e-05 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575192+0000", 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.174e-06 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575199+0000", 2024-09-21T10:14:44.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5039999999999999e-06 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.743933+0000", 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16873438900000001 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.743939+0000", 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7640000000000002e-06 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.699731+0000", 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.955791909 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.699794+0000", 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2682999999999996e-05 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51622 10.5 10:a753f686:::benchmark_data_smithi017_268344_object51621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.575207+0000", 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 40.270350847000003, 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0984178950000001, 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51622 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575207+0000", 2024-09-21T10:14:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575207+0000", 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575208+0000", 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.37e-07 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.918 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575229+0000", 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0642999999999999e-05 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575239+0000", 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0176e-05 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575244+0000", 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6219999999999997e-06 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.696992+0000", 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.121747442 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.697007+0000", 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4749999999999999e-05 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673550+0000", 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97654350499999998 2024-09-21T10:14:44.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673625+0000", 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5220000000000001e-05 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51623 10.9 10:9231d598:::benchmark_data_smithi017_268344_object51622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:44.922 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.575253+0000", 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 40.270305428, 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.107427027, 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51623 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575253+0000", 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575253+0000", 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575253+0000", 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8800000000000002e-07 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575276+0000", 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2578999999999998e-05 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575283+0000", 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.401e-06 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575290+0000", 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.122e-06 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.743672+0000", 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16838228799999999 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.743681+0000", 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4739999999999994e-06 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.675671+0000", 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.93198983499999999 2024-09-21T10:14:44.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.676076+0000", 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040438200000000001 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.682148+0000", 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060727150000000002 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.682680+0000", 2024-09-21T10:14:44.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00053164300000000005 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51627 10.19 10:99d5db52:::benchmark_data_smithi017_268344_object51626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.575298+0000", 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 40.270260487999998, 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1298630730000001, 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.928 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51627 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575298+0000", 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575298+0000", 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575298+0000", 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9200000000000001e-07 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575343+0000", 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4567999999999997e-05 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575351+0000", 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9289999999999998e-06 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575357+0000", 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1789999999999996e-06 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.743958+0000", 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16860141200000001 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.743964+0000", 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.913e-06 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.699819+0000", 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95585481999999999 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.700614+0000", 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079471200000000004 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.705083+0000", 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044692940000000004 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.705161+0000", 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7754000000000001e-05 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51642 10.5 10:a2b11751:::benchmark_data_smithi017_268344_object51641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.783440+0000", 2024-09-21T10:14:44.933 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 40.062118062000003, 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89913019800000005, 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51642 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.783440+0000", 2024-09-21T10:14:44.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.783440+0000", 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.783442+0000", 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6559999999999999e-06 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.935 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.783478+0000", 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5605000000000002e-05 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.783501+0000", 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3591000000000002e-05 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.783513+0000", 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2045000000000001e-05 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.936 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.783589+0000", 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6276e-05 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.783599+0000", 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7380000000000006e-06 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673649+0000", 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89004942200000003 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673762+0000", 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113156 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673877+0000", 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115326 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.675641+0000", 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017643960000000001 2024-09-21T10:14:44.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.678863+0000", 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003221528 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.682463+0000", 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035997609999999999 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.682532+0000", 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9202999999999995e-05 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.682570+0000", 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8495000000000003e-05 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:93112 10.3 10:c3584504:::benchmark_data_smithi017_268344_object93111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:38.960649+0000", 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 5.8849092450000002, 2024-09-21T10:14:44.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31493255100000001, 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 93112 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.960649+0000", 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.960649+0000", 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.960650+0000", 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0279999999999999e-06 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.960823+0000", 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017325899999999999 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.960843+0000", 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9511e-05 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.960851+0000", 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6620000000000005e-06 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.960917+0000", 2024-09-21T10:14:44.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6509999999999998e-05 2024-09-21T10:14:44.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961074+0000", 2024-09-21T10:14:44.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015703399999999999 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961209+0000", 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013514500000000001 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.963299+0000", 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020900290000000002 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.963802+0000", 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00050299299999999997 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:39.275479+0000", 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.311677229 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:39.275535+0000", 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.592e-05 2024-09-21T10:14:44.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:39.275582+0000", 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6230999999999998e-05 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:93116 10.13 10:c85a59e9:::benchmark_data_smithi017_268344_object93115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:38.961381+0000", 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 5.88417689, 2024-09-21T10:14:44.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31425663999999998, 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 93116 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961381+0000", 2024-09-21T10:14:44.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961381+0000", 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961383+0000", 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1790000000000001e-06 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961438+0000", 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5757000000000002e-05 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961450+0000", 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.128e-05 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961464+0000", 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3944999999999999e-05 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961521+0000", 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7237000000000001e-05 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961715+0000", 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000194169 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.961861+0000", 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014595600000000001 2024-09-21T10:14:44.952 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.963386+0000", 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001525233 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.963890+0000", 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00050339299999999998 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:39.275522+0000", 2024-09-21T10:14:44.953 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31163246900000002 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:39.275576+0000", 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4088999999999997e-05 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:39.275638+0000", 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1933000000000005e-05 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.954 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:93119 10.5 10:a430c909:::benchmark_data_smithi017_268344_object93118:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:38.966946+0000", 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 5.878612135, 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30725648700000002, 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 93119 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:14:44.955 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.966946+0000", 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.966946+0000", 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.966947+0000", 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3289999999999999e-06 2024-09-21T10:14:44.956 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.966995+0000", 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7051000000000001e-05 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.967007+0000", 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2146e-05 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.967012+0000", 2024-09-21T10:14:44.957 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.66e-06 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.967061+0000", 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8282999999999998e-05 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.967162+0000", 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010149000000000001 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:14:44.958 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.967250+0000", 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7595000000000004e-05 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.968799+0000", 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015490689999999999 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:38.968994+0000", 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000195546 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:44.959 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:39.274097+0000", 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.305103134 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:39.274156+0000", 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8581e-05 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:39.274203+0000", 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6603000000000001e-05 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.960 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:14:44.961 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:14:44.961 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:14:44.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:44.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:45.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:45.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:45.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:45.204 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:45.234 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:45.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:45.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:45.373 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:45.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:45.475 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:45.507 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:45.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:45.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 100416 100400 95.0571 77.9375 0.00705404 0.0105156 2024-09-21T10:14:45.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:45.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:45.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:45 smithi017 ceph-mon[102913]: pgmap v2013: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 4020/288327 objects degraded (1.394%); 2387/288327 objects misplaced (0.828%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:14:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:45 smithi017 ceph-mon[102913]: osdmap e1305: 8 total, 8 up, 8 in 2024-09-21T10:14:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:45 smithi017 ceph-mon[105151]: pgmap v2013: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 4020/288327 objects degraded (1.394%); 2387/288327 objects misplaced (0.828%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:14:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:45 smithi017 ceph-mon[105151]: osdmap e1305: 8 total, 8 up, 8 in 2024-09-21T10:14:45.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:45.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:45.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:45.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:45.935 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:45.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:45 smithi134 ceph-mon[90085]: pgmap v2013: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 4020/288327 objects degraded (1.394%); 2387/288327 objects misplaced (0.828%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:14:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:45 smithi134 ceph-mon[90085]: osdmap e1305: 8 total, 8 up, 8 in 2024-09-21T10:14:46.036 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:46.072 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:46.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:46.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:46.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:46.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:46.315 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:46.354 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:46.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:46.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:46.497 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:46.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 101782 101766 94.9125 85.375 0.0147746 0.0105312 2024-09-21T10:14:46.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:46.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:14:46.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.657 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:46.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:46.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:46.785 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:46.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:46.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:46.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:46.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:14:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:14:46.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:14:47.022 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:47.046 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:47.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:47.148 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:47.169 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:47.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:47.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:47.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:47.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:47.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:47.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:47.524 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:47.545 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:47.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:47.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 102304 102288 93.9966 32.625 0.0108408 0.0105284 2024-09-21T10:14:47.647 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:47.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:47.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:47.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:47.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:47.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:47.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:47.919 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:47.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:47.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:47 smithi017 ceph-mon[102913]: pgmap v2015: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 6.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 5748/299166 objects misplaced (1.921%); 7.6 MiB/s, 122 objects/s recovering 2024-09-21T10:14:47.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:47 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4020/288327 objects degraded (1.394%), 1 pg degraded) 2024-09-21T10:14:47.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:14:47.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:47 smithi017 ceph-mon[102913]: osdmap e1306: 8 total, 8 up, 8 in 2024-09-21T10:14:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:47 smithi017 ceph-mon[105151]: pgmap v2015: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 6.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 5748/299166 objects misplaced (1.921%); 7.6 MiB/s, 122 objects/s recovering 2024-09-21T10:14:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:47 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4020/288327 objects degraded (1.394%), 1 pg degraded) 2024-09-21T10:14:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:14:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:47 smithi017 ceph-mon[105151]: osdmap e1306: 8 total, 8 up, 8 in 2024-09-21T10:14:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:47 smithi134 ceph-mon[90085]: pgmap v2015: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 6.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 5748/299166 objects misplaced (1.921%); 7.6 MiB/s, 122 objects/s recovering 2024-09-21T10:14:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:47 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4020/288327 objects degraded (1.394%), 1 pg degraded) 2024-09-21T10:14:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:14:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:47 smithi134 ceph-mon[90085]: osdmap e1306: 8 total, 8 up, 8 in 2024-09-21T10:14:48.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:48.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:48.146 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:48.167 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:48.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:48.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:48.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:48.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:48.394 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:48.415 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:48.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:48.516 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:48.538 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:48.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:48.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 102304 102288 92.6345 0 - 0.0105284 2024-09-21T10:14:48.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:48.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:48.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:48.793 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:48.895 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:48.917 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:48.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:48.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:48 smithi017 ceph-mon[102913]: osdmap e1307: 8 total, 8 up, 8 in 2024-09-21T10:14:48.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:48 smithi017 ceph-mon[105151]: osdmap e1307: 8 total, 8 up, 8 in 2024-09-21T10:14:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:48 smithi134 ceph-mon[90085]: osdmap e1307: 8 total, 8 up, 8 in 2024-09-21T10:14:49.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:49.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:49.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:49.143 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:49.165 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:49.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:49.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:49.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:49.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:49.442 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:49.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:49.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:49.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 102687 102671 91.6532 11.9688 0.00336867 0.0109052 2024-09-21T10:14:49.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:49.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:49.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:49.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:49.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:49.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:49.848 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:49.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:49.950 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:49 smithi017 ceph-mon[102913]: pgmap v2018: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 117 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 5748/301881 objects misplaced (1.904%); 6.0 MiB/s, 96 objects/s recovering 2024-09-21T10:14:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:49 smithi017 ceph-mon[102913]: osdmap e1308: 8 total, 8 up, 8 in 2024-09-21T10:14:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:49 smithi017 ceph-mon[105151]: pgmap v2018: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 117 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 5748/301881 objects misplaced (1.904%); 6.0 MiB/s, 96 objects/s recovering 2024-09-21T10:14:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:49 smithi017 ceph-mon[105151]: osdmap e1308: 8 total, 8 up, 8 in 2024-09-21T10:14:49.986 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:49.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:49 smithi134 ceph-mon[90085]: pgmap v2018: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 117 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 5748/301881 objects misplaced (1.904%); 6.0 MiB/s, 96 objects/s recovering 2024-09-21T10:14:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:49 smithi134 ceph-mon[90085]: osdmap e1308: 8 total, 8 up, 8 in 2024-09-21T10:14:50.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:50.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:50.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:50.269 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:50.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:50.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:50.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:50.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:14:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:14:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:14:50.512 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:50.542 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:50.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:50.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 104007 103991 91.5242 82.5 0.00463842 0.0109212 2024-09-21T10:14:50.644 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:50.686 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:50.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:50.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:50.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:50.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:50.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:50.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:51.070 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:51.104 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:51.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:51.206 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:51.226 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:51.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:51.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:51.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:51.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:51.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:51.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:51.565 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 105183 105167 91.2737 73.5 0.0202083 0.0109512 2024-09-21T10:14:51.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:51.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:51.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:51.771 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:51.832 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:51.838 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:51.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:51.885 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:51 smithi017 ceph-mon[102913]: pgmap v2020: 119 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 117 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s; 4776/305847 objects misplaced (1.562%); 8.5 MiB/s, 135 objects/s recovering 2024-09-21T10:14:51.885 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:51 smithi017 ceph-mon[105151]: pgmap v2020: 119 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 117 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s; 4776/305847 objects misplaced (1.562%); 8.5 MiB/s, 135 objects/s recovering 2024-09-21T10:14:51.919 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:51.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:51.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:51 smithi134 ceph-mon[90085]: pgmap v2020: 119 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 117 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s; 4776/305847 objects misplaced (1.562%); 8.5 MiB/s, 135 objects/s recovering 2024-09-21T10:14:52.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:52.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:52.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:52.216 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:52.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:52.318 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:52.347 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:52.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:52.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:52.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:52.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 106356 106340 91.0276 73.3125 0.0156521 0.0109808 2024-09-21T10:14:52.590 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:52.619 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:52.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:52.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.771 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:52.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:52.872 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:52.906 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:52.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:53.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:53.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:53.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:53.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:53.180 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:53.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:53.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:53.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:53.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:53.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:53.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:53.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 107534 107518 90.7924 73.625 0.0104746 0.0110095 2024-09-21T10:14:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:53.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:53.684 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:53.715 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:53.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:53.817 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:53.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:53.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:53.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:14:53.910 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:53 smithi017 ceph-mon[105151]: pgmap v2021: 119 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 117 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s; 4776/310458 objects misplaced (1.538%); 3.4 MiB/s, 53 objects/s recovering 2024-09-21T10:14:53.910 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:53 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2702197533' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.6210299007075906} : dispatch 2024-09-21T10:14:53.910 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:53 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.6210299007075906} : dispatch 2024-09-21T10:14:53.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:53 smithi017 ceph-mon[102913]: pgmap v2021: 119 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 117 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s; 4776/310458 objects misplaced (1.538%); 3.4 MiB/s, 53 objects/s recovering 2024-09-21T10:14:53.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:53 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2702197533' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.6210299007075906} : dispatch 2024-09-21T10:14:53.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:53 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.6210299007075906} : dispatch 2024-09-21T10:14:53.939 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:53.960 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:53.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:53 smithi134 ceph-mon[90085]: pgmap v2021: 119 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 117 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s; 4776/310458 objects misplaced (1.538%); 3.4 MiB/s, 53 objects/s recovering 2024-09-21T10:14:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:53 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2702197533' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.6210299007075906} : dispatch 2024-09-21T10:14:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:53 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.6210299007075906} : dispatch 2024-09-21T10:14:54.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:54.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:54.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:54.187 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:54.208 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:54.310 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:54.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:54.433 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:54.454 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:54.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:54.556 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:54.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 107774 107758 89.7819 15 0.00290136 0.0110092 2024-09-21T10:14:54.577 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:54.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:54.679 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:54.709 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:54.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:54.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:54.942 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.6 primary-affinity to 0.62103 (8406992) 2024-09-21T10:14:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:54 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.6210299007075906}]': finished 2024-09-21T10:14:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:54 smithi017 ceph-mon[102913]: osdmap e1309: 8 total, 8 up, 8 in 2024-09-21T10:14:54.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:54 smithi017 ceph-mon[102913]: osdmap e1310: 8 total, 8 up, 8 in 2024-09-21T10:14:54.944 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:14:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:14:54] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-09-21T10:14:54.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:54 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.6210299007075906}]': finished 2024-09-21T10:14:54.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:54 smithi017 ceph-mon[105151]: osdmap e1309: 8 total, 8 up, 8 in 2024-09-21T10:14:54.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:54 smithi017 ceph-mon[105151]: osdmap e1310: 8 total, 8 up, 8 in 2024-09-21T10:14:54.953 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:54.980 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:54 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.6210299007075906}]': finished 2024-09-21T10:14:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:54 smithi134 ceph-mon[90085]: osdmap e1309: 8 total, 8 up, 8 in 2024-09-21T10:14:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:54 smithi134 ceph-mon[90085]: osdmap e1310: 8 total, 8 up, 8 in 2024-09-21T10:14:55.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:55.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:55.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:55.223 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:55.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.258 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:55.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:55.359 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:55.390 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:55.492 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:55.524 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:55.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:55.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 108934 108918 89.5544 72.5 0.00669934 0.0111615 2024-09-21T10:14:55.626 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:55.656 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:55.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:55.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:55.790 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:55.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:55 smithi017 ceph-mon[102913]: pgmap v2023: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 6624/315474 objects degraded (2.100%); 4776/315474 objects misplaced (1.514%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:14:55.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:55 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 6624/315474 objects degraded (2.100%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:55.849 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:55.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:55.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:55 smithi017 ceph-mon[105151]: pgmap v2023: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 6624/315474 objects degraded (2.100%); 4776/315474 objects misplaced (1.514%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:14:55.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:55 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 6624/315474 objects degraded (2.100%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:55.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:55.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:55.892 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:55.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:55.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:55 smithi134 ceph-mon[90085]: pgmap v2023: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 6624/315474 objects degraded (2.100%); 4776/315474 objects misplaced (1.514%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:14:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:55 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 6624/315474 objects degraded (2.100%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:14:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:14:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:56.023 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:56.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:56.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:56.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:56.184 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:56.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:56.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:56.317 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:56.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:56.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:56.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:56.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:56.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:56.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 110221 110205 89.4359 80.4375 0.00355572 0.0111754 2024-09-21T10:14:56.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:56.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:56.685 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:14:56.685 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:56.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:56.725 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:14:56.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:56.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:56.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:56.761 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:56 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1711014507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:56.761 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:56 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1711014507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:56.761 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:14:56.762 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:56.797 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:14:56.797 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:56.829 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:14:56.829 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:56.857 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:56.861 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:14:56.861 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:56.886 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:56.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:56.894 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:14:56.895 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:56.928 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:14:56.929 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:56.987 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:56.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:56 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1711014507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:57.021 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:57.123 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:57.149 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:57.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:57.305 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:57.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:57.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:57.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:57.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:57.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:57.566 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 111434 111418 89.2607 75.8125 0.0175571 0.0111981 2024-09-21T10:14:57.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:57.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:57.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:57.707 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:57.743 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:57.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:57.845 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:57.880 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:57.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:57.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:57 smithi017 ceph-mon[102913]: pgmap v2025: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.3 KiB/s rd, 56 MiB/s wr, 903 op/s; 6624/323112 objects degraded (2.050%); 4065/323112 objects misplaced (1.258%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:14:57.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:57 smithi017 ceph-mon[105151]: pgmap v2025: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.3 KiB/s rd, 56 MiB/s wr, 903 op/s; 6624/323112 objects degraded (2.050%); 4065/323112 objects misplaced (1.258%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:14:57.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:57 smithi134 ceph-mon[90085]: pgmap v2025: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 4.3 KiB/s rd, 56 MiB/s wr, 903 op/s; 6624/323112 objects degraded (2.050%); 4065/323112 objects misplaced (1.258%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:14:58.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:58.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:58.138 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:58.173 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:58.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:58.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:58.319 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:58.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:58.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:58.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:58.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 112688 112672 89.1228 78.375 0.00586777 0.0112154 2024-09-21T10:14:58.569 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:58.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:58.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:58.709 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:58.742 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:58.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:58.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:58.882 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:58.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:58.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:58.991 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:58.991 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:59.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:59.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:59.125 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:59.157 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:59.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:59.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:59.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:59.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:59.398 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:59.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.431 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:59.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:59.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:59.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:14:59.565778+0000 min lat: 0.00108589 max lat: 2.55598 avg lat: 0.0112285 2024-09-21T10:14:59.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:14:59.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 113981 113965 89.0189 80.8125 0.0178968 0.0112285 2024-09-21T10:14:59.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:59.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:59.688 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:14:59 smithi017 ceph-mon[102913]: pgmap v2026: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 6.7 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 8.5 KiB/s rd, 58 MiB/s wr, 928 op/s; 6624/328017 objects degraded (2.019%); 4065/328017 objects misplaced (1.239%); 1.9 MiB/s, 29 objects/s recovering 2024-09-21T10:14:59.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:14:59.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:59.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:59.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:59.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.864 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:14:59.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:59.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:14:59 smithi017 ceph-mon[105151]: pgmap v2026: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 6.7 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 8.5 KiB/s rd, 58 MiB/s wr, 928 op/s; 6624/328017 objects degraded (2.019%); 4065/328017 objects misplaced (1.239%); 1.9 MiB/s, 29 objects/s recovering 2024-09-21T10:14:59.965 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:59.990 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:14:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:14:59 smithi134 ceph-mon[90085]: pgmap v2026: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 117 active+clean; 6.7 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 8.5 KiB/s rd, 58 MiB/s wr, 928 op/s; 6624/328017 objects degraded (2.019%); 4065/328017 objects misplaced (1.239%); 1.9 MiB/s, 29 objects/s recovering 2024-09-21T10:15:00.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:00.119 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:00.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:00.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:00.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:00.364 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:00.395 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:00.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:15:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:15:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:15:00.497 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:00.524 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:00.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:00.567 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 115188 115172 88.8511 75.4375 0.0182943 0.0112497 2024-09-21T10:15:00.619 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:15:00.619 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:15:00.619 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2024-09-21T10:15:00.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:15:00.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:00.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:00.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:15:00.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:00.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:00.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:00.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:00 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6624/336399 objects degraded (1.969%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:00.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:00 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6624/336399 objects degraded (1.969%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:00.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:00.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:00 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6624/336399 objects degraded (1.969%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:01.026 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:01.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:01.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:01.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:01.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:01.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:01.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:01.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.472 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:01.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:01.499 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:01.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:01.574 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 116247 116231 88.5711 66.1875 0.0161519 0.0112856 2024-09-21T10:15:01.597 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:01.605 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:01.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:01.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:01.764 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:01.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:01.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:01.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:01.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:01 smithi017 ceph-mon[102913]: pgmap v2027: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 8.5 KiB/s rd, 68 MiB/s wr, 1.08k op/s; 6624/336399 objects degraded (1.969%); 3525/336399 objects misplaced (1.048%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:15:01.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:01 smithi017 ceph-mon[105151]: pgmap v2027: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 8.5 KiB/s rd, 68 MiB/s wr, 1.08k op/s; 6624/336399 objects degraded (1.969%); 3525/336399 objects misplaced (1.048%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:15:01.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:01.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:01 smithi134 ceph-mon[90085]: pgmap v2027: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 8.5 KiB/s rd, 68 MiB/s wr, 1.08k op/s; 6624/336399 objects degraded (1.969%); 3525/336399 objects misplaced (1.048%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:15:02.064 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:02.098 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:02.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:02.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:02.236 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:02.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:02.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:02.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:02.485 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:02.515 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:02.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 15 117387 117372 88.3631 71.3125 0.00831685 0.0113122 2024-09-21T10:15:02.617 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:02.645 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:02.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:02.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:02.768 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:15:02.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:02.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:02.929 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:02.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:03.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:03.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:03.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:03.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:02 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1734881301' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:03.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:02 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:02 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1734881301' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:03.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:02 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:03.207 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:03.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:02 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1734881301' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:03.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:02 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:03.308 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:03.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:03.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:03.443 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:03.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:03.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 118679 118663 88.2716 80.6875 0.00521353 0.0113239 2024-09-21T10:15:03.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:03.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:03.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:03.721 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:03.752 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:03.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:03.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:03.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:03.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:03.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:04.010 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:03 smithi017 ceph-mon[102913]: pgmap v2028: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 7.7 KiB/s rd, 61 MiB/s wr, 987 op/s; 6624/341553 objects degraded (1.939%); 3525/341553 objects misplaced (1.032%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:15:04.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:03 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:04.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:03 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:04.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:03 smithi017 ceph-mon[102913]: osdmap e1311: 8 total, 8 up, 8 in 2024-09-21T10:15:04.011 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:03 smithi017 ceph-mon[105151]: pgmap v2028: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 7.7 KiB/s rd, 61 MiB/s wr, 987 op/s; 6624/341553 objects degraded (1.939%); 3525/341553 objects misplaced (1.032%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:15:04.011 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:03 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:04.011 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:03 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:04.011 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:03 smithi017 ceph-mon[105151]: osdmap e1311: 8 total, 8 up, 8 in 2024-09-21T10:15:04.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:04.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:04.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:04.178 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:04.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:04.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:03 smithi134 ceph-mon[90085]: pgmap v2028: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 7.7 KiB/s rd, 61 MiB/s wr, 987 op/s; 6624/341553 objects degraded (1.939%); 3525/341553 objects misplaced (1.032%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:15:04.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:03 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:04.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:03 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:04.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:03 smithi134 ceph-mon[90085]: osdmap e1311: 8 total, 8 up, 8 in 2024-09-21T10:15:04.280 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:04.314 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:04.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:04.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:04.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:04.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:04.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:04.570 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 119976 119960 88.1868 81.0625 0.00742904 0.0113342 2024-09-21T10:15:04.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:04.704 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:04.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:04.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:04.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:04.865 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:15:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:15:04] "GET /metrics HTTP/1.1" 200 36653 "" "Prometheus/2.51.0" 2024-09-21T10:15:04.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:04.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:04.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:05.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:05.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:05.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:05.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:05.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:05.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:05.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:05.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:05.425 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:05.465 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:05.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:05.567 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:05.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 121210 121194 88.0581 77.125 0.0099273 0.0113514 2024-09-21T10:15:05.597 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:05.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:05.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:05.741 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:05.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:15:05.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:05.863 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:05 smithi017 ceph-mon[102913]: pgmap v2030: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 6.9 KiB/s rd, 78 MiB/s wr, 1.25k op/s; 6624/352329 objects degraded (1.880%); 3216/352329 objects misplaced (0.913%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:15:05.863 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:05 smithi017 ceph-mon[105151]: pgmap v2030: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 6.9 KiB/s rd, 78 MiB/s wr, 1.25k op/s; 6624/352329 objects degraded (1.880%); 3216/352329 objects misplaced (0.913%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:15:05.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.919 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:05.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:06.020 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:06.045 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:05 smithi134 ceph-mon[90085]: pgmap v2030: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 6.9 KiB/s rd, 78 MiB/s wr, 1.25k op/s; 6624/352329 objects degraded (1.880%); 3216/352329 objects misplaced (0.913%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:15:06.050 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:06.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:06.152 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:06.185 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:06.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:06.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:06.320 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:06.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:06.352 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:15:06.352 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:06.395 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:15:06.395 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:06.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:06.438 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:15:06.438 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:06.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.474 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:06.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:06.486 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:15:06.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:15:06.531 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:15:06.531 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:06.566 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:15:06.566 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:06.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 122298 122282 87.8275 68 0.00732044 0.0113809 2024-09-21T10:15:06.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:06.609 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:15:06.609 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:06.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:06.646 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:15:06.647 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:06.720 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:06.757 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:06.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:06.859 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:06.894 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:06.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:06.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:06 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2895947036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:06.995 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:07.043 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:07.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:07.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:06 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2895947036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:07.071 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:06 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2895947036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:07.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:07.184 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:07.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:07.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.325 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:07.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:07.426 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:07.460 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:07.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:07.550 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:15:07.551 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:15:07.551 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:15:07.551 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:15:07.551 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:15:07.563 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:07.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 123515 123499 87.6937 76.0625 0.0151841 0.0113984 2024-09-21T10:15:07.600 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:07.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:07.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:07.736 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:07.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:07.838 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:07.871 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:07.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:08.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:08.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:08.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:08.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:07 smithi017 ceph-mon[102913]: pgmap v2031: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 3.4 KiB/s rd, 72 MiB/s wr, 1.15k op/s; 6624/357468 objects degraded (1.853%); 3216/357468 objects misplaced (0.900%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:15:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:07 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6624/357468 objects degraded (1.853%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:07 smithi017 ceph-mon[105151]: pgmap v2031: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 3.4 KiB/s rd, 72 MiB/s wr, 1.15k op/s; 6624/357468 objects degraded (1.853%); 3216/357468 objects misplaced (0.900%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:15:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:07 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6624/357468 objects degraded (1.853%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:07 smithi134 ceph-mon[90085]: pgmap v2031: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 3.4 KiB/s rd, 72 MiB/s wr, 1.15k op/s; 6624/357468 objects degraded (1.853%); 3216/357468 objects misplaced (0.900%); 1.8 MiB/s, 28 objects/s recovering 2024-09-21T10:15:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:07 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6624/357468 objects degraded (1.853%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:08.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:08.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:08.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:08.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:08.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:08.543 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:08.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 124771 124755 87.5903 78.5 0.0214187 0.0114118 2024-09-21T10:15:08.577 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:08.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:08.679 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:08.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:08.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:08.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:08.839 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:08.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:08.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:15:08.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.998 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:08.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:09.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:09.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:09.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:09.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:09.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:09.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:09.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:09.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:09.547 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:09.571 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:09.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0049 2024-09-21T10:15:09.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 125928 2024-09-21T10:15:09.571 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 87.4453 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 40.98 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 212.938 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1399 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 655.701 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3407 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0114315 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0490756 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 2.55598 2024-09-21T10:15:09.572 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00108589 2024-09-21T10:15:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:09.576 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:15:09.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:09.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:09.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:09.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:09.841 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:09.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:09.942 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:09.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.983 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:09.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:10.084 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:10.117 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:10.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:09 smithi017 ceph-mon[102913]: pgmap v2032: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 6624/367098 objects degraded (1.804%); 2595/367098 objects misplaced (0.707%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:15:10.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:09 smithi017 ceph-mon[105151]: pgmap v2032: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 6624/367098 objects degraded (1.804%); 2595/367098 objects misplaced (0.707%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:15:10.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:09 smithi134 ceph-mon[90085]: pgmap v2032: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 6624/367098 objects degraded (1.804%); 2595/367098 objects misplaced (0.707%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:15:10.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:10.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:10.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:10.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:10.401 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:10.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:15:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:15:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:15:10.503 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:10.532 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:10.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:10.623 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:10.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:10.666 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:10.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:10.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:10.785 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:15:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:10.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:10.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:10.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:10.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:11.042 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:11.076 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:11.126 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:11.127 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:15:11.127 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:11.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:11.205 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:15:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:11.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:11.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:11.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:11.461 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:11.490 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:11.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:11.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:11.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:11.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:11.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:11.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:11.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:11.921 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:11.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:11.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:11 smithi017 ceph-mon[102913]: pgmap v2033: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 6624/370674 objects degraded (1.787%); 2595/370674 objects misplaced (0.700%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:15:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:11 smithi017 ceph-mon[105151]: pgmap v2033: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 6624/370674 objects degraded (1.787%); 2595/370674 objects misplaced (0.700%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:15:12.023 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:12.052 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:12.069 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:15:12.071 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:15:12.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:12.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:12.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:11 smithi134 ceph-mon[90085]: pgmap v2033: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 6624/370674 objects degraded (1.787%); 2595/370674 objects misplaced (0.700%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:15:12.292 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:12.321 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:12.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:12.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:12.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:12.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:15:12.572 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:12.599 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:12.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:12.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:12.849 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:12.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.879 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:12.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:12.945 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:12.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:13.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:13.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:13.107 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:12 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6624/373194 objects degraded (1.775%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:13.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:12 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6624/373194 objects degraded (1.775%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:13.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:13.182 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:13.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:13.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:12 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6624/373194 objects degraded (1.775%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:13.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:13.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.322 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:13.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:13.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:13.461 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:13.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:13.563 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:13.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.594 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:13.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:13.695 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:13.729 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:13.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:13.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:13.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:13.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:13.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:14.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:14.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:14.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:13 smithi017 ceph-mon[102913]: from='client.36297 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:14.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:13 smithi017 ceph-mon[102913]: pgmap v2034: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.6 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 68 MiB/s wr, 1.14k op/s; 6624/373194 objects degraded (1.775%); 2595/373194 objects misplaced (0.695%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:15:14.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:13 smithi017 ceph-mon[105151]: from='client.36297 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:14.102 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:13 smithi017 ceph-mon[105151]: pgmap v2034: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.6 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 68 MiB/s wr, 1.14k op/s; 6624/373194 objects degraded (1.775%); 2595/373194 objects misplaced (0.695%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:15:14.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:14.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:14.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:14.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:13 smithi134 ceph-mon[90085]: from='client.36297 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:14.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:13 smithi134 ceph-mon[90085]: pgmap v2034: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.6 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 68 MiB/s wr, 1.14k op/s; 6624/373194 objects degraded (1.775%); 2595/373194 objects misplaced (0.695%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:15:14.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:14.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:14.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:14.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:14.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.426 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:14.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:14.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:14.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:14.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:14.663 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:14.692 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:14.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:14.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:14.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:14.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:14.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:14.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:14.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:15.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:15.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:15.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:15.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:15:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:15:14] "GET /metrics HTTP/1.1" 200 36664 "" "Prometheus/2.51.0" 2024-09-21T10:15:15.207 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:15.239 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:15.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:15.341 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:15.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.365 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:15.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:15.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:15.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:15.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:15.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:15.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:15.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:15.731 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:15.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:15.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:15.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:15.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:15.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:15.942 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":2033,"stamp":"2024-09-21T10:15:10.492976+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8096475507,"num_objects":123558,"num_object_clones":0,"num_object_copies":370674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6624,"num_objects_misplaced":2595,"num_objects_unfound":0,"num_objects_dirty":123558,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":111675,"num_write_kb":7142547,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4989,"num_bytes_recovered":323499128,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":144535,"ondisk_log_size":144535,"up":357,"acting":355,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":370,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":30434520,"kb_used_data":24203352,"kb_used_omap":411,"kb_used_meta":6230692,"kb_avail":719526696,"statfs":{"total":767960285184,"available":736795336704,"internally_reserved":0,"allocated":24784232448,"data_stored":24758857375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":420990,"internal_metadata":6380229506},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":748748800,"num_objects":11425,"num_object_clones":0,"num_object_copies":34275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-930,"num_objects_unfound":0,"num_objects_dirty":11425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11425,"num_write_kb":731200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":310,"num_bytes_recovered":20316160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005754"},"pg_stats":[{"pgid":"1.11","version":"288'100","reported_seq":784,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.768014+0000","last_change":"2024-09-21T10:12:50.589581+0000","last_active":"2024-09-21T10:15:02.768014+0000","last_peered":"2024-09-21T10:15:02.768014+0000","last_clean":"2024-09-21T10:15:02.768014+0000","last_became_active":"2024-09-21T10:12:50.589222+0000","last_became_peered":"2024-09-21T10:12:50.589222+0000","last_unstale":"2024-09-21T10:15:02.768014+0000","last_undegraded":"2024-09-21T10:15:02.768014+0000","last_fullsized":"2024-09-21T10:15:02.768014+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.888350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.888350+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:37:10.080751+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1992,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.764289+0000","last_change":"2024-09-21T10:12:25.715091+0000","last_active":"2024-09-21T10:15:02.764289+0000","last_peered":"2024-09-21T10:15:02.764289+0000","last_clean":"2024-09-21T10:15:02.764289+0000","last_became_active":"2024-09-21T10:12:25.711946+0000","last_became_peered":"2024-09-21T10:12:25.711946+0000","last_unstale":"2024-09-21T10:15:02.764289+0000","last_undegraded":"2024-09-21T10:15:02.764289+0000","last_fullsized":"2024-09-21T10:15:02.764289+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:08:07.743522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":59,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.650180+0000","last_change":"2024-09-21T10:12:25.720808+0000","last_active":"2024-09-21T10:14:54.650180+0000","last_peered":"2024-09-21T10:14:54.650180+0000","last_clean":"2024-09-21T10:14:54.650180+0000","last_became_active":"2024-09-21T10:12:25.706699+0000","last_became_peered":"2024-09-21T10:12:25.706699+0000","last_unstale":"2024-09-21T10:14:54.650180+0000","last_undegraded":"2024-09-21T10:14:54.650180+0000","last_fullsized":"2024-09-21T10:14:54.650180+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:29.044701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"288'100","reported_seq":73,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.649230+0000","last_change":"2024-09-21T10:13:22.620513+0000","last_active":"2024-09-21T10:14:54.649230+0000","last_peered":"2024-09-21T10:14:54.649230+0000","last_clean":"2024-09-21T10:14:54.649230+0000","last_became_active":"2024-09-21T10:12:50.581542+0000","last_became_peered":"2024-09-21T10:12:50.581542+0000","last_unstale":"2024-09-21T10:14:54.649230+0000","last_undegraded":"2024-09-21T10:14:54.649230+0000","last_fullsized":"2024-09-21T10:14:54.649230+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:22.620446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:22.620446+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:13.111755+0000","scrub_duration":0.0014078199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":63,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.650792+0000","last_change":"2024-09-21T10:12:50.584513+0000","last_active":"2024-09-21T10:14:54.650792+0000","last_peered":"2024-09-21T10:14:54.650792+0000","last_clean":"2024-09-21T10:14:54.650792+0000","last_became_active":"2024-09-21T10:12:50.584107+0000","last_became_peered":"2024-09-21T10:12:50.584107+0000","last_unstale":"2024-09-21T10:14:54.650792+0000","last_undegraded":"2024-09-21T10:14:54.650792+0000","last_fullsized":"2024-09-21T10:14:54.650792+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:23.047403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2061,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.765775+0000","last_change":"2024-09-21T10:12:25.700249+0000","last_active":"2024-09-21T10:15:02.765775+0000","last_peered":"2024-09-21T10:15:02.765775+0000","last_clean":"2024-09-21T10:15:02.765775+0000","last_became_active":"2024-09-21T10:12:25.699912+0000","last_became_peered":"2024-09-21T10:12:25.699912+0000","last_unstale":"2024-09-21T10:15:02.765775+0000","last_undegraded":"2024-09-21T10:15:02.765775+0000","last_fullsized":"2024-09-21T10:15:02.765775+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:47:59.054215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.18","version":"1311'5455","reported_seq":5562,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.001650+0000","last_change":"2024-09-21T10:14:13.444475+0000","last_active":"2024-09-21T10:15:08.001650+0000","last_peered":"2024-09-21T10:15:08.001650+0000","last_clean":"2024-09-21T10:15:08.001650+0000","last_became_active":"2024-09-21T10:14:05.668168+0000","last_became_peered":"2024-09-21T10:14:05.668168+0000","last_unstale":"2024-09-21T10:15:08.001650+0000","last_undegraded":"2024-09-21T10:15:08.001650+0000","last_fullsized":"2024-09-21T10:15:08.001650+0000","mapping_epoch":1292,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"1296'3597","last_scrub_stamp":"2024-09-21T10:14:13.444340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:13.444340+0000","objects_scrubbed":1948,"log_size":5455,"log_dups_size":0,"ondisk_log_size":5455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:08:41.128004+0000","scrub_duration":424,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249692160,"num_objects":3810,"num_object_clones":0,"num_object_copies":11430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1862,"num_write_kb":119168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"288'100","reported_seq":170,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.772658+0000","last_change":"2024-09-21T10:13:16.405658+0000","last_active":"2024-09-21T10:15:02.772658+0000","last_peered":"2024-09-21T10:15:02.772658+0000","last_clean":"2024-09-21T10:15:02.772658+0000","last_became_active":"2024-09-21T10:08:45.521658+0000","last_became_peered":"2024-09-21T10:08:45.521658+0000","last_unstale":"2024-09-21T10:15:02.772658+0000","last_undegraded":"2024-09-21T10:15:02.772658+0000","last_fullsized":"2024-09-21T10:15:02.772658+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:16.405582+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:16.405582+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:32.832948+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2063,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.763663+0000","last_change":"2024-09-21T10:12:25.703395+0000","last_active":"2024-09-21T10:15:02.763663+0000","last_peered":"2024-09-21T10:15:02.763663+0000","last_clean":"2024-09-21T10:15:02.763663+0000","last_became_active":"2024-09-21T10:12:25.701460+0000","last_became_peered":"2024-09-21T10:12:25.701460+0000","last_unstale":"2024-09-21T10:15:02.763663+0000","last_undegraded":"2024-09-21T10:15:02.763663+0000","last_fullsized":"2024-09-21T10:15:02.763663+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:27:12.806968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.649468+0000","last_change":"2024-09-21T10:12:25.700080+0000","last_active":"2024-09-21T10:14:54.649468+0000","last_peered":"2024-09-21T10:14:54.649468+0000","last_clean":"2024-09-21T10:14:54.649468+0000","last_became_active":"2024-09-21T10:12:25.699762+0000","last_became_peered":"2024-09-21T10:12:25.699762+0000","last_unstale":"2024-09-21T10:14:54.649468+0000","last_undegraded":"2024-09-21T10:14:54.649468+0000","last_fullsized":"2024-09-21T10:14:54.649468+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:41.509175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.19","version":"1311'5436","reported_seq":5519,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:07.862371+0000","last_change":"2024-09-21T10:14:17.575627+0000","last_active":"2024-09-21T10:15:07.862371+0000","last_peered":"2024-09-21T10:15:07.862371+0000","last_clean":"2024-09-21T10:15:07.862371+0000","last_became_active":"2024-09-21T10:14:05.697205+0000","last_became_peered":"2024-09-21T10:14:05.697205+0000","last_unstale":"2024-09-21T10:15:07.862371+0000","last_undegraded":"2024-09-21T10:15:07.862371+0000","last_fullsized":"2024-09-21T10:15:07.862371+0000","mapping_epoch":1292,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5436,"log_dups_size":0,"ondisk_log_size":5436,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249888768,"num_objects":3813,"num_object_clones":0,"num_object_copies":11439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3813,"num_write_kb":244032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"288'100","reported_seq":191,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.765999+0000","last_change":"2024-09-21T10:13:28.346056+0000","last_active":"2024-09-21T10:15:02.765999+0000","last_peered":"2024-09-21T10:15:02.765999+0000","last_clean":"2024-09-21T10:15:02.765999+0000","last_became_active":"2024-09-21T10:12:50.582710+0000","last_became_peered":"2024-09-21T10:12:50.582710+0000","last_unstale":"2024-09-21T10:15:02.765999+0000","last_undegraded":"2024-09-21T10:15:02.765999+0000","last_fullsized":"2024-09-21T10:15:02.765999+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:28.346003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:28.346003+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.707712+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1991,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.767961+0000","last_change":"2024-09-21T10:12:25.707969+0000","last_active":"2024-09-21T10:15:02.767961+0000","last_peered":"2024-09-21T10:15:02.767961+0000","last_clean":"2024-09-21T10:15:02.767961+0000","last_became_active":"2024-09-21T10:12:25.707531+0000","last_became_peered":"2024-09-21T10:12:25.707531+0000","last_unstale":"2024-09-21T10:15:02.767961+0000","last_undegraded":"2024-09-21T10:15:02.767961+0000","last_fullsized":"2024-09-21T10:15:02.767961+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:08:47.289964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":98,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.646132+0000","last_change":"2024-09-21T10:12:50.620444+0000","last_active":"2024-09-21T10:14:54.646132+0000","last_peered":"2024-09-21T10:14:54.646132+0000","last_clean":"2024-09-21T10:14:54.646132+0000","last_became_active":"2024-09-21T10:12:50.589267+0000","last_became_peered":"2024-09-21T10:12:50.589267+0000","last_unstale":"2024-09-21T10:14:54.646132+0000","last_undegraded":"2024-09-21T10:14:54.646132+0000","last_fullsized":"2024-09-21T10:14:54.646132+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:42:11.055834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"1311'5486","reported_seq":5566,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.046324+0000","last_change":"2024-09-21T10:14:05.660673+0000","last_active":"2024-09-21T10:15:08.046324+0000","last_peered":"2024-09-21T10:15:08.046324+0000","last_clean":"2024-09-21T10:15:08.046324+0000","last_became_active":"2024-09-21T10:14:05.658576+0000","last_became_peered":"2024-09-21T10:14:05.658576+0000","last_unstale":"2024-09-21T10:15:08.046324+0000","last_undegraded":"2024-09-21T10:15:08.046324+0000","last_fullsized":"2024-09-21T10:15:08.046324+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252510208,"num_objects":3853,"num_object_clones":0,"num_object_copies":11559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3853,"num_write_kb":246592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"288'100","reported_seq":63,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.650736+0000","last_change":"2024-09-21T10:13:16.697889+0000","last_active":"2024-09-21T10:14:54.650736+0000","last_peered":"2024-09-21T10:14:54.650736+0000","last_clean":"2024-09-21T10:14:54.650736+0000","last_became_active":"2024-09-21T10:12:25.699431+0000","last_became_peered":"2024-09-21T10:12:25.699431+0000","last_unstale":"2024-09-21T10:14:54.650736+0000","last_undegraded":"2024-09-21T10:14:54.650736+0000","last_fullsized":"2024-09-21T10:14:54.650736+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:16.697812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:16.697812+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:18.395903+0000","scrub_duration":0.001732562,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":57,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.651099+0000","last_change":"2024-09-21T10:12:25.764496+0000","last_active":"2024-09-21T10:14:54.651099+0000","last_peered":"2024-09-21T10:14:54.651099+0000","last_clean":"2024-09-21T10:14:54.651099+0000","last_became_active":"2024-09-21T10:12:25.707263+0000","last_became_peered":"2024-09-21T10:12:25.707263+0000","last_unstale":"2024-09-21T10:14:54.651099+0000","last_undegraded":"2024-09-21T10:14:54.651099+0000","last_fullsized":"2024-09-21T10:14:54.651099+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:45.728577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":63,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.649302+0000","last_change":"2024-09-21T10:12:50.573976+0000","last_active":"2024-09-21T10:14:54.649302+0000","last_peered":"2024-09-21T10:14:54.649302+0000","last_clean":"2024-09-21T10:14:54.649302+0000","last_became_active":"2024-09-21T10:12:50.573657+0000","last_became_peered":"2024-09-21T10:12:50.573657+0000","last_unstale":"2024-09-21T10:14:54.649302+0000","last_undegraded":"2024-09-21T10:14:54.649302+0000","last_fullsized":"2024-09-21T10:14:54.649302+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:44:44.245943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"1311'5575","reported_seq":5632,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.098901+0000","last_change":"2024-09-21T10:14:54.654921+0000","last_active":"2024-09-21T10:15:08.098901+0000","last_peered":"2024-09-21T10:15:08.098901+0000","last_clean":"2024-09-21T10:15:08.098901+0000","last_became_active":"2024-09-21T10:14:54.650488+0000","last_became_peered":"2024-09-21T10:14:54.650488+0000","last_unstale":"2024-09-21T10:15:08.098901+0000","last_undegraded":"2024-09-21T10:15:08.098901+0000","last_fullsized":"2024-09-21T10:15:08.098901+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:15:56.216936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257916928,"num_objects":3935,"num_object_clones":0,"num_object_copies":11805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3935,"num_write_kb":251872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"288'100","reported_seq":453,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.764154+0000","last_change":"2024-09-21T10:10:13.678895+0000","last_active":"2024-09-21T10:15:02.764154+0000","last_peered":"2024-09-21T10:15:02.764154+0000","last_clean":"2024-09-21T10:15:02.764154+0000","last_became_active":"2024-09-21T10:10:13.678219+0000","last_became_peered":"2024-09-21T10:10:13.678219+0000","last_unstale":"2024-09-21T10:15:02.764154+0000","last_undegraded":"2024-09-21T10:15:02.764154+0000","last_fullsized":"2024-09-21T10:15:02.764154+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:27.395700+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:27.395700+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:26.476978+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":57,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.650963+0000","last_change":"2024-09-21T10:12:25.711574+0000","last_active":"2024-09-21T10:14:54.650963+0000","last_peered":"2024-09-21T10:14:54.650963+0000","last_clean":"2024-09-21T10:14:54.650963+0000","last_became_active":"2024-09-21T10:12:25.699847+0000","last_became_peered":"2024-09-21T10:12:25.699847+0000","last_unstale":"2024-09-21T10:14:54.650963+0000","last_undegraded":"2024-09-21T10:14:54.650963+0000","last_fullsized":"2024-09-21T10:14:54.650963+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:09:56.556064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1276,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644078+0000","last_change":"2024-09-21T10:08:45.515858+0000","last_active":"2024-09-21T10:14:54.644078+0000","last_peered":"2024-09-21T10:14:54.644078+0000","last_clean":"2024-09-21T10:14:54.644078+0000","last_became_active":"2024-09-21T10:08:45.515543+0000","last_became_peered":"2024-09-21T10:08:45.515543+0000","last_unstale":"2024-09-21T10:14:54.644078+0000","last_undegraded":"2024-09-21T10:14:54.644078+0000","last_fullsized":"2024-09-21T10:14:54.644078+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:29.565740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"1311'5527","reported_seq":5606,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.106171+0000","last_change":"2024-09-21T10:14:05.649563+0000","last_active":"2024-09-21T10:15:08.106171+0000","last_peered":"2024-09-21T10:15:08.106171+0000","last_clean":"2024-09-21T10:15:08.106171+0000","last_became_active":"2024-09-21T10:14:05.647194+0000","last_became_peered":"2024-09-21T10:14:05.647194+0000","last_unstale":"2024-09-21T10:15:08.106171+0000","last_undegraded":"2024-09-21T10:15:08.106171+0000","last_fullsized":"2024-09-21T10:15:08.106171+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5527,"log_dups_size":0,"ondisk_log_size":5527,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255885312,"num_objects":3904,"num_object_clones":0,"num_object_copies":11712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3904,"num_write_kb":249888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"288'100","reported_seq":171,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.648282+0000","last_change":"2024-09-21T10:14:54.647582+0000","last_active":"2024-09-21T10:14:54.648282+0000","last_peered":"2024-09-21T10:14:54.648282+0000","last_clean":"2024-09-21T10:14:54.648282+0000","last_became_active":"2024-09-21T10:14:54.647213+0000","last_became_peered":"2024-09-21T10:14:54.647213+0000","last_unstale":"2024-09-21T10:14:54.648282+0000","last_undegraded":"2024-09-21T10:14:54.648282+0000","last_fullsized":"2024-09-21T10:14:54.648282+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:23.746751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:23.746751+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:04.424278+0000","scrub_duration":0.0021986229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":71,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.650580+0000","last_change":"2024-09-21T10:12:50.623002+0000","last_active":"2024-09-21T10:14:54.650580+0000","last_peered":"2024-09-21T10:14:54.650580+0000","last_clean":"2024-09-21T10:14:54.650580+0000","last_became_active":"2024-09-21T10:12:50.588355+0000","last_became_peered":"2024-09-21T10:12:50.588355+0000","last_unstale":"2024-09-21T10:14:54.650580+0000","last_undegraded":"2024-09-21T10:14:54.650580+0000","last_fullsized":"2024-09-21T10:14:54.650580+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:22.564459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.646393+0000","last_change":"2024-09-21T10:12:25.707043+0000","last_active":"2024-09-21T10:14:54.646393+0000","last_peered":"2024-09-21T10:14:54.646393+0000","last_clean":"2024-09-21T10:14:54.646393+0000","last_became_active":"2024-09-21T10:12:25.706636+0000","last_became_peered":"2024-09-21T10:12:25.706636+0000","last_unstale":"2024-09-21T10:14:54.646393+0000","last_undegraded":"2024-09-21T10:14:54.646393+0000","last_fullsized":"2024-09-21T10:14:54.646393+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:38.994917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"1311'5436","reported_seq":5551,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:07.893927+0000","last_change":"2024-09-21T10:14:33.475485+0000","last_active":"2024-09-21T10:15:07.893927+0000","last_peered":"2024-09-21T10:15:07.893927+0000","last_clean":"2024-09-21T10:15:07.893927+0000","last_became_active":"2024-09-21T10:14:05.671586+0000","last_became_peered":"2024-09-21T10:14:05.671586+0000","last_unstale":"2024-09-21T10:15:07.893927+0000","last_undegraded":"2024-09-21T10:15:07.893927+0000","last_fullsized":"2024-09-21T10:15:07.893927+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"1303'4238","last_scrub_stamp":"2024-09-21T10:14:33.475181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:33.475181+0000","objects_scrubbed":2651,"log_size":5436,"log_dups_size":0,"ondisk_log_size":5436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:52.230611+0000","scrub_duration":1000,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253362176,"num_objects":3866,"num_object_clones":0,"num_object_copies":11598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1215,"num_write_kb":77760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"288'100","reported_seq":117,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644823+0000","last_change":"2024-09-21T10:12:25.697192+0000","last_active":"2024-09-21T10:14:54.644823+0000","last_peered":"2024-09-21T10:14:54.644823+0000","last_clean":"2024-09-21T10:14:54.644823+0000","last_became_active":"2024-09-21T10:12:25.696789+0000","last_became_peered":"2024-09-21T10:12:25.696789+0000","last_unstale":"2024-09-21T10:14:54.644823+0000","last_undegraded":"2024-09-21T10:14:54.644823+0000","last_fullsized":"2024-09-21T10:14:54.644823+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:03.581657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:03.581657+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:24.147270+0000","scrub_duration":0.0038308109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":991,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.764401+0000","last_change":"2024-09-21T10:10:13.678563+0000","last_active":"2024-09-21T10:15:02.764401+0000","last_peered":"2024-09-21T10:15:02.764401+0000","last_clean":"2024-09-21T10:15:02.764401+0000","last_became_active":"2024-09-21T10:10:13.676594+0000","last_became_peered":"2024-09-21T10:10:13.676594+0000","last_unstale":"2024-09-21T10:15:02.764401+0000","last_undegraded":"2024-09-21T10:15:02.764401+0000","last_fullsized":"2024-09-21T10:15:02.764401+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:34.275762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2075,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.767992+0000","last_change":"2024-09-21T10:08:45.528426+0000","last_active":"2024-09-21T10:15:02.767992+0000","last_peered":"2024-09-21T10:15:02.767992+0000","last_clean":"2024-09-21T10:15:02.767992+0000","last_became_active":"2024-09-21T10:08:45.528049+0000","last_became_peered":"2024-09-21T10:08:45.528049+0000","last_unstale":"2024-09-21T10:15:02.767992+0000","last_undegraded":"2024-09-21T10:15:02.767992+0000","last_fullsized":"2024-09-21T10:15:02.767992+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:10:10.687040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"1311'5501","reported_seq":5580,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.351812+0000","last_change":"2024-09-21T10:14:15.602000+0000","last_active":"2024-09-21T10:15:08.351812+0000","last_peered":"2024-09-21T10:15:08.351812+0000","last_clean":"2024-09-21T10:15:08.351812+0000","last_became_active":"2024-09-21T10:14:05.651497+0000","last_became_peered":"2024-09-21T10:14:05.651497+0000","last_unstale":"2024-09-21T10:15:08.351812+0000","last_undegraded":"2024-09-21T10:15:08.351812+0000","last_fullsized":"2024-09-21T10:15:08.351812+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5501,"log_dups_size":0,"ondisk_log_size":5501,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:15:49.235197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253198336,"num_objects":3863,"num_object_clones":0,"num_object_copies":11589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3863,"num_write_kb":247264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":63,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.649668+0000","last_change":"2024-09-21T10:13:15.663291+0000","last_active":"2024-09-21T10:14:54.649668+0000","last_peered":"2024-09-21T10:14:54.649668+0000","last_clean":"2024-09-21T10:14:54.649668+0000","last_became_active":"2024-09-21T10:12:25.712199+0000","last_became_peered":"2024-09-21T10:12:25.712199+0000","last_unstale":"2024-09-21T10:14:54.649668+0000","last_undegraded":"2024-09-21T10:14:54.649668+0000","last_fullsized":"2024-09-21T10:14:54.649668+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:15.663228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:15.663228+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:44.919423+0000","scrub_duration":0.0014833489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":1985,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.762171+0000","last_change":"2024-09-21T10:08:25.436371+0000","last_active":"2024-09-21T10:15:02.762171+0000","last_peered":"2024-09-21T10:15:02.762171+0000","last_clean":"2024-09-21T10:15:02.762171+0000","last_became_active":"2024-09-21T10:08:25.435921+0000","last_became_peered":"2024-09-21T10:08:25.435921+0000","last_unstale":"2024-09-21T10:15:02.762171+0000","last_undegraded":"2024-09-21T10:15:02.762171+0000","last_fullsized":"2024-09-21T10:15:02.762171+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:27.612386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1253,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644905+0000","last_change":"2024-09-21T10:10:13.661328+0000","last_active":"2024-09-21T10:14:54.644905+0000","last_peered":"2024-09-21T10:14:54.644905+0000","last_clean":"2024-09-21T10:14:54.644905+0000","last_became_active":"2024-09-21T10:10:13.661028+0000","last_became_peered":"2024-09-21T10:10:13.661028+0000","last_unstale":"2024-09-21T10:14:54.644905+0000","last_undegraded":"2024-09-21T10:14:54.644905+0000","last_fullsized":"2024-09-21T10:14:54.644905+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:36.965214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"1311'5604","reported_seq":5718,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:07.930156+0000","last_change":"2024-09-21T10:14:28.156006+0000","last_active":"2024-09-21T10:15:07.930156+0000","last_peered":"2024-09-21T10:15:07.930156+0000","last_clean":"2024-09-21T10:15:07.930156+0000","last_became_active":"2024-09-21T10:14:05.674553+0000","last_became_peered":"2024-09-21T10:14:05.674553+0000","last_unstale":"2024-09-21T10:15:07.930156+0000","last_undegraded":"2024-09-21T10:15:07.930156+0000","last_fullsized":"2024-09-21T10:15:07.930156+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"1303'4190","last_scrub_stamp":"2024-09-21T10:14:28.155685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:28.155685+0000","objects_scrubbed":2487,"log_size":5604,"log_dups_size":0,"ondisk_log_size":5604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:33.678082+0000","scrub_duration":718,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256835584,"num_objects":3919,"num_object_clones":0,"num_object_copies":11757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1432,"num_write_kb":91648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":432,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.777823+0000","last_change":"2024-09-21T10:13:15.622804+0000","last_active":"2024-09-21T10:15:02.777823+0000","last_peered":"2024-09-21T10:15:02.777823+0000","last_clean":"2024-09-21T10:15:02.777823+0000","last_became_active":"2024-09-21T10:12:50.586357+0000","last_became_peered":"2024-09-21T10:12:50.586357+0000","last_unstale":"2024-09-21T10:15:02.777823+0000","last_undegraded":"2024-09-21T10:15:02.777823+0000","last_fullsized":"2024-09-21T10:15:02.777823+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1281,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:28.894448+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":495,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.648702+0000","last_change":"2024-09-21T10:14:54.648488+0000","last_active":"2024-09-21T10:14:54.648702+0000","last_peered":"2024-09-21T10:14:54.648702+0000","last_clean":"2024-09-21T10:14:54.648702+0000","last_became_active":"2024-09-21T10:14:54.647310+0000","last_became_peered":"2024-09-21T10:14:54.647310+0000","last_unstale":"2024-09-21T10:14:54.648702+0000","last_undegraded":"2024-09-21T10:14:54.648702+0000","last_fullsized":"2024-09-21T10:14:54.648702+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:55.531730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1276,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.645772+0000","last_change":"2024-09-21T10:08:45.524238+0000","last_active":"2024-09-21T10:14:54.645772+0000","last_peered":"2024-09-21T10:14:54.645772+0000","last_clean":"2024-09-21T10:14:54.645772+0000","last_became_active":"2024-09-21T10:08:45.523517+0000","last_became_peered":"2024-09-21T10:08:45.523517+0000","last_unstale":"2024-09-21T10:14:54.645772+0000","last_undegraded":"2024-09-21T10:14:54.645772+0000","last_fullsized":"2024-09-21T10:14:54.645772+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:13.958459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"1311'5405","reported_seq":5451,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:05.836423+0000","last_change":"2024-09-21T10:14:05.642668+0000","last_active":"2024-09-21T10:15:05.836423+0000","last_peered":"2024-09-21T10:15:05.836423+0000","last_clean":"2024-09-21T10:15:05.836423+0000","last_became_active":"2024-09-21T10:14:05.640797+0000","last_became_peered":"2024-09-21T10:14:05.640797+0000","last_unstale":"2024-09-21T10:15:05.836423+0000","last_undegraded":"2024-09-21T10:15:05.836423+0000","last_fullsized":"2024-09-21T10:15:05.836423+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5405,"log_dups_size":0,"ondisk_log_size":5405,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250019840,"num_objects":3815,"num_object_clones":0,"num_object_copies":11445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3815,"num_write_kb":244160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"288'100","reported_seq":177,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.772470+0000","last_change":"2024-09-21T10:13:30.346091+0000","last_active":"2024-09-21T10:15:02.772470+0000","last_peered":"2024-09-21T10:15:02.772470+0000","last_clean":"2024-09-21T10:15:02.772470+0000","last_became_active":"2024-09-21T10:12:50.586593+0000","last_became_peered":"2024-09-21T10:12:50.586593+0000","last_unstale":"2024-09-21T10:15:02.772470+0000","last_undegraded":"2024-09-21T10:15:02.772470+0000","last_fullsized":"2024-09-21T10:15:02.772470+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:30.346035+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:30.346035+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:01.387926+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1267,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.643881+0000","last_change":"2024-09-21T10:12:25.697464+0000","last_active":"2024-09-21T10:14:54.643881+0000","last_peered":"2024-09-21T10:14:54.643881+0000","last_clean":"2024-09-21T10:14:54.643881+0000","last_became_active":"2024-09-21T10:12:25.697050+0000","last_became_peered":"2024-09-21T10:12:25.697050+0000","last_unstale":"2024-09-21T10:14:54.643881+0000","last_undegraded":"2024-09-21T10:14:54.643881+0000","last_fullsized":"2024-09-21T10:14:54.643881+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:50.687008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1299,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644992+0000","last_change":"2024-09-21T10:10:13.641608+0000","last_active":"2024-09-21T10:14:54.644992+0000","last_peered":"2024-09-21T10:14:54.644992+0000","last_clean":"2024-09-21T10:14:54.644992+0000","last_became_active":"2024-09-21T10:10:13.640299+0000","last_became_peered":"2024-09-21T10:10:13.640299+0000","last_unstale":"2024-09-21T10:14:54.644992+0000","last_undegraded":"2024-09-21T10:14:54.644992+0000","last_fullsized":"2024-09-21T10:14:54.644992+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:46.978168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"1311'5441","reported_seq":5497,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.628262+0000","last_change":"2024-09-21T10:14:42.115355+0000","last_active":"2024-09-21T10:15:08.628262+0000","last_peered":"2024-09-21T10:15:08.628262+0000","last_clean":"2024-09-21T10:15:08.628262+0000","last_became_active":"2024-09-21T10:14:05.637987+0000","last_became_peered":"2024-09-21T10:14:05.637987+0000","last_unstale":"2024-09-21T10:15:08.628262+0000","last_undegraded":"2024-09-21T10:15:08.628262+0000","last_fullsized":"2024-09-21T10:15:08.628262+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5441,"log_dups_size":0,"ondisk_log_size":5441,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253132800,"num_objects":3862,"num_object_clones":0,"num_object_copies":11586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3862,"num_write_kb":247200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":433,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.777472+0000","last_change":"2024-09-21T10:10:13.622553+0000","last_active":"2024-09-21T10:15:02.777472+0000","last_peered":"2024-09-21T10:15:02.777472+0000","last_clean":"2024-09-21T10:15:02.777472+0000","last_became_active":"2024-09-21T10:10:13.622268+0000","last_became_peered":"2024-09-21T10:10:13.622268+0000","last_unstale":"2024-09-21T10:15:02.777472+0000","last_undegraded":"2024-09-21T10:15:02.777472+0000","last_fullsized":"2024-09-21T10:15:02.777472+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:24.686892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:24.686892+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:55.474818+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2005,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.762371+0000","last_change":"2024-09-21T10:12:25.709463+0000","last_active":"2024-09-21T10:15:02.762371+0000","last_peered":"2024-09-21T10:15:02.762371+0000","last_clean":"2024-09-21T10:15:02.762371+0000","last_became_active":"2024-09-21T10:12:25.706354+0000","last_became_peered":"2024-09-21T10:12:25.706354+0000","last_unstale":"2024-09-21T10:15:02.762371+0000","last_undegraded":"2024-09-21T10:15:02.762371+0000","last_fullsized":"2024-09-21T10:15:02.762371+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:37:56.890445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":162,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.765657+0000","last_change":"2024-09-21T10:08:45.523818+0000","last_active":"2024-09-21T10:15:02.765657+0000","last_peered":"2024-09-21T10:15:02.765657+0000","last_clean":"2024-09-21T10:15:02.765657+0000","last_became_active":"2024-09-21T10:08:45.523507+0000","last_became_peered":"2024-09-21T10:08:45.523507+0000","last_unstale":"2024-09-21T10:15:02.765657+0000","last_undegraded":"2024-09-21T10:15:02.765657+0000","last_fullsized":"2024-09-21T10:15:02.765657+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:32.923210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"1311'7569","reported_seq":7605,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:05.854585+0000","last_change":"2024-09-21T10:13:28.745891+0000","last_active":"2024-09-21T10:15:05.854585+0000","last_peered":"2024-09-21T10:15:05.854585+0000","last_clean":"2024-09-21T10:15:05.854585+0000","last_became_active":"2024-09-21T10:13:28.745565+0000","last_became_peered":"2024-09-21T10:13:28.745565+0000","last_unstale":"2024-09-21T10:15:05.854585+0000","last_undegraded":"2024-09-21T10:15:05.854585+0000","last_fullsized":"2024-09-21T10:15:05.854585+0000","mapping_epoch":1283,"log_start":"1289'1900","ondisk_log_start":"1289'1900","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5669,"log_dups_size":0,"ondisk_log_size":5669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:18.910062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":496041984,"num_objects":7569,"num_object_clones":0,"num_object_copies":22707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7569,"num_write_kb":484416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":63,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.649072+0000","last_change":"2024-09-21T10:13:20.647735+0000","last_active":"2024-09-21T10:14:54.649072+0000","last_peered":"2024-09-21T10:14:54.649072+0000","last_clean":"2024-09-21T10:14:54.649072+0000","last_became_active":"2024-09-21T10:12:25.708657+0000","last_became_peered":"2024-09-21T10:12:25.708657+0000","last_unstale":"2024-09-21T10:14:54.649072+0000","last_undegraded":"2024-09-21T10:14:54.649072+0000","last_fullsized":"2024-09-21T10:14:54.649072+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:20.647659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:20.647659+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:09.770331+0000","scrub_duration":0.0016005629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":484,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.778125+0000","last_change":"2024-09-21T10:12:56.081277+0000","last_active":"2024-09-21T10:15:02.778125+0000","last_peered":"2024-09-21T10:15:02.778125+0000","last_clean":"2024-09-21T10:15:02.778125+0000","last_became_active":"2024-09-21T10:12:50.586317+0000","last_became_peered":"2024-09-21T10:12:50.586317+0000","last_unstale":"2024-09-21T10:15:02.778125+0000","last_undegraded":"2024-09-21T10:15:02.778125+0000","last_fullsized":"2024-09-21T10:15:02.778125+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:57.776335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1983,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.762080+0000","last_change":"2024-09-21T10:12:50.577072+0000","last_active":"2024-09-21T10:15:02.762080+0000","last_peered":"2024-09-21T10:15:02.762080+0000","last_clean":"2024-09-21T10:15:02.762080+0000","last_became_active":"2024-09-21T10:12:50.576731+0000","last_became_peered":"2024-09-21T10:12:50.576731+0000","last_unstale":"2024-09-21T10:15:02.762080+0000","last_undegraded":"2024-09-21T10:15:02.762080+0000","last_fullsized":"2024-09-21T10:15:02.762080+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:56:06.937599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":170,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.650584+0000","last_change":"2024-09-21T10:14:54.650325+0000","last_active":"2024-09-21T10:14:54.650584+0000","last_peered":"2024-09-21T10:14:54.650584+0000","last_clean":"2024-09-21T10:14:54.650584+0000","last_became_active":"2024-09-21T10:14:54.650048+0000","last_became_peered":"2024-09-21T10:14:54.650048+0000","last_unstale":"2024-09-21T10:14:54.650584+0000","last_undegraded":"2024-09-21T10:14:54.650584+0000","last_fullsized":"2024-09-21T10:14:54.650584+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:47.899727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"1311'5451","reported_seq":5531,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.062302+0000","last_change":"2024-09-21T10:14:05.650295+0000","last_active":"2024-09-21T10:15:08.062302+0000","last_peered":"2024-09-21T10:15:08.062302+0000","last_clean":"2024-09-21T10:15:08.062302+0000","last_became_active":"2024-09-21T10:14:05.647420+0000","last_became_peered":"2024-09-21T10:14:05.647420+0000","last_unstale":"2024-09-21T10:15:08.062302+0000","last_undegraded":"2024-09-21T10:15:08.062302+0000","last_fullsized":"2024-09-21T10:15:08.062302+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5451,"log_dups_size":0,"ondisk_log_size":5451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250970112,"num_objects":3829,"num_object_clones":0,"num_object_copies":11487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3829,"num_write_kb":245088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":185,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.768224+0000","last_change":"2024-09-21T10:13:18.313624+0000","last_active":"2024-09-21T10:15:02.768224+0000","last_peered":"2024-09-21T10:15:02.768224+0000","last_clean":"2024-09-21T10:15:02.768224+0000","last_became_active":"2024-09-21T10:10:13.637267+0000","last_became_peered":"2024-09-21T10:10:13.637267+0000","last_unstale":"2024-09-21T10:15:02.768224+0000","last_undegraded":"2024-09-21T10:15:02.768224+0000","last_fullsized":"2024-09-21T10:15:02.768224+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:18.313538+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:18.313538+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:11.379319+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":53,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.649126+0000","last_change":"2024-09-21T10:12:25.708338+0000","last_active":"2024-09-21T10:14:54.649126+0000","last_peered":"2024-09-21T10:14:54.649126+0000","last_clean":"2024-09-21T10:14:54.649126+0000","last_became_active":"2024-09-21T10:12:25.708041+0000","last_became_peered":"2024-09-21T10:12:25.708041+0000","last_unstale":"2024-09-21T10:14:54.649126+0000","last_undegraded":"2024-09-21T10:14:54.649126+0000","last_fullsized":"2024-09-21T10:14:54.649126+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:07:48.291125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2060,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.767913+0000","last_change":"2024-09-21T10:08:25.429287+0000","last_active":"2024-09-21T10:15:02.767913+0000","last_peered":"2024-09-21T10:15:02.767913+0000","last_clean":"2024-09-21T10:15:02.767913+0000","last_became_active":"2024-09-21T10:08:25.428923+0000","last_became_peered":"2024-09-21T10:08:25.428923+0000","last_unstale":"2024-09-21T10:15:02.767913+0000","last_undegraded":"2024-09-21T10:15:02.767913+0000","last_fullsized":"2024-09-21T10:15:02.767913+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:09.255665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2075,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.760834+0000","last_change":"2024-09-21T10:08:45.532253+0000","last_active":"2024-09-21T10:15:02.760834+0000","last_peered":"2024-09-21T10:15:02.760834+0000","last_clean":"2024-09-21T10:15:02.760834+0000","last_became_active":"2024-09-21T10:08:45.531519+0000","last_became_peered":"2024-09-21T10:08:45.531519+0000","last_unstale":"2024-09-21T10:15:02.760834+0000","last_undegraded":"2024-09-21T10:15:02.760834+0000","last_fullsized":"2024-09-21T10:15:02.760834+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:25.378783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"1311'5457","reported_seq":5538,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:07.929299+0000","last_change":"2024-09-21T10:14:22.570382+0000","last_active":"2024-09-21T10:15:07.929299+0000","last_peered":"2024-09-21T10:15:07.929299+0000","last_clean":"2024-09-21T10:15:07.929299+0000","last_became_active":"2024-09-21T10:14:05.671753+0000","last_became_peered":"2024-09-21T10:14:05.671753+0000","last_unstale":"2024-09-21T10:15:07.929299+0000","last_undegraded":"2024-09-21T10:15:07.929299+0000","last_fullsized":"2024-09-21T10:15:07.929299+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5457,"log_dups_size":0,"ondisk_log_size":5457,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251265024,"num_objects":3834,"num_object_clones":0,"num_object_copies":11502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3834,"num_write_kb":245376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":63,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.649819+0000","last_change":"2024-09-21T10:13:17.734264+0000","last_active":"2024-09-21T10:14:54.649819+0000","last_peered":"2024-09-21T10:14:54.649819+0000","last_clean":"2024-09-21T10:14:54.649819+0000","last_became_active":"2024-09-21T10:12:25.707372+0000","last_became_peered":"2024-09-21T10:12:25.707372+0000","last_unstale":"2024-09-21T10:14:54.649819+0000","last_undegraded":"2024-09-21T10:14:54.649819+0000","last_fullsized":"2024-09-21T10:14:54.649819+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:17.734169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:17.734169+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:59.937743+0000","scrub_duration":0.0017947760000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2023,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.777929+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:15:02.777929+0000","last_peered":"2024-09-21T10:15:02.777929+0000","last_clean":"2024-09-21T10:15:02.777929+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:15:02.777929+0000","last_undegraded":"2024-09-21T10:15:02.777929+0000","last_fullsized":"2024-09-21T10:15:02.777929+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":177,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.760671+0000","last_change":"2024-09-21T10:10:13.635570+0000","last_active":"2024-09-21T10:15:02.760671+0000","last_peered":"2024-09-21T10:15:02.760671+0000","last_clean":"2024-09-21T10:15:02.760671+0000","last_became_active":"2024-09-21T10:10:13.635099+0000","last_became_peered":"2024-09-21T10:10:13.635099+0000","last_unstale":"2024-09-21T10:15:02.760671+0000","last_undegraded":"2024-09-21T10:15:02.760671+0000","last_fullsized":"2024-09-21T10:15:02.760671+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:43.686232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":169,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.765572+0000","last_change":"2024-09-21T10:12:50.581491+0000","last_active":"2024-09-21T10:15:02.765572+0000","last_peered":"2024-09-21T10:15:02.765572+0000","last_clean":"2024-09-21T10:15:02.765572+0000","last_became_active":"2024-09-21T10:12:50.580970+0000","last_became_peered":"2024-09-21T10:12:50.580970+0000","last_unstale":"2024-09-21T10:15:02.765572+0000","last_undegraded":"2024-09-21T10:15:02.765572+0000","last_fullsized":"2024-09-21T10:15:02.765572+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:17.961335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"1311'7667","reported_seq":7703,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:05.843666+0000","last_change":"2024-09-21T10:13:28.746797+0000","last_active":"2024-09-21T10:15:05.843666+0000","last_peered":"2024-09-21T10:15:05.843666+0000","last_clean":"2024-09-21T10:15:05.843666+0000","last_became_active":"2024-09-21T10:13:28.746510+0000","last_became_peered":"2024-09-21T10:13:28.746510+0000","last_unstale":"2024-09-21T10:15:05.843666+0000","last_undegraded":"2024-09-21T10:15:05.843666+0000","last_fullsized":"2024-09-21T10:15:05.843666+0000","mapping_epoch":1283,"log_start":"1289'2000","ondisk_log_start":"1289'2000","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5667,"log_dups_size":0,"ondisk_log_size":5667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:12:58.123721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":502464512,"num_objects":7667,"num_object_clones":0,"num_object_copies":23001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7667,"num_write_kb":490688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":779,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.767804+0000","last_change":"2024-09-21T10:12:50.578738+0000","last_active":"2024-09-21T10:15:02.767804+0000","last_peered":"2024-09-21T10:15:02.767804+0000","last_clean":"2024-09-21T10:15:02.767804+0000","last_became_active":"2024-09-21T10:12:50.578363+0000","last_became_peered":"2024-09-21T10:12:50.578363+0000","last_unstale":"2024-09-21T10:15:02.767804+0000","last_undegraded":"2024-09-21T10:15:02.767804+0000","last_fullsized":"2024-09-21T10:15:02.767804+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:19.901507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:19.901507+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:08:56.749215+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":178,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.766139+0000","last_change":"2024-09-21T10:10:13.636061+0000","last_active":"2024-09-21T10:15:02.766139+0000","last_peered":"2024-09-21T10:15:02.766139+0000","last_clean":"2024-09-21T10:15:02.766139+0000","last_became_active":"2024-09-21T10:10:13.635006+0000","last_became_peered":"2024-09-21T10:10:13.635006+0000","last_unstale":"2024-09-21T10:15:02.766139+0000","last_undegraded":"2024-09-21T10:15:02.766139+0000","last_fullsized":"2024-09-21T10:15:02.766139+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:24.272055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1254,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644106+0000","last_change":"2024-09-21T10:10:13.686243+0000","last_active":"2024-09-21T10:14:54.644106+0000","last_peered":"2024-09-21T10:14:54.644106+0000","last_clean":"2024-09-21T10:14:54.644106+0000","last_became_active":"2024-09-21T10:10:13.685894+0000","last_became_peered":"2024-09-21T10:10:13.685894+0000","last_unstale":"2024-09-21T10:14:54.644106+0000","last_undegraded":"2024-09-21T10:14:54.644106+0000","last_fullsized":"2024-09-21T10:14:54.644106+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:12.394732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":163,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.765247+0000","last_change":"2024-09-21T10:08:45.525546+0000","last_active":"2024-09-21T10:15:02.765247+0000","last_peered":"2024-09-21T10:15:02.765247+0000","last_clean":"2024-09-21T10:15:02.765247+0000","last_became_active":"2024-09-21T10:08:45.525116+0000","last_became_peered":"2024-09-21T10:08:45.525116+0000","last_unstale":"2024-09-21T10:15:02.765247+0000","last_undegraded":"2024-09-21T10:15:02.765247+0000","last_fullsized":"2024-09-21T10:15:02.765247+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:58.739744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"1311'7495","reported_seq":7531,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:05.824382+0000","last_change":"2024-09-21T10:13:28.747073+0000","last_active":"2024-09-21T10:15:05.824382+0000","last_peered":"2024-09-21T10:15:05.824382+0000","last_clean":"2024-09-21T10:15:05.824382+0000","last_became_active":"2024-09-21T10:13:28.746745+0000","last_became_peered":"2024-09-21T10:13:28.746745+0000","last_unstale":"2024-09-21T10:15:05.824382+0000","last_undegraded":"2024-09-21T10:15:05.824382+0000","last_fullsized":"2024-09-21T10:15:05.824382+0000","mapping_epoch":1283,"log_start":"1290'2120","ondisk_log_start":"1290'2120","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5375,"log_dups_size":0,"ondisk_log_size":5375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:09:26.352129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":491192320,"num_objects":7495,"num_object_clones":0,"num_object_copies":22485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7495,"num_write_kb":479680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":56,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.767860+0000","last_change":"2024-09-21T10:14:54.655208+0000","last_active":"2024-09-21T10:15:02.767860+0000","last_peered":"2024-09-21T10:15:02.767860+0000","last_clean":"2024-09-21T10:15:02.767860+0000","last_became_active":"2024-09-21T10:14:54.654060+0000","last_became_peered":"2024-09-21T10:14:54.654060+0000","last_unstale":"2024-09-21T10:15:02.767860+0000","last_undegraded":"2024-09-21T10:15:02.767860+0000","last_fullsized":"2024-09-21T10:15:02.767860+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:17.223016+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:17.223016+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:15:58.230812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":34,"num_write":6,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":987,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.763252+0000","last_change":"2024-09-21T10:12:50.579273+0000","last_active":"2024-09-21T10:15:02.763252+0000","last_peered":"2024-09-21T10:15:02.763252+0000","last_clean":"2024-09-21T10:15:02.763252+0000","last_became_active":"2024-09-21T10:12:50.578806+0000","last_became_peered":"2024-09-21T10:12:50.578806+0000","last_unstale":"2024-09-21T10:15:02.763252+0000","last_undegraded":"2024-09-21T10:15:02.763252+0000","last_fullsized":"2024-09-21T10:15:02.763252+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:04:53.664017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":177,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.765158+0000","last_change":"2024-09-21T10:12:25.705453+0000","last_active":"2024-09-21T10:15:02.765158+0000","last_peered":"2024-09-21T10:15:02.765158+0000","last_clean":"2024-09-21T10:15:02.765158+0000","last_became_active":"2024-09-21T10:12:25.705105+0000","last_became_peered":"2024-09-21T10:12:25.705105+0000","last_unstale":"2024-09-21T10:15:02.765158+0000","last_undegraded":"2024-09-21T10:15:02.765158+0000","last_fullsized":"2024-09-21T10:15:02.765158+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:07:02.419776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":170,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.765943+0000","last_change":"2024-09-21T10:12:50.585127+0000","last_active":"2024-09-21T10:15:02.765943+0000","last_peered":"2024-09-21T10:15:02.765943+0000","last_clean":"2024-09-21T10:15:02.765943+0000","last_became_active":"2024-09-21T10:12:50.584761+0000","last_became_peered":"2024-09-21T10:12:50.584761+0000","last_unstale":"2024-09-21T10:15:02.765943+0000","last_undegraded":"2024-09-21T10:15:02.765943+0000","last_fullsized":"2024-09-21T10:15:02.765943+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:15.425510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"1311'7936","reported_seq":7981,"reported_epoch":1311,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:15:08.617614+0000","last_change":"2024-09-21T10:14:54.735767+0000","last_active":"2024-09-21T10:15:08.617614+0000","last_peered":"2024-09-21T10:15:08.617614+0000","last_clean":"2024-09-21T10:14:53.627961+0000","last_became_active":"2024-09-21T10:14:54.661670+0000","last_became_peered":"2024-09-21T10:14:54.661670+0000","last_unstale":"2024-09-21T10:15:08.617614+0000","last_undegraded":"2024-09-21T10:15:08.617614+0000","last_fullsized":"2024-09-21T10:15:08.617614+0000","mapping_epoch":1309,"log_start":"1290'2796","ondisk_log_start":"1290'2796","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5140,"log_dups_size":0,"ondisk_log_size":5140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:47:06.826746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":520093696,"num_objects":7936,"num_object_clones":0,"num_object_copies":23808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7936,"num_write_kb":507904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":7936}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":402,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.760660+0000","last_change":"2024-09-21T10:14:54.652063+0000","last_active":"2024-09-21T10:15:02.760660+0000","last_peered":"2024-09-21T10:15:02.760660+0000","last_clean":"2024-09-21T10:15:02.760660+0000","last_became_active":"2024-09-21T10:14:54.651668+0000","last_became_peered":"2024-09-21T10:14:54.651668+0000","last_unstale":"2024-09-21T10:15:02.760660+0000","last_undegraded":"2024-09-21T10:15:02.760660+0000","last_fullsized":"2024-09-21T10:15:02.760660+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:10:37.256819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:10:37.256819+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":34,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":170,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.765071+0000","last_change":"2024-09-21T10:12:50.585333+0000","last_active":"2024-09-21T10:15:02.765071+0000","last_peered":"2024-09-21T10:15:02.765071+0000","last_clean":"2024-09-21T10:15:02.765071+0000","last_became_active":"2024-09-21T10:12:50.584978+0000","last_became_peered":"2024-09-21T10:12:50.584978+0000","last_unstale":"2024-09-21T10:15:02.765071+0000","last_undegraded":"2024-09-21T10:15:02.765071+0000","last_fullsized":"2024-09-21T10:15:02.765071+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:16:10.033667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":53,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.650619+0000","last_change":"2024-09-21T10:12:25.709819+0000","last_active":"2024-09-21T10:14:54.650619+0000","last_peered":"2024-09-21T10:14:54.650619+0000","last_clean":"2024-09-21T10:14:54.650619+0000","last_became_active":"2024-09-21T10:12:25.709548+0000","last_became_peered":"2024-09-21T10:12:25.709548+0000","last_unstale":"2024-09-21T10:14:54.650619+0000","last_undegraded":"2024-09-21T10:14:54.650619+0000","last_fullsized":"2024-09-21T10:14:54.650619+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:35.986614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2068,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.768084+0000","last_change":"2024-09-21T10:13:41.795984+0000","last_active":"2024-09-21T10:15:02.768084+0000","last_peered":"2024-09-21T10:15:02.768084+0000","last_clean":"2024-09-21T10:15:02.768084+0000","last_became_active":"2024-09-21T10:13:41.795641+0000","last_became_peered":"2024-09-21T10:13:41.795641+0000","last_unstale":"2024-09-21T10:15:02.768084+0000","last_undegraded":"2024-09-21T10:15:02.768084+0000","last_fullsized":"2024-09-21T10:15:02.768084+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:58:05.578670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"1311'7864","reported_seq":7900,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.622275+0000","last_change":"2024-09-21T10:13:28.743256+0000","last_active":"2024-09-21T10:15:08.622275+0000","last_peered":"2024-09-21T10:15:08.622275+0000","last_clean":"2024-09-21T10:15:08.622275+0000","last_became_active":"2024-09-21T10:13:28.742998+0000","last_became_peered":"2024-09-21T10:13:28.742998+0000","last_unstale":"2024-09-21T10:15:08.622275+0000","last_undegraded":"2024-09-21T10:15:08.622275+0000","last_fullsized":"2024-09-21T10:15:08.622275+0000","mapping_epoch":1283,"log_start":"1290'2700","ondisk_log_start":"1290'2700","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5164,"log_dups_size":0,"ondisk_log_size":5164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:13:15.751354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":515375104,"num_objects":7864,"num_object_clones":0,"num_object_copies":23592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7864,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":211,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644488+0000","last_change":"2024-09-21T10:10:13.623475+0000","last_active":"2024-09-21T10:14:54.644488+0000","last_peered":"2024-09-21T10:14:54.644488+0000","last_clean":"2024-09-21T10:14:54.644488+0000","last_became_active":"2024-09-21T10:10:13.622580+0000","last_became_peered":"2024-09-21T10:10:13.622580+0000","last_unstale":"2024-09-21T10:14:54.644488+0000","last_undegraded":"2024-09-21T10:14:54.644488+0000","last_fullsized":"2024-09-21T10:14:54.644488+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:36.469530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:08:36.469530+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:47.757854+0000","scrub_duration":0.001636942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":1997,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.768178+0000","last_change":"2024-09-21T10:12:25.718713+0000","last_active":"2024-09-21T10:15:02.768178+0000","last_peered":"2024-09-21T10:15:02.768178+0000","last_clean":"2024-09-21T10:15:02.768178+0000","last_became_active":"2024-09-21T10:12:25.713007+0000","last_became_peered":"2024-09-21T10:12:25.713007+0000","last_unstale":"2024-09-21T10:15:02.768178+0000","last_undegraded":"2024-09-21T10:15:02.768178+0000","last_fullsized":"2024-09-21T10:15:02.768178+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:09.753443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":176,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.772291+0000","last_change":"2024-09-21T10:10:13.637236+0000","last_active":"2024-09-21T10:15:02.772291+0000","last_peered":"2024-09-21T10:15:02.772291+0000","last_clean":"2024-09-21T10:15:02.772291+0000","last_became_active":"2024-09-21T10:10:13.636907+0000","last_became_peered":"2024-09-21T10:10:13.636907+0000","last_unstale":"2024-09-21T10:15:02.772291+0000","last_undegraded":"2024-09-21T10:15:02.772291+0000","last_fullsized":"2024-09-21T10:15:02.772291+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:33.507370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1987,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.761726+0000","last_change":"2024-09-21T10:12:25.709467+0000","last_active":"2024-09-21T10:15:02.761726+0000","last_peered":"2024-09-21T10:15:02.761726+0000","last_clean":"2024-09-21T10:15:02.761726+0000","last_became_active":"2024-09-21T10:12:25.706170+0000","last_became_peered":"2024-09-21T10:12:25.706170+0000","last_unstale":"2024-09-21T10:15:02.761726+0000","last_undegraded":"2024-09-21T10:15:02.761726+0000","last_fullsized":"2024-09-21T10:15:02.761726+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:30.335358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"1311'7686","reported_seq":7721,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:05.838063+0000","last_change":"2024-09-21T10:13:28.789659+0000","last_active":"2024-09-21T10:15:05.838063+0000","last_peered":"2024-09-21T10:15:05.838063+0000","last_clean":"2024-09-21T10:15:05.838063+0000","last_became_active":"2024-09-21T10:13:28.789382+0000","last_became_peered":"2024-09-21T10:13:28.789382+0000","last_unstale":"2024-09-21T10:15:05.838063+0000","last_undegraded":"2024-09-21T10:15:05.838063+0000","last_fullsized":"2024-09-21T10:15:05.838063+0000","mapping_epoch":1283,"log_start":"1289'2000","ondisk_log_start":"1289'2000","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:51.179733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":503709696,"num_objects":7686,"num_object_clones":0,"num_object_copies":23058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7686,"num_write_kb":491904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":466,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.764056+0000","last_change":"2024-09-21T10:12:50.578581+0000","last_active":"2024-09-21T10:15:02.764056+0000","last_peered":"2024-09-21T10:15:02.764056+0000","last_clean":"2024-09-21T10:15:02.764056+0000","last_became_active":"2024-09-21T10:12:50.578246+0000","last_became_peered":"2024-09-21T10:12:50.578246+0000","last_unstale":"2024-09-21T10:15:02.764056+0000","last_undegraded":"2024-09-21T10:15:02.764056+0000","last_fullsized":"2024-09-21T10:15:02.764056+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.423664+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.423664+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:38.323326+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":2096,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.763756+0000","last_change":"2024-09-21T10:13:41.806309+0000","last_active":"2024-09-21T10:15:02.763756+0000","last_peered":"2024-09-21T10:15:02.763756+0000","last_clean":"2024-09-21T10:15:02.763756+0000","last_became_active":"2024-09-21T10:13:41.794296+0000","last_became_peered":"2024-09-21T10:13:41.794296+0000","last_unstale":"2024-09-21T10:15:02.763756+0000","last_undegraded":"2024-09-21T10:15:02.763756+0000","last_fullsized":"2024-09-21T10:15:02.763756+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:09:59.470176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":78,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.647082+0000","last_change":"2024-09-21T10:12:25.704090+0000","last_active":"2024-09-21T10:14:54.647082+0000","last_peered":"2024-09-21T10:14:54.647082+0000","last_clean":"2024-09-21T10:14:54.647082+0000","last_became_active":"2024-09-21T10:12:25.702945+0000","last_became_peered":"2024-09-21T10:12:25.702945+0000","last_unstale":"2024-09-21T10:14:54.647082+0000","last_undegraded":"2024-09-21T10:14:54.647082+0000","last_fullsized":"2024-09-21T10:14:54.647082+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:29.609613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1246,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.643896+0000","last_change":"2024-09-21T10:12:50.571672+0000","last_active":"2024-09-21T10:14:54.643896+0000","last_peered":"2024-09-21T10:14:54.643896+0000","last_clean":"2024-09-21T10:14:54.643896+0000","last_became_active":"2024-09-21T10:12:50.571358+0000","last_became_peered":"2024-09-21T10:12:50.571358+0000","last_unstale":"2024-09-21T10:14:54.643896+0000","last_undegraded":"2024-09-21T10:14:54.643896+0000","last_fullsized":"2024-09-21T10:14:54.643896+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:38.440163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1299,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.643914+0000","last_change":"2024-09-21T10:10:13.640158+0000","last_active":"2024-09-21T10:14:54.643914+0000","last_peered":"2024-09-21T10:14:54.643914+0000","last_clean":"2024-09-21T10:14:54.643914+0000","last_became_active":"2024-09-21T10:10:13.639801+0000","last_became_peered":"2024-09-21T10:10:13.639801+0000","last_unstale":"2024-09-21T10:14:54.643914+0000","last_undegraded":"2024-09-21T10:14:54.643914+0000","last_fullsized":"2024-09-21T10:14:54.643914+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:25.153985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1339,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644782+0000","last_change":"2024-09-21T10:12:25.713745+0000","last_active":"2024-09-21T10:14:54.644782+0000","last_peered":"2024-09-21T10:14:54.644782+0000","last_clean":"2024-09-21T10:14:54.644782+0000","last_became_active":"2024-09-21T10:12:25.698011+0000","last_became_peered":"2024-09-21T10:12:25.698011+0000","last_unstale":"2024-09-21T10:14:54.644782+0000","last_undegraded":"2024-09-21T10:14:54.644782+0000","last_fullsized":"2024-09-21T10:14:54.644782+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:45.276913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23720,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":176,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.765822+0000","last_change":"2024-09-21T10:10:13.635438+0000","last_active":"2024-09-21T10:15:02.765822+0000","last_peered":"2024-09-21T10:15:02.765822+0000","last_clean":"2024-09-21T10:15:02.765822+0000","last_became_active":"2024-09-21T10:10:13.634268+0000","last_became_peered":"2024-09-21T10:10:13.634268+0000","last_unstale":"2024-09-21T10:15:02.765822+0000","last_undegraded":"2024-09-21T10:15:02.765822+0000","last_fullsized":"2024-09-21T10:15:02.765822+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:00:07.541446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":59,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.650542+0000","last_change":"2024-09-21T10:12:25.717602+0000","last_active":"2024-09-21T10:14:54.650542+0000","last_peered":"2024-09-21T10:14:54.650542+0000","last_clean":"2024-09-21T10:14:54.650542+0000","last_became_active":"2024-09-21T10:12:25.705211+0000","last_became_peered":"2024-09-21T10:12:25.705211+0000","last_unstale":"2024-09-21T10:14:54.650542+0000","last_undegraded":"2024-09-21T10:14:54.650542+0000","last_fullsized":"2024-09-21T10:14:54.650542+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:30.822884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"288'100","reported_seq":182,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.783258+0000","last_change":"2024-09-21T10:12:25.698905+0000","last_active":"2024-09-21T10:15:02.783258+0000","last_peered":"2024-09-21T10:15:02.783258+0000","last_clean":"2024-09-21T10:15:02.783258+0000","last_became_active":"2024-09-21T10:12:25.698552+0000","last_became_peered":"2024-09-21T10:12:25.698552+0000","last_unstale":"2024-09-21T10:15:02.783258+0000","last_undegraded":"2024-09-21T10:15:02.783258+0000","last_fullsized":"2024-09-21T10:15:02.783258+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.389965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.389965+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:36.307701+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":63,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.650989+0000","last_change":"2024-09-21T10:12:50.588976+0000","last_active":"2024-09-21T10:14:54.650989+0000","last_peered":"2024-09-21T10:14:54.650989+0000","last_clean":"2024-09-21T10:14:54.650989+0000","last_became_active":"2024-09-21T10:12:50.588232+0000","last_became_peered":"2024-09-21T10:12:50.588232+0000","last_unstale":"2024-09-21T10:14:54.650989+0000","last_undegraded":"2024-09-21T10:14:54.650989+0000","last_fullsized":"2024-09-21T10:14:54.650989+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:44:25.302039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1989,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.761854+0000","last_change":"2024-09-21T10:12:25.707393+0000","last_active":"2024-09-21T10:15:02.761854+0000","last_peered":"2024-09-21T10:15:02.761854+0000","last_clean":"2024-09-21T10:15:02.761854+0000","last_became_active":"2024-09-21T10:12:25.706457+0000","last_became_peered":"2024-09-21T10:12:25.706457+0000","last_unstale":"2024-09-21T10:15:02.761854+0000","last_undegraded":"2024-09-21T10:15:02.761854+0000","last_fullsized":"2024-09-21T10:15:02.761854+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:56.520799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2054,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.760048+0000","last_change":"2024-09-21T10:12:50.591554+0000","last_active":"2024-09-21T10:15:02.760048+0000","last_peered":"2024-09-21T10:15:02.760048+0000","last_clean":"2024-09-21T10:15:02.760048+0000","last_became_active":"2024-09-21T10:12:50.591171+0000","last_became_peered":"2024-09-21T10:12:50.591171+0000","last_unstale":"2024-09-21T10:15:02.760048+0000","last_undegraded":"2024-09-21T10:15:02.760048+0000","last_fullsized":"2024-09-21T10:15:02.760048+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:13:22.080704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2024,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.761624+0000","last_change":"2024-09-21T10:13:41.788034+0000","last_active":"2024-09-21T10:15:02.761624+0000","last_peered":"2024-09-21T10:15:02.761624+0000","last_clean":"2024-09-21T10:15:02.761624+0000","last_became_active":"2024-09-21T10:13:41.787698+0000","last_became_peered":"2024-09-21T10:13:41.787698+0000","last_unstale":"2024-09-21T10:15:02.761624+0000","last_undegraded":"2024-09-21T10:15:02.761624+0000","last_fullsized":"2024-09-21T10:15:02.761624+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:08.268799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.11","version":"1311'5438","reported_seq":9214,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.087312+0000","last_change":"2024-09-21T10:14:44.616174+0000","last_active":"2024-09-21T10:15:08.087312+0000","last_peered":"2024-09-21T10:15:08.087312+0000","last_clean":"2024-09-21T10:15:08.087312+0000","last_became_active":"2024-09-21T10:14:44.611934+0000","last_became_peered":"2024-09-21T10:14:44.611934+0000","last_unstale":"2024-09-21T10:15:08.087312+0000","last_undegraded":"2024-09-21T10:15:08.087312+0000","last_fullsized":"2024-09-21T10:15:08.087312+0000","mapping_epoch":1304,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1305,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5438,"log_dups_size":0,"ondisk_log_size":5438,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252936192,"num_objects":3860,"num_object_clones":0,"num_object_copies":11580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3860,"num_write_kb":247008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1851,"num_bytes_recovered":121307136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.647197+0000","last_change":"2024-09-21T10:10:13.634975+0000","last_active":"2024-09-21T10:14:54.647197+0000","last_peered":"2024-09-21T10:14:54.647197+0000","last_clean":"2024-09-21T10:14:54.647197+0000","last_became_active":"2024-09-21T10:10:13.634667+0000","last_became_peered":"2024-09-21T10:10:13.634667+0000","last_unstale":"2024-09-21T10:14:54.647197+0000","last_undegraded":"2024-09-21T10:14:54.647197+0000","last_fullsized":"2024-09-21T10:14:54.647197+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:52.296990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1995,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.762291+0000","last_change":"2024-09-21T10:08:45.520524+0000","last_active":"2024-09-21T10:15:02.762291+0000","last_peered":"2024-09-21T10:15:02.762291+0000","last_clean":"2024-09-21T10:15:02.762291+0000","last_became_active":"2024-09-21T10:08:45.520119+0000","last_became_peered":"2024-09-21T10:08:45.520119+0000","last_unstale":"2024-09-21T10:15:02.762291+0000","last_undegraded":"2024-09-21T10:15:02.762291+0000","last_fullsized":"2024-09-21T10:15:02.762291+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:55.559438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"1311'5456","reported_seq":9380,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.588014+0000","last_change":"2024-09-21T10:14:41.105467+0000","last_active":"2024-09-21T10:15:08.588014+0000","last_peered":"2024-09-21T10:15:08.588014+0000","last_clean":"2024-09-21T10:15:08.588014+0000","last_became_active":"2024-09-21T10:14:25.660440+0000","last_became_peered":"2024-09-21T10:14:25.660440+0000","last_unstale":"2024-09-21T10:15:08.588014+0000","last_undegraded":"2024-09-21T10:15:08.588014+0000","last_fullsized":"2024-09-21T10:15:08.588014+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253362176,"num_objects":3866,"num_object_clones":0,"num_object_copies":11598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3866,"num_write_kb":247424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1926,"num_bytes_recovered":126222336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2066,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.764030+0000","last_change":"2024-09-21T10:12:25.703790+0000","last_active":"2024-09-21T10:15:02.764030+0000","last_peered":"2024-09-21T10:15:02.764030+0000","last_clean":"2024-09-21T10:15:02.764030+0000","last_became_active":"2024-09-21T10:12:25.702989+0000","last_became_peered":"2024-09-21T10:12:25.702989+0000","last_unstale":"2024-09-21T10:15:02.764030+0000","last_undegraded":"2024-09-21T10:15:02.764030+0000","last_fullsized":"2024-09-21T10:15:02.764030+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:15.338067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":162,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.760248+0000","last_change":"2024-09-21T10:08:45.523144+0000","last_active":"2024-09-21T10:15:02.760248+0000","last_peered":"2024-09-21T10:15:02.760248+0000","last_clean":"2024-09-21T10:15:02.760248+0000","last_became_active":"2024-09-21T10:08:45.522526+0000","last_became_peered":"2024-09-21T10:08:45.522526+0000","last_unstale":"2024-09-21T10:15:02.760248+0000","last_undegraded":"2024-09-21T10:15:02.760248+0000","last_fullsized":"2024-09-21T10:15:02.760248+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:45.926968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.13","version":"1311'5618","reported_seq":5729,"reported_epoch":1311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:15:07.930075+0000","last_change":"2024-09-21T10:14:49.255431+0000","last_active":"2024-09-21T10:15:07.930075+0000","last_peered":"2024-09-21T10:15:07.930075+0000","last_clean":"2024-09-21T10:14:46.612265+0000","last_became_active":"2024-09-21T10:14:49.249254+0000","last_became_peered":"2024-09-21T10:14:49.249254+0000","last_unstale":"2024-09-21T10:15:07.930075+0000","last_undegraded":"2024-09-21T10:14:48.745923+0000","last_fullsized":"2024-09-21T10:14:48.630141+0000","mapping_epoch":1307,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"1300'3938","last_scrub_stamp":"2024-09-21T10:14:22.265794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:22.265794+0000","objects_scrubbed":2294,"log_size":5618,"log_dups_size":0,"ondisk_log_size":5618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":688,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261816320,"num_objects":3995,"num_object_clones":0,"num_object_copies":11985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1701,"num_write_kb":108864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[2,1],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"1,2","objects":683},{"shards":"2,3,5","objects":3312}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1988,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.763042+0000","last_change":"2024-09-21T10:08:45.515262+0000","last_active":"2024-09-21T10:15:02.763042+0000","last_peered":"2024-09-21T10:15:02.763042+0000","last_clean":"2024-09-21T10:15:02.763042+0000","last_became_active":"2024-09-21T10:08:45.514926+0000","last_became_peered":"2024-09-21T10:08:45.514926+0000","last_unstale":"2024-09-21T10:15:02.763042+0000","last_undegraded":"2024-09-21T10:15:02.763042+0000","last_fullsized":"2024-09-21T10:15:02.763042+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:49.954351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1278,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644081+0000","last_change":"2024-09-21T10:08:45.524864+0000","last_active":"2024-09-21T10:14:54.644081+0000","last_peered":"2024-09-21T10:14:54.644081+0000","last_clean":"2024-09-21T10:14:54.644081+0000","last_became_active":"2024-09-21T10:08:45.524533+0000","last_became_peered":"2024-09-21T10:08:45.524533+0000","last_unstale":"2024-09-21T10:14:54.644081+0000","last_undegraded":"2024-09-21T10:14:54.644081+0000","last_fullsized":"2024-09-21T10:14:54.644081+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:52.547302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"1311'5548","reported_seq":10210,"reported_epoch":1311,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-21T10:15:08.097930+0000","last_change":"2024-09-21T10:14:43.408784+0000","last_active":"2024-09-21T10:15:08.097930+0000","last_peered":"2024-09-21T10:15:08.097930+0000","last_clean":"2024-09-21T10:14:13.393048+0000","last_became_active":"2024-09-21T10:14:16.711167+0000","last_became_peered":"2024-09-21T10:14:16.711167+0000","last_unstale":"2024-09-21T10:15:08.097930+0000","last_undegraded":"2024-09-21T10:15:08.097930+0000","last_fullsized":"2024-09-21T10:14:16.599839+0000","mapping_epoch":1299,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256278528,"num_objects":3911,"num_object_clones":0,"num_object_copies":11733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2595,"num_objects_unfound":0,"num_objects_dirty":3911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3911,"num_write_kb":250272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1146,"num_bytes_recovered":75104256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[0,2],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":865},{"shards":"0,2","objects":3046}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2015,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.783437+0000","last_change":"2024-09-21T10:08:45.525340+0000","last_active":"2024-09-21T10:15:02.783437+0000","last_peered":"2024-09-21T10:15:02.783437+0000","last_clean":"2024-09-21T10:15:02.783437+0000","last_became_active":"2024-09-21T10:08:45.525023+0000","last_became_peered":"2024-09-21T10:08:45.525023+0000","last_unstale":"2024-09-21T10:15:02.783437+0000","last_undegraded":"2024-09-21T10:15:02.783437+0000","last_fullsized":"2024-09-21T10:15:02.783437+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:37.674092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":2065,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.768216+0000","last_change":"2024-09-21T10:12:25.700804+0000","last_active":"2024-09-21T10:15:02.768216+0000","last_peered":"2024-09-21T10:15:02.768216+0000","last_clean":"2024-09-21T10:15:02.768216+0000","last_became_active":"2024-09-21T10:12:25.700390+0000","last_became_peered":"2024-09-21T10:12:25.700390+0000","last_unstale":"2024-09-21T10:15:02.768216+0000","last_undegraded":"2024-09-21T10:15:02.768216+0000","last_fullsized":"2024-09-21T10:15:02.768216+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:07:23.705541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.15","version":"1311'5444","reported_seq":5557,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:07.918286+0000","last_change":"2024-09-21T10:14:31.335316+0000","last_active":"2024-09-21T10:15:07.918286+0000","last_peered":"2024-09-21T10:15:07.918286+0000","last_clean":"2024-09-21T10:15:07.918286+0000","last_became_active":"2024-09-21T10:14:05.676464+0000","last_became_peered":"2024-09-21T10:14:05.676464+0000","last_unstale":"2024-09-21T10:15:07.918286+0000","last_undegraded":"2024-09-21T10:15:07.918286+0000","last_fullsized":"2024-09-21T10:15:07.918286+0000","mapping_epoch":1292,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"1303'4138","last_scrub_stamp":"2024-09-21T10:14:31.335247+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:31.335247+0000","objects_scrubbed":2553,"log_size":5444,"log_dups_size":0,"ondisk_log_size":5444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:11:57.788287+0000","scrub_duration":943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254017536,"num_objects":3876,"num_object_clones":0,"num_object_copies":11628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1323,"num_write_kb":84672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1286,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.643829+0000","last_change":"2024-09-21T10:13:41.789883+0000","last_active":"2024-09-21T10:14:54.643829+0000","last_peered":"2024-09-21T10:14:54.643829+0000","last_clean":"2024-09-21T10:14:54.643829+0000","last_became_active":"2024-09-21T10:13:41.789620+0000","last_became_peered":"2024-09-21T10:13:41.789620+0000","last_unstale":"2024-09-21T10:14:54.643829+0000","last_undegraded":"2024-09-21T10:14:54.643829+0000","last_fullsized":"2024-09-21T10:14:54.643829+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:36.869540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":248,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644696+0000","last_change":"2024-09-21T10:08:45.535971+0000","last_active":"2024-09-21T10:14:54.644696+0000","last_peered":"2024-09-21T10:14:54.644696+0000","last_clean":"2024-09-21T10:14:54.644696+0000","last_became_active":"2024-09-21T10:08:45.530341+0000","last_became_peered":"2024-09-21T10:08:45.530341+0000","last_unstale":"2024-09-21T10:14:54.644696+0000","last_undegraded":"2024-09-21T10:14:54.644696+0000","last_fullsized":"2024-09-21T10:14:54.644696+0000","mapping_epoch":1236,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:58:34.894434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"1311'5375","reported_seq":5454,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.046651+0000","last_change":"2024-09-21T10:14:05.662622+0000","last_active":"2024-09-21T10:15:08.046651+0000","last_peered":"2024-09-21T10:15:08.046651+0000","last_clean":"2024-09-21T10:15:08.046651+0000","last_became_active":"2024-09-21T10:14:05.660813+0000","last_became_peered":"2024-09-21T10:14:05.660813+0000","last_unstale":"2024-09-21T10:15:08.046651+0000","last_undegraded":"2024-09-21T10:15:08.046651+0000","last_fullsized":"2024-09-21T10:15:08.046651+0000","mapping_epoch":1292,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5375,"log_dups_size":0,"ondisk_log_size":5375,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245923840,"num_objects":3753,"num_object_clones":0,"num_object_copies":11259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3753,"num_write_kb":240160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1256,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.643881+0000","last_change":"2024-09-21T10:08:25.430667+0000","last_active":"2024-09-21T10:14:54.643881+0000","last_peered":"2024-09-21T10:14:54.643881+0000","last_clean":"2024-09-21T10:14:54.643881+0000","last_became_active":"2024-09-21T10:08:25.430357+0000","last_became_peered":"2024-09-21T10:08:25.430357+0000","last_unstale":"2024-09-21T10:14:54.643881+0000","last_undegraded":"2024-09-21T10:14:54.643881+0000","last_fullsized":"2024-09-21T10:14:54.643881+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:19.972020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":250,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.644859+0000","last_change":"2024-09-21T10:10:13.639231+0000","last_active":"2024-09-21T10:14:54.644859+0000","last_peered":"2024-09-21T10:14:54.644859+0000","last_clean":"2024-09-21T10:14:54.644859+0000","last_became_active":"2024-09-21T10:10:13.622497+0000","last_became_peered":"2024-09-21T10:10:13.622497+0000","last_unstale":"2024-09-21T10:14:54.644859+0000","last_undegraded":"2024-09-21T10:14:54.644859+0000","last_fullsized":"2024-09-21T10:14:54.644859+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:31:46.263003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"1311'5520","reported_seq":5576,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:08.086254+0000","last_change":"2024-09-21T10:14:54.654799+0000","last_active":"2024-09-21T10:15:08.086254+0000","last_peered":"2024-09-21T10:15:08.086254+0000","last_clean":"2024-09-21T10:15:08.086254+0000","last_became_active":"2024-09-21T10:14:54.651071+0000","last_became_peered":"2024-09-21T10:14:54.651071+0000","last_unstale":"2024-09-21T10:15:08.086254+0000","last_undegraded":"2024-09-21T10:15:08.086254+0000","last_fullsized":"2024-09-21T10:15:08.086254+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5520,"log_dups_size":0,"ondisk_log_size":5520,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:15:55.251758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254312448,"num_objects":3881,"num_object_clones":0,"num_object_copies":11643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3881,"num_write_kb":248352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1973,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.762195+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:15:02.762195+0000","last_peered":"2024-09-21T10:15:02.762195+0000","last_clean":"2024-09-21T10:15:02.762195+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:15:02.762195+0000","last_undegraded":"2024-09-21T10:15:02.762195+0000","last_fullsized":"2024-09-21T10:15:02.762195+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":1989,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.762877+0000","last_change":"2024-09-21T10:10:13.651599+0000","last_active":"2024-09-21T10:15:02.762877+0000","last_peered":"2024-09-21T10:15:02.762877+0000","last_clean":"2024-09-21T10:15:02.762877+0000","last_became_active":"2024-09-21T10:10:13.651250+0000","last_became_peered":"2024-09-21T10:10:13.651250+0000","last_unstale":"2024-09-21T10:15:02.762877+0000","last_undegraded":"2024-09-21T10:15:02.762877+0000","last_fullsized":"2024-09-21T10:15:02.762877+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:34.100056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"1311'5503","reported_seq":5583,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:07.986444+0000","last_change":"2024-09-21T10:14:05.670190+0000","last_active":"2024-09-21T10:15:07.986444+0000","last_peered":"2024-09-21T10:15:07.986444+0000","last_clean":"2024-09-21T10:15:07.986444+0000","last_became_active":"2024-09-21T10:14:05.665682+0000","last_became_peered":"2024-09-21T10:14:05.665682+0000","last_unstale":"2024-09-21T10:15:07.986444+0000","last_undegraded":"2024-09-21T10:15:07.986444+0000","last_fullsized":"2024-09-21T10:15:07.986444+0000","mapping_epoch":1292,"log_start":"0'0","ondisk_log_start":"0'0","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253624320,"num_objects":3870,"num_object_clones":0,"num_object_copies":11610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3870,"num_write_kb":247680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1299,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-09-21T10:14:54.643664+0000","last_change":"2024-09-21T10:10:13.639970+0000","last_active":"2024-09-21T10:14:54.643664+0000","last_peered":"2024-09-21T10:14:54.643664+0000","last_clean":"2024-09-21T10:14:54.643664+0000","last_became_active":"2024-09-21T10:10:13.639671+0000","last_became_peered":"2024-09-21T10:10:13.639671+0000","last_unstale":"2024-09-21T10:14:54.643664+0000","last_undegraded":"2024-09-21T10:14:54.643664+0000","last_fullsized":"2024-09-21T10:14:54.643664+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:53.941605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":2069,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-09-21T10:15:02.760136+0000","last_change":"2024-09-21T10:12:50.594933+0000","last_active":"2024-09-21T10:15:02.760136+0000","last_peered":"2024-09-21T10:15:02.760136+0000","last_clean":"2024-09-21T10:15:02.760136+0000","last_became_active":"2024-09-21T10:12:50.578183+0000","last_became_peered":"2024-09-21T10:12:50.578183+0000","last_unstale":"2024-09-21T10:15:02.760136+0000","last_undegraded":"2024-09-21T10:15:02.760136+0000","last_fullsized":"2024-09-21T10:15:02.760136+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:17.508182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":8095825920,"num_objects":123532,"num_object_clones":0,"num_object_copies":370596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6624,"num_objects_misplaced":2595,"num_objects_unfound":0,"num_objects_dirty":123532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":111599,"num_write_kb":7142368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4926,"num_bytes_recovered":322830336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24734007296,"data_stored":24734007296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142377,"ondisk_log_size":142377,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":24716,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34026,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":26,"num_read_kb":68,"num_write":13,"num_write_kb":148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":644038,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397991,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3408152,"kb_used_data":2503504,"kb_used_omap":1,"kb_used_meta":904638,"kb_avail":90337000,"statfs":{"total":95995035648,"available":92505088000,"internally_reserved":0,"allocated":2563588096,"data_stored":2560405579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":926349656},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594220,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4717784,"kb_used_data":3826064,"kb_used_omap":1,"kb_used_meta":891710,"kb_avail":89027368,"statfs":{"total":95995035648,"available":91164024832,"internally_reserved":0,"allocated":3917889536,"data_stored":3914707990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913111384},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790449,"num_pgs":10,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1856256,"kb_used_data":957944,"kb_used_omap":1,"kb_used_meta":898302,"kb_avail":91888896,"statfs":{"total":95995035648,"available":94094229504,"internally_reserved":0,"allocated":980934656,"data_stored":977771758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919861592},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1224,"seq":5257039970386,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4513812,"kb_used_data":4173132,"kb_used_omap":290,"kb_used_meta":340381,"kb_avail":89231340,"statfs":{"total":95995035648,"available":91372892160,"internally_reserved":0,"allocated":4273287168,"data_stored":4270108661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":297377,"internal_metadata":348550751},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999318,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3390024,"kb_used_data":2444032,"kb_used_omap":25,"kb_used_meta":945958,"kb_avail":90355128,"statfs":{"total":95995035648,"available":92523651072,"internally_reserved":0,"allocated":2502688768,"data_stored":2499518383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":968661959},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111261,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4605000,"kb_used_data":3673088,"kb_used_omap":31,"kb_used_meta":931872,"kb_avail":89140152,"statfs":{"total":95995035648,"available":91279515648,"internally_reserved":0,"allocated":3761242112,"data_stored":3758072171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":954237160},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":513,"seq":2203318223204,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2197632,"kb_used_data":1708280,"kb_used_omap":26,"kb_used_meta":489317,"kb_avail":91547520,"statfs":{"total":95995035648,"available":93744660480,"internally_reserved":0,"allocated":1749278720,"data_stored":1746115842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27636,"internal_metadata":501060620},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335147,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5745860,"kb_used_data":4917308,"kb_used_omap":31,"kb_used_meta":828512,"kb_avail":87999292,"statfs":{"total":95995035648,"available":90111275008,"internally_reserved":0,"allocated":5035323392,"data_stored":5032156991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":848396384},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":5070585856,"data_stored":5070585856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1744175104,"data_stored":1744175104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3765829632,"data_stored":3765829632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2514157568,"data_stored":2514157568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4282318848,"data_stored":4282318848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":972161024,"data_stored":972161024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3940548608,"data_stored":3940548608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2444230656,"data_stored":2444230656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:15:15.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_5 pgp_num 26 2024-09-21T10:15:16.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:16.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:16.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:16.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:16.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:15 smithi017 ceph-mon[102913]: pgmap v2035: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 87 B/s rd, 64 MiB/s wr, 1.30k op/s; 6400/368382 objects degraded (1.737%); 1950/368382 objects misplaced (0.529%); 3.4 MiB/s, 54 objects/s recovering 2024-09-21T10:15:16.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:15 smithi017 ceph-mon[105151]: pgmap v2035: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 87 B/s rd, 64 MiB/s wr, 1.30k op/s; 6400/368382 objects degraded (1.737%); 1950/368382 objects misplaced (0.529%); 3.4 MiB/s, 54 objects/s recovering 2024-09-21T10:15:16.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:16.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:15 smithi134 ceph-mon[90085]: pgmap v2035: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 87 B/s rd, 64 MiB/s wr, 1.30k op/s; 6400/368382 objects degraded (1.737%); 1950/368382 objects misplaced (0.529%); 3.4 MiB/s, 54 objects/s recovering 2024-09-21T10:15:16.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:16.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:16.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:16.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:16.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:16.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:16.615 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:16.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:16.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:16.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:16.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:16.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:16.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:16.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:16.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:17.039 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:17.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:17.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.174 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:17.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:17.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:17.311 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:17.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:17.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:17.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:17.550 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:17.590 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:17.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:17.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:17.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:17.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:17.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:17.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:17.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:17.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:18.018 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:18.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:18.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:18.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:17 smithi017 ceph-mon[102913]: pgmap v2036: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 1.09k op/s; 6400/363951 objects degraded (1.758%); 1950/363951 objects misplaced (0.536%); 2.8 MiB/s, 44 objects/s recovering 2024-09-21T10:15:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:17 smithi017 ceph-mon[105151]: pgmap v2036: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 1.09k op/s; 6400/363951 objects degraded (1.758%); 1950/363951 objects misplaced (0.536%); 2.8 MiB/s, 44 objects/s recovering 2024-09-21T10:15:18.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:17 smithi134 ceph-mon[90085]: pgmap v2036: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.4 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 1.09k op/s; 6400/363951 objects degraded (1.758%); 1950/363951 objects misplaced (0.536%); 2.8 MiB/s, 44 objects/s recovering 2024-09-21T10:15:18.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:18.294 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:18.395 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:18.426 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:18.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:18.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:18.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:18.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:18.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:18.709 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:18.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:18.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:18.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:18.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:18.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:18 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6400/363951 objects degraded (1.758%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:19.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.150 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:19.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:18 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6400/363951 objects degraded (1.758%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:18 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6400/363951 objects degraded (1.758%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:15:19.251 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:19.282 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:19.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:19.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:19.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:19.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:19.568 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:19.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:19.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:19.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:19.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:19.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:19.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:19.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.980 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:19.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:20.080 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:20.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:20.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:19 smithi017 ceph-mon[102913]: pgmap v2037: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 1.18k op/s; 6032/355614 objects degraded (1.696%); 1292/355614 objects misplaced (0.363%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T10:15:20.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:19 smithi017 ceph-mon[105151]: pgmap v2037: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 1.18k op/s; 6032/355614 objects degraded (1.696%); 1292/355614 objects misplaced (0.363%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T10:15:20.214 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:20.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:19 smithi134 ceph-mon[90085]: pgmap v2037: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 1.18k op/s; 6032/355614 objects degraded (1.696%); 1292/355614 objects misplaced (0.363%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T10:15:20.259 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:20.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:20.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:20.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:20.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:15:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:15:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:15:20.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:20.536 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:20.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:20.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:20.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:20.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:20.786 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:20.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:20.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:20.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:20.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:20.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:21.058 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:21.088 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:21.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:21.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:21.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:21.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:21.329 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:21.361 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:21.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:21 smithi017 ceph-mon[102913]: pgmap v2038: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 998 op/s; 6032/352515 objects degraded (1.711%); 1292/352515 objects misplaced (0.367%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:15:21.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:21.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:21.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:21 smithi017 ceph-mon[105151]: pgmap v2038: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 998 op/s; 6032/352515 objects degraded (1.711%); 1292/352515 objects misplaced (0.367%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:15:21.463 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:21 smithi134 ceph-mon[90085]: pgmap v2038: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 998 op/s; 6032/352515 objects degraded (1.711%); 1292/352515 objects misplaced (0.367%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:15:21.496 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:21.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:21.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:21.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:21.739 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:21.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:21.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:21.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:21.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:22.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:22.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:22.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:22.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:22.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:22.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:22.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:22.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:22.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:22.416 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:22.444 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:22.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:22.545 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:22.575 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:22.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:22.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:22.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:22.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:22.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:22.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:22.960 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:22.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.989 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:22.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:23.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:23.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:23.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:23.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:23.271 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:23.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:23.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:23.412 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:23.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:23.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:23.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:23.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:23.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:23.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:23.799 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:23.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.835 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:23.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:23.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:23 smithi017 ceph-mon[102913]: pgmap v2039: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 1.03k op/s; 6032/347766 objects degraded (1.734%); 1292/347766 objects misplaced (0.372%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:15:23.903 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:23 smithi017 ceph-mon[105151]: pgmap v2039: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 1.03k op/s; 6032/347766 objects degraded (1.734%); 1292/347766 objects misplaced (0.372%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:15:23.936 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:23.965 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:23.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:23 smithi134 ceph-mon[90085]: pgmap v2039: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 1.03k op/s; 6032/347766 objects degraded (1.734%); 1292/347766 objects misplaced (0.372%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:15:24.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:24.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:24.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:24.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:24.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:24.383 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.413 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:24.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:24.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:24.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:24.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:24.662 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:24.689 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:24.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:24 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6032/347766 objects degraded (1.734%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:24.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:24 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6032/347766 objects degraded (1.734%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:24.713 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:15:24.713 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:15:24.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:15:24.713 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:15:24.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.713 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263821 8.f 8:f02fefb5:::benchmark_data_smithi017_209235_object130915:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1223)", 2024-09-21T10:15:24.713 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.418315+0000", 2024-09-21T10:15:24.713 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 420.28756492000002, 2024-09-21T10:15:24.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2255271790000002, 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263821 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418315+0000", 2024-09-21T10:15:24.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418315+0000", 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418317+0000", 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.468e-06 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418321+0000", 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7570000000000004e-06 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418333+0000", 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1931e-05 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418341+0000", 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5329999999999997e-06 2024-09-21T10:15:24.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418739+0000", 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00039836800000000002 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.418859+0000", 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119894 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.419410+0000", 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00055092599999999998 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.423594+0000", 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041834790000000004 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.436988+0000", 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013394368 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.437000+0000", 2024-09-21T10:15:24.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2503999999999999e-05 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.428163+0000", 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99116300300000004 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.428175+0000", 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1568e-05 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.643733+0000", 2024-09-21T10:15:24.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2155582730000001 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.643842+0000", 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109107 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263824 8.f 8:fed575a8:::benchmark_data_smithi017_209235_object130918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1223)", 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.424585+0000", 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 420.28129557300002, 2024-09-21T10:15:24.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2195379640000001, 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263824 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424585+0000", 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.721 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424585+0000", 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424587+0000", 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0449999999999999e-06 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424592+0000", 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4709999999999998e-06 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424613+0000", 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1331e-05 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424622+0000", 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3769999999999996e-06 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424765+0000", 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014287600000000001 2024-09-21T10:15:24.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.424915+0000", 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015034299999999999 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2", 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.425037+0000", 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012184399999999999 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.437031+0000", 2024-09-21T10:15:24.724 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011994461999999999 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.437040+0000", 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7909999999999993e-06 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.428208+0000", 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99116771800000003 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.725 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.428217+0000", 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1079999999999999e-06 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.644076+0000", 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.215859064 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.644123+0000", 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6533999999999999e-05 2024-09-21T10:15:24.726 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263830 8.b 8:dc611968:::benchmark_data_smithi017_209235_object130924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.441220+0000", 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 420.26466004899999, 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1669228899999999, 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:15:24.727 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263830 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441220+0000", 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441220+0000", 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441222+0000", 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.762e-06 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441227+0000", 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1259999999999997e-06 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441241+0000", 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3661e-05 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.729 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441249+0000", 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.422e-06 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441313+0000", 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3777999999999998e-05 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.441325+0000", 2024-09-21T10:15:24.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2349e-05 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.426634+0000", 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98530858499999996 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.426652+0000", 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8334000000000001e-05 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.595823+0000", 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.169171245 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.595979+0000", 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015592700000000001 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.596130+0000", 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015037399999999999 2024-09-21T10:15:24.732 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.599707+0000", 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035773689999999999 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.603014+0000", 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033070360000000002 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.606794+0000", 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003779691 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608021+0000", 2024-09-21T10:15:24.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001227676 2024-09-21T10:15:24.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608143+0000", 2024-09-21T10:15:24.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121555 2024-09-21T10:15:24.734 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263835 8.b 8:d7eca9d4:::benchmark_data_smithi017_209235_object130929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.454536+0000", 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 420.251343905, 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1538747279999999, 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.735 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.736 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.736 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.736 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:15:24.736 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:15:24.736 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263835 2024-09-21T10:15:24.736 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.736 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.736 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454536+0000", 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454536+0000", 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454538+0000", 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0049999999999999e-06 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454544+0000", 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4260000000000002e-06 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454558+0000", 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4073000000000001e-05 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454564+0000", 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2480000000000001e-06 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454623+0000", 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8721999999999998e-05 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.454631+0000", 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2749999999999995e-06 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.427335+0000", 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97270446799999999 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.427346+0000", 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0023999999999999e-05 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.599745+0000", 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1723999119999999 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.599872+0000", 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126871 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:15:24.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.600047+0000", 2024-09-21T10:15:24.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017517100000000001 2024-09-21T10:15:24.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608161+0000", 2024-09-21T10:15:24.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081138900000000003 2024-09-21T10:15:24.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608210+0000", 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9044e-05 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608261+0000", 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0562000000000001e-05 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608317+0000", 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5630000000000001e-05 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608411+0000", 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4407000000000006e-05 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263838 8.b 8:d9aed5fa:::benchmark_data_smithi017_209235_object130932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.456576+0000", 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 420.249304671, 2024-09-21T10:15:24.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1539254689999998, 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263838 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456576+0000", 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456576+0000", 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456577+0000", 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.251e-06 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456582+0000", 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7790000000000003e-06 2024-09-21T10:15:24.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456594+0000", 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2867000000000001e-05 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456602+0000", 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9640000000000003e-06 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456677+0000", 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4278999999999998e-05 2024-09-21T10:15:24.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.456686+0000", 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4469999999999995e-06 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.427371+0000", 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97068533099999998 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.427380+0000", 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3790000000000006e-06 2024-09-21T10:15:24.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.603501+0000", 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1761207929999999 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.603651+0000", 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015053700000000001 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.603792+0000", 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014081200000000001 2024-09-21T10:15:24.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608224+0000", 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044316620000000003 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.608466+0000", 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024204399999999999 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.610334+0000", 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001868607 2024-09-21T10:15:24.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.610399+0000", 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4605999999999998e-05 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.610501+0000", 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102111 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57258 9.c 9:337f61f9:::benchmark_data_smithi017_237924_object57257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.412546+0000", 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 360.29333406000001, 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2051300349999998, 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57258 2024-09-21T10:15:24.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412546+0000", 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412546+0000", 2024-09-21T10:15:24.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412548+0000", 2024-09-21T10:15:24.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.654e-06 2024-09-21T10:15:24.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412567+0000", 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9585999999999998e-05 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412580+0000", 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2644e-05 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.412588+0000", 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.746e-06 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.414696+0000", 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021078949999999998 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.414847+0000", 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000151731 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:15:24.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.414992+0000", 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144349 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.421028+0000", 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060366220000000002 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450408+0000", 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029379563000000001 2024-09-21T10:15:24.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450432+0000", 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.393e-05 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.422399+0000", 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.971966679 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.422410+0000", 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1615e-05 2024-09-21T10:15:24.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.617503+0000", 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1950931549999999 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.617676+0000", 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017286600000000001 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57263 9.c 9:35231ded:::benchmark_data_smithi017_237924_object57262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.415713+0000", 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 360.29016764199997, 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2043443279999999, 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57263 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415713+0000", 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415713+0000", 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415714+0000", 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3149999999999999e-06 2024-09-21T10:15:24.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415737+0000", 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3323999999999999e-05 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415748+0000", 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0557000000000001e-05 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.415755+0000", 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3209999999999999e-06 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.417937+0000", 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021822230000000001 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.418089+0000", 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015200700000000001 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:15:24.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.418233+0000", 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014391100000000001 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.421035+0000", 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002802157 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450466+0000", 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029430464999999999 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450474+0000", 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3310000000000004e-06 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.422444+0000", 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97196981699999996 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.422453+0000", 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3000000000000007e-06 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.619820+0000", 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1973671800000001 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.620057+0000", 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023641999999999999 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57272 9.c 9:3bb2c359:::benchmark_data_smithi017_237924_object57271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:15:24.765 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.424002+0000", 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 360.281878393, 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1984504490000001, 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57272 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424002+0000", 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424002+0000", 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424004+0000", 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9489999999999999e-06 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424030+0000", 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6444999999999998e-05 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424049+0000", 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8681999999999998e-05 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424058+0000", 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1819999999999995e-06 2024-09-21T10:15:24.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424142+0000", 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3628999999999995e-05 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424420+0000", 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027829800000000001 2024-09-21T10:15:24.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:15:24.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424592+0000", 2024-09-21T10:15:24.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000171931 2024-09-21T10:15:24.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450503+0000", 2024-09-21T10:15:24.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025911338999999999 2024-09-21T10:15:24.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450511+0000", 2024-09-21T10:15:24.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9719999999999992e-06 2024-09-21T10:15:24.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.424670+0000", 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97415910100000003 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.424683+0000", 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2357e-05 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.620183+0000", 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1954999120000001 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.622452+0000", 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002269652 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57278 9.c 9:32156efe:::benchmark_data_smithi017_237924_object57277:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.439362+0000", 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 360.26651778199999, 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.195950984, 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.774 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57278 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439362+0000", 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439362+0000", 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439364+0000", 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2720000000000001e-06 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439395+0000", 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1324e-05 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439408+0000", 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2524000000000001e-05 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439416+0000", 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9730000000000003e-06 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450534+0000", 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011117392 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.450541+0000", 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5429999999999996e-06 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.424766+0000", 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97422412000000003 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.424774+0000", 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9139999999999997e-06 2024-09-21T10:15:24.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.622588+0000", 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1978139999999999 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.622840+0000", 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000251473 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.623023+0000", 2024-09-21T10:15:24.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018283899999999999 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.627435+0000", 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044125550000000003 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.631224+0000", 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037888930000000002 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.632444+0000", 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001219695 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.632578+0000", 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013418999999999999 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.635313+0000", 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027352769999999999 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.781 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57268 9.3 9:ce6a408f:::benchmark_data_smithi017_237924_object57267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.439429+0000", 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 360.26645147200003, 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2849673930000001, 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:15:24.782 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57268 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439429+0000", 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439429+0000", 2024-09-21T10:15:24.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439429+0000", 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.82e-07 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439727+0000", 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029737900000000001 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439741+0000", 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4508999999999999e-05 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439748+0000", 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2389999999999996e-06 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.440593+0000", 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00084440299999999997 2024-09-21T10:15:24.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.440602+0000", 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4879999999999996e-06 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.421006+0000", 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98040421300000002 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.421015+0000", 2024-09-21T10:15:24.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8769999999999999e-06 2024-09-21T10:15:24.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.651013+0000", 2024-09-21T10:15:24.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2299973500000001 2024-09-21T10:15:24.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:15:24.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.651585+0000", 2024-09-21T10:15:24.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00057230499999999995 2024-09-21T10:15:24.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.724307+0000", 2024-09-21T10:15:24.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.072722014000000001 2024-09-21T10:15:24.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.724396+0000", 2024-09-21T10:15:24.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9134000000000004e-05 2024-09-21T10:15:24.789 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.789 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57280 9.7 9:e169e579:::benchmark_data_smithi017_237924_object57279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.439765+0000", 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 360.26611550199999, 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.353750952, 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.790 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57280 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439765+0000", 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439765+0000", 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439765+0000", 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4099999999999998e-07 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439789+0000", 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3578999999999999e-05 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439796+0000", 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5480000000000004e-06 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.439802+0000", 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4709999999999998e-06 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.440744+0000", 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00094240600000000002 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.440755+0000", 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0886e-05 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.423951+0000", 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.983195504 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.423961+0000", 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9450000000000005e-06 2024-09-21T10:15:24.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.768747+0000", 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.344786491 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.770629+0000", 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001881364 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.772941+0000", 2024-09-21T10:15:24.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023123430000000001 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.785246+0000", 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012304864 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.788330+0000", 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030839629999999999 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.791240+0000", 2024-09-21T10:15:24.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002909867 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.791286+0000", 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6421000000000002e-05 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.793516+0000", 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022296590000000002 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57290 9.7 9:eed5bd25:::benchmark_data_smithi017_237924_object57289:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.459346+0000", 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 360.24653414900001, 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.339554245, 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57290 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459346+0000", 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459346+0000", 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459347+0000", 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.421e-06 2024-09-21T10:15:24.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459376+0000", 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8235e-05 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459386+0000", 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8360000000000004e-06 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459392+0000", 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1260000000000003e-06 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459552+0000", 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016084000000000001 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.459572+0000", 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9230999999999999e-05 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.423986+0000", 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96441438400000001 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.423994+0000", 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2779999999999993e-06 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.777718+0000", 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3537239160000001 2024-09-21T10:15:24.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.777871+0000", 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000152242 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.778023+0000", 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000151989 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.788345+0000", 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010322964 2024-09-21T10:15:24.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.788384+0000", 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8717999999999998e-05 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.796197+0000", 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0078130860000000003 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.796402+0000", 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020457 2024-09-21T10:15:24.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.798900+0000", 2024-09-21T10:15:24.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002498409 2024-09-21T10:15:24.805 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.805 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.805 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.806 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57295 9.1 9:8fc19906:::benchmark_data_smithi017_237924_object57294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:15:24.806 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.464232+0000", 2024-09-21T10:15:24.806 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 360.24164849499999, 2024-09-21T10:15:24.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.230825754, 2024-09-21T10:15:24.806 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.806 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.806 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.806 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.807 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.807 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.807 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57295 2024-09-21T10:15:24.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.807 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464232+0000", 2024-09-21T10:15:24.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464232+0000", 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464234+0000", 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8700000000000001e-06 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464255+0000", 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1012000000000001e-05 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464272+0000", 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7889000000000001e-05 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464281+0000", 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7649999999999999e-06 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464371+0000", 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9474000000000004e-05 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.464384+0000", 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3433e-05 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.420648+0000", 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95626375299999999 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.420659+0000", 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1449e-05 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.685864+0000", 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.265204947 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.686055+0000", 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000190352 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.686232+0000", 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000177193 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.690565+0000", 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043328280000000004 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.690589+0000", 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4719000000000002e-05 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.692160+0000", 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001570335 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.692225+0000", 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5074999999999996e-05 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.695057+0000", 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028326599999999999 2024-09-21T10:15:24.813 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:170355 9.3 9:c1c098f1:::benchmark_data_smithi017_237924_object77351:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:19.663514+0000", 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 185.042366402, 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0621249859999999, 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.815 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 170355 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.663514+0000", 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.663514+0000", 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.663516+0000", 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7889999999999999e-06 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.663521+0000", 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2349999999999999e-06 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.663532+0000", 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0789e-05 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.663538+0000", 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7390000000000002e-06 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.663644+0000", 2024-09-21T10:15:24.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105758 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.663652+0000", 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3930000000000008e-06 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.671700+0000", 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080471080000000007 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.671711+0000", 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0991e-05 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.722344+0000", 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.050633062 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.722424+0000", 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0175999999999999e-05 2024-09-21T10:15:24.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.722499+0000", 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5131999999999998e-05 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.725486+0000", 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029867309999999998 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.725503+0000", 2024-09-21T10:15:24.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7439e-05 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.725542+0000", 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8788999999999997e-05 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.725592+0000", 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9978000000000002e-05 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.725639+0000", 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6876999999999999e-05 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:170373 9.3 9:c65b5485:::benchmark_data_smithi017_237924_object77369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:19.704654+0000", 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 185.00122642100001, 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0277790170000001, 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.823 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 170373 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.704654+0000", 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.704654+0000", 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.704656+0000", 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.813e-06 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.704660+0000", 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2100000000000003e-06 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.704673+0000", 2024-09-21T10:15:24.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3478e-05 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.704681+0000", 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2029999999999992e-06 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.704746+0000", 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4678000000000007e-05 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.704757+0000", 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0496000000000001e-05 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.725681+0000", 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.020924379 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.725753+0000", 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2086000000000001e-05 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:15:24.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.725846+0000", 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2576000000000003e-05 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.732189+0000", 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063433659999999996 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.732268+0000", 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8453000000000003e-05 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.732327+0000", 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9676000000000003e-05 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.732390+0000", 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2467999999999999e-05 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.732433+0000", 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3135e-05 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.829 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:170377 9.3 9:cd83ec5f:::benchmark_data_smithi017_237924_object77373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:19.708140+0000", 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 184.99774047299999, 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.027076734, 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.830 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 170377 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.708140+0000", 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.708140+0000", 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.708142+0000", 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.813e-06 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.708147+0000", 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6409999999999999e-06 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.708161+0000", 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3903000000000001e-05 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.708170+0000", 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4989999999999994e-06 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.708273+0000", 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103417 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:19.708283+0000", 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0399e-05 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.727784+0000", 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0195010410000001 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.727863+0000", 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8102000000000005e-05 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.727929+0000", 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6428999999999997e-05 2024-09-21T10:15:24.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.732250+0000", 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043207760000000001 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.732274+0000", 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3768999999999999e-05 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.735114+0000", 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028404139999999999 2024-09-21T10:15:24.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.735169+0000", 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4882000000000001e-05 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:20.735216+0000", 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7648999999999997e-05 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173493 9.1 9:838aea2c:::benchmark_data_smithi017_237924_object80489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.675265+0000", 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 180.030614969, 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.426658829, 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173493 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675265+0000", 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675265+0000", 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675266+0000", 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0500000000000004e-07 2024-09-21T10:15:24.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675268+0000", 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.774e-06 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675279+0000", 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0543999999999999e-05 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675285+0000", 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8549999999999998e-06 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.681973+0000", 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066879540000000003 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682093+0000", 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120488 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:15:24.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682203+0000", 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109916 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703356+0000", 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021152608999999999 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703367+0000", 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0736e-05 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695780+0000", 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99241349599999995 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695792+0000", 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1928e-05 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917420+0000", 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2216284660000001 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917536+0000", 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011577400000000001 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.101842+0000", 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18430565700000001 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.101924+0000", 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2126999999999995e-05 2024-09-21T10:15:24.843 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173503 9.1 9:8faf9e71:::benchmark_data_smithi017_237924_object80499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.688561+0000", 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 180.01731912100001, 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4121262919999999, 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.844 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173503 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688561+0000", 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688561+0000", 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688563+0000", 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5820000000000001e-06 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688567+0000", 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4089999999999996e-06 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688580+0000", 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3081999999999999e-05 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688590+0000", 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7960000000000007e-06 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703388+0000", 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014798443999999999 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703395+0000", 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4420000000000003e-06 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695908+0000", 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99251281199999997 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695917+0000", 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9579999999999996e-06 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.924501+0000", 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2285848580000001 2024-09-21T10:15:24.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.924586+0000", 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4695000000000001e-05 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.924673+0000", 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6879999999999995e-05 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929451+0000", 2024-09-21T10:15:24.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047781400000000002 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929490+0000", 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8472e-05 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.932192+0000", 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027021039999999999 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100434+0000", 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.168242063 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100534+0000", 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9809000000000006e-05 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100687+0000", 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015374599999999999 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.851 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173507 9.1 9:8f5fdfea:::benchmark_data_smithi017_237924_object80503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.702211+0000", 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 180.00366948000001, 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3987571480000001, 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173507 2024-09-21T10:15:24.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702211+0000", 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702211+0000", 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702212+0000", 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5659999999999999e-06 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702217+0000", 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9010000000000002e-06 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702230+0000", 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2894000000000001e-05 2024-09-21T10:15:24.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702239+0000", 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9600000000000006e-06 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703414+0000", 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001175075 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703420+0000", 2024-09-21T10:15:24.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3559999999999998e-06 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695939+0000", 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99251901499999995 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695947+0000", 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2259999999999999e-06 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929558+0000", 2024-09-21T10:15:24.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2336112850000001 2024-09-21T10:15:24.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929714+0000", 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015599900000000001 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929843+0000", 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012873499999999999 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.935021+0000", 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051778010000000001 2024-09-21T10:15:24.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.935122+0000", 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101073 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.097445+0000", 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16232375099999999 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100785+0000", 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033395149999999999 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100844+0000", 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9632000000000001e-05 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100968+0000", 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012336399999999999 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.860 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173514 9.7 9:ecfd9956:::benchmark_data_smithi017_237924_object80510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.723765+0000", 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 179.982115669, 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1725742700000001, 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:15:24.861 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173514 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723765+0000", 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723765+0000", 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723766+0000", 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6390000000000001e-06 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723773+0000", 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3810000000000004e-06 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723790+0000", 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7751999999999999e-05 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723800+0000", 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2159999999999995e-06 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723863+0000", 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3297000000000001e-05 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723873+0000", 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8500000000000006e-06 2024-09-21T10:15:24.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.687969+0000", 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96409637599999998 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.687981+0000", 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1549000000000001e-05 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.884912+0000", 2024-09-21T10:15:24.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.196931465 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.885045+0000", 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013280099999999999 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5,7", 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.885179+0000", 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013392699999999999 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.890945+0000", 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057662329999999999 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.891473+0000", 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052811299999999998 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.892458+0000", 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0009847320000000001 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.895831+0000", 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033727309999999999 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.895893+0000", 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2228000000000001e-05 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.896339+0000", 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00044598000000000001 2024-09-21T10:15:24.868 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.869 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.869 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.869 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:15:24.869 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:15:24.869 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:15:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.870 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:24.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:24.917 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:24.971 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:24 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6032/347766 objects degraded (1.734%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:25.004 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:25.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:25.106 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:25.137 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:25.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:25.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:15:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:15:24] "GET /metrics HTTP/1.1" 200 36664 "" "Prometheus/2.51.0" 2024-09-21T10:15:25.239 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:25.263 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:25.364 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:25.399 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:25.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:25.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:25.528 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:25.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:25.594 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:15:25.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:25.673 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:25.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:25.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:25.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.815 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:25.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:25.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:25 smithi017 ceph-mon[102913]: pgmap v2040: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 5.8 MiB/s wr, 1.24k op/s; 5670/335187 objects degraded (1.692%); 480/335187 objects misplaced (0.143%); 6.2 MiB/s, 98 objects/s recovering 2024-09-21T10:15:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:25 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/77322024' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:25 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:25 smithi017 ceph-mon[105151]: pgmap v2040: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 5.8 MiB/s wr, 1.24k op/s; 5670/335187 objects degraded (1.692%); 480/335187 objects misplaced (0.143%); 6.2 MiB/s, 98 objects/s recovering 2024-09-21T10:15:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:25 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/77322024' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:25 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:25.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:25.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:25 smithi134 ceph-mon[90085]: pgmap v2040: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 5.8 MiB/s wr, 1.24k op/s; 5670/335187 objects degraded (1.692%); 480/335187 objects misplaced (0.143%); 6.2 MiB/s, 98 objects/s recovering 2024-09-21T10:15:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:25 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/77322024' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:25 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:26.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:26.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.105 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:26.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:26.205 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:26.235 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:26.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:26.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:26.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:26.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:26.478 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:26.508 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:26.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:26.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:26.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:26.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:26.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:26.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:26.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:26.925 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:26.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:26 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:26 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:26.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:26 smithi017 ceph-mon[102913]: osdmap e1312: 8 total, 8 up, 8 in 2024-09-21T10:15:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:26 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:26 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:26.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:26 smithi017 ceph-mon[105151]: osdmap e1312: 8 total, 8 up, 8 in 2024-09-21T10:15:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:26 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:26 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:26 smithi134 ceph-mon[90085]: osdmap e1312: 8 total, 8 up, 8 in 2024-09-21T10:15:27.028 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:27.052 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:27.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:27.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:27.183 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:27.273 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:15:27.284 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:27.337 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:27.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:27.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:27.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:27.531 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:27.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:27.586 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:27.590 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:27.692 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:27.726 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:27.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:27.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:27.858 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:27 smithi134 ceph-mon[90085]: pgmap v2042: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 5670/330387 objects degraded (1.716%); 480/330387 objects misplaced (0.145%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:15:27.858 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:27 smithi134 ceph-mon[90085]: osdmap e1313: 8 total, 8 up, 8 in 2024-09-21T10:15:27.858 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:27 smithi134 ceph-mon[90085]: 10.6 scrub starts 2024-09-21T10:15:27.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:27.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:27 smithi017 ceph-mon[102913]: pgmap v2042: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 5670/330387 objects degraded (1.716%); 480/330387 objects misplaced (0.145%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:15:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:27 smithi017 ceph-mon[102913]: osdmap e1313: 8 total, 8 up, 8 in 2024-09-21T10:15:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:27 smithi017 ceph-mon[102913]: 10.6 scrub starts 2024-09-21T10:15:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:27 smithi017 ceph-mon[105151]: pgmap v2042: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 116 active+clean; 6.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 5670/330387 objects degraded (1.716%); 480/330387 objects misplaced (0.145%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:15:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:27 smithi017 ceph-mon[105151]: osdmap e1313: 8 total, 8 up, 8 in 2024-09-21T10:15:27.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:27 smithi017 ceph-mon[105151]: 10.6 scrub starts 2024-09-21T10:15:27.970 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:27.996 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:27.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:28.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:28.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:28.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:28.238 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:28.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.275 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:28.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:28.376 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:28.407 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:28.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:28.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:28.542 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:28.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:28.644 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:28.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:28.779 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:28.803 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:28.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:28.905 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:28.925 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:28 smithi017 ceph-mon[102913]: 10.6 scrub ok 2024-09-21T10:15:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:28 smithi017 ceph-mon[102913]: osdmap e1314: 8 total, 8 up, 8 in 2024-09-21T10:15:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:28 smithi017 ceph-mon[102913]: 10.16 scrub starts 2024-09-21T10:15:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:28 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:15:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:28 smithi017 ceph-mon[105151]: 10.6 scrub ok 2024-09-21T10:15:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:28 smithi017 ceph-mon[105151]: osdmap e1314: 8 total, 8 up, 8 in 2024-09-21T10:15:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:28 smithi017 ceph-mon[105151]: 10.16 scrub starts 2024-09-21T10:15:28.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:28 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:15:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:28 smithi134 ceph-mon[90085]: 10.6 scrub ok 2024-09-21T10:15:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:28 smithi134 ceph-mon[90085]: osdmap e1314: 8 total, 8 up, 8 in 2024-09-21T10:15:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:28 smithi134 ceph-mon[90085]: 10.16 scrub starts 2024-09-21T10:15:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:28 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:15:29.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:29.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:29.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:29.173 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:29.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:29.275 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:29.299 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:29.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:29.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:29.427 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:29.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:29.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:29.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:29.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:29.674 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[105151]: 10.1 scrub starts 2024-09-21T10:15:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[105151]: pgmap v2045: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 115 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5764/322767 objects misplaced (1.786%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:15:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:15:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[105151]: osdmap e1315: 8 total, 8 up, 8 in 2024-09-21T10:15:29.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[102913]: 10.1 scrub starts 2024-09-21T10:15:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[102913]: pgmap v2045: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 115 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5764/322767 objects misplaced (1.786%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:15:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:15:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[102913]: osdmap e1315: 8 total, 8 up, 8 in 2024-09-21T10:15:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:29 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:29.776 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:29.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:29.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.921 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:29.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:29 smithi134 ceph-mon[90085]: 10.1 scrub starts 2024-09-21T10:15:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:29 smithi134 ceph-mon[90085]: pgmap v2045: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 115 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5764/322767 objects misplaced (1.786%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:15:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:29 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:15:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:29 smithi134 ceph-mon[90085]: osdmap e1315: 8 total, 8 up, 8 in 2024-09-21T10:15:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:29 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:30.022 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:30.043 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:30.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:30.144 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:30.165 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:30.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:30.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:30.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:30.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:30.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:30.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:15:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:15:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:15:30.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:30.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:30.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:30.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:30.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:30.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:30.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:30.793 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:30 smithi134 ceph-mon[90085]: 10.16 scrub ok 2024-09-21T10:15:30.793 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:30 smithi134 ceph-mon[90085]: 10.10 scrub starts 2024-09-21T10:15:30.793 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:30 smithi134 ceph-mon[90085]: osdmap e1316: 8 total, 8 up, 8 in 2024-09-21T10:15:30.793 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:15:30.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:30.903 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:30.933 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:30.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:30 smithi017 ceph-mon[102913]: 10.16 scrub ok 2024-09-21T10:15:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:30 smithi017 ceph-mon[102913]: 10.10 scrub starts 2024-09-21T10:15:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:30 smithi017 ceph-mon[102913]: osdmap e1316: 8 total, 8 up, 8 in 2024-09-21T10:15:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:15:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:30 smithi017 ceph-mon[105151]: 10.16 scrub ok 2024-09-21T10:15:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:30 smithi017 ceph-mon[105151]: 10.10 scrub starts 2024-09-21T10:15:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:30 smithi017 ceph-mon[105151]: osdmap e1316: 8 total, 8 up, 8 in 2024-09-21T10:15:30.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:15:31.035 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:31.070 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:31.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:31.071 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:15:31.071 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:15:31.071 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 0 2024-09-21T10:15:31.071 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:15:31.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:31.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:31.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:31.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:31.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:31.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:31.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:31.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:31.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:31.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:31.576 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:31.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:31.700 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:31.741 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:31 smithi134 ceph-mon[90085]: pgmap v2048: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 116 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 5764/320013 objects misplaced (1.801%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:15:31.741 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:15:31.741 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:31 smithi134 ceph-mon[90085]: osdmap e1317: 8 total, 8 up, 8 in 2024-09-21T10:15:31.764 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:15:31.802 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:31.827 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:31.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:31.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:31 smithi017 ceph-mon[102913]: pgmap v2048: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 116 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 5764/320013 objects misplaced (1.801%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:15:31.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:15:31.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:31 smithi017 ceph-mon[102913]: osdmap e1317: 8 total, 8 up, 8 in 2024-09-21T10:15:31.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:31 smithi017 ceph-mon[105151]: pgmap v2048: 119 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 116 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 5764/320013 objects misplaced (1.801%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T10:15:31.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:15:31.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:31 smithi017 ceph-mon[105151]: osdmap e1317: 8 total, 8 up, 8 in 2024-09-21T10:15:31.929 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:31.950 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:31.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:32.013 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:32.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:32.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:32.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:15:32.177 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:32.199 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:32.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:32.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:32.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:32.426 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:32.447 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:32.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:32.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:32.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:32.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:32 smithi017 ceph-mon[102913]: osdmap e1318: 8 total, 8 up, 8 in 2024-09-21T10:15:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:15:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:32 smithi017 ceph-mon[105151]: osdmap e1318: 8 total, 8 up, 8 in 2024-09-21T10:15:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:15:32.696 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:32 smithi134 ceph-mon[90085]: osdmap e1318: 8 total, 8 up, 8 in 2024-09-21T10:15:32.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:15:32.798 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:32.822 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:32.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:32.924 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:32.953 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:33.055 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:33.082 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:33.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:33.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:33.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:33.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:33.236 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:15:33.236 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:33.259 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:15:33.260 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:33.285 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:15:33.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:33.308 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:15:33.308 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:15:33.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:33.330 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:15:33.330 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:33.351 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:15:33.352 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:33.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:33.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:33.375 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:15:33.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:33.398 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:15:33.398 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:33.457 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:33.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:33.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:33.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:33.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:33.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:33.706 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:33.728 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:33.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:33 smithi134 ceph-mon[90085]: pgmap v2051: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 5284/316914 objects misplaced (1.667%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:15:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:33 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-09-21T10:15:33.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:33 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:15:33.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:33 smithi134 ceph-mon[90085]: osdmap e1319: 8 total, 8 up, 8 in 2024-09-21T10:15:33.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:33 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2568232797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:33.745 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:33 smithi134 ceph-mon[90085]: osdmap e1320: 8 total, 8 up, 8 in 2024-09-21T10:15:33.748 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:33.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:33.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:33.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:33.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[102913]: pgmap v2051: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 5284/316914 objects misplaced (1.667%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:15:33.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-09-21T10:15:33.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:15:33.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[102913]: osdmap e1319: 8 total, 8 up, 8 in 2024-09-21T10:15:33.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2568232797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:33.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[102913]: osdmap e1320: 8 total, 8 up, 8 in 2024-09-21T10:15:33.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[105151]: pgmap v2051: 119 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 116 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 5284/316914 objects misplaced (1.667%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:15:33.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-09-21T10:15:33.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:15:33.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[105151]: osdmap e1319: 8 total, 8 up, 8 in 2024-09-21T10:15:33.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2568232797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:33.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:33 smithi017 ceph-mon[105151]: osdmap e1320: 8 total, 8 up, 8 in 2024-09-21T10:15:33.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:33.976 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:34.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:34.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:34.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:34.128 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:34.128 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:34.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:34.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:34.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:34.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:34.455 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:34.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:34.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:34.602 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:34.627 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 10 pgp_num to 26 2024-09-21T10:15:34.704 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:34.726 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/796438797' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-09-21T10:15:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:34 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-09-21T10:15:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:34 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2024-09-21T10:15:34.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:34 smithi134 ceph-mon[90085]: osdmap e1321: 8 total, 8 up, 8 in 2024-09-21T10:15:34.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:34.865 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:34.910 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:15:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/796438797' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-09-21T10:15:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:34 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-09-21T10:15:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:34 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2024-09-21T10:15:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:34 smithi017 ceph-mon[102913]: osdmap e1321: 8 total, 8 up, 8 in 2024-09-21T10:15:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/796438797' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-09-21T10:15:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:34 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-09-21T10:15:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:34 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2024-09-21T10:15:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:34 smithi017 ceph-mon[105151]: osdmap e1321: 8 total, 8 up, 8 in 2024-09-21T10:15:34.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:15:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:15:34] "GET /metrics HTTP/1.1" 200 36668 "" "Prometheus/2.51.0" 2024-09-21T10:15:34.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:35.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:35.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:35.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:35.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:35.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:35.263 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:35.306 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:35.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:35.324 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:35.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:35.436 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:35.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:35.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:35.572 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:35.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:35.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:35.707 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:35 smithi134 ceph-mon[90085]: pgmap v2054: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 7762/313419 objects degraded (2.477%); 8354/313419 objects misplaced (2.665%); 8.5 MiB/s, 135 objects/s recovering 2024-09-21T10:15:35.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:35 smithi134 ceph-mon[90085]: Health check failed: Degraded data redundancy: 7762/313419 objects degraded (2.477%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:35.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:35.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.845 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:35.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:35.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:35 smithi017 ceph-mon[102913]: pgmap v2054: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 7762/313419 objects degraded (2.477%); 8354/313419 objects misplaced (2.665%); 8.5 MiB/s, 135 objects/s recovering 2024-09-21T10:15:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:35 smithi017 ceph-mon[102913]: Health check failed: Degraded data redundancy: 7762/313419 objects degraded (2.477%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:35 smithi017 ceph-mon[105151]: pgmap v2054: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 7762/313419 objects degraded (2.477%); 8354/313419 objects misplaced (2.665%); 8.5 MiB/s, 135 objects/s recovering 2024-09-21T10:15:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:35 smithi017 ceph-mon[105151]: Health check failed: Degraded data redundancy: 7762/313419 objects degraded (2.477%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:35.985 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:35.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:36.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:36.115 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:36.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:36.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:36.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:15:36.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:36.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:36.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:36.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:36.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:36.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:36.663 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.690 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:36.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:36.791 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:36.825 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:36.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:36.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:36.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:36.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:37.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.102 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:37.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:37.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:37.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:37.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:37.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:37.367 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:37.367 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:37.396 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:37.498 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:37.532 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:37.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:37.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:37.666 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:37.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:37.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:37.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:37.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:37.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:37.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:37 smithi017 ceph-mon[102913]: pgmap v2056: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 7762/311424 objects degraded (2.492%); 8354/311424 objects misplaced (2.683%); 4.9 MiB/s, 77 objects/s recovering 2024-09-21T10:15:37.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:37 smithi017 ceph-mon[105151]: pgmap v2056: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 7762/311424 objects degraded (2.492%); 8354/311424 objects misplaced (2.683%); 4.9 MiB/s, 77 objects/s recovering 2024-09-21T10:15:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:37 smithi134 ceph-mon[90085]: pgmap v2056: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 7762/311424 objects degraded (2.492%); 8354/311424 objects misplaced (2.683%); 4.9 MiB/s, 77 objects/s recovering 2024-09-21T10:15:38.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:38.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:38.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:38.106 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:15:38.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:38.209 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:38.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:38.311 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:38.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.357 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:38.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:38.458 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:38.464 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:38.508 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:38.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:38.610 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:38.643 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:38.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:38.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:38.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:38.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:38.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:38.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:39.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:39.097 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:39.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:39.198 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:39.236 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:39.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:39.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:39.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:39.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:39.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.529 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:39.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:39.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:39.659 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:39.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:39.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:39.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:39.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:39.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:39.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:39 smithi017 ceph-mon[102913]: pgmap v2057: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 7762/305682 objects degraded (2.539%); 11711/305682 objects misplaced (3.831%); 6.8 MiB/s, 109 objects/s recovering 2024-09-21T10:15:39.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:39 smithi017 ceph-mon[105151]: pgmap v2057: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 7762/305682 objects degraded (2.539%); 11711/305682 objects misplaced (3.831%); 6.8 MiB/s, 109 objects/s recovering 2024-09-21T10:15:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:39.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:39 smithi134 ceph-mon[90085]: pgmap v2057: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 7762/305682 objects degraded (2.539%); 11711/305682 objects misplaced (3.831%); 6.8 MiB/s, 109 objects/s recovering 2024-09-21T10:15:40.053 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:40.093 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:40.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:40.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:40.237 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:40.309 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:15:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:15:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:15:40.338 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:40.379 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:40.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:40.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:40.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:40.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:40.657 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:40.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:40.783 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:40.809 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:40.809 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:40.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:40.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:41.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:41.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:41.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:41.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:41.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:41.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:41.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:41.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:41.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:41.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:41.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:41.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:41.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:15:41.593 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:41.624 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:41.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:41.725 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:41.759 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:41.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:41.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:41.891 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:41.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.902 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:41.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:41 smithi017 ceph-mon[102913]: 10.1 scrub starts 2024-09-21T10:15:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:41 smithi017 ceph-mon[102913]: pgmap v2058: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 7762/303303 objects degraded (2.559%); 11711/303303 objects misplaced (3.861%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:15:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:41 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7762/303303 objects degraded (2.559%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:41.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:41 smithi017 ceph-mon[105151]: 10.1 scrub starts 2024-09-21T10:15:41.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:41 smithi017 ceph-mon[105151]: pgmap v2058: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 7762/303303 objects degraded (2.559%); 11711/303303 objects misplaced (3.861%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:15:41.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:41 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7762/303303 objects degraded (2.559%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:41 smithi134 ceph-mon[90085]: 10.1 scrub starts 2024-09-21T10:15:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:41 smithi134 ceph-mon[90085]: pgmap v2058: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 7762/303303 objects degraded (2.559%); 11711/303303 objects misplaced (3.861%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:15:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:41 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7762/303303 objects degraded (2.559%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:42.003 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:42.032 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:42.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:42.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:42.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:42.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:42.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:42.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:42.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:42.432 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:42.458 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:42.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:42.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:42.599 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:42.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:42.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:42.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:42.842 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:42.869 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:42.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:42.971 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:42.997 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:42.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:43.099 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:43.130 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:43.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:43.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:43.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:43.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:43.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:43.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:43.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:43.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:43.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:43.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:43.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:43.796 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:43 smithi134 ceph-mon[90085]: 10.10 scrub starts 2024-09-21T10:15:43.796 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:43 smithi134 ceph-mon[90085]: pgmap v2059: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 7762/298044 objects degraded (2.604%); 11711/298044 objects misplaced (3.929%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:15:43.843 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:43.871 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:43.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:43.915 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:43.915 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:43 smithi017 ceph-mon[102913]: 10.10 scrub starts 2024-09-21T10:15:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:43 smithi017 ceph-mon[102913]: pgmap v2059: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 7762/298044 objects degraded (2.604%); 11711/298044 objects misplaced (3.929%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:15:43.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:43 smithi017 ceph-mon[105151]: 10.10 scrub starts 2024-09-21T10:15:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:43 smithi017 ceph-mon[105151]: pgmap v2059: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 7762/298044 objects degraded (2.604%); 11711/298044 objects misplaced (3.929%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:15:43.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:44.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:44.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:44.116 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:44.142 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:44.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:44.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:44.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:44.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:44.387 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:44.410 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:44.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:44.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:44.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:44.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:44.654 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:15:44.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:44.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.699 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:44.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:44.800 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:44.820 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:44.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:44.922 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:44.944 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:44.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:44.962 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:45.046 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:45.066 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:15:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:15:44] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.51.0" 2024-09-21T10:15:45.067 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:45.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:45.169 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:45.198 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:45.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:45.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:45.326 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:45.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:45.428 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:45.465 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:45.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:45.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:45.619 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:45.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:45.767 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:45.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:45.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:45.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:45.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:45 smithi017 ceph-mon[102913]: pgmap v2060: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 7762/287748 objects degraded (2.697%); 11383/287748 objects misplaced (3.956%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:15:45.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:45 smithi017 ceph-mon[105151]: pgmap v2060: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 7762/287748 objects degraded (2.697%); 11383/287748 objects misplaced (3.956%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:15:45.981 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:45 smithi134 ceph-mon[90085]: pgmap v2060: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 402 op/s; 7762/287748 objects degraded (2.697%); 11383/287748 objects misplaced (3.956%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:15:46.011 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:46.038 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:46.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:46.140 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:46.167 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:46.269 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:46.300 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:46.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:46.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:46.445 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:46.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:46.546 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:46.570 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:46.571 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:46.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:46.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:46.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:46.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:46.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:46.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:46 smithi017 ceph-mon[102913]: 10.1 scrub starts 2024-09-21T10:15:46.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:46 smithi017 ceph-mon[105151]: 10.1 scrub starts 2024-09-21T10:15:46.986 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:46 smithi134 ceph-mon[90085]: 10.1 scrub starts 2024-09-21T10:15:46.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:47.012 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:47.012 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:47.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.054 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:47.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:47.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:47.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:47.289 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:47.315 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:47.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:47.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:47.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:47.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:47.576 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:47.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:47.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:47.678 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:15:47.678 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:15:47.678 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-21T10:15:47.678 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.824119 2024-09-21T10:15:47.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 2 0.8241192064686736 2024-09-21T10:15:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:47.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:47.829 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:47.853 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:47.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:47.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[105151]: 10.10 scrub starts 2024-09-21T10:15:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[105151]: pgmap v2061: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 7762/283497 objects degraded (2.738%); 11383/283497 objects misplaced (4.015%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:15:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7762/283497 objects degraded (2.738%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:47.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[102913]: 10.10 scrub starts 2024-09-21T10:15:47.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[102913]: pgmap v2061: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 7762/283497 objects degraded (2.738%); 11383/283497 objects misplaced (4.015%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:15:47.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7762/283497 objects degraded (2.738%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:47.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:47.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:47.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:47 smithi134 ceph-mon[90085]: 10.10 scrub starts 2024-09-21T10:15:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:47 smithi134 ceph-mon[90085]: pgmap v2061: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 7762/283497 objects degraded (2.738%); 11383/283497 objects misplaced (4.015%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:15:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:47 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7762/283497 objects degraded (2.738%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:48.006 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:48.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:48.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:48.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:48.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:48.270 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:48.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:48.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.443 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:48.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:48.455 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:15:48.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:48.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:48.699 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:48.736 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:48.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:48.811 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:48.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:48.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:15:48.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:15:48.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:48.852 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:15:48.853 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:15:48.853 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:48.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:48.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:15:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:15:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:49.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:49.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:49.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:49.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:49.203 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:49.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:49.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:49.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:49.424 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:49.452 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:49.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:49.553 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:49.576 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:49.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:49.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:49.701 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:15:49.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:49.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:49.816 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:49 smithi017 ceph-mon[102913]: pgmap v2062: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 7762/276291 objects degraded (2.809%); 11081/276291 objects misplaced (4.011%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:15:49.816 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:49 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2256030545' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:49.816 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:49 smithi017 ceph-mon[105151]: pgmap v2062: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 7762/276291 objects degraded (2.809%); 11081/276291 objects misplaced (4.011%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:15:49.816 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:49 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2256030545' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:49.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:49.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:49.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:49.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:49 smithi134 ceph-mon[90085]: pgmap v2062: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 7762/276291 objects degraded (2.809%); 11081/276291 objects misplaced (4.011%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:15:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:49 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2256030545' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:50.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:50.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:50.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:50.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:50.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:50.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:50.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:50.463 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:50.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:15:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:15:50.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:15:50.565 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:50.599 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:50.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:50.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:50.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:50.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:50.798 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:50 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:15:50.799 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:50 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2256030545' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:50.799 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:50 smithi134 ceph-mon[90085]: osdmap e1322: 8 total, 8 up, 8 in 2024-09-21T10:15:50.799 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:50 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3772051095' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:50.829 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:50.829 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:50.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:50.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:50.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:50.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:51.018 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:51.120 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:51.157 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:51.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:50 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:15:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:50 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2256030545' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:50 smithi017 ceph-mon[102913]: osdmap e1322: 8 total, 8 up, 8 in 2024-09-21T10:15:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:50 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3772051095' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:50 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:15:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:50 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2256030545' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:50 smithi017 ceph-mon[105151]: osdmap e1322: 8 total, 8 up, 8 in 2024-09-21T10:15:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:50 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3772051095' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:51.259 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:51.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.287 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:51.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:51.388 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:51.422 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:51.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:51.524 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:51.554 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:51.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:51.584 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:15:51.656 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:51.696 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:51.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:51.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:51.834 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:51.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:51.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:51.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:51.997 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:51 smithi134 ceph-mon[90085]: pgmap v2064: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 987 op/s; 7762/273678 objects degraded (2.836%); 11081/273678 objects misplaced (4.049%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:15:52.020 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:52.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:52.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:52.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:52.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:51 smithi017 ceph-mon[102913]: pgmap v2064: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 987 op/s; 7762/273678 objects degraded (2.836%); 11081/273678 objects misplaced (4.049%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:15:52.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:51 smithi017 ceph-mon[105151]: pgmap v2064: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 987 op/s; 7762/273678 objects degraded (2.836%); 11081/273678 objects misplaced (4.049%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:15:52.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:52.237 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:52.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:52.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:52.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:52.481 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:52.510 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:52.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:52.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:52.752 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:52.780 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:52.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:52.870 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:15:52.870 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:15:52.870 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:15:52.870 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:15:52.871 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:52.871 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:52.871 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:52.871 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:15:52.871 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:15:52.871 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:15:52.871 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:15:52.871 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:15:52.872 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:52.872 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:52.872 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:52.872 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:52.881 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:52.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:53.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:53.044 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:53.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:53.185 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:52 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7762/268779 objects degraded (2.888%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:53.186 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:52 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7762/268779 objects degraded (2.888%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:53.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:53.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:52 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7762/268779 objects degraded (2.888%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:53.295 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:53.324 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:53.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:53.426 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:53.451 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:53.553 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:53.580 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:53.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:53.682 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:53.715 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:53.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:53.817 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:53.845 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:53.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:53.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:53 smithi017 ceph-mon[102913]: pgmap v2065: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 7762/268779 objects degraded (2.888%); 11081/268779 objects misplaced (4.123%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:15:53.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:53 smithi017 ceph-mon[102913]: 10.1 deep-scrub starts 2024-09-21T10:15:53.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:53 smithi017 ceph-mon[105151]: pgmap v2065: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 7762/268779 objects degraded (2.888%); 11081/268779 objects misplaced (4.123%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:15:53.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:53 smithi017 ceph-mon[105151]: 10.1 deep-scrub starts 2024-09-21T10:15:54.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:54.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:54.105 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:54.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:54.238 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:53 smithi134 ceph-mon[90085]: pgmap v2065: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 7762/268779 objects degraded (2.888%); 11081/268779 objects misplaced (4.123%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:15:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:53 smithi134 ceph-mon[90085]: 10.1 deep-scrub starts 2024-09-21T10:15:54.262 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:54.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:54.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:54.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:54.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:54.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:54.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:54.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:54.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:54.798 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:54.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:15:54.826 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:54.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:54.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:54.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:15:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:15:54] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.51.0" 2024-09-21T10:15:54.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:55.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:55.127 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:55.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:55.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:55.298 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:55.299 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:55.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:55.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:55.424 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:55.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:55.559 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:55.661 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:55.689 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:55.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:55.797 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.2 primary-affinity to 0.824119 (8540092) 2024-09-21T10:15:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:55.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:55.934 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:55 smithi017 ceph-mon[102913]: pgmap v2066: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 974 op/s; 7762/258507 objects degraded (3.003%); 10747/258507 objects misplaced (4.157%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:15:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:55 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2788590531' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.8241192064686736} : dispatch 2024-09-21T10:15:55.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:55 smithi017 ceph-mon[105151]: pgmap v2066: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 974 op/s; 7762/258507 objects degraded (3.003%); 10747/258507 objects misplaced (4.157%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:15:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:55 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2788590531' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.8241192064686736} : dispatch 2024-09-21T10:15:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:55.959 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:56.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:56.090 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:56.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:56.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:15:56.191 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:56.224 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:56.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:55 smithi134 ceph-mon[90085]: pgmap v2066: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 974 op/s; 7762/258507 objects degraded (3.003%); 10747/258507 objects misplaced (4.157%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T10:15:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:55 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2788590531' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.8241192064686736} : dispatch 2024-09-21T10:15:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:15:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:56.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:56.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:56.463 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:56.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:56.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:56.586 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:56.609 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:56.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:56.711 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:56.740 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:56.842 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:56.874 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:56.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:56.975 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:57.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.006 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:57.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:57.107 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:57.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.140 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:57.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:57.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:56 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2788590531' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.8241192064686736}]': finished 2024-09-21T10:15:57.152 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:56 smithi134 ceph-mon[90085]: osdmap e1323: 8 total, 8 up, 8 in 2024-09-21T10:15:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:56 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2788590531' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.8241192064686736}]': finished 2024-09-21T10:15:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:56 smithi017 ceph-mon[102913]: osdmap e1323: 8 total, 8 up, 8 in 2024-09-21T10:15:57.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:56 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2788590531' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.8241192064686736}]': finished 2024-09-21T10:15:57.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:56 smithi017 ceph-mon[105151]: osdmap e1323: 8 total, 8 up, 8 in 2024-09-21T10:15:57.241 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:57.276 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:57.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:57.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:57.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:57.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:57.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:57.562 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:57.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:57.664 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:57.695 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:57.796 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:57.834 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:57.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:57.904 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:57 smithi134 ceph-mon[90085]: pgmap v2068: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 7762/253770 objects degraded (3.059%); 10747/253770 objects misplaced (4.235%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:15:57.918 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:57.936 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:57.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:57.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:58.030 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:15:58.031 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:15:58.031 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:15:58.031 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:15:58.031 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:15:58.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:58.091 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:57 smithi017 ceph-mon[102913]: pgmap v2068: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 7762/253770 objects degraded (3.059%); 10747/253770 objects misplaced (4.235%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:15:58.092 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:57 smithi017 ceph-mon[105151]: pgmap v2068: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 7762/253770 objects degraded (3.059%); 10747/253770 objects misplaced (4.235%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:15:58.110 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:58.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:58.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:58.241 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:58.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:58.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:58.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:58.499 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:58.526 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:58.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:58.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:58.671 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:58.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:58.760 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:15:58.773 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:58.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.848 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:58.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:58.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:58.986 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:58.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:58 smithi134 ceph-mon[90085]: 10.10 scrub starts 2024-09-21T10:15:58.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:58 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7762/253770 objects degraded (3.059%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:59.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:59.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:59.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:59.132 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:58 smithi017 ceph-mon[102913]: 10.10 scrub starts 2024-09-21T10:15:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:58 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7762/253770 objects degraded (3.059%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:58 smithi017 ceph-mon[105151]: 10.10 scrub starts 2024-09-21T10:15:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:58 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7762/253770 objects degraded (3.059%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:15:59.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:59.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:59.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:59.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:59.410 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:59.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:59.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:59.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:59.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:59.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:59.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:59.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:59.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:15:59.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:59.935 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:59.963 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:15:59.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:59.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:15:59 smithi134 ceph-mon[90085]: pgmap v2069: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 7762/246669 objects degraded (3.147%); 10331/246669 objects misplaced (4.188%); 2.7 MiB/s, 42 objects/s recovering 2024-09-21T10:16:00.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:00.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:00.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:00.134 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:15:59 smithi017 ceph-mon[102913]: pgmap v2069: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 7762/246669 objects degraded (3.147%); 10331/246669 objects misplaced (4.188%); 2.7 MiB/s, 42 objects/s recovering 2024-09-21T10:16:00.134 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:15:59 smithi017 ceph-mon[105151]: pgmap v2069: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 7762/246669 objects degraded (3.147%); 10331/246669 objects misplaced (4.188%); 2.7 MiB/s, 42 objects/s recovering 2024-09-21T10:16:00.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:00.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:00.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:00.375 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:00.407 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:00.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:00.445 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:16:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:16:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:16:00.503 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:16:00.503 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:16:00.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:00.541 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:16:00.542 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:00.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:00.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:00.584 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:16:00.584 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:00.619 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:16:00.619 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:16:00.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:00.653 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:16:00.653 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:00.675 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:00.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:00.683 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:16:00.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:00.719 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:16:00.719 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:00.751 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:16:00.751 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:00.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:00.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:00.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:00.856 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:16:00.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:00.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:00.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:00.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:00 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1701088262' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:00.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:00 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1701088262' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:01.050 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:01.085 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:01.099 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:00 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1701088262' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:01.099 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:00 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3886983792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:01.129 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:01.129 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:01.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:01.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:01.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:01.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:01.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:01.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:01.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3886983792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:01.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3886983792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:01.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:01.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:01.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:01.604 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:01.631 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:01.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:01.732 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:01.761 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:01.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:01.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:01.903 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:01.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:01.946 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:16:02.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:02.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:02.139 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:02.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:02.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[102913]: pgmap v2070: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 996 op/s; 7762/243783 objects degraded (3.184%); 10331/243783 objects misplaced (4.238%); 2.3 MiB/s, 37 objects/s recovering 2024-09-21T10:16:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1701088262' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:02.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[102913]: osdmap e1324: 8 total, 8 up, 8 in 2024-09-21T10:16:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[105151]: pgmap v2070: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 996 op/s; 7762/243783 objects degraded (3.184%); 10331/243783 objects misplaced (4.238%); 2.3 MiB/s, 37 objects/s recovering 2024-09-21T10:16:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1701088262' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:01 smithi017 ceph-mon[105151]: osdmap e1324: 8 total, 8 up, 8 in 2024-09-21T10:16:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:01 smithi134 ceph-mon[90085]: pgmap v2070: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 996 op/s; 7762/243783 objects degraded (3.184%); 10331/243783 objects misplaced (4.238%); 2.3 MiB/s, 37 objects/s recovering 2024-09-21T10:16:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:01 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:01 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1701088262' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:01 smithi134 ceph-mon[90085]: osdmap e1324: 8 total, 8 up, 8 in 2024-09-21T10:16:02.278 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:02.317 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:02.345 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:16:02.407 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:16:02.407 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:16:02.408 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-21T10:16:02.408 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2024-09-21T10:16:02.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 0 0 2024-09-21T10:16:02.419 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.448 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:02.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:02.550 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:02.576 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:02.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:02.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.725 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:02.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:02.826 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:02.856 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:02.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:02.958 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:02.986 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:03.088 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:03.117 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:03.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:03.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:03.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:03.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:03.357 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:03.385 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:03.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:03.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:03.524 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:03.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:03.625 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:03.655 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:03.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:03.733 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:03.757 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:03.783 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:03.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:03.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:03 smithi017 ceph-mon[102913]: pgmap v2072: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 7762/238815 objects degraded (3.250%); 10331/238815 objects misplaced (4.326%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:16:03.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:03 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7762/238815 objects degraded (3.250%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:03.885 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:03.917 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:04.019 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:04.046 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:04.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:04.148 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:04.159 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:03 smithi134 ceph-mon[90085]: pgmap v2072: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 7762/238815 objects degraded (3.250%); 10331/238815 objects misplaced (4.326%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:16:04.160 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:03 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7762/238815 objects degraded (3.250%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:04.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:03 smithi017 ceph-mon[105151]: pgmap v2072: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 7762/238815 objects degraded (3.250%); 10331/238815 objects misplaced (4.326%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:16:04.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:03 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7762/238815 objects degraded (3.250%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:04.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:04.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:04.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:04.423 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:04.456 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:04.557 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:04.597 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:04.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:04.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:04.740 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:04.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:04.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:04.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:04.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:04.986 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:05.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:16:05.017 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:05.048 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:16:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:16:04] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-09-21T10:16:05.119 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:05.149 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:05.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:05.172 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:05.173 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:05.250 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:05.276 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:05.378 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:05.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.409 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:05.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:05.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:05.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:05.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:05.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:16:05.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:05.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.735 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:05.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:05.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:05.861 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:05.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:05.909 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:16:05.924 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:05.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:05 smithi017 ceph-mon[102913]: pgmap v2073: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 7762/229095 objects degraded (3.388%); 9941/229095 objects misplaced (4.339%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:16:05.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:05 smithi017 ceph-mon[105151]: pgmap v2073: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 7762/229095 objects degraded (3.388%); 9941/229095 objects misplaced (4.339%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:16:05.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:06.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:06.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:06.123 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:06.149 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:06.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:06.239 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:16:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:05 smithi134 ceph-mon[90085]: pgmap v2073: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 7762/229095 objects degraded (3.388%); 9941/229095 objects misplaced (4.339%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:16:06.251 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:06.276 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:06.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:06.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:06.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:06.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:06.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:06.559 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:06.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:06.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:06.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:06.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:06.839 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:06.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:06.910 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.0 primary-affinity to 0 (802) 2024-09-21T10:16:06.941 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:06.965 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:06.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:07.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:07.090 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:07.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:07.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:06 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/219818032' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.0} : dispatch 2024-09-21T10:16:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:06 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.0} : dispatch 2024-09-21T10:16:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:06 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/219818032' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.0} : dispatch 2024-09-21T10:16:07.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:06 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.0} : dispatch 2024-09-21T10:16:07.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:07.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:07.235 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:06 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/219818032' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.0} : dispatch 2024-09-21T10:16:07.235 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:06 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.0} : dispatch 2024-09-21T10:16:07.317 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:07.338 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:07.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:07.462 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:07.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:07.564 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:07.587 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:07.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:07.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:07.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:07.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:07.836 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:07.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:07.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:08.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:08.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.103 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:08.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:08.141 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:07 smithi134 ceph-mon[90085]: pgmap v2074: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 982 op/s; 7762/224295 objects degraded (3.461%); 9941/224295 objects misplaced (4.432%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:16:08.141 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:07 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2024-09-21T10:16:08.141 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:07 smithi134 ceph-mon[90085]: osdmap e1325: 8 total, 8 up, 8 in 2024-09-21T10:16:08.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:07 smithi017 ceph-mon[102913]: pgmap v2074: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 982 op/s; 7762/224295 objects degraded (3.461%); 9941/224295 objects misplaced (4.432%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:16:08.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:07 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2024-09-21T10:16:08.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:07 smithi017 ceph-mon[102913]: osdmap e1325: 8 total, 8 up, 8 in 2024-09-21T10:16:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:07 smithi017 ceph-mon[105151]: pgmap v2074: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 114 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 982 op/s; 7762/224295 objects degraded (3.461%); 9941/224295 objects misplaced (4.432%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:16:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:07 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2024-09-21T10:16:08.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:07 smithi017 ceph-mon[105151]: osdmap e1325: 8 total, 8 up, 8 in 2024-09-21T10:16:08.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:08.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:08.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:08.251 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:08.251 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:08.328 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:08.358 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:08.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:08.489 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:08.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:08.590 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:08.621 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:08.723 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:08.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.759 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:08.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:08.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:08.895 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:08.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:08.997 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:09.018 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:09.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:09.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:09.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:08 smithi017 ceph-mon[102913]: osdmap e1326: 8 total, 8 up, 8 in 2024-09-21T10:16:09.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:08 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7762/224295 objects degraded (3.461%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:08 smithi017 ceph-mon[105151]: osdmap e1326: 8 total, 8 up, 8 in 2024-09-21T10:16:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:08 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7762/224295 objects degraded (3.461%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:08 smithi134 ceph-mon[90085]: osdmap e1326: 8 total, 8 up, 8 in 2024-09-21T10:16:09.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:08 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7762/224295 objects degraded (3.461%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:09.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:09.300 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:09.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:09.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:09.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:09.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:09.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:09.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.566 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:09.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:09.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:09.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:09.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:09.810 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:09.849 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:09.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:09.950 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:09.986 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:09.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:10.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:10.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:09 smithi017 ceph-mon[102913]: pgmap v2077: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 111 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 7762/219390 objects degraded (3.538%); 9237/219390 objects misplaced (4.210%); 4.5 MiB/s, 70 objects/s recovering 2024-09-21T10:16:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:09 smithi017 ceph-mon[102913]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:16:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:09 smithi017 ceph-mon[105151]: pgmap v2077: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 111 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 7762/219390 objects degraded (3.538%); 9237/219390 objects misplaced (4.210%); 4.5 MiB/s, 70 objects/s recovering 2024-09-21T10:16:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:09 smithi017 ceph-mon[105151]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:16:10.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:09 smithi134 ceph-mon[90085]: pgmap v2077: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 111 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 7762/219390 objects degraded (3.538%); 9237/219390 objects misplaced (4.210%); 4.5 MiB/s, 70 objects/s recovering 2024-09-21T10:16:10.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:09 smithi134 ceph-mon[90085]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:16:10.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:10.258 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:10.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:10.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:10.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:10.410 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:16:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:16:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:16:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:16:10.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:10.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:10.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:10.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:10.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:10.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:10.784 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:16:10.789 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:10.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:10.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:10.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:10.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:10.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:11.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:11.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:11.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:11.202 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:11.232 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:11.333 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:11.368 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:11.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:11.470 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:11.496 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:11.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:11.597 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:11.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.632 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:11.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:11.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:11.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:11 smithi017 ceph-mon[102913]: pgmap v2078: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 110 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 7762/216357 objects degraded (3.588%); 9237/216357 objects misplaced (4.269%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T10:16:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:11.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:11 smithi017 ceph-mon[105151]: pgmap v2078: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 110 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 7762/216357 objects degraded (3.588%); 9237/216357 objects misplaced (4.269%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T10:16:11.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:11 smithi134 ceph-mon[90085]: pgmap v2078: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 110 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 7762/216357 objects degraded (3.588%); 9237/216357 objects misplaced (4.269%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T10:16:11.771 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:11.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:11.873 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:11.913 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:12.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:12.061 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:12.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:12.163 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.211 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:12.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:12.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:12.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:12.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:12.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:12.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:12.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:12.590 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:12.614 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:12.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:12.715 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:12.746 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:12.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:12.836 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:12 smithi134 ceph-mon[90085]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-21T10:16:12.844 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:12.844 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:12.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:12.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:12 smithi017 ceph-mon[102913]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-21T10:16:12.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:12 smithi017 ceph-mon[105151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-21T10:16:12.996 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:13.023 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:13.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:13.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:13.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:13.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:13.263 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:13.290 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:13.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:13.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:13.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:13.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:13.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:13.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:16:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:13.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:13.676 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:13.699 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:13.801 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:13.825 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:13.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:13.927 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:13.951 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:13.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:13.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:13 smithi017 ceph-mon[102913]: pgmap v2079: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 7762/209898 objects degraded (3.698%); 8911/209898 objects misplaced (4.245%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T10:16:13.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:13 smithi017 ceph-mon[105151]: pgmap v2079: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 7762/209898 objects degraded (3.698%); 8911/209898 objects misplaced (4.245%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T10:16:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:13 smithi134 ceph-mon[90085]: pgmap v2079: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 7762/209898 objects degraded (3.698%); 8911/209898 objects misplaced (4.245%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T10:16:14.053 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:14.084 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:14.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:14.113 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:14.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:14.252 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:14.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:14.354 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:14.383 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:14.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:14.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:14.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:14.638 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:14.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:14.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:14.725 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:14 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7762/209898 objects degraded (3.698%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:14.725 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:14 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7762/209898 objects degraded (3.698%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:14.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:14.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:14.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:14.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:14.974 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:14.981 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:16:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:16:14] "GET /metrics HTTP/1.1" 200 36736 "" "Prometheus/2.51.0" 2024-09-21T10:16:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:14 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7762/209898 objects degraded (3.698%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:15.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:15.101 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:15.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:15.169 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:16:15.170 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:16:15.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:15.207 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:16:15.208 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:15.248 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:15.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:15.250 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:16:15.250 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:15.284 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:16:15.284 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:16:15.315 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:16:15.315 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:15.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:16:15.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:15.350 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:15.382 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:16:15.382 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:15.384 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:15.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:15.417 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:16:15.417 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:15.486 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:15.512 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:15.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:15.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:15.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:15.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:15.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:15.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:15.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:15.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:15.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:15.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:15.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:15 smithi017 ceph-mon[102913]: pgmap v2080: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 7762/203664 objects degraded (3.811%); 8911/203664 objects misplaced (4.375%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T10:16:15.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:15 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2537813641' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:15.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:15 smithi017 ceph-mon[105151]: pgmap v2080: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 7762/203664 objects degraded (3.811%); 8911/203664 objects misplaced (4.375%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T10:16:15.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:15 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2537813641' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:15 smithi134 ceph-mon[90085]: pgmap v2080: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 4.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 7762/203664 objects degraded (3.811%); 8911/203664 objects misplaced (4.375%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T10:16:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:15 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2537813641' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:16.052 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:16.094 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:16.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:16.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:16.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:16.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:16.366 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:16.403 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:16.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:16.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:16.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:16.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:16.645 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:16.675 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:16.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:16.776 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:16.803 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:16.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:16.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:16.943 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:16.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:17.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:17.080 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:17.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:17.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:17.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:17.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:17.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:17.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:17.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:17.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:17.608 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:17.644 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:17.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:17.746 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:17.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:17.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:17.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:17.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:17.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:17 smithi017 ceph-mon[102913]: pgmap v2081: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 7762/195516 objects degraded (3.970%); 8435/195516 objects misplaced (4.314%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T10:16:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:17 smithi017 ceph-mon[105151]: pgmap v2081: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 7762/195516 objects degraded (3.970%); 8435/195516 objects misplaced (4.314%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T10:16:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:17 smithi134 ceph-mon[90085]: pgmap v2081: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 7762/195516 objects degraded (3.970%); 8435/195516 objects misplaced (4.314%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T10:16:18.015 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:18.017 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 1, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:16:18.018 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:16:18.018 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 2, 3, 4, 5, 1, 6, 7] 2024-09-21T10:16:18.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd out 1 2024-09-21T10:16:18.042 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:18.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:18.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:18.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:18.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:18.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:18.351 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:18.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:18.454 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:18.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:18.579 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:18.611 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:18.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:18.675 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:16:18.712 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:18.746 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:18.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:18.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:16:18.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:18.888 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:18.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:18 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/242578017' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:18.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:18 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:18.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:18 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7762/195516 objects degraded (3.970%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:18.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:18 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/242578017' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:18.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:18 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:18.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:18 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7762/195516 objects degraded (3.970%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:18.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:18 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/242578017' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:18 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:18 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7762/195516 objects degraded (3.970%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:19.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:19.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:19.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:19.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:19.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:19.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:19.326 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:19.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:19.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:19.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.481 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:19.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:19.582 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:19.611 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:19.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:19.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:19.741 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:19.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:19.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:19.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:19 smithi017 ceph-mon[102913]: pgmap v2082: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 973 op/s; 7762/192834 objects degraded (4.025%); 8435/192834 objects misplaced (4.374%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T10:16:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:19 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:19 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:19 smithi017 ceph-mon[102913]: osdmap e1327: 8 total, 8 up, 8 in 2024-09-21T10:16:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:19 smithi017 ceph-mon[105151]: pgmap v2082: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 973 op/s; 7762/192834 objects degraded (4.025%); 8435/192834 objects misplaced (4.374%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T10:16:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:19 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:19 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:19 smithi017 ceph-mon[105151]: osdmap e1327: 8 total, 8 up, 8 in 2024-09-21T10:16:19.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:19 smithi134 ceph-mon[90085]: pgmap v2082: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 973 op/s; 7762/192834 objects degraded (4.025%); 8435/192834 objects misplaced (4.374%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T10:16:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:19 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:19 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:19 smithi134 ceph-mon[90085]: osdmap e1327: 8 total, 8 up, 8 in 2024-09-21T10:16:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:20.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:20.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:20.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:20.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:20.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:20.413 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:20.449 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:20.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:16:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:16:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:16:20.552 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:20.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:20.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:20.685 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:20.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:20.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:20.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:20.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:20.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:20.962 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:20.995 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:21.014 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:21.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:21.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:21.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:21.250 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.279 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:21.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:21.380 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.410 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:21.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:21.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:21.569 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:21.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:21.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:21.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:21.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:21.816 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:21.843 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:21.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:21.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:21 smithi017 ceph-mon[102913]: pgmap v2084: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 906 op/s; 7762/189153 objects degraded (4.104%); 8435/189153 objects misplaced (4.459%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:16:21.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:21 smithi017 ceph-mon[105151]: pgmap v2084: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 906 op/s; 7762/189153 objects degraded (4.104%); 8435/189153 objects misplaced (4.459%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:16:21.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.977 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:21.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:21 smithi134 ceph-mon[90085]: pgmap v2084: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 906 op/s; 7762/189153 objects degraded (4.104%); 8435/189153 objects misplaced (4.459%); 2.5 MiB/s, 40 objects/s recovering 2024-09-21T10:16:22.025 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:16:22.025 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:16:22.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.026 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46058.0:193842 10.c e1327/1283 10:3991dc70:::benchmark_data_smithi017_268344_object67911:head v 1327'11932, mlcod=1327'11930)", 2024-09-21T10:16:22.026 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:16:22.010360+0000", 2024-09-21T10:16:22.026 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.011612384, 2024-09-21T10:16:22.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011639346, 2024-09-21T10:16:22.026 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:22.027 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:22.027 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:16:22.027 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:22.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:22.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010360+0000", 2024-09-21T10:16:22.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:22.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010360+0000", 2024-09-21T10:16:22.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:22.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010361+0000", 2024-09-21T10:16:22.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3099999999999997e-07 2024-09-21T10:16:22.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:22.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010368+0000", 2024-09-21T10:16:22.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8709999999999999e-06 2024-09-21T10:16:22.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:22.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010378+0000", 2024-09-21T10:16:22.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0533000000000001e-05 2024-09-21T10:16:22.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:22.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010385+0000", 2024-09-21T10:16:22.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5860000000000001e-06 2024-09-21T10:16:22.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:22.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010563+0000", 2024-09-21T10:16:22.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017837699999999999 2024-09-21T10:16:22.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:22.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010583+0000", 2024-09-21T10:16:22.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0009000000000001e-05 2024-09-21T10:16:22.033 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.033 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:22.033 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.034 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46058.0:193839 10.4 e1327/1325 10:275ef3a5:::benchmark_data_smithi017_268344_object67908:head v 1327'7727, mlcod=1327'7725)", 2024-09-21T10:16:22.034 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:16:22.010422+0000", 2024-09-21T10:16:22.034 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.011550902, 2024-09-21T10:16:22.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011636741000000001, 2024-09-21T10:16:22.034 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:22.034 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:22.034 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:16:22.034 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:22.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:22.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010422+0000", 2024-09-21T10:16:22.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:22.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010422+0000", 2024-09-21T10:16:22.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010422+0000", 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9299999999999998e-07 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010426+0000", 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5090000000000002e-06 2024-09-21T10:16:22.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010433+0000", 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2400000000000001e-06 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010438+0000", 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7260000000000002e-06 2024-09-21T10:16:22.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:22.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.018004+0000", 2024-09-21T10:16:22.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007566466 2024-09-21T10:16:22.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:22.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.018025+0000", 2024-09-21T10:16:22.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0642e-05 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46058.0:193823 10.14 e1327/1316 ondisk, result = 0)", 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:16:22.010879+0000", 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.011093224, 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011228348000000001, 2024-09-21T10:16:22.039 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010879+0000", 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:22.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010879+0000", 2024-09-21T10:16:22.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:22.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010880+0000", 2024-09-21T10:16:22.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6400000000000002e-07 2024-09-21T10:16:22.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:22.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010884+0000", 2024-09-21T10:16:22.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9369999999999997e-06 2024-09-21T10:16:22.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:22.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010894+0000", 2024-09-21T10:16:22.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.533e-06 2024-09-21T10:16:22.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010900+0000", 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0719999999999996e-06 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.020914+0000", 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010014145 2024-09-21T10:16:22.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:22.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.020939+0000", 2024-09-21T10:16:22.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5460999999999999e-05 2024-09-21T10:16:22.044 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.044 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:22.044 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46058.0:193847 10.c e1327/1283 10:3a54f189:::benchmark_data_smithi017_268344_object67916:head v 1327'11933, mlcod=1327'11930)", 2024-09-21T10:16:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:16:22.017559+0000", 2024-09-21T10:16:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0044134650000000001, 2024-09-21T10:16:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.004591052, 2024-09-21T10:16:22.045 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:22.046 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:22.046 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:16:22.046 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:22.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:22.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.017559+0000", 2024-09-21T10:16:22.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:22.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.017559+0000", 2024-09-21T10:16:22.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.017560+0000", 2024-09-21T10:16:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0440000000000001e-06 2024-09-21T10:16:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.017567+0000", 2024-09-21T10:16:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6880000000000002e-06 2024-09-21T10:16:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.017579+0000", 2024-09-21T10:16:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2153e-05 2024-09-21T10:16:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:22.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.017587+0000", 2024-09-21T10:16:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7670000000000003e-06 2024-09-21T10:16:22.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:22.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.021678+0000", 2024-09-21T10:16:22.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040909969999999999 2024-09-21T10:16:22.052 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:193850 10.14 10.47e4c3b4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-09-21T10:16:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:16:22.021517+0000", 2024-09-21T10:16:22.053 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00045561799999999997, 2024-09-21T10:16:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00067527500000000003, 2024-09-21T10:16:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:16:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:22.054 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 193850 2024-09-21T10:16:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.021517+0000", 2024-09-21T10:16:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:22.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.021517+0000", 2024-09-21T10:16:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.021519+0000", 2024-09-21T10:16:22.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.443e-06 2024-09-21T10:16:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.021523+0000", 2024-09-21T10:16:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4739999999999999e-06 2024-09-21T10:16:22.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.021535+0000", 2024-09-21T10:16:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1636e-05 2024-09-21T10:16:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.021542+0000", 2024-09-21T10:16:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9870000000000002e-06 2024-09-21T10:16:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:193823 10.14 10:2dc12089:::benchmark_data_smithi017_268344_object67892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-09-21T10:16:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:16:21.987031+0000", 2024-09-21T10:16:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.034942195000000002, 2024-09-21T10:16:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035207797999999998, 2024-09-21T10:16:22.061 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:22.062 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:22.062 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:22.062 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:22.062 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:22.062 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:22.062 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 193823 2024-09-21T10:16:22.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.063 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:22.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:22.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:21.987031+0000", 2024-09-21T10:16:22.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:22.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:21.987031+0000", 2024-09-21T10:16:22.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:22.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:21.987032+0000", 2024-09-21T10:16:22.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.378e-06 2024-09-21T10:16:22.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:22.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:21.987037+0000", 2024-09-21T10:16:22.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3680000000000001e-06 2024-09-21T10:16:22.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:22.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:21.987052+0000", 2024-09-21T10:16:22.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5172e-05 2024-09-21T10:16:22.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:22.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:21.987059+0000", 2024-09-21T10:16:22.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9369999999999998e-06 2024-09-21T10:16:22.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:22.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:21.992232+0000", 2024-09-21T10:16:22.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051725160000000003 2024-09-21T10:16:22.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:22.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:21.992391+0000", 2024-09-21T10:16:22.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015914999999999999 2024-09-21T10:16:22.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2,4", 2024-09-21T10:16:22.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:21.992998+0000", 2024-09-21T10:16:22.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060728799999999999 2024-09-21T10:16:22.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:22.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.000265+0000", 2024-09-21T10:16:22.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072665170000000001 2024-09-21T10:16:22.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:22.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.000315+0000", 2024-09-21T10:16:22.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9910999999999997e-05 2024-09-21T10:16:22.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:22.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.009188+0000", 2024-09-21T10:16:22.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.008873433 2024-09-21T10:16:22.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:22.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.020961+0000", 2024-09-21T10:16:22.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011772754999999999 2024-09-21T10:16:22.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:22.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.021600+0000", 2024-09-21T10:16:22.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00063954400000000003 2024-09-21T10:16:22.074 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.074 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:22.074 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.075 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:193840 10.16 10:6a6dc1e3:::benchmark_data_smithi017_268344_object67909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-09-21T10:16:22.075 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:16:22.009983+0000", 2024-09-21T10:16:22.075 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.011989432, 2024-09-21T10:16:22.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012326244, 2024-09-21T10:16:22.075 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:22.075 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:22.076 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:22.076 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:22.076 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:22.076 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:22.076 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 193840 2024-09-21T10:16:22.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.077 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:22.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:22.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.009983+0000", 2024-09-21T10:16:22.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:22.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.009983+0000", 2024-09-21T10:16:22.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:22.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.009985+0000", 2024-09-21T10:16:22.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4419999999999999e-06 2024-09-21T10:16:22.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:22.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.009988+0000", 2024-09-21T10:16:22.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6990000000000001e-06 2024-09-21T10:16:22.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:22.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010000+0000", 2024-09-21T10:16:22.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1634000000000001e-05 2024-09-21T10:16:22.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:22.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010007+0000", 2024-09-21T10:16:22.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.255e-06 2024-09-21T10:16:22.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:22.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.012163+0000", 2024-09-21T10:16:22.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021552780000000001 2024-09-21T10:16:22.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:22.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.012315+0000", 2024-09-21T10:16:22.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015287300000000001 2024-09-21T10:16:22.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:16:22.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.012840+0000", 2024-09-21T10:16:22.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000524747 2024-09-21T10:16:22.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:22.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.019670+0000", 2024-09-21T10:16:22.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068302340000000001 2024-09-21T10:16:22.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:22.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.022172+0000", 2024-09-21T10:16:22.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025019090000000001 2024-09-21T10:16:22.085 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.085 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:22.085 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.085 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:193841 10.16 10:6ff30d3a:::benchmark_data_smithi017_268344_object67910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-09-21T10:16:22.085 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:16:22.010022+0000", 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.011950486999999999, 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012347785, 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 193841 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010022+0000", 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010022+0000", 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010023+0000", 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2899999999999999e-07 2024-09-21T10:16:22.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010025+0000", 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0310000000000001e-06 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010031+0000", 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9009999999999999e-06 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:22.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.010036+0000", 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9649999999999999e-06 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.015859+0000", 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058233900000000003 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.016018+0000", 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015927000000000001 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.016487+0000", 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00046865900000000001 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.020488+0000", 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040006360000000001 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:16:22.022211+0000", 2024-09-21T10:16:22.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001723528 2024-09-21T10:16:22.091 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.091 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:22.091 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.091 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:22.091 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-21T10:16:22.091 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 8 2024-09-21T10:16:22.091 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:16:22.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:22.130 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:22.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:22.258 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:22.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:22.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:22.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:22.503 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:22.534 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:22.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:16:22.636 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:22.667 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:22.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:22.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:22.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:22.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:22.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:23.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:23.104 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:23.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:23.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:23.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:23.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:23.339 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:23.361 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:23.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:23.455 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:23.463 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:23.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:23.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:23.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:23.606 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:23.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:23.708 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:23.729 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:23.831 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:23.860 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:23.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:16:23.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:23 smithi017 ceph-mon[102913]: pgmap v2085: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 7762/179958 objects degraded (4.313%); 7861/179958 objects misplaced (4.368%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:16:23.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:23 smithi017 ceph-mon[102913]: osdmap e1328: 8 total, 8 up, 8 in 2024-09-21T10:16:23.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:23 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 7762/179958 objects degraded (4.313%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:23.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:23 smithi017 ceph-mon[105151]: pgmap v2085: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 7762/179958 objects degraded (4.313%); 7861/179958 objects misplaced (4.368%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:16:23.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:23 smithi017 ceph-mon[105151]: osdmap e1328: 8 total, 8 up, 8 in 2024-09-21T10:16:23.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:23 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 7762/179958 objects degraded (4.313%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:23.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:23 smithi134 ceph-mon[90085]: pgmap v2085: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 7762/179958 objects degraded (4.313%); 7861/179958 objects misplaced (4.368%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:16:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:23 smithi134 ceph-mon[90085]: osdmap e1328: 8 total, 8 up, 8 in 2024-09-21T10:16:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:23 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 7762/179958 objects degraded (4.313%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:24.006 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:24.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:24.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:24.212 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:24.258 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:24.282 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:24.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:24.384 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:24.413 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:24.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:24.514 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:24.546 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:24.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:24.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:24.678 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:24.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:24.821 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:24.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:24.923 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:24.949 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:24.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:24.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:24 smithi017 ceph-mon[102913]: osdmap e1329: 8 total, 8 up, 8 in 2024-09-21T10:16:24.953 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:24 smithi017 ceph-mon[105151]: osdmap e1329: 8 total, 8 up, 8 in 2024-09-21T10:16:24.953 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:16:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:16:24] "GET /metrics HTTP/1.1" 200 36736 "" "Prometheus/2.51.0" 2024-09-21T10:16:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:24 smithi134 ceph-mon[90085]: osdmap e1329: 8 total, 8 up, 8 in 2024-09-21T10:16:25.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:25.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:25.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.254 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:25.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:25.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:25.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:25.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:25.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:25.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:25.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:25.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:25.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:25.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:25.921 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:25.950 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:16:25.630+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1329 definitely_dead 0 2024-09-21T10:16:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[102913]: pgmap v2088: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 5478/174315 objects degraded (3.143%); 8865/174315 objects misplaced (5.086%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:16:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/517757204' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:16:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[102913]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:16:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[105151]: pgmap v2088: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 5478/174315 objects degraded (3.143%); 8865/174315 objects misplaced (5.086%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:16:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/517757204' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:16:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:25 smithi017 ceph-mon[105151]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:16:25.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:25 smithi134 ceph-mon[90085]: pgmap v2088: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait, 113 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 5478/174315 objects degraded (3.143%); 8865/174315 objects misplaced (5.086%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:16:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:25 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/517757204' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-21T10:16:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:25 smithi134 ceph-mon[90085]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-21T10:16:26.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:26.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:26.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:26.192 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:26.224 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:26.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:26.303 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.1. 2024-09-21T10:16:26.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:26.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:26.463 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:26.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:26.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:26.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:26.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:26.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:26.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:26.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:26.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:26.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:26.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:26.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:26.981 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:27.082 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:27.105 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:27.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:27.206 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:27.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.229 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:27.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:27.330 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:27.354 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:27.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:27 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/517757204' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:16:27.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:27 smithi017 ceph-mon[105151]: osdmap e1330: 8 total, 8 up, 7 in 2024-09-21T10:16:27.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:27 smithi017 ceph-mon[105151]: pgmap v2090: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 5478/167061 objects degraded (3.279%); 8865/167061 objects misplaced (5.306%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:16:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:27 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/517757204' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:16:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:27 smithi017 ceph-mon[102913]: osdmap e1330: 8 total, 8 up, 7 in 2024-09-21T10:16:27.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:27 smithi017 ceph-mon[102913]: pgmap v2090: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 5478/167061 objects degraded (3.279%); 8865/167061 objects misplaced (5.306%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:16:27.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:27.477 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:27.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:27.579 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:27.600 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:27.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:27.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:27.728 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:27 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/517757204' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-21T10:16:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:27 smithi134 ceph-mon[90085]: osdmap e1330: 8 total, 8 up, 7 in 2024-09-21T10:16:27.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:27 smithi134 ceph-mon[90085]: pgmap v2090: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 5478/167061 objects degraded (3.279%); 8865/167061 objects misplaced (5.306%); 3.3 MiB/s, 52 objects/s recovering 2024-09-21T10:16:27.830 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:27.852 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:27.954 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:27.975 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:27.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:28.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:28.101 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:28.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:28.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:28.226 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:28.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:28.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:28.363 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:28.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:28.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:28.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:28.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:28.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:28.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:28.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:28 smithi017 ceph-mon[102913]: osdmap e1331: 8 total, 8 up, 7 in 2024-09-21T10:16:28.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:28 smithi017 ceph-mon[105151]: osdmap e1331: 8 total, 8 up, 7 in 2024-09-21T10:16:28.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:28.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:28 smithi134 ceph-mon[90085]: osdmap e1331: 8 total, 8 up, 7 in 2024-09-21T10:16:28.749 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:28.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:28.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:28.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:28.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:29.003 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:29.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:29.105 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:29.129 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:29.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:29.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:29.254 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:29.356 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:29.379 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:29.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:29.480 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:29.504 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:29.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:29.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:29.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[102913]: osdmap e1332: 8 total, 8 up, 7 in 2024-09-21T10:16:29.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[102913]: 10.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1329'7826 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[102913]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 1329'7746 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[102913]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 1329'12307 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[102913]: 10.16 starting backfill to osd.2 from (0'0,0'0] MAX to 1329'7781 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[102913]: pgmap v2093: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.8 KiB/s rd, 10 KiB/s wr, 553 op/s; 1548/166344 objects degraded (0.931%); 8865/166344 objects misplaced (5.329%); 3.6 KiB/s, 49 objects/s recovering 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5478/167061 objects degraded (3.279%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[105151]: osdmap e1332: 8 total, 8 up, 7 in 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[105151]: 10.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1329'7826 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[105151]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 1329'7746 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[105151]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 1329'12307 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[105151]: 10.16 starting backfill to osd.2 from (0'0,0'0] MAX to 1329'7781 2024-09-21T10:16:29.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[105151]: pgmap v2093: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.8 KiB/s rd, 10 KiB/s wr, 553 op/s; 1548/166344 objects degraded (0.931%); 8865/166344 objects misplaced (5.329%); 3.6 KiB/s, 49 objects/s recovering 2024-09-21T10:16:29.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:29 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5478/167061 objects degraded (3.279%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:29.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:29 smithi134 ceph-mon[90085]: osdmap e1332: 8 total, 8 up, 7 in 2024-09-21T10:16:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:29 smithi134 ceph-mon[90085]: 10.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1329'7826 2024-09-21T10:16:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:29 smithi134 ceph-mon[90085]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 1329'7746 2024-09-21T10:16:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:29 smithi134 ceph-mon[90085]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 1329'12307 2024-09-21T10:16:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:29 smithi134 ceph-mon[90085]: 10.16 starting backfill to osd.2 from (0'0,0'0] MAX to 1329'7781 2024-09-21T10:16:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:29 smithi134 ceph-mon[90085]: pgmap v2093: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 114 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 4.8 KiB/s rd, 10 KiB/s wr, 553 op/s; 1548/166344 objects degraded (0.931%); 8865/166344 objects misplaced (5.329%); 3.6 KiB/s, 49 objects/s recovering 2024-09-21T10:16:29.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:29 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5478/167061 objects degraded (3.279%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:16:29.755 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:29.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:29.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:29.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:29.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:29.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:30.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:30.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:30.106 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:30.130 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:30.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:30.255 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:30.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:30.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:30.383 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:30.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:30.483 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:16:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:16:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:16:30.485 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:30.507 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:30.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:30.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:30 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3788559333' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:30.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:30 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3788559333' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:30.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:30.633 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:30.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:30.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:30.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:30 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3788559333' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:30.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:30.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:30.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:30.885 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:30.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:30.987 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:31.010 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:31.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:31.109 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:16:31.109 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:16:31.109 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:16:31.109 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:16:31.109 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:16:31.110 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:16:31.110 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:31.110 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:31.110 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:16:31.110 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:31.110 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:16:31.110 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:16:31.110 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:16:31.111 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:31.111 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:31.111 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:31.112 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:31.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:31.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:31.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:31.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:31.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:31.363 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:31.386 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:31.488 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:31.511 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:31.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:31.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:31.637 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:31.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:31.739 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:31.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:31 smithi134 ceph-mon[90085]: pgmap v2094: 119 pgs: 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 113 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 7.5 KiB/s rd, 16 KiB/s wr, 565 op/s; 5437/164217 objects degraded (3.311%); 10509/164217 objects misplaced (6.399%); 5.8 KiB/s, 39 objects/s recovering 2024-09-21T10:16:31.762 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:31.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:31.864 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:31.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:31 smithi017 ceph-mon[102913]: pgmap v2094: 119 pgs: 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 113 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 7.5 KiB/s rd, 16 KiB/s wr, 565 op/s; 5437/164217 objects degraded (3.311%); 10509/164217 objects misplaced (6.399%); 5.8 KiB/s, 39 objects/s recovering 2024-09-21T10:16:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:31 smithi017 ceph-mon[105151]: pgmap v2094: 119 pgs: 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 113 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 7.5 KiB/s rd, 16 KiB/s wr, 565 op/s; 5437/164217 objects degraded (3.311%); 10509/164217 objects misplaced (6.399%); 5.8 KiB/s, 39 objects/s recovering 2024-09-21T10:16:31.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:32.012 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:32.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:32.114 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:32.137 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:32.207 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:16:32.207 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:16:32.207 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-21T10:16:32.207 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:16:32.207 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:16:32.208 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-21T10:16:32.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:32.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:32.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:32.357 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:32.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:32.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:32.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:32.521 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:32.547 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:16:32.622 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:32.647 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:32.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:32.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:32.771 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:32.873 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:32.900 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:32.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:33.002 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:33.026 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:33.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:33.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:33.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:33.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:33.253 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:33.278 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:33.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:33.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:33.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:33.499 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:16:33.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:33.530 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:33.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:33.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:33.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:33.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:33.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:33 smithi017 ceph-mon[102913]: osdmap e1333: 8 total, 8 up, 7 in 2024-09-21T10:16:33.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:33 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2093913396' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:16:33.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:33 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:16:33.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:33 smithi017 ceph-mon[102913]: pgmap v2096: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 activating+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 111 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 250 op/s; 14020/162738 objects degraded (8.615%); 8296/162738 objects misplaced (5.098%); 26 MiB/s, 461 objects/s recovering 2024-09-21T10:16:33.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:33 smithi017 ceph-mon[105151]: osdmap e1333: 8 total, 8 up, 7 in 2024-09-21T10:16:33.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:33 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2093913396' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:16:33.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:33 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:16:33.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:33 smithi017 ceph-mon[105151]: pgmap v2096: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 activating+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 111 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 250 op/s; 14020/162738 objects degraded (8.615%); 8296/162738 objects misplaced (5.098%); 26 MiB/s, 461 objects/s recovering 2024-09-21T10:16:33.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:33.782 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:33.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:33.852 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:33 smithi134 ceph-mon[90085]: osdmap e1333: 8 total, 8 up, 7 in 2024-09-21T10:16:33.852 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:33 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2093913396' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:16:33.852 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:33 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:16:33.852 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:33 smithi134 ceph-mon[90085]: pgmap v2096: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 activating+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 111 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 17 KiB/s rd, 37 KiB/s wr, 250 op/s; 14020/162738 objects degraded (8.615%); 8296/162738 objects misplaced (5.098%); 26 MiB/s, 461 objects/s recovering 2024-09-21T10:16:33.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:33.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:33.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:34.012 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:34.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:16:34.039 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:34.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:34.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:34.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:34.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:34.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:34.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:34.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:34.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:34.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:34.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:34.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:34.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:34.693 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:34.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:34.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:34.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:34.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:34.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:34 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:34 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:16:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:34 smithi017 ceph-mon[102913]: osdmap e1334: 8 total, 8 up, 7 in 2024-09-21T10:16:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:34 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 14020/162738 objects degraded (8.615%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:34 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:34 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:16:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:34 smithi017 ceph-mon[105151]: osdmap e1334: 8 total, 8 up, 7 in 2024-09-21T10:16:34.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:34 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 14020/162738 objects degraded (8.615%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:34.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:16:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:16:34] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-21T10:16:34.962 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:34 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:34 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:16:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:34 smithi134 ceph-mon[90085]: osdmap e1334: 8 total, 8 up, 7 in 2024-09-21T10:16:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:34 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 14020/162738 objects degraded (8.615%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:35.064 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:35.096 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:35.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:35.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:35.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:35.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:35.328 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:35.360 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:35.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:35.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:35.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:35.588 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:35.614 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:35.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:35.715 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:35.744 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:35.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:35.846 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:35.872 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:35.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:35.936 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:35 smithi017 ceph-mon[102913]: pgmap v2098: 119 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 activating+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 111 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 13 KiB/s rd, 28 KiB/s wr, 202 op/s; 14020/162705 objects degraded (8.617%); 8293/162705 objects misplaced (5.097%); 25 MiB/s, 619 objects/s recovering 2024-09-21T10:16:35.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:35 smithi017 ceph-mon[105151]: pgmap v2098: 119 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 activating+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 111 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 13 KiB/s rd, 28 KiB/s wr, 202 op/s; 14020/162705 objects degraded (8.617%); 8293/162705 objects misplaced (5.097%); 25 MiB/s, 619 objects/s recovering 2024-09-21T10:16:35.974 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:35 smithi134 ceph-mon[90085]: pgmap v2098: 119 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 activating+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 111 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 13 KiB/s rd, 28 KiB/s wr, 202 op/s; 14020/162705 objects degraded (8.617%); 8293/162705 objects misplaced (5.097%); 25 MiB/s, 619 objects/s recovering 2024-09-21T10:16:35.996 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:35.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:36.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:36.124 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:36.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:36.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:36.250 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:36.352 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:36.377 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:36.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:36.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:36.506 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:36.608 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:36.638 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:36.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:36.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:36.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:36.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:36.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:36.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:36.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:37.005 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:37.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:37.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:37.134 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:37.162 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:37.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:37.264 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:37.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:37.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:37.398 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:37.431 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:37.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:37.532 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:37.558 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:37.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:37.659 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:37.684 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:37.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:37.747 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:37 smithi017 ceph-mon[102913]: pgmap v2099: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 111 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 9.9 KiB/s rd, 22 KiB/s wr, 174 op/s; 10944/162297 objects degraded (6.743%); 8621/162297 objects misplaced (5.312%); 23 MiB/s, 535 objects/s recovering 2024-09-21T10:16:37.747 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:37 smithi017 ceph-mon[105151]: pgmap v2099: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 111 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 9.9 KiB/s rd, 22 KiB/s wr, 174 op/s; 10944/162297 objects degraded (6.743%); 8621/162297 objects misplaced (5.312%); 23 MiB/s, 535 objects/s recovering 2024-09-21T10:16:37.786 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:37.811 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:37.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:37.852 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:37.852 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:37.913 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.940 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:37.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:37 smithi134 ceph-mon[90085]: pgmap v2099: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 activating+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 111 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 9.9 KiB/s rd, 22 KiB/s wr, 174 op/s; 10944/162297 objects degraded (6.743%); 8621/162297 objects misplaced (5.312%); 23 MiB/s, 535 objects/s recovering 2024-09-21T10:16:38.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:38.068 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:38.133 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:38.171 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:38.201 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:38.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:38.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:38.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:38.430 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:38.457 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:38.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:38.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:38.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:38.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:38.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:38.692 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:16:38.692 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "" 2024-09-21T10:16:38.692 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:16:38.718 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:38.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:38.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:16:38.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:38.847 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:38.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:38.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:38.980 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:38.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:39.082 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:39.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:39.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:39.233 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:39.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:39.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:39.358 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:39.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:39.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:39.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:39.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:39.590 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:39.617 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:39.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:39.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:39.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:16:39.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:39.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:39.863 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:39.891 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:39 smithi017 ceph-mon[102913]: pgmap v2100: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 7.1 KiB/s rd, 15 KiB/s wr, 211 op/s; 10845/159258 objects degraded (6.810%); 7663/159258 objects misplaced (4.812%); 30 MiB/s, 653 objects/s recovering 2024-09-21T10:16:39.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:39 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 10845/159258 objects degraded (6.810%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:39 smithi017 ceph-mon[105151]: pgmap v2100: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 7.1 KiB/s rd, 15 KiB/s wr, 211 op/s; 10845/159258 objects degraded (6.810%); 7663/159258 objects misplaced (4.812%); 30 MiB/s, 653 objects/s recovering 2024-09-21T10:16:39.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:39 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 10845/159258 objects degraded (6.810%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:39.993 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:39 smithi134 ceph-mon[90085]: pgmap v2100: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 7.1 KiB/s rd, 15 KiB/s wr, 211 op/s; 10845/159258 objects degraded (6.810%); 7663/159258 objects misplaced (4.812%); 30 MiB/s, 653 objects/s recovering 2024-09-21T10:16:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:39 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 10845/159258 objects degraded (6.810%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:40.023 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:40.125 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:40.150 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:40.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:40.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:40.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:40.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:40.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:40.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:40.488 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:16:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:16:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:16:40.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:40.528 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:40.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:40.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:40.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:40.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:40.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:40.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:40.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:40.791 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:40.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:40.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:40.924 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:40.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:41.026 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:41.053 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:41.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:41.178 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:41.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:41.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:41.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:41.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:41.317 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:16:41.318 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:16:41.344 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:16:41.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:41.380 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:16:41.380 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:41.402 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:16:41.403 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:16:41.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:41.423 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:16:41.423 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:41.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:41.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:41.458 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:16:41.459 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:41.486 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:16:41.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:41.512 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:16:41.512 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:41.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:41.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:41.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:41.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:41.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:41.817 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:41.844 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:41.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:41.945 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:41 smithi017 ceph-mon[102913]: pgmap v2101: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 10845/157113 objects degraded (6.903%); 7583/157113 objects misplaced (4.826%); 11 MiB/s, 334 objects/s recovering 2024-09-21T10:16:41.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:41 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3936329632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:41 smithi017 ceph-mon[105151]: pgmap v2101: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 10845/157113 objects degraded (6.903%); 7583/157113 objects misplaced (4.826%); 11 MiB/s, 334 objects/s recovering 2024-09-21T10:16:41.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:41 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3936329632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:41.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:41.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:41 smithi134 ceph-mon[90085]: pgmap v2101: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 233 op/s; 10845/157113 objects degraded (6.903%); 7583/157113 objects misplaced (4.826%); 11 MiB/s, 334 objects/s recovering 2024-09-21T10:16:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:41 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3936329632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:42.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:42.115 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:42.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:42.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:42.249 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:42.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:42.351 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:42.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:42.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:42.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:42.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:42.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:42.649 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:42.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:42.751 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:42.779 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:42.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:42.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:42.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:42.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:43.007 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:43.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:43.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:43.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:43.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:43.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:43.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:43.296 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:43.321 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:43.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:43.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:43.452 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:43.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:43.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:43.579 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:43.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:43.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:43.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:43.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:43.835 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:43 smithi017 ceph-mon[102913]: pgmap v2102: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 322 op/s; 10711/153057 objects degraded (6.998%); 7099/153057 objects misplaced (4.638%); 11 MiB/s, 341 objects/s recovering 2024-09-21T10:16:43.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:43 smithi017 ceph-mon[105151]: pgmap v2102: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 322 op/s; 10711/153057 objects degraded (6.998%); 7099/153057 objects misplaced (4.638%); 11 MiB/s, 341 objects/s recovering 2024-09-21T10:16:43.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:43.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:43.939 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:43.968 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:43 smithi134 ceph-mon[90085]: pgmap v2102: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 322 op/s; 10711/153057 objects degraded (6.998%); 7099/153057 objects misplaced (4.638%); 11 MiB/s, 341 objects/s recovering 2024-09-21T10:16:44.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:44.096 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:44.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:44.197 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:44.224 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:44.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:44.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:44.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:44.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:44.485 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:44.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:44.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:16:44.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:44.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:44.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:44 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 10711/153057 objects degraded (6.998%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/286290954' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:44.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:44 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:44 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 10711/153057 objects degraded (6.998%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/286290954' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:44.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:44 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:44.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:44.758 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:44.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:44.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:44.888 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:44 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 10711/153057 objects degraded (6.998%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:44.888 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/286290954' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:44.888 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:44 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:44.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.889 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:44.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:44.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:45.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:45.034 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:16:45.036 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:16:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:16:44] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-21T10:16:45.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:45.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:45.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:45.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:45.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:45.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:45.387 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:45.409 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:45.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:45.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:45.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:45.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:45.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:45.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:45 smithi017 ceph-mon[102913]: pgmap v2103: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 373 op/s; 10711/150363 objects degraded (7.123%); 7036/150363 objects misplaced (4.679%); 10 MiB/s, 190 objects/s recovering 2024-09-21T10:16:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:45 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:16:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:45 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:45 smithi017 ceph-mon[102913]: osdmap e1335: 8 total, 8 up, 7 in 2024-09-21T10:16:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:45 smithi017 ceph-mon[105151]: pgmap v2103: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 373 op/s; 10711/150363 objects degraded (7.123%); 7036/150363 objects misplaced (4.679%); 10 MiB/s, 190 objects/s recovering 2024-09-21T10:16:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:45 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:16:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:45 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:45 smithi017 ceph-mon[105151]: osdmap e1335: 8 total, 8 up, 7 in 2024-09-21T10:16:45.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:45.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:45.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:45.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:45.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:45 smithi134 ceph-mon[90085]: pgmap v2103: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 373 op/s; 10711/150363 objects degraded (7.123%); 7036/150363 objects misplaced (4.679%); 10 MiB/s, 190 objects/s recovering 2024-09-21T10:16:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:45 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:16:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:45 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:45.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:45 smithi134 ceph-mon[90085]: osdmap e1335: 8 total, 8 up, 7 in 2024-09-21T10:16:46.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:46.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:46.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:46.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:46.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:46.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:46.292 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:46.315 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:46.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:46.365 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:16:46.365 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:16:46.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263842 8.e 8:74c05c70:::benchmark_data_smithi017_209235_object130936:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.460994+0000", 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 501.89844735000003, 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1964085500000001, 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:16:46.366 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263842 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.460994+0000", 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.460994+0000", 2024-09-21T10:16:46.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.460995+0000", 2024-09-21T10:16:46.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9300000000000006e-07 2024-09-21T10:16:46.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.460999+0000", 2024-09-21T10:16:46.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9559999999999999e-06 2024-09-21T10:16:46.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.461009+0000", 2024-09-21T10:16:46.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6919999999999997e-06 2024-09-21T10:16:46.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.461014+0000", 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.938e-06 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.461072+0000", 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7902999999999998e-05 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.461083+0000", 2024-09-21T10:16:46.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0606e-05 2024-09-21T10:16:46.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.420582+0000", 2024-09-21T10:16:46.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95949984399999999 2024-09-21T10:16:46.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.420593+0000", 2024-09-21T10:16:46.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0464e-05 2024-09-21T10:16:46.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.642934+0000", 2024-09-21T10:16:46.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.222341165 2024-09-21T10:16:46.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.643068+0000", 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013448799999999999 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.643209+0000", 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014089300000000001 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.646941+0000", 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037320299999999999 2024-09-21T10:16:46.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.653755+0000", 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068134429999999998 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657243+0000", 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034884680000000002 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657293+0000", 2024-09-21T10:16:46.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9672999999999997e-05 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657403+0000", 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109994 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45620.0:263850 8.e 8:73dca25d:::benchmark_data_smithi017_209235_object130944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:08:24.477238+0000", 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 501.88220335699998, 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1804316350000001, 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.376 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45620", 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3204292246", 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 263850 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477238+0000", 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477238+0000", 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477240+0000", 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.604e-06 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477246+0000", 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4470000000000003e-06 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477262+0000", 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5455999999999999e-05 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477269+0000", 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.311e-06 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477333+0000", 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3355999999999994e-05 2024-09-21T10:16:46.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:24.477343+0000", 2024-09-21T10:16:46.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0161e-05 2024-09-21T10:16:46.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421884+0000", 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94454095100000002 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:25.421895+0000", 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1535999999999999e-05 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.646983+0000", 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.225087781 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.647090+0000", 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000107135 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:16:46.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.647161+0000", 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1082999999999997e-05 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.653697+0000", 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00653588 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657450+0000", 2024-09-21T10:16:46.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037531679999999999 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657463+0000", 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2928999999999999e-05 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657549+0000", 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6212999999999996e-05 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:08:26.657670+0000", 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120624 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57281 9.9 9:92fb5fa7:::benchmark_data_smithi017_237924_object57280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.448907+0000", 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 441.91053495800003, 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2562489019999998, 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.385 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57281 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448907+0000", 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448907+0000", 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448909+0000", 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7600000000000001e-06 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448954+0000", 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5025999999999999e-05 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448968+0000", 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4243000000000001e-05 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.448996+0000", 2024-09-21T10:16:46.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8161e-05 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.449193+0000", 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019727500000000001 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.449205+0000", 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1917e-05 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.458659+0000", 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0094541319999999 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.458672+0000", 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2559e-05 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.687854+0000", 2024-09-21T10:16:46.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.229181708 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.687998+0000", 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014425500000000001 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.688153+0000", 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015539199999999999 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.693095+0000", 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049422550000000004 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.693218+0000", 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122077 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.702384+0000", 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0091666160000000007 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.702457+0000", 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2898999999999994e-05 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.705156+0000", 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026986269999999999 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.393 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57298 9.5 9:a7783708:::benchmark_data_smithi017_237924_object57297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.469242+0000", 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 441.89019971300002, 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.293953723, 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57298 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469242+0000", 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469242+0000", 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469243+0000", 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.344e-06 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469266+0000", 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2154000000000002e-05 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469281+0000", 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5262000000000001e-05 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469290+0000", 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7800000000000006e-06 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469364+0000", 2024-09-21T10:16:46.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4004999999999994e-05 2024-09-21T10:16:46.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.469375+0000", 2024-09-21T10:16:46.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.17e-05 2024-09-21T10:16:46.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.434823+0000", 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96544806999999999 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.434837+0000", 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3692e-05 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.741938+0000", 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.307101399 2024-09-21T10:16:46.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.742085+0000", 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000146911 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.742239+0000", 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015382899999999999 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.747492+0000", 2024-09-21T10:16:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052529810000000003 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.752487+0000", 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049951450000000003 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.756963+0000", 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044756479999999996 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.757079+0000", 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116134 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.763196+0000", 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061166689999999999 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173500 9.5 9:adc0c03e:::benchmark_data_smithi017_237924_object80496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.683064+0000", 2024-09-21T10:16:46.401 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 261.67637805599998, 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2520501240000002, 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173500 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683064+0000", 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683064+0000", 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683066+0000", 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8190000000000001e-06 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683072+0000", 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9789999999999998e-06 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683083+0000", 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1467e-05 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683093+0000", 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.594e-06 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686032+0000", 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029398359999999999 2024-09-21T10:16:46.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686156+0000", 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012317 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686284+0000", 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012887799999999999 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.695220+0000", 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0089355019999999997 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.700524+0000", 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053036070000000001 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.719426+0000", 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018902077 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.719439+0000", 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3179999999999999e-05 2024-09-21T10:16:46.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.693124+0000", 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.973685315 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.693142+0000", 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8020999999999999e-05 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917018+0000", 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2238759670000001 2024-09-21T10:16:46.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917129+0000", 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111148 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.935025+0000", 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017895409000000001 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.935114+0000", 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9154999999999996e-05 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.408 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173510 9.5 9:aed0fd9d:::benchmark_data_smithi017_237924_object80506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.718247+0000", 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 261.64119512399998, 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2162618580000002, 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173510 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.409 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718247+0000", 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718247+0000", 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718249+0000", 2024-09-21T10:16:46.411 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:46.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2330000000000001e-06 2024-09-21T10:16:46.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718255+0000", 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1990000000000003e-06 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718269+0000", 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.375e-05 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718279+0000", 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.679e-06 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.721856+0000", 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035772970000000001 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.721865+0000", 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1989999999999995e-06 2024-09-21T10:16:46.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.693179+0000", 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97131427199999998 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.693188+0000", 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2259999999999994e-06 2024-09-21T10:16:46.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917704+0000", 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2245152340000001 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917832+0000", 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128565 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5,7", 2024-09-21T10:16:46.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917978+0000", 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014551200000000001 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.920920+0000", 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029422060000000002 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.921118+0000", 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000197648 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.921345+0000", 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022725100000000001 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.934019+0000", 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012674505000000001 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.934102+0000", 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2096999999999997e-05 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.934508+0000", 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040698500000000001 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173512 9.9 9:99bb9c6d:::benchmark_data_smithi017_237924_object80508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.718318+0000", 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 261.64112422199997, 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.211803164, 2024-09-21T10:16:46.418 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173512 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718318+0000", 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718318+0000", 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718318+0000", 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2800000000000003e-07 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718320+0000", 2024-09-21T10:16:46.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6929999999999999e-06 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718325+0000", 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4310000000000002e-06 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.718330+0000", 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.515e-06 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.724088+0000", 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057589590000000001 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.724097+0000", 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5499999999999995e-06 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.696969+0000", 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97287164800000003 2024-09-21T10:16:46.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.696980+0000", 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1403000000000001e-05 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.887650+0000", 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1906697939999999 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.887743+0000", 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3257000000000005e-05 2024-09-21T10:16:46.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.887820+0000", 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6514000000000006e-05 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894542+0000", 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067218820000000002 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894590+0000", 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8013000000000003e-05 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894646+0000", 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6104999999999999e-05 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929714+0000", 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035068662 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929780+0000", 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5324e-05 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.930121+0000", 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034108599999999999 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173518 9.9 9:935b61bf:::benchmark_data_smithi017_237924_object80514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.729828+0000", 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 261.62961391900001, 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2030523099999999, 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173518 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729828+0000", 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729828+0000", 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729830+0000", 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.756e-06 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729836+0000", 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3099999999999997e-06 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729853+0000", 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7350000000000002e-05 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729862+0000", 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9190000000000005e-06 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729929+0000", 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7291000000000003e-05 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.729937+0000", 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3529999999999997e-06 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.698207+0000", 2024-09-21T10:16:46.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96827046900000002 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.698217+0000", 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9399999999999997e-06 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.891189+0000", 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.192971875 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.891337+0000", 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000147396 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.891474+0000", 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013725 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894704+0000", 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032297950000000001 2024-09-21T10:16:46.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894749+0000", 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5677000000000003e-05 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.894763+0000", 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4248e-05 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.932394+0000", 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.037630080000000003 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.932448+0000", 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4033999999999998e-05 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.932880+0000", 2024-09-21T10:16:46.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00043256700000000002 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62494 10.11 10:8d7e2c9e:::benchmark_data_smithi017_268344_object62493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.077717+0000", 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.28172437800001, 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5820376550000002, 2024-09-21T10:16:46.435 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62494 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.077717+0000", 2024-09-21T10:16:46.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.077717+0000", 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.077718+0000", 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.026e-06 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.077742+0000", 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3693000000000002e-05 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.077760+0000", 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8073000000000001e-05 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.077768+0000", 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2879999999999993e-06 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.077861+0000", 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2870999999999999e-05 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.077874+0000", 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2452999999999999e-05 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.597968+0000", 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.52009461899999998 2024-09-21T10:16:46.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.597983+0000", 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4100999999999999e-05 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.650856+0000", 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.052873189 2024-09-21T10:16:46.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.650984+0000", 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127825 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.651111+0000", 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012710900000000001 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.653938+0000", 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028274609999999999 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.655524+0000", 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001586173 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.658667+0000", 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031423929999999998 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.658704+0000", 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7412e-05 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.659755+0000", 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0010509689999999999 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62575 10.11 10:8be77efe:::benchmark_data_smithi017_268344_object62574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.143236+0000", 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.216205401, 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5180160840000001, 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.443 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62575 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.143236+0000", 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.143236+0000", 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.143238+0000", 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.145e-06 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.143265+0000", 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7500000000000001e-05 2024-09-21T10:16:46.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.143277+0000", 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1503e-05 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.143283+0000", 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9e-06 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.144728+0000", 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014447100000000001 2024-09-21T10:16:46.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.144738+0000", 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0087e-05 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598366+0000", 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.45362788100000001 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598375+0000", 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4379999999999995e-06 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.652453+0000", 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.054078047 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.652548+0000", 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5610000000000001e-05 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.652671+0000", 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012322799999999999 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.655513+0000", 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028418829999999999 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.655530+0000", 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7020999999999998e-05 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.659915+0000", 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043844590000000003 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.659980+0000", 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5236999999999999e-05 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.661252+0000", 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012724349999999999 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62631 10.11 10:88c1a7b1:::benchmark_data_smithi017_268344_object62630:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.191339+0000", 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.168103183, 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.473759166, 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62631 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.191339+0000", 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.191339+0000", 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.191340+0000", 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2559999999999999e-06 2024-09-21T10:16:46.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.191366+0000", 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5749999999999999e-05 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.191378+0000", 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.238e-05 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.453 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.191387+0000", 2024-09-21T10:16:46.454 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7069999999999998e-06 2024-09-21T10:16:46.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:46.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.191560+0000", 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017355800000000001 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.191571+0000", 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0705e-05 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598396+0000", 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.40682548699999999 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598402+0000", 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.169e-06 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.653242+0000", 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0548400330000001 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.653327+0000", 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5173999999999995e-05 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.653434+0000", 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000107173 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.655561+0000", 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021266480000000001 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.655574+0000", 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3404000000000001e-05 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.664262+0000", 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0086884590000000008 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.664302+0000", 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9282000000000001e-05 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.665098+0000", 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079598099999999997 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62659 10.11 10:8f50109f:::benchmark_data_smithi017_268344_object62658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.209230+0000", 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.15021133499999, 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4613908420000001, 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.460 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62659 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.209230+0000", 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.209230+0000", 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.209232+0000", 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1850000000000001e-06 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.209252+0000", 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.084e-05 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.209261+0000", 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2339999999999994e-06 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.209266+0000", 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6049999999999997e-06 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.212823+0000", 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003556371 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.212834+0000", 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1551e-05 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598422+0000", 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.38558734 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598426+0000", 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0589999999999996e-06 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.653983+0000", 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0555578539999999 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.654096+0000", 2024-09-21T10:16:46.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112131 2024-09-21T10:16:46.466 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.466 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.466 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.466 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.654192+0000", 2024-09-21T10:16:46.466 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6810000000000003e-05 2024-09-21T10:16:46.466 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.466 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.466 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.466 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.656968+0000", 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027757509999999999 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.657767+0000", 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079867200000000005 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.669342+0000", 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011575369 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.669407+0000", 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5287e-05 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.670621+0000", 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001213783 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.468 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62675 10.11 10:8e749d0d:::benchmark_data_smithi017_268344_object62674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.231239+0000", 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.12820256699999, 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.440449291, 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62675 2024-09-21T10:16:46.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.231239+0000", 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.231239+0000", 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.231241+0000", 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0899999999999999e-06 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.231357+0000", 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116019 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.231373+0000", 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6011e-05 2024-09-21T10:16:46.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.231382+0000", 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0729999999999994e-06 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.231448+0000", 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5815999999999995e-05 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.231459+0000", 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0402e-05 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598445+0000", 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36698634000000002 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598449+0000", 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.985e-06 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.654749+0000", 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.056300271 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.654842+0000", 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2681000000000003e-05 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.654939+0000", 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6997999999999995e-05 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.657771+0000", 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028323799999999998 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.658527+0000", 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075529600000000003 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.670714+0000", 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012187699999999999 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.670773+0000", 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9098000000000002e-05 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.671688+0000", 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00091513100000000002 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62709 10.11 10:8b362c16:::benchmark_data_smithi017_268344_object62708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.476 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.257183+0000", 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.102258984, 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.415972507, 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62709 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.257183+0000", 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.257183+0000", 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.257184+0000", 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.08e-06 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.257208+0000", 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4301999999999999e-05 2024-09-21T10:16:46.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.257218+0000", 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0018e-05 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.257226+0000", 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6420000000000007e-06 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.257401+0000", 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017537400000000001 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.257411+0000", 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6460000000000005e-06 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598468+0000", 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34105720299999998 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598472+0000", 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9380000000000002e-06 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.655591+0000", 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.057118711 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.655688+0000", 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7114000000000006e-05 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.655782+0000", 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3740999999999996e-05 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.658537+0000", 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027550019999999999 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.658575+0000", 2024-09-21T10:16:46.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8717999999999998e-05 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.671840+0000", 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013264531 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.671900+0000", 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9926000000000002e-05 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.673155+0000", 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012555610000000001 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62712 10.11 10:8d4d381a:::benchmark_data_smithi017_268344_object62711:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.260547+0000", 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.09889517799999, 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4134918679999999, 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.484 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62712 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.260547+0000", 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.260547+0000", 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.260548+0000", 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.767e-06 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.260571+0000", 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3019000000000001e-05 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.260584+0000", 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2539e-05 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.260592+0000", 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9999999999999996e-06 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.260670+0000", 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7810000000000002e-05 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.260680+0000", 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0657e-05 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598491+0000", 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33781060400000001 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598495+0000", 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8460000000000001e-06 2024-09-21T10:16:46.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.656269+0000", 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.057774422 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.656367+0000", 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7777999999999998e-05 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.656471+0000", 2024-09-21T10:16:46.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010443299999999999 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.658542+0000", 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020702329999999999 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.658618+0000", 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6415000000000006e-05 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.673211+0000", 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014592647 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.673271+0000", 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9880999999999998e-05 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.674038+0000", 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00076781699999999996 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.491 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62719 10.11 10:8de4f4ad:::benchmark_data_smithi017_268344_object62718:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.265537+0000", 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.093905076, 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4094240259999999, 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.492 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62719 2024-09-21T10:16:46.493 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.493 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.493 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.493 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.493 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.265537+0000", 2024-09-21T10:16:46.493 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.494 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.494 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.494 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.494 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.265537+0000", 2024-09-21T10:16:46.494 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.494 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.265538+0000", 2024-09-21T10:16:46.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2440000000000001e-06 2024-09-21T10:16:46.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.265557+0000", 2024-09-21T10:16:46.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9145000000000001e-05 2024-09-21T10:16:46.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.497 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.265569+0000", 2024-09-21T10:16:46.497 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1762e-05 2024-09-21T10:16:46.497 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.497 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.497 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.265576+0000", 2024-09-21T10:16:46.497 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2010000000000002e-06 2024-09-21T10:16:46.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.498 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.498 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.265808+0000", 2024-09-21T10:16:46.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000231516 2024-09-21T10:16:46.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.265819+0000", 2024-09-21T10:16:46.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1328999999999999e-05 2024-09-21T10:16:46.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598514+0000", 2024-09-21T10:16:46.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33269490000000002 2024-09-21T10:16:46.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598518+0000", 2024-09-21T10:16:46.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7689999999999998e-06 2024-09-21T10:16:46.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.657008+0000", 2024-09-21T10:16:46.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.05849079 2024-09-21T10:16:46.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.502 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.502 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.502 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.657101+0000", 2024-09-21T10:16:46.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2448000000000007e-05 2024-09-21T10:16:46.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.503 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.503 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.657208+0000", 2024-09-21T10:16:46.503 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010688000000000001 2024-09-21T10:16:46.503 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.504 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.658630+0000", 2024-09-21T10:16:46.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014227210000000001 2024-09-21T10:16:46.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.659836+0000", 2024-09-21T10:16:46.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012053859999999999 2024-09-21T10:16:46.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.674109+0000", 2024-09-21T10:16:46.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014273460999999999 2024-09-21T10:16:46.506 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.506 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.506 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.506 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.674168+0000", 2024-09-21T10:16:46.506 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9020000000000001e-05 2024-09-21T10:16:46.506 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.507 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.507 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.507 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.674961+0000", 2024-09-21T10:16:46.507 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00079245400000000005 2024-09-21T10:16:46.507 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.507 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.507 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62742 10.11 10:8d2619ad:::benchmark_data_smithi017_268344_object62741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.280972+0000", 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.078469367, 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.404117668, 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.508 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62742 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.280972+0000", 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.280972+0000", 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.280974+0000", 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.511e-06 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.280997+0000", 2024-09-21T10:16:46.510 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3589999999999999e-05 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.281010+0000", 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2622e-05 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.281018+0000", 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5360000000000004e-06 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.511 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.281129+0000", 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111049 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.281140+0000", 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1357e-05 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598536+0000", 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.317396122 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598540+0000", 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8190000000000002e-06 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.657802+0000", 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.059262326 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.513 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.657895+0000", 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2275999999999995e-05 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.657992+0000", 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6992999999999998e-05 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.659854+0000", 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018620450000000001 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.514 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.661320+0000", 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014660960000000001 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.683885+0000", 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022564810000000001 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.683947+0000", 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2617e-05 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.685090+0000", 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001142899 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62756 10.11 10:883b80ea:::benchmark_data_smithi017_268344_object62755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.292969+0000", 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.06647257500001, 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3981548670000001, 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.516 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62756 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.292969+0000", 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.292969+0000", 2024-09-21T10:16:46.517 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.292971+0000", 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.516e-06 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.292992+0000", 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1427e-05 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.518 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.293005+0000", 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2797e-05 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.293012+0000", 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5220000000000001e-06 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.293080+0000", 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7454999999999997e-05 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.293090+0000", 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9040000000000004e-06 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598559+0000", 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30546879700000001 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.520 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598562+0000", 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8260000000000003e-06 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.661358+0000", 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0627956759999999 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.661519+0000", 2024-09-21T10:16:46.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016138900000000001 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.661705+0000", 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000185775 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.667822+0000", 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061171749999999999 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.669193+0000", 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001370762 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.689266+0000", 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020072601999999998 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.689330+0000", 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4369000000000007e-05 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.691124+0000", 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017938749999999999 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62768 10.11 10:8cefeafa:::benchmark_data_smithi017_268344_object62767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.524 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.303400+0000", 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.05604155099999, 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.388467685, 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62768 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.525 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.303400+0000", 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.303400+0000", 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.303402+0000", 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.818e-06 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.303434+0000", 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2020999999999997e-05 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.303446+0000", 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2296999999999999e-05 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.303454+0000", 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8599999999999993e-06 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.303567+0000", 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011263 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.303578+0000", 2024-09-21T10:16:46.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0730999999999999e-05 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598581+0000", 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.29500372200000002 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598585+0000", 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7450000000000001e-06 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.529 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.662985+0000", 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0643995559999999 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.663150+0000", 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016568999999999999 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.663335+0000", 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018482899999999999 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.667830+0000", 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044952689999999997 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.669248+0000", 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014177199999999999 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.691183+0000", 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021935335 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.691222+0000", 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8600000000000003e-05 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.691868+0000", 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00064586199999999998 2024-09-21T10:16:46.532 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:62819 10.11 10:8d876127:::benchmark_data_smithi017_268344_object62818:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1296)", 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:13.352807+0000", 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 153.00663496799999, 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.342508598, 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:46.533 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 62819 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.352807+0000", 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.534 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.352807+0000", 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.352808+0000", 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5230000000000001e-06 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.352834+0000", 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5432e-05 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.352846+0000", 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2112e-05 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.352851+0000", 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2209999999999996e-06 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.352916+0000", 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4896999999999999e-05 2024-09-21T10:16:46.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.352927+0000", 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1182999999999999e-05 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598617+0000", 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.245690139 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:13.598623+0000", 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2700000000000004e-06 2024-09-21T10:16:46.537 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.665155+0000", 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.066532212 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.665321+0000", 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166064 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:16:46.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.665526+0000", 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020553499999999999 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.669202+0000", 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036759750000000002 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.669295+0000", 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2682000000000005e-05 2024-09-21T10:16:46.539 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.693719+0000", 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.024424444 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.693782+0000", 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2080000000000002e-05 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:14.695315+0000", 2024-09-21T10:16:46.540 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001533829 2024-09-21T10:16:46.541 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.541 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.541 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.541 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:16:46.541 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:16:46.541 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:16:46.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:46.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:46.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:46.704 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:46.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:46.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:46.836 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:46.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:46.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:46.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:47.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:47.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:47.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:47.210 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:47.235 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:47.337 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:47.363 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:47.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:47.464 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:47.487 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:47.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:47.589 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:47.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:16:47.617 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:47.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:47.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:47.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:47.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:47.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:16:47.847 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:47.875 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:47.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:47.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:47 smithi017 ceph-mon[102913]: pgmap v2105: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 10711/149127 objects degraded (7.182%); 7036/149127 objects misplaced (4.718%); 8.8 MiB/s, 241 objects/s recovering 2024-09-21T10:16:47.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:47 smithi017 ceph-mon[105151]: pgmap v2105: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 10711/149127 objects degraded (7.182%); 7036/149127 objects misplaced (4.718%); 8.8 MiB/s, 241 objects/s recovering 2024-09-21T10:16:47.977 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:47.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:47 smithi134 ceph-mon[90085]: pgmap v2105: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 10711/149127 objects degraded (7.182%); 7036/149127 objects misplaced (4.718%); 8.8 MiB/s, 241 objects/s recovering 2024-09-21T10:16:48.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:48.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:48.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:48.152 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:48.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:48.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:48.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:48.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:48.389 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:48.420 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:48.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:48.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:48.557 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:48.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:48.657 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:48.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:48.712 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:48.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:48.815 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:48.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:16:48.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:16:48.851 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:48.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:48.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:48.994 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:16:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:49.096 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:49.123 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:49.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:49.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:49.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:49.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:49.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:49.387 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:49.432 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:16:49.432 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:16:49.452 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:49.473 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:16:49.474 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:49.489 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:49.507 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:16:49.508 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:49.516 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:49.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:49.538 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:16:49.539 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:16:49.568 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:16:49.569 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:49.595 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:16:49.595 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:49.616 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:16:49.616 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:49.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:49.641 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:16:49.641 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:49.651 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:49.753 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:49.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.782 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:49.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:49.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:49.913 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:49.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:49.927 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:16:49.928 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:16:49.928 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-21T10:16:49.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:16:49.931 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:49 smithi134 ceph-mon[90085]: pgmap v2106: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 10544/145428 objects degraded (7.250%); 6882/145428 objects misplaced (4.732%); 3.9 MiB/s, 163 objects/s recovering 2024-09-21T10:16:49.931 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:49 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 10544/145428 objects degraded (7.250%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:49.931 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:49 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2699544679' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:49 smithi017 ceph-mon[105151]: pgmap v2106: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 10544/145428 objects degraded (7.250%); 6882/145428 objects misplaced (4.732%); 3.9 MiB/s, 163 objects/s recovering 2024-09-21T10:16:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:49 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 10544/145428 objects degraded (7.250%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:49.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:49 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2699544679' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:49 smithi017 ceph-mon[102913]: pgmap v2106: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 10544/145428 objects degraded (7.250%); 6882/145428 objects misplaced (4.732%); 3.9 MiB/s, 163 objects/s recovering 2024-09-21T10:16:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:49 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 10544/145428 objects degraded (7.250%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:49 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2699544679' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:50.015 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:50.043 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:50.144 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:50.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:50.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:50.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:50.295 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:50.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:50.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:50.422 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:50.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:16:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:16:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:16:50.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:50.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:50.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:50.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:50.682 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:50.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:50.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:50.808 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:50.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:50.909 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:50.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:50.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:51.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:51.063 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:51.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:51.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:51.192 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:51.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:51.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:51.325 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:51.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:51.427 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:51.458 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:51.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:51.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:51.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:51.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.726 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:51.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:51.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:51.857 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:51.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:51.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:51 smithi017 ceph-mon[102913]: pgmap v2107: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 10544/142332 objects degraded (7.408%); 6806/142332 objects misplaced (4.782%); 3.9 MiB/s, 163 objects/s recovering 2024-09-21T10:16:51.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:51 smithi017 ceph-mon[105151]: pgmap v2107: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 10544/142332 objects degraded (7.408%); 6806/142332 objects misplaced (4.782%); 3.9 MiB/s, 163 objects/s recovering 2024-09-21T10:16:51.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:51.960 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:51 smithi134 ceph-mon[90085]: pgmap v2107: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 10544/142332 objects degraded (7.408%); 6806/142332 objects misplaced (4.782%); 3.9 MiB/s, 163 objects/s recovering 2024-09-21T10:16:51.993 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:51.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:52.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:52.123 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:52.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:52.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:52.247 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:52.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:52.256 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:52.350 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:52.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:52.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:52.478 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:52.503 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:52.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:52.605 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:52.633 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:52.735 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:52.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:52.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:52.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:52.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:52.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:52.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:53.023 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:53.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:16:53.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:53.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:53.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:53.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:53.294 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:53.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:53.419 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:53.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:16:53.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:53.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:53.675 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:53.714 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:53.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:53.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:53 smithi017 ceph-mon[102913]: pgmap v2108: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 10381/136827 objects degraded (7.587%); 6622/136827 objects misplaced (4.840%); 2.1 MiB/s, 203 objects/s recovering 2024-09-21T10:16:53.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:53 smithi017 ceph-mon[102913]: osdmap e1336: 8 total, 8 up, 7 in 2024-09-21T10:16:53.791 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:53 smithi017 ceph-mon[105151]: pgmap v2108: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 10381/136827 objects degraded (7.587%); 6622/136827 objects misplaced (4.840%); 2.1 MiB/s, 203 objects/s recovering 2024-09-21T10:16:53.791 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:53 smithi017 ceph-mon[105151]: osdmap e1336: 8 total, 8 up, 7 in 2024-09-21T10:16:53.798 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:53.816 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:53.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.845 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:53.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:53.946 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:53.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:53.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:53 smithi134 ceph-mon[90085]: pgmap v2108: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 10381/136827 objects degraded (7.587%); 6622/136827 objects misplaced (4.840%); 2.1 MiB/s, 203 objects/s recovering 2024-09-21T10:16:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:53 smithi134 ceph-mon[90085]: osdmap e1336: 8 total, 8 up, 7 in 2024-09-21T10:16:54.075 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:54.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:54.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:54.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:54.241 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:54.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:54.343 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:54.372 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:54.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:54.507 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:54.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:54.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:54.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:54.743 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:54 smithi134 ceph-mon[90085]: osdmap e1337: 8 total, 8 up, 7 in 2024-09-21T10:16:54.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:54 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 10381/136827 objects degraded (7.587%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:54.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:54.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:54 smithi017 ceph-mon[102913]: osdmap e1337: 8 total, 8 up, 7 in 2024-09-21T10:16:54.813 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:54 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 10381/136827 objects degraded (7.587%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:54.814 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:16:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:16:54] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-21T10:16:54.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:54 smithi017 ceph-mon[105151]: osdmap e1337: 8 total, 8 up, 7 in 2024-09-21T10:16:54.814 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:54 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 10381/136827 objects degraded (7.587%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:54.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:54.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:54.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:55.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:55.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:55.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:55.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:55.211 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:55.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:55.313 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:55.343 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:55.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:55.445 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:55.474 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:55.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:55.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:55.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:55.710 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:55.738 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:55.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:55.752 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:55 smithi017 ceph-mon[102913]: pgmap v2111: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 6820/134553 objects degraded (5.069%); 7784/134553 objects misplaced (5.785%); 3.5 MiB/s, 175 objects/s recovering 2024-09-21T10:16:55.752 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:55.752 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:55.753 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:55 smithi017 ceph-mon[105151]: pgmap v2111: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 6820/134553 objects degraded (5.069%); 7784/134553 objects misplaced (5.785%); 3.5 MiB/s, 175 objects/s recovering 2024-09-21T10:16:55.753 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:55.753 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:55.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:55.877 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:55.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:55.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:55 smithi134 ceph-mon[90085]: pgmap v2111: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 112 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 6820/134553 objects degraded (5.069%); 7784/134553 objects misplaced (5.785%); 3.5 MiB/s, 175 objects/s recovering 2024-09-21T10:16:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:56.009 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:56.111 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:56.136 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:56.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:56.238 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:56.279 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:56.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:56.381 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:56.416 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:56.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:56.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:56.548 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:56.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:56.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.684 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:56.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:56.785 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:56.810 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:56.912 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:56.939 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:56.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:57.041 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:57.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.068 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:57.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:57.103 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:16:57.169 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:57.199 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:57.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:57.301 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:57.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.340 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:57.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:57.352 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[102913]: 10.11 deep-scrub starts 2024-09-21T10:16:57.352 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:57.352 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:57.352 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1819106342' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:57.352 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:57.352 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[102913]: pgmap v2112: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 6820/132603 objects degraded (5.143%); 7784/132603 objects misplaced (5.870%); 2.2 MiB/s, 198 objects/s recovering 2024-09-21T10:16:57.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[105151]: 10.11 deep-scrub starts 2024-09-21T10:16:57.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:57.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:57.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1819106342' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:57.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:57.353 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:57 smithi017 ceph-mon[105151]: pgmap v2112: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 6820/132603 objects degraded (5.143%); 7784/132603 objects misplaced (5.870%); 2.2 MiB/s, 198 objects/s recovering 2024-09-21T10:16:57.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:57.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:57.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:57 smithi134 ceph-mon[90085]: 10.11 deep-scrub starts 2024-09-21T10:16:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:16:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1819106342' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:57 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:57 smithi134 ceph-mon[90085]: pgmap v2112: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 6820/132603 objects degraded (5.143%); 7784/132603 objects misplaced (5.870%); 2.2 MiB/s, 198 objects/s recovering 2024-09-21T10:16:57.569 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:57.597 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:57.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:57.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:57.832 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:57.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:57.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:57.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:57.989 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:57.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:58.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:58.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:58.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:58.247 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:58.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:58.349 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:58.375 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:58.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:58 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:58.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:58 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:58.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:58 smithi017 ceph-mon[102913]: osdmap e1338: 8 total, 8 up, 7 in 2024-09-21T10:16:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:58 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:58 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:58.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:58 smithi017 ceph-mon[105151]: osdmap e1338: 8 total, 8 up, 7 in 2024-09-21T10:16:58.476 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:58 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:58 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:58 smithi134 ceph-mon[90085]: osdmap e1338: 8 total, 8 up, 7 in 2024-09-21T10:16:58.501 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:58.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:58.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:58.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:58.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:58.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:58.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:58.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:58.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:58.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:59.016 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:59.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:59.151 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:59.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:59.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:59.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:59.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:59.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:59.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:59 smithi017 ceph-mon[102913]: pgmap v2114: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 6658/128031 objects degraded (5.200%); 7595/128031 objects misplaced (5.932%); 2.9 MiB/s, 105 objects/s recovering 2024-09-21T10:16:59.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:16:59 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6820/132603 objects degraded (5.143%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:59 smithi017 ceph-mon[105151]: pgmap v2114: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 6658/128031 objects degraded (5.200%); 7595/128031 objects misplaced (5.932%); 2.9 MiB/s, 105 objects/s recovering 2024-09-21T10:16:59.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:16:59 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6820/132603 objects degraded (5.143%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:59 smithi134 ceph-mon[90085]: pgmap v2114: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 6658/128031 objects degraded (5.200%); 7595/128031 objects misplaced (5.932%); 2.9 MiB/s, 105 objects/s recovering 2024-09-21T10:16:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:16:59 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6820/132603 objects degraded (5.143%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:59.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:16:59.541 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:59.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:16:59.575 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:59.581 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:59.644 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:59.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:59.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:59.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:59.803 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:16:59.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:59.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:59.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:16:59.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:00.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:00.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:00.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:00.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:00.212 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:00.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:00.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.348 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:00.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:00.449 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:00.477 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:17:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:17:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:17:00.579 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:00.616 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:00.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:00.717 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:00.753 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:00.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:00.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:00.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:00.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:00.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:01.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:01.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:01.145 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:01.247 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:01.272 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:01.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:01.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:17:01.374 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:01.400 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:01.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:01.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:01.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:01.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:01.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:01.802 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:01.833 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:01.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:01.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:01 smithi017 ceph-mon[102913]: pgmap v2115: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 6658/124509 objects degraded (5.347%); 7053/124509 objects misplaced (5.665%); 5.6 MiB/s, 134 objects/s recovering 2024-09-21T10:17:01.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:01 smithi017 ceph-mon[105151]: pgmap v2115: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 6658/124509 objects degraded (5.347%); 7053/124509 objects misplaced (5.665%); 5.6 MiB/s, 134 objects/s recovering 2024-09-21T10:17:01.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:01.960 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:01.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:01 smithi134 ceph-mon[90085]: pgmap v2115: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 6658/124509 objects degraded (5.347%); 7053/124509 objects misplaced (5.665%); 5.6 MiB/s, 134 objects/s recovering 2024-09-21T10:17:02.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:02.086 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:02.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:02.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:02.212 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:02.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:02.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:02.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:02.442 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:02.465 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:02.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:02.570 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:02.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:02.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:02.709 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:02.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:02.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:02.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:02.869 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:02.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:02.883 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:17:02.885 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:17:02.971 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:02.999 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:03.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:03.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:03.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:03.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:03.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:03.252 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:03.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:03.353 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:03.383 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:03.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:03.485 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:03.518 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:03.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:03.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:03.649 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:03.751 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:03.784 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:03.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:03.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:03.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:03 smithi017 ceph-mon[102913]: pgmap v2116: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 6470/118482 objects degraded (5.461%); 6849/118482 objects misplaced (5.781%); 6.3 MiB/s, 141 objects/s recovering 2024-09-21T10:17:03.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:03 smithi017 ceph-mon[102913]: from='client.36396 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:17:03.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:03 smithi017 ceph-mon[105151]: pgmap v2116: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 6470/118482 objects degraded (5.461%); 6849/118482 objects misplaced (5.781%); 6.3 MiB/s, 141 objects/s recovering 2024-09-21T10:17:03.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:03 smithi017 ceph-mon[105151]: from='client.36396 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:17:03.910 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:03.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:03 smithi134 ceph-mon[90085]: pgmap v2116: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 6470/118482 objects degraded (5.461%); 6849/118482 objects misplaced (5.781%); 6.3 MiB/s, 141 objects/s recovering 2024-09-21T10:17:03.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:03 smithi134 ceph-mon[90085]: from='client.36396 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:17:04.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:04.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:04.144 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:04.172 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:04.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:04.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:04.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:04.402 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:04.427 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:04.529 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:04.555 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:04.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:04.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:04.686 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:04.788 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:04.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:04.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:04.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:04.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:04 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6470/118482 objects degraded (5.461%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:04.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:17:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:17:04] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.51.0" 2024-09-21T10:17:04.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:04 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6470/118482 objects degraded (5.461%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:04.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:04.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:04 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6470/118482 objects degraded (5.461%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:05.056 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:05.087 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:05.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:05.189 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:05.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:05.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:05.347 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:05.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:05.475 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:05.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:05.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:05.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:05 smithi017 ceph-mon[102913]: pgmap v2117: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 6470/116070 objects degraded (5.574%); 6277/116070 objects misplaced (5.408%); 7.9 MiB/s, 161 objects/s recovering 2024-09-21T10:17:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:05 smithi017 ceph-mon[105151]: pgmap v2117: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 6470/116070 objects degraded (5.574%); 6277/116070 objects misplaced (5.408%); 7.9 MiB/s, 161 objects/s recovering 2024-09-21T10:17:05.710 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:05.741 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:05.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:05.880 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:05.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:05.967 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:17:05.967 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:05.982 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:17:05.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:05.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:05 smithi134 ceph-mon[90085]: pgmap v2117: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 6470/116070 objects degraded (5.574%); 6277/116070 objects misplaced (5.408%); 7.9 MiB/s, 161 objects/s recovering 2024-09-21T10:17:06.004 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:06.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:17:06.012 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:17:06.035 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:17:06.059 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:06.083 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:17:06.084 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:06.106 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:17:06.112 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:06.132 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:06.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:17:06.134 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:06.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:06.262 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:06.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:06.364 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:06.397 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:06.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:06.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:06.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:06.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:06.648 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:06.658 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:17:06.658 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:17:06.658 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:17:06.659 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:17:06.659 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:17:06.692 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:06.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:06.753 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:06 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1318654187' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:06.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:06 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1318654187' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:06.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:06.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:06.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:06.894 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:06 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1318654187' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:06.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:06.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:06.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:07.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:07.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:07.195 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:07.226 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:07.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:07.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:07.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:07.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:07.494 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:07.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:07.596 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:07.634 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:07.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:07.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:07 smithi017 ceph-mon[102913]: pgmap v2118: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 6470/114192 objects degraded (5.666%); 6277/114192 objects misplaced (5.497%); 7.9 MiB/s, 126 objects/s recovering 2024-09-21T10:17:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:07 smithi017 ceph-mon[105151]: pgmap v2118: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 6470/114192 objects degraded (5.666%); 6277/114192 objects misplaced (5.497%); 7.9 MiB/s, 126 objects/s recovering 2024-09-21T10:17:07.736 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:07.766 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:07.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:07.867 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:07.892 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":2116,"stamp":"2024-09-21T10:17:02.534353+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2587206660,"num_objects":39494,"num_object_clones":0,"num_object_copies":118482,"num_objects_missing_on_primary":6714,"num_objects_missing":6714,"num_objects_degraded":6470,"num_objects_misplaced":6849,"num_objects_unfound":0,"num_objects_dirty":39494,"num_whiteouts":0,"num_read":68,"num_read_kb":171,"num_write":191445,"num_write_kb":6793522,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18609,"num_bytes_recovered":961965220,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147436,"ondisk_log_size":147436,"up":357,"acting":355,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13827908,"kb_used_data":7874436,"kb_used_omap":384,"kb_used_meta":5953023,"kb_avail":642388156,"statfs":{"total":671965249536,"available":657805471744,"internally_reserved":0,"allocated":8063422464,"data_stored":8040418257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":393354,"internal_metadata":6095896438},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-400752640,"num_objects":-6115,"num_object_clones":0,"num_object_copies":-18345,"num_objects_missing_on_primary":-351,"num_objects_missing":-351,"num_objects_degraded":-3911,"num_objects_misplaced":227,"num_objects_unfound":0,"num_objects_dirty":-6115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6115,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1257,"num_bytes_recovered":59244544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.905723"},"pg_stats":[{"pgid":"1.11","version":"288'100","reported_seq":655,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.108967+0000","last_change":"2024-09-21T10:16:27.324371+0000","last_active":"2024-09-21T10:16:57.108967+0000","last_peered":"2024-09-21T10:16:57.108967+0000","last_clean":"2024-09-21T10:16:57.108967+0000","last_became_active":"2024-09-21T10:16:27.323938+0000","last_became_peered":"2024-09-21T10:16:27.323938+0000","last_unstale":"2024-09-21T10:16:57.108967+0000","last_undegraded":"2024-09-21T10:16:57.108967+0000","last_fullsized":"2024-09-21T10:16:57.108967+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.888350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.888350+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:04.482506+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2053,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110226+0000","last_change":"2024-09-21T10:16:27.318532+0000","last_active":"2024-09-21T10:16:57.110226+0000","last_peered":"2024-09-21T10:16:57.110226+0000","last_clean":"2024-09-21T10:16:57.110226+0000","last_became_active":"2024-09-21T10:16:27.317049+0000","last_became_peered":"2024-09-21T10:16:27.317049+0000","last_unstale":"2024-09-21T10:16:57.110226+0000","last_undegraded":"2024-09-21T10:16:57.110226+0000","last_fullsized":"2024-09-21T10:16:57.110226+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:43.431430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":86,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633497+0000","last_change":"2024-09-21T10:12:25.720808+0000","last_active":"2024-09-21T10:16:53.633497+0000","last_peered":"2024-09-21T10:16:53.633497+0000","last_clean":"2024-09-21T10:16:53.633497+0000","last_became_active":"2024-09-21T10:12:25.706699+0000","last_became_peered":"2024-09-21T10:12:25.706699+0000","last_unstale":"2024-09-21T10:16:53.633497+0000","last_undegraded":"2024-09-21T10:16:53.633497+0000","last_fullsized":"2024-09-21T10:16:53.633497+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:29.044701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"288'100","reported_seq":100,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633925+0000","last_change":"2024-09-21T10:13:22.620513+0000","last_active":"2024-09-21T10:16:53.633925+0000","last_peered":"2024-09-21T10:16:53.633925+0000","last_clean":"2024-09-21T10:16:53.633925+0000","last_became_active":"2024-09-21T10:12:50.581542+0000","last_became_peered":"2024-09-21T10:12:50.581542+0000","last_unstale":"2024-09-21T10:16:53.633925+0000","last_undegraded":"2024-09-21T10:16:53.633925+0000","last_fullsized":"2024-09-21T10:16:53.633925+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:22.620446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:22.620446+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:13.111755+0000","scrub_duration":0.0014078199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":90,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634404+0000","last_change":"2024-09-21T10:12:50.584513+0000","last_active":"2024-09-21T10:16:53.634404+0000","last_peered":"2024-09-21T10:16:53.634404+0000","last_clean":"2024-09-21T10:16:53.634404+0000","last_became_active":"2024-09-21T10:12:50.584107+0000","last_became_peered":"2024-09-21T10:12:50.584107+0000","last_unstale":"2024-09-21T10:16:53.634404+0000","last_undegraded":"2024-09-21T10:16:53.634404+0000","last_fullsized":"2024-09-21T10:16:53.634404+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:23.047403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1989,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633156+0000","last_change":"2024-09-21T10:16:07.907197+0000","last_active":"2024-09-21T10:16:53.633156+0000","last_peered":"2024-09-21T10:16:53.633156+0000","last_clean":"2024-09-21T10:16:53.633156+0000","last_became_active":"2024-09-21T10:16:07.906770+0000","last_became_peered":"2024-09-21T10:16:07.906770+0000","last_unstale":"2024-09-21T10:16:53.633156+0000","last_undegraded":"2024-09-21T10:16:53.633156+0000","last_fullsized":"2024-09-21T10:16:53.633156+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:55:53.815656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"1338'8192","reported_seq":8364,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:01.936452+0000","last_change":"2024-09-21T10:16:07.966200+0000","last_active":"2024-09-21T10:17:01.936452+0000","last_peered":"2024-09-21T10:17:01.936452+0000","last_clean":"2024-09-21T10:17:01.936452+0000","last_became_active":"2024-09-21T10:16:07.927337+0000","last_became_peered":"2024-09-21T10:16:07.927337+0000","last_unstale":"2024-09-21T10:17:01.936452+0000","last_undegraded":"2024-09-21T10:17:01.936452+0000","last_fullsized":"2024-09-21T10:17:01.936452+0000","mapping_epoch":1325,"log_start":"1289'1102","ondisk_log_start":"1289'1102","created":1292,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":5,"last_scrub":"1296'3597","last_scrub_stamp":"2024-09-21T10:14:13.444340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:13.444340+0000","objects_scrubbed":1948,"log_size":7090,"log_dups_size":0,"ondisk_log_size":7090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:09:42.139051+0000","scrub_duration":424,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77398016,"num_objects":1181,"num_object_clones":0,"num_object_copies":3543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4599,"num_write_kb":122624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"288'100","reported_seq":40,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.109639+0000","last_change":"2024-09-21T10:16:27.331860+0000","last_active":"2024-09-21T10:16:57.109639+0000","last_peered":"2024-09-21T10:16:57.109639+0000","last_clean":"2024-09-21T10:16:57.109639+0000","last_became_active":"2024-09-21T10:16:27.331320+0000","last_became_peered":"2024-09-21T10:16:27.331320+0000","last_unstale":"2024-09-21T10:16:57.109639+0000","last_undegraded":"2024-09-21T10:16:57.109639+0000","last_fullsized":"2024-09-21T10:16:57.109639+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:16.405582+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:16.405582+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":23,"num_write":5,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2003,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110528+0000","last_change":"2024-09-21T10:16:07.912706+0000","last_active":"2024-09-21T10:16:57.110528+0000","last_peered":"2024-09-21T10:16:57.110528+0000","last_clean":"2024-09-21T10:16:57.110528+0000","last_became_active":"2024-09-21T10:16:07.912247+0000","last_became_peered":"2024-09-21T10:16:07.912247+0000","last_unstale":"2024-09-21T10:16:57.110528+0000","last_undegraded":"2024-09-21T10:16:57.110528+0000","last_fullsized":"2024-09-21T10:16:57.110528+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:52.397464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":80,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633990+0000","last_change":"2024-09-21T10:12:25.700080+0000","last_active":"2024-09-21T10:16:53.633990+0000","last_peered":"2024-09-21T10:16:53.633990+0000","last_clean":"2024-09-21T10:16:53.633990+0000","last_became_active":"2024-09-21T10:12:25.699762+0000","last_became_peered":"2024-09-21T10:12:25.699762+0000","last_unstale":"2024-09-21T10:16:53.633990+0000","last_undegraded":"2024-09-21T10:16:53.633990+0000","last_fullsized":"2024-09-21T10:16:53.633990+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:41.509175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.19","version":"1338'8264","reported_seq":8406,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:01.896191+0000","last_change":"2024-09-21T10:14:17.575627+0000","last_active":"2024-09-21T10:17:01.896191+0000","last_peered":"2024-09-21T10:17:01.896191+0000","last_clean":"2024-09-21T10:17:01.896191+0000","last_became_active":"2024-09-21T10:14:05.697205+0000","last_became_peered":"2024-09-21T10:14:05.697205+0000","last_unstale":"2024-09-21T10:17:01.896191+0000","last_undegraded":"2024-09-21T10:17:01.896191+0000","last_fullsized":"2024-09-21T10:17:01.896191+0000","mapping_epoch":1292,"log_start":"1289'1370","ondisk_log_start":"1289'1370","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":6894,"log_dups_size":0,"ondisk_log_size":6894,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71892992,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6641,"num_write_kb":247616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"288'100","reported_seq":68,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.630533+0000","last_change":"2024-09-21T10:16:27.337521+0000","last_active":"2024-09-21T10:16:53.630533+0000","last_peered":"2024-09-21T10:16:53.630533+0000","last_clean":"2024-09-21T10:16:53.630533+0000","last_became_active":"2024-09-21T10:16:27.337115+0000","last_became_peered":"2024-09-21T10:16:27.337115+0000","last_unstale":"2024-09-21T10:16:53.630533+0000","last_undegraded":"2024-09-21T10:16:53.630533+0000","last_fullsized":"2024-09-21T10:16:53.630533+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:28.346003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:28.346003+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":14,"num_read_kb":34,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2053,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110401+0000","last_change":"2024-09-21T10:16:27.324409+0000","last_active":"2024-09-21T10:16:57.110401+0000","last_peered":"2024-09-21T10:16:57.110401+0000","last_clean":"2024-09-21T10:16:57.110401+0000","last_became_active":"2024-09-21T10:16:27.321554+0000","last_became_peered":"2024-09-21T10:16:27.321554+0000","last_unstale":"2024-09-21T10:16:57.110401+0000","last_undegraded":"2024-09-21T10:16:57.110401+0000","last_fullsized":"2024-09-21T10:16:57.110401+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:56.209301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":125,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.630314+0000","last_change":"2024-09-21T10:12:50.620444+0000","last_active":"2024-09-21T10:16:53.630314+0000","last_peered":"2024-09-21T10:16:53.630314+0000","last_clean":"2024-09-21T10:16:53.630314+0000","last_became_active":"2024-09-21T10:12:50.589267+0000","last_became_peered":"2024-09-21T10:12:50.589267+0000","last_unstale":"2024-09-21T10:16:53.630314+0000","last_undegraded":"2024-09-21T10:16:53.630314+0000","last_fullsized":"2024-09-21T10:16:53.630314+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:42:11.055834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"1338'8230","reported_seq":8418,"reported_epoch":1338,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:16:58.679057+0000","last_change":"2024-09-21T10:16:32.359174+0000","last_active":"2024-09-21T10:16:58.679057+0000","last_peered":"2024-09-21T10:16:58.679057+0000","last_clean":"2024-09-21T10:16:06.899689+0000","last_became_active":"2024-09-21T10:16:32.336354+0000","last_became_peered":"2024-09-21T10:16:32.336354+0000","last_unstale":"2024-09-21T10:16:58.679057+0000","last_undegraded":"2024-09-21T10:16:58.679057+0000","last_fullsized":"2024-09-21T10:16:58.679057+0000","mapping_epoch":1331,"log_start":"1293'3276","ondisk_log_start":"1293'3276","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"1313'6176","last_scrub_stamp":"2024-09-21T10:15:27.508272+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:15:27.508272+0000","objects_scrubbed":3295,"log_size":4954,"log_dups_size":0,"ondisk_log_size":4954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:48:57.185973+0000","scrub_duration":454,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81264640,"num_objects":1240,"num_object_clones":0,"num_object_copies":3720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1240,"num_objects_unfound":0,"num_objects_dirty":1240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2055,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1240}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"288'100","reported_seq":100,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634377+0000","last_change":"2024-09-21T10:16:27.315882+0000","last_active":"2024-09-21T10:16:53.634377+0000","last_peered":"2024-09-21T10:16:53.634377+0000","last_clean":"2024-09-21T10:16:53.634377+0000","last_became_active":"2024-09-21T10:16:27.315502+0000","last_became_peered":"2024-09-21T10:16:27.315502+0000","last_unstale":"2024-09-21T10:16:53.634377+0000","last_undegraded":"2024-09-21T10:16:53.634377+0000","last_fullsized":"2024-09-21T10:16:53.634377+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:16.697812+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:16.697812+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:18.770038+0000","scrub_duration":0.001732562,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":84,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634315+0000","last_change":"2024-09-21T10:12:25.764496+0000","last_active":"2024-09-21T10:16:53.634315+0000","last_peered":"2024-09-21T10:16:53.634315+0000","last_clean":"2024-09-21T10:16:53.634315+0000","last_became_active":"2024-09-21T10:12:25.707263+0000","last_became_peered":"2024-09-21T10:12:25.707263+0000","last_unstale":"2024-09-21T10:16:53.634315+0000","last_undegraded":"2024-09-21T10:16:53.634315+0000","last_fullsized":"2024-09-21T10:16:53.634315+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:45.728577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":90,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634006+0000","last_change":"2024-09-21T10:12:50.573976+0000","last_active":"2024-09-21T10:16:53.634006+0000","last_peered":"2024-09-21T10:16:53.634006+0000","last_clean":"2024-09-21T10:16:53.634006+0000","last_became_active":"2024-09-21T10:12:50.573657+0000","last_became_peered":"2024-09-21T10:12:50.573657+0000","last_unstale":"2024-09-21T10:16:53.634006+0000","last_undegraded":"2024-09-21T10:16:53.634006+0000","last_fullsized":"2024-09-21T10:16:53.634006+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:44:44.245943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"1338'8364","reported_seq":8480,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:59.902671+0000","last_change":"2024-09-21T10:15:30.627957+0000","last_active":"2024-09-21T10:16:59.902671+0000","last_peered":"2024-09-21T10:16:59.902671+0000","last_clean":"2024-09-21T10:16:59.902671+0000","last_became_active":"2024-09-21T10:14:54.650488+0000","last_became_peered":"2024-09-21T10:14:54.650488+0000","last_unstale":"2024-09-21T10:16:59.902671+0000","last_undegraded":"2024-09-21T10:16:59.902671+0000","last_fullsized":"2024-09-21T10:16:59.902671+0000","mapping_epoch":1309,"log_start":"1293'3443","ondisk_log_start":"1293'3443","created":1283,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4921,"log_dups_size":0,"ondisk_log_size":4921,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81166336,"num_objects":1238,"num_object_clones":0,"num_object_copies":3714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6724,"num_write_kb":254816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"288'100","reported_seq":508,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110133+0000","last_change":"2024-09-21T10:10:13.678895+0000","last_active":"2024-09-21T10:16:57.110133+0000","last_peered":"2024-09-21T10:16:57.110133+0000","last_clean":"2024-09-21T10:16:57.110133+0000","last_became_active":"2024-09-21T10:10:13.678219+0000","last_became_peered":"2024-09-21T10:10:13.678219+0000","last_unstale":"2024-09-21T10:16:57.110133+0000","last_undegraded":"2024-09-21T10:16:57.110133+0000","last_fullsized":"2024-09-21T10:16:57.110133+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:27.395700+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:27.395700+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:26.476978+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":98,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633867+0000","last_change":"2024-09-21T10:16:27.338520+0000","last_active":"2024-09-21T10:16:53.633867+0000","last_peered":"2024-09-21T10:16:53.633867+0000","last_clean":"2024-09-21T10:16:53.633867+0000","last_became_active":"2024-09-21T10:16:27.322695+0000","last_became_peered":"2024-09-21T10:16:27.322695+0000","last_unstale":"2024-09-21T10:16:53.633867+0000","last_undegraded":"2024-09-21T10:16:53.633867+0000","last_fullsized":"2024-09-21T10:16:53.633867+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:11:08.696733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1313,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633767+0000","last_change":"2024-09-21T10:16:27.321387+0000","last_active":"2024-09-21T10:16:53.633767+0000","last_peered":"2024-09-21T10:16:53.633767+0000","last_clean":"2024-09-21T10:16:53.633767+0000","last_became_active":"2024-09-21T10:16:27.321042+0000","last_became_peered":"2024-09-21T10:16:27.321042+0000","last_unstale":"2024-09-21T10:16:53.633767+0000","last_undegraded":"2024-09-21T10:16:53.633767+0000","last_fullsized":"2024-09-21T10:16:53.633767+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:31.231738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"1338'8263","reported_seq":8396,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.657943+0000","last_change":"2024-09-21T10:16:07.941435+0000","last_active":"2024-09-21T10:16:58.657943+0000","last_peered":"2024-09-21T10:16:58.657943+0000","last_clean":"2024-09-21T10:16:58.657943+0000","last_became_active":"2024-09-21T10:16:07.910214+0000","last_became_peered":"2024-09-21T10:16:07.910214+0000","last_unstale":"2024-09-21T10:16:58.657943+0000","last_undegraded":"2024-09-21T10:16:58.657943+0000","last_fullsized":"2024-09-21T10:16:58.657943+0000","mapping_epoch":1325,"log_start":"1293'3305","ondisk_log_start":"1293'3305","created":1283,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4958,"log_dups_size":0,"ondisk_log_size":4958,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82739200,"num_objects":1262,"num_object_clones":0,"num_object_copies":3786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6640,"num_write_kb":252896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"288'100","reported_seq":198,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631889+0000","last_change":"2024-09-21T10:14:54.647582+0000","last_active":"2024-09-21T10:16:53.631889+0000","last_peered":"2024-09-21T10:16:53.631889+0000","last_clean":"2024-09-21T10:16:53.631889+0000","last_became_active":"2024-09-21T10:14:54.647213+0000","last_became_peered":"2024-09-21T10:14:54.647213+0000","last_unstale":"2024-09-21T10:16:53.631889+0000","last_undegraded":"2024-09-21T10:16:53.631889+0000","last_fullsized":"2024-09-21T10:16:53.631889+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:23.746751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:23.746751+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:04.424278+0000","scrub_duration":0.0021986229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":98,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633369+0000","last_change":"2024-09-21T10:12:50.623002+0000","last_active":"2024-09-21T10:16:53.633369+0000","last_peered":"2024-09-21T10:16:53.633369+0000","last_clean":"2024-09-21T10:16:53.633369+0000","last_became_active":"2024-09-21T10:12:50.588355+0000","last_became_peered":"2024-09-21T10:12:50.588355+0000","last_unstale":"2024-09-21T10:16:53.633369+0000","last_undegraded":"2024-09-21T10:16:53.633369+0000","last_fullsized":"2024-09-21T10:16:53.633369+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:22.564459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":115,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.630721+0000","last_change":"2024-09-21T10:16:27.315202+0000","last_active":"2024-09-21T10:16:53.630721+0000","last_peered":"2024-09-21T10:16:53.630721+0000","last_clean":"2024-09-21T10:16:53.630721+0000","last_became_active":"2024-09-21T10:16:27.314827+0000","last_became_peered":"2024-09-21T10:16:27.314827+0000","last_unstale":"2024-09-21T10:16:53.630721+0000","last_undegraded":"2024-09-21T10:16:53.630721+0000","last_fullsized":"2024-09-21T10:16:53.630721+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:31.494018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"1338'8192","reported_seq":8366,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:01.891624+0000","last_change":"2024-09-21T10:14:33.475485+0000","last_active":"2024-09-21T10:17:01.891624+0000","last_peered":"2024-09-21T10:17:01.891624+0000","last_clean":"2024-09-21T10:17:01.891624+0000","last_became_active":"2024-09-21T10:14:05.671586+0000","last_became_peered":"2024-09-21T10:14:05.671586+0000","last_unstale":"2024-09-21T10:17:01.891624+0000","last_undegraded":"2024-09-21T10:17:01.891624+0000","last_fullsized":"2024-09-21T10:17:01.891624+0000","mapping_epoch":1283,"log_start":"1289'1230","ondisk_log_start":"1289'1230","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"1303'4238","last_scrub_stamp":"2024-09-21T10:14:33.475181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:33.475181+0000","objects_scrubbed":2651,"log_size":6962,"log_dups_size":0,"ondisk_log_size":6962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:10:12.942432+0000","scrub_duration":1000,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80084992,"num_objects":1222,"num_object_clones":0,"num_object_copies":3666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3971,"num_write_kb":81344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"288'100","reported_seq":144,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.632018+0000","last_change":"2024-09-21T10:12:25.697192+0000","last_active":"2024-09-21T10:16:53.632018+0000","last_peered":"2024-09-21T10:16:53.632018+0000","last_clean":"2024-09-21T10:16:53.632018+0000","last_became_active":"2024-09-21T10:12:25.696789+0000","last_became_peered":"2024-09-21T10:12:25.696789+0000","last_unstale":"2024-09-21T10:16:53.632018+0000","last_undegraded":"2024-09-21T10:16:53.632018+0000","last_fullsized":"2024-09-21T10:16:53.632018+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:09:03.581657+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:09:03.581657+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:24.147270+0000","scrub_duration":0.0038308109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1046,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110428+0000","last_change":"2024-09-21T10:10:13.678563+0000","last_active":"2024-09-21T10:16:57.110428+0000","last_peered":"2024-09-21T10:16:57.110428+0000","last_clean":"2024-09-21T10:16:57.110428+0000","last_became_active":"2024-09-21T10:10:13.676594+0000","last_became_peered":"2024-09-21T10:10:13.676594+0000","last_unstale":"2024-09-21T10:16:57.110428+0000","last_undegraded":"2024-09-21T10:16:57.110428+0000","last_fullsized":"2024-09-21T10:16:57.110428+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:34.275762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1937,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.632072+0000","last_change":"2024-09-21T10:16:27.325294+0000","last_active":"2024-09-21T10:16:53.632072+0000","last_peered":"2024-09-21T10:16:53.632072+0000","last_clean":"2024-09-21T10:16:53.632072+0000","last_became_active":"2024-09-21T10:16:27.324974+0000","last_became_peered":"2024-09-21T10:16:27.324974+0000","last_unstale":"2024-09-21T10:16:53.632072+0000","last_undegraded":"2024-09-21T10:16:53.632072+0000","last_fullsized":"2024-09-21T10:16:53.632072+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:58.600136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.2","version":"1338'8192","reported_seq":9439,"reported_epoch":1338,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:16:58.719594+0000","last_change":"2024-09-21T10:16:52.410526+0000","last_active":"2024-09-21T10:16:58.719594+0000","last_peered":"2024-09-21T10:16:58.719594+0000","last_clean":"2024-09-21T10:16:26.171053+0000","last_became_active":"2024-09-21T10:16:28.621940+0000","last_became_peered":"2024-09-21T10:16:28.621940+0000","last_unstale":"2024-09-21T10:16:58.719594+0000","last_undegraded":"2024-09-21T10:16:28.342612+0000","last_fullsized":"2024-09-21T10:16:28.306679+0000","mapping_epoch":1331,"log_start":"1291'3198","ondisk_log_start":"1291'3198","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4994,"log_dups_size":0,"ondisk_log_size":4994,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81690624,"num_objects":1246,"num_object_clones":0,"num_object_copies":3738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":328,"num_objects_misplaced":802,"num_objects_unfound":0,"num_objects_dirty":1246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6554,"num_write_kb":249632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":554,"num_bytes_recovered":36306944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":802},{"shards":"0,4","objects":116}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":90,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634097+0000","last_change":"2024-09-21T10:13:15.663291+0000","last_active":"2024-09-21T10:16:53.634097+0000","last_peered":"2024-09-21T10:16:53.634097+0000","last_clean":"2024-09-21T10:16:53.634097+0000","last_became_active":"2024-09-21T10:12:25.712199+0000","last_became_peered":"2024-09-21T10:12:25.712199+0000","last_unstale":"2024-09-21T10:16:53.634097+0000","last_undegraded":"2024-09-21T10:16:53.634097+0000","last_fullsized":"2024-09-21T10:16:53.634097+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:15.663228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:15.663228+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:44.919423+0000","scrub_duration":0.0014833489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":2040,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.109778+0000","last_change":"2024-09-21T10:08:25.436371+0000","last_active":"2024-09-21T10:16:57.109778+0000","last_peered":"2024-09-21T10:16:57.109778+0000","last_clean":"2024-09-21T10:16:57.109778+0000","last_became_active":"2024-09-21T10:08:25.435921+0000","last_became_peered":"2024-09-21T10:08:25.435921+0000","last_unstale":"2024-09-21T10:16:57.109778+0000","last_undegraded":"2024-09-21T10:16:57.109778+0000","last_fullsized":"2024-09-21T10:16:57.109778+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:27.612386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1280,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631540+0000","last_change":"2024-09-21T10:10:13.661328+0000","last_active":"2024-09-21T10:16:53.631540+0000","last_peered":"2024-09-21T10:16:53.631540+0000","last_clean":"2024-09-21T10:16:53.631540+0000","last_became_active":"2024-09-21T10:10:13.661028+0000","last_became_peered":"2024-09-21T10:10:13.661028+0000","last_unstale":"2024-09-21T10:16:53.631540+0000","last_undegraded":"2024-09-21T10:16:53.631540+0000","last_fullsized":"2024-09-21T10:16:53.631540+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:36.965214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"1338'8430","reported_seq":8603,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:01.762796+0000","last_change":"2024-09-21T10:14:28.156006+0000","last_active":"2024-09-21T10:17:01.762796+0000","last_peered":"2024-09-21T10:17:01.762796+0000","last_clean":"2024-09-21T10:17:01.762796+0000","last_became_active":"2024-09-21T10:14:05.674553+0000","last_became_peered":"2024-09-21T10:14:05.674553+0000","last_unstale":"2024-09-21T10:17:01.762796+0000","last_undegraded":"2024-09-21T10:17:01.762796+0000","last_fullsized":"2024-09-21T10:17:01.762796+0000","mapping_epoch":1283,"log_start":"1289'1533","ondisk_log_start":"1289'1533","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"1303'4190","last_scrub_stamp":"2024-09-21T10:14:28.155685+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:28.155685+0000","objects_scrubbed":2487,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:24.938089+0000","scrub_duration":718,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80543744,"num_objects":1229,"num_object_clones":0,"num_object_copies":3687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4258,"num_write_kb":96000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":486,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.101997+0000","last_change":"2024-09-21T10:13:15.622804+0000","last_active":"2024-09-21T10:16:57.101997+0000","last_peered":"2024-09-21T10:16:57.101997+0000","last_clean":"2024-09-21T10:16:57.101997+0000","last_became_active":"2024-09-21T10:12:50.586357+0000","last_became_peered":"2024-09-21T10:12:50.586357+0000","last_unstale":"2024-09-21T10:16:57.101997+0000","last_undegraded":"2024-09-21T10:16:57.101997+0000","last_fullsized":"2024-09-21T10:16:57.101997+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1281,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:28.894448+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":532,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634103+0000","last_change":"2024-09-21T10:16:27.318938+0000","last_active":"2024-09-21T10:16:53.634103+0000","last_peered":"2024-09-21T10:16:53.634103+0000","last_clean":"2024-09-21T10:16:53.634103+0000","last_became_active":"2024-09-21T10:16:27.318623+0000","last_became_peered":"2024-09-21T10:16:27.318623+0000","last_unstale":"2024-09-21T10:16:53.634103+0000","last_undegraded":"2024-09-21T10:16:53.634103+0000","last_fullsized":"2024-09-21T10:16:53.634103+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:21.912596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1313,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.632791+0000","last_change":"2024-09-21T10:16:27.320140+0000","last_active":"2024-09-21T10:16:53.632791+0000","last_peered":"2024-09-21T10:16:53.632791+0000","last_clean":"2024-09-21T10:16:53.632791+0000","last_became_active":"2024-09-21T10:16:27.319787+0000","last_became_peered":"2024-09-21T10:16:27.319787+0000","last_unstale":"2024-09-21T10:16:53.632791+0000","last_undegraded":"2024-09-21T10:16:53.632791+0000","last_fullsized":"2024-09-21T10:16:53.632791+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:11.703898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"1338'8218","reported_seq":8302,"reported_epoch":1338,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:17:02.284820+0000","last_change":"2024-09-21T10:16:28.458976+0000","last_active":"2024-09-21T10:17:02.284820+0000","last_peered":"2024-09-21T10:17:02.284820+0000","last_clean":"2024-09-21T10:16:26.297502+0000","last_became_active":"2024-09-21T10:16:28.434292+0000","last_became_peered":"2024-09-21T10:16:28.434292+0000","last_unstale":"2024-09-21T10:17:02.284820+0000","last_undegraded":"2024-09-21T10:17:02.284820+0000","last_fullsized":"2024-09-21T10:17:02.284820+0000","mapping_epoch":1331,"log_start":"1293'3408","ondisk_log_start":"1293'3408","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4810,"log_dups_size":0,"ondisk_log_size":4810,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755008,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1278,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6628,"num_write_kb":252992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":1278}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"288'100","reported_seq":115,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.101823+0000","last_change":"2024-09-21T10:16:27.328874+0000","last_active":"2024-09-21T10:16:57.101823+0000","last_peered":"2024-09-21T10:16:57.101823+0000","last_clean":"2024-09-21T10:16:57.101823+0000","last_became_active":"2024-09-21T10:16:27.328463+0000","last_became_peered":"2024-09-21T10:16:27.328463+0000","last_unstale":"2024-09-21T10:16:57.101823+0000","last_undegraded":"2024-09-21T10:16:57.101823+0000","last_fullsized":"2024-09-21T10:16:57.101823+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:30.346035+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:30.346035+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":23,"num_write":5,"num_write_kb":50,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1294,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633206+0000","last_change":"2024-09-21T10:12:25.697464+0000","last_active":"2024-09-21T10:16:53.633206+0000","last_peered":"2024-09-21T10:16:53.633206+0000","last_clean":"2024-09-21T10:16:53.633206+0000","last_became_active":"2024-09-21T10:12:25.697050+0000","last_became_peered":"2024-09-21T10:12:25.697050+0000","last_unstale":"2024-09-21T10:16:53.633206+0000","last_undegraded":"2024-09-21T10:16:53.633206+0000","last_fullsized":"2024-09-21T10:16:53.633206+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:50.687008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1336,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.632995+0000","last_change":"2024-09-21T10:16:27.321771+0000","last_active":"2024-09-21T10:16:53.632995+0000","last_peered":"2024-09-21T10:16:53.632995+0000","last_clean":"2024-09-21T10:16:53.632995+0000","last_became_active":"2024-09-21T10:16:27.321423+0000","last_became_peered":"2024-09-21T10:16:27.321423+0000","last_unstale":"2024-09-21T10:16:53.632995+0000","last_undegraded":"2024-09-21T10:16:53.632995+0000","last_fullsized":"2024-09-21T10:16:53.632995+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:04.652246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"1338'8105","reported_seq":8200,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.452729+0000","last_change":"2024-09-21T10:15:53.125229+0000","last_active":"2024-09-21T10:16:58.452729+0000","last_peered":"2024-09-21T10:16:58.452729+0000","last_clean":"2024-09-21T10:16:58.452729+0000","last_became_active":"2024-09-21T10:14:05.637987+0000","last_became_peered":"2024-09-21T10:14:05.637987+0000","last_unstale":"2024-09-21T10:16:58.452729+0000","last_undegraded":"2024-09-21T10:16:58.452729+0000","last_fullsized":"2024-09-21T10:16:58.452729+0000","mapping_epoch":1283,"log_start":"1293'3165","ondisk_log_start":"1293'3165","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4940,"log_dups_size":0,"ondisk_log_size":4940,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82739200,"num_objects":1262,"num_object_clones":0,"num_object_copies":3786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6526,"num_write_kb":249248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":487,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.100214+0000","last_change":"2024-09-21T10:10:13.622553+0000","last_active":"2024-09-21T10:16:57.100214+0000","last_peered":"2024-09-21T10:16:57.100214+0000","last_clean":"2024-09-21T10:16:57.100214+0000","last_became_active":"2024-09-21T10:10:13.622268+0000","last_became_peered":"2024-09-21T10:10:13.622268+0000","last_unstale":"2024-09-21T10:16:57.100214+0000","last_undegraded":"2024-09-21T10:16:57.100214+0000","last_fullsized":"2024-09-21T10:16:57.100214+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:24.686892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:24.686892+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:55.474818+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2066,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.108859+0000","last_change":"2024-09-21T10:16:27.330267+0000","last_active":"2024-09-21T10:16:57.108859+0000","last_peered":"2024-09-21T10:16:57.108859+0000","last_clean":"2024-09-21T10:16:57.108859+0000","last_became_active":"2024-09-21T10:16:27.329320+0000","last_became_peered":"2024-09-21T10:16:27.329320+0000","last_unstale":"2024-09-21T10:16:57.108859+0000","last_undegraded":"2024-09-21T10:16:57.108859+0000","last_fullsized":"2024-09-21T10:16:57.108859+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:37.062508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":760,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110546+0000","last_change":"2024-09-21T10:16:27.333765+0000","last_active":"2024-09-21T10:16:57.110546+0000","last_peered":"2024-09-21T10:16:57.110546+0000","last_clean":"2024-09-21T10:16:57.110546+0000","last_became_active":"2024-09-21T10:16:27.331548+0000","last_became_peered":"2024-09-21T10:16:27.331548+0000","last_unstale":"2024-09-21T10:16:57.110546+0000","last_undegraded":"2024-09-21T10:16:57.110546+0000","last_fullsized":"2024-09-21T10:16:57.110546+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:28:55.065363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.b","version":"1338'13286","reported_seq":19844,"reported_epoch":1338,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:17:02.329778+0000","last_change":"2024-09-21T10:16:33.704542+0000","last_active":"2024-09-21T10:17:02.329778+0000","last_peered":"2024-09-21T10:17:02.329778+0000","last_clean":"2024-09-21T10:16:26.290239+0000","last_became_active":"2024-09-21T10:16:28.415892+0000","last_became_peered":"2024-09-21T10:16:28.415892+0000","last_unstale":"2024-09-21T10:17:02.329778+0000","last_undegraded":"2024-09-21T10:17:02.329778+0000","last_fullsized":"2024-09-21T10:17:02.329778+0000","mapping_epoch":1331,"log_start":"1311'8469","ondisk_log_start":"1311'8469","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4817,"log_dups_size":0,"ondisk_log_size":4817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:00.414935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158466048,"num_objects":2418,"num_object_clones":0,"num_object_copies":7254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":117,"num_objects_unfound":0,"num_objects_dirty":2418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13286,"num_write_kb":502528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3240,"num_bytes_recovered":212336640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,1],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":2418}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":90,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633645+0000","last_change":"2024-09-21T10:13:20.647735+0000","last_active":"2024-09-21T10:16:53.633645+0000","last_peered":"2024-09-21T10:16:53.633645+0000","last_clean":"2024-09-21T10:16:53.633645+0000","last_became_active":"2024-09-21T10:12:25.708657+0000","last_became_peered":"2024-09-21T10:12:25.708657+0000","last_unstale":"2024-09-21T10:16:53.633645+0000","last_undegraded":"2024-09-21T10:16:53.633645+0000","last_fullsized":"2024-09-21T10:16:53.633645+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:20.647659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:20.647659+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:09.770331+0000","scrub_duration":0.0016005629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":539,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.101995+0000","last_change":"2024-09-21T10:12:56.081277+0000","last_active":"2024-09-21T10:16:57.101995+0000","last_peered":"2024-09-21T10:16:57.101995+0000","last_clean":"2024-09-21T10:16:57.101995+0000","last_became_active":"2024-09-21T10:12:50.586317+0000","last_became_peered":"2024-09-21T10:12:50.586317+0000","last_unstale":"2024-09-21T10:16:57.101995+0000","last_undegraded":"2024-09-21T10:16:57.101995+0000","last_fullsized":"2024-09-21T10:16:57.101995+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:57.776335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2038,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.109359+0000","last_change":"2024-09-21T10:12:50.577072+0000","last_active":"2024-09-21T10:16:57.109359+0000","last_peered":"2024-09-21T10:16:57.109359+0000","last_clean":"2024-09-21T10:16:57.109359+0000","last_became_active":"2024-09-21T10:12:50.576731+0000","last_became_peered":"2024-09-21T10:12:50.576731+0000","last_unstale":"2024-09-21T10:16:57.109359+0000","last_undegraded":"2024-09-21T10:16:57.109359+0000","last_fullsized":"2024-09-21T10:16:57.109359+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:56:06.937599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":197,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.632635+0000","last_change":"2024-09-21T10:14:54.650325+0000","last_active":"2024-09-21T10:16:53.632635+0000","last_peered":"2024-09-21T10:16:53.632635+0000","last_clean":"2024-09-21T10:16:53.632635+0000","last_became_active":"2024-09-21T10:14:54.650048+0000","last_became_peered":"2024-09-21T10:14:54.650048+0000","last_unstale":"2024-09-21T10:16:53.632635+0000","last_undegraded":"2024-09-21T10:16:53.632635+0000","last_fullsized":"2024-09-21T10:16:53.632635+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:47.899727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"1338'8236","reported_seq":8382,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:01.909883+0000","last_change":"2024-09-21T10:16:07.927724+0000","last_active":"2024-09-21T10:17:01.909883+0000","last_peered":"2024-09-21T10:17:01.909883+0000","last_clean":"2024-09-21T10:17:01.909883+0000","last_became_active":"2024-09-21T10:16:07.911810+0000","last_became_peered":"2024-09-21T10:16:07.911810+0000","last_unstale":"2024-09-21T10:17:01.909883+0000","last_undegraded":"2024-09-21T10:17:01.909883+0000","last_fullsized":"2024-09-21T10:17:01.909883+0000","mapping_epoch":1325,"log_start":"1289'1270","ondisk_log_start":"1289'1270","created":1283,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":6966,"log_dups_size":0,"ondisk_log_size":6966,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76054528,"num_objects":1160,"num_object_clones":0,"num_object_copies":3480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6614,"num_write_kb":248800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":21,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631999+0000","last_change":"2024-09-21T10:16:27.328640+0000","last_active":"2024-09-21T10:16:53.631999+0000","last_peered":"2024-09-21T10:16:53.631999+0000","last_clean":"2024-09-21T10:16:53.631999+0000","last_became_active":"2024-09-21T10:16:27.328295+0000","last_became_peered":"2024-09-21T10:16:27.328295+0000","last_unstale":"2024-09-21T10:16:53.631999+0000","last_undegraded":"2024-09-21T10:16:53.631999+0000","last_fullsized":"2024-09-21T10:16:53.631999+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:18.313538+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:18.313538+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":22,"num_write":4,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":90,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633788+0000","last_change":"2024-09-21T10:16:27.320147+0000","last_active":"2024-09-21T10:16:53.633788+0000","last_peered":"2024-09-21T10:16:53.633788+0000","last_clean":"2024-09-21T10:16:53.633788+0000","last_became_active":"2024-09-21T10:16:27.319802+0000","last_became_peered":"2024-09-21T10:16:27.319802+0000","last_unstale":"2024-09-21T10:16:53.633788+0000","last_undegraded":"2024-09-21T10:16:53.633788+0000","last_fullsized":"2024-09-21T10:16:53.633788+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:31.946542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1900,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633560+0000","last_change":"2024-09-21T10:16:07.907738+0000","last_active":"2024-09-21T10:16:53.633560+0000","last_peered":"2024-09-21T10:16:53.633560+0000","last_clean":"2024-09-21T10:16:53.633560+0000","last_became_active":"2024-09-21T10:16:07.907415+0000","last_became_peered":"2024-09-21T10:16:07.907415+0000","last_unstale":"2024-09-21T10:16:53.633560+0000","last_undegraded":"2024-09-21T10:16:53.633560+0000","last_fullsized":"2024-09-21T10:16:53.633560+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:44.971174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1944,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.101954+0000","last_change":"2024-09-21T10:16:27.323221+0000","last_active":"2024-09-21T10:16:57.101954+0000","last_peered":"2024-09-21T10:16:57.101954+0000","last_clean":"2024-09-21T10:16:57.101954+0000","last_became_active":"2024-09-21T10:16:27.322846+0000","last_became_peered":"2024-09-21T10:16:27.322846+0000","last_unstale":"2024-09-21T10:16:57.101954+0000","last_undegraded":"2024-09-21T10:16:57.101954+0000","last_fullsized":"2024-09-21T10:16:57.101954+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:50.647831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"1338'8116","reported_seq":8256,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:01.863849+0000","last_change":"2024-09-21T10:14:22.570382+0000","last_active":"2024-09-21T10:17:01.863849+0000","last_peered":"2024-09-21T10:17:01.863849+0000","last_clean":"2024-09-21T10:17:01.863849+0000","last_became_active":"2024-09-21T10:14:05.671753+0000","last_became_peered":"2024-09-21T10:14:05.671753+0000","last_unstale":"2024-09-21T10:17:01.863849+0000","last_undegraded":"2024-09-21T10:17:01.863849+0000","last_fullsized":"2024-09-21T10:17:01.863849+0000","mapping_epoch":1283,"log_start":"1289'836","ondisk_log_start":"1289'836","created":1283,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":7280,"log_dups_size":0,"ondisk_log_size":7280,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84082688,"num_objects":1283,"num_object_clones":0,"num_object_copies":3849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6493,"num_write_kb":248832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":90,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633236+0000","last_change":"2024-09-21T10:13:17.734264+0000","last_active":"2024-09-21T10:16:53.633236+0000","last_peered":"2024-09-21T10:16:53.633236+0000","last_clean":"2024-09-21T10:16:53.633236+0000","last_became_active":"2024-09-21T10:12:25.707372+0000","last_became_peered":"2024-09-21T10:12:25.707372+0000","last_unstale":"2024-09-21T10:16:53.633236+0000","last_undegraded":"2024-09-21T10:16:53.633236+0000","last_fullsized":"2024-09-21T10:16:53.633236+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:17.734169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:17.734169+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:59.937743+0000","scrub_duration":0.0017947760000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2078,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.102533+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:16:57.102533+0000","last_peered":"2024-09-21T10:16:57.102533+0000","last_clean":"2024-09-21T10:16:57.102533+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:16:57.102533+0000","last_undegraded":"2024-09-21T10:16:57.102533+0000","last_fullsized":"2024-09-21T10:16:57.102533+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":68,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.630124+0000","last_change":"2024-09-21T10:16:27.322540+0000","last_active":"2024-09-21T10:16:53.630124+0000","last_peered":"2024-09-21T10:16:53.630124+0000","last_clean":"2024-09-21T10:16:53.630124+0000","last_became_active":"2024-09-21T10:16:27.322003+0000","last_became_peered":"2024-09-21T10:16:27.322003+0000","last_unstale":"2024-09-21T10:16:53.630124+0000","last_undegraded":"2024-09-21T10:16:53.630124+0000","last_fullsized":"2024-09-21T10:16:53.630124+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:13.361229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":108,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634012+0000","last_change":"2024-09-21T10:16:27.335535+0000","last_active":"2024-09-21T10:16:53.634012+0000","last_peered":"2024-09-21T10:16:53.634012+0000","last_clean":"2024-09-21T10:16:53.634012+0000","last_became_active":"2024-09-21T10:16:27.335258+0000","last_became_peered":"2024-09-21T10:16:27.335258+0000","last_unstale":"2024-09-21T10:16:53.634012+0000","last_undegraded":"2024-09-21T10:16:53.634012+0000","last_fullsized":"2024-09-21T10:16:53.634012+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:16.786401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.a","version":"1338'13426","reported_seq":13489,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:02.278561+0000","last_change":"2024-09-21T10:13:28.746797+0000","last_active":"2024-09-21T10:17:02.278561+0000","last_peered":"2024-09-21T10:17:02.278561+0000","last_clean":"2024-09-21T10:17:02.278561+0000","last_became_active":"2024-09-21T10:13:28.746510+0000","last_became_peered":"2024-09-21T10:13:28.746510+0000","last_unstale":"2024-09-21T10:17:02.278561+0000","last_undegraded":"2024-09-21T10:17:02.278561+0000","last_fullsized":"2024-09-21T10:17:02.278561+0000","mapping_epoch":1283,"log_start":"1311'8583","ondisk_log_start":"1311'8583","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4843,"log_dups_size":0,"ondisk_log_size":4843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:12:58.123721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161349632,"num_objects":2462,"num_object_clones":0,"num_object_copies":7386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13426,"num_write_kb":508416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":723,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633641+0000","last_change":"2024-09-21T10:16:07.911971+0000","last_active":"2024-09-21T10:16:53.633641+0000","last_peered":"2024-09-21T10:16:53.633641+0000","last_clean":"2024-09-21T10:16:53.633641+0000","last_became_active":"2024-09-21T10:16:07.911538+0000","last_became_peered":"2024-09-21T10:16:07.911538+0000","last_unstale":"2024-09-21T10:16:53.633641+0000","last_undegraded":"2024-09-21T10:16:53.633641+0000","last_fullsized":"2024-09-21T10:16:53.633641+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:19.901507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:19.901507+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:29.782749+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1082,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.651619+0000","last_change":"2024-09-21T10:16:27.343418+0000","last_active":"2024-09-21T10:16:53.651619+0000","last_peered":"2024-09-21T10:16:53.651619+0000","last_clean":"2024-09-21T10:16:53.651619+0000","last_became_active":"2024-09-21T10:16:27.339592+0000","last_became_peered":"2024-09-21T10:16:27.339592+0000","last_unstale":"2024-09-21T10:16:53.651619+0000","last_undegraded":"2024-09-21T10:16:53.651619+0000","last_fullsized":"2024-09-21T10:16:53.651619+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:42.895218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1281,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.632649+0000","last_change":"2024-09-21T10:10:13.686243+0000","last_active":"2024-09-21T10:16:53.632649+0000","last_peered":"2024-09-21T10:16:53.632649+0000","last_clean":"2024-09-21T10:16:53.632649+0000","last_became_active":"2024-09-21T10:10:13.685894+0000","last_became_peered":"2024-09-21T10:10:13.685894+0000","last_unstale":"2024-09-21T10:16:53.632649+0000","last_undegraded":"2024-09-21T10:16:53.632649+0000","last_fullsized":"2024-09-21T10:16:53.632649+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:12.394732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":759,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.145403+0000","last_change":"2024-09-21T10:16:27.315307+0000","last_active":"2024-09-21T10:16:57.145403+0000","last_peered":"2024-09-21T10:16:57.145403+0000","last_clean":"2024-09-21T10:16:57.145403+0000","last_became_active":"2024-09-21T10:16:27.314999+0000","last_became_peered":"2024-09-21T10:16:27.314999+0000","last_unstale":"2024-09-21T10:16:57.145403+0000","last_undegraded":"2024-09-21T10:16:57.145403+0000","last_fullsized":"2024-09-21T10:16:57.145403+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:34.047746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"1338'13104","reported_seq":13167,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:00.755808+0000","last_change":"2024-09-21T10:13:28.747073+0000","last_active":"2024-09-21T10:17:00.755808+0000","last_peered":"2024-09-21T10:17:00.755808+0000","last_clean":"2024-09-21T10:17:00.755808+0000","last_became_active":"2024-09-21T10:13:28.746745+0000","last_became_peered":"2024-09-21T10:13:28.746745+0000","last_unstale":"2024-09-21T10:17:00.755808+0000","last_undegraded":"2024-09-21T10:17:00.755808+0000","last_fullsized":"2024-09-21T10:17:00.755808+0000","mapping_epoch":1283,"log_start":"1311'7525","ondisk_log_start":"1311'7525","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5579,"log_dups_size":0,"ondisk_log_size":5579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:09:26.352129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158203904,"num_objects":2414,"num_object_clones":0,"num_object_copies":7242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13104,"num_write_kb":496576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":110,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110332+0000","last_change":"2024-09-21T10:14:54.655208+0000","last_active":"2024-09-21T10:16:57.110332+0000","last_peered":"2024-09-21T10:16:57.110332+0000","last_clean":"2024-09-21T10:16:57.110332+0000","last_became_active":"2024-09-21T10:14:54.654060+0000","last_became_peered":"2024-09-21T10:14:54.654060+0000","last_unstale":"2024-09-21T10:16:57.110332+0000","last_undegraded":"2024-09-21T10:16:57.110332+0000","last_fullsized":"2024-09-21T10:16:57.110332+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:17.223016+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:17.223016+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:17:27.136197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":34,"num_write":6,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1042,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110916+0000","last_change":"2024-09-21T10:12:50.579273+0000","last_active":"2024-09-21T10:16:57.110916+0000","last_peered":"2024-09-21T10:16:57.110916+0000","last_clean":"2024-09-21T10:16:57.110916+0000","last_became_active":"2024-09-21T10:12:50.578806+0000","last_became_peered":"2024-09-21T10:12:50.578806+0000","last_unstale":"2024-09-21T10:16:57.110916+0000","last_undegraded":"2024-09-21T10:16:57.110916+0000","last_fullsized":"2024-09-21T10:16:57.110916+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:04:53.664017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":99,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634106+0000","last_change":"2024-09-21T10:16:27.328517+0000","last_active":"2024-09-21T10:16:53.634106+0000","last_peered":"2024-09-21T10:16:53.634106+0000","last_clean":"2024-09-21T10:16:53.634106+0000","last_became_active":"2024-09-21T10:16:27.328203+0000","last_became_peered":"2024-09-21T10:16:27.328203+0000","last_unstale":"2024-09-21T10:16:53.634106+0000","last_undegraded":"2024-09-21T10:16:53.634106+0000","last_fullsized":"2024-09-21T10:16:53.634106+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:54.497430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":544,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634174+0000","last_change":"2024-09-21T10:16:27.337427+0000","last_active":"2024-09-21T10:16:53.634174+0000","last_peered":"2024-09-21T10:16:53.634174+0000","last_clean":"2024-09-21T10:16:53.634174+0000","last_became_active":"2024-09-21T10:16:27.336986+0000","last_became_peered":"2024-09-21T10:16:27.336986+0000","last_unstale":"2024-09-21T10:16:53.634174+0000","last_undegraded":"2024-09-21T10:16:53.634174+0000","last_fullsized":"2024-09-21T10:16:53.634174+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:51:38.804787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.d","version":"1338'13361","reported_seq":13438,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.654174+0000","last_change":"2024-09-21T10:15:25.772575+0000","last_active":"2024-09-21T10:16:58.654174+0000","last_peered":"2024-09-21T10:16:58.654174+0000","last_clean":"2024-09-21T10:16:58.654174+0000","last_became_active":"2024-09-21T10:14:54.661670+0000","last_became_peered":"2024-09-21T10:14:54.661670+0000","last_unstale":"2024-09-21T10:16:58.654174+0000","last_undegraded":"2024-09-21T10:16:58.654174+0000","last_fullsized":"2024-09-21T10:16:58.654174+0000","mapping_epoch":1309,"log_start":"1311'8415","ondisk_log_start":"1311'8415","created":1283,"last_epoch_clean":1312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4946,"log_dups_size":0,"ondisk_log_size":4946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:47:06.826746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174719000,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13361,"num_write_kb":512833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":384,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110373+0000","last_change":"2024-09-21T10:16:07.913120+0000","last_active":"2024-09-21T10:16:57.110373+0000","last_peered":"2024-09-21T10:16:57.110373+0000","last_clean":"2024-09-21T10:16:57.110373+0000","last_became_active":"2024-09-21T10:16:07.912620+0000","last_became_peered":"2024-09-21T10:16:07.912620+0000","last_unstale":"2024-09-21T10:16:57.110373+0000","last_undegraded":"2024-09-21T10:16:57.110373+0000","last_fullsized":"2024-09-21T10:16:57.110373+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:10:37.256819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:10:37.256819+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":13,"num_read_kb":34,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":685,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.145259+0000","last_change":"2024-09-21T10:16:27.329301+0000","last_active":"2024-09-21T10:16:57.145259+0000","last_peered":"2024-09-21T10:16:57.145259+0000","last_clean":"2024-09-21T10:16:57.145259+0000","last_became_active":"2024-09-21T10:16:27.328337+0000","last_became_peered":"2024-09-21T10:16:27.328337+0000","last_unstale":"2024-09-21T10:16:57.145259+0000","last_undegraded":"2024-09-21T10:16:57.145259+0000","last_fullsized":"2024-09-21T10:16:57.145259+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:40.669541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":80,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634293+0000","last_change":"2024-09-21T10:12:25.709819+0000","last_active":"2024-09-21T10:16:53.634293+0000","last_peered":"2024-09-21T10:16:53.634293+0000","last_clean":"2024-09-21T10:16:53.634293+0000","last_became_active":"2024-09-21T10:12:25.709548+0000","last_became_peered":"2024-09-21T10:12:25.709548+0000","last_unstale":"2024-09-21T10:16:53.634293+0000","last_undegraded":"2024-09-21T10:16:53.634293+0000","last_fullsized":"2024-09-21T10:16:53.634293+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:35.986614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2038,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633296+0000","last_change":"2024-09-21T10:16:07.913449+0000","last_active":"2024-09-21T10:16:53.633296+0000","last_peered":"2024-09-21T10:16:53.633296+0000","last_clean":"2024-09-21T10:16:53.633296+0000","last_became_active":"2024-09-21T10:16:07.913117+0000","last_became_peered":"2024-09-21T10:16:07.913117+0000","last_unstale":"2024-09-21T10:16:53.633296+0000","last_undegraded":"2024-09-21T10:16:53.633296+0000","last_fullsized":"2024-09-21T10:16:53.633296+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:24.442444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"1338'13281","reported_seq":13343,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.679263+0000","last_change":"2024-09-21T10:13:28.743256+0000","last_active":"2024-09-21T10:16:58.679263+0000","last_peered":"2024-09-21T10:16:58.679263+0000","last_clean":"2024-09-21T10:16:58.679263+0000","last_became_active":"2024-09-21T10:13:28.742998+0000","last_became_peered":"2024-09-21T10:13:28.742998+0000","last_unstale":"2024-09-21T10:16:58.679263+0000","last_undegraded":"2024-09-21T10:16:58.679263+0000","last_fullsized":"2024-09-21T10:16:58.679263+0000","mapping_epoch":1283,"log_start":"1311'8317","ondisk_log_start":"1311'8317","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4964,"log_dups_size":0,"ondisk_log_size":4964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:13:15.751354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168361984,"num_objects":2569,"num_object_clones":0,"num_object_copies":7707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13281,"num_write_kb":507200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":238,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631640+0000","last_change":"2024-09-21T10:10:13.623475+0000","last_active":"2024-09-21T10:16:53.631640+0000","last_peered":"2024-09-21T10:16:53.631640+0000","last_clean":"2024-09-21T10:16:53.631640+0000","last_became_active":"2024-09-21T10:10:13.622580+0000","last_became_peered":"2024-09-21T10:10:13.622580+0000","last_unstale":"2024-09-21T10:16:53.631640+0000","last_undegraded":"2024-09-21T10:16:53.631640+0000","last_fullsized":"2024-09-21T10:16:53.631640+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:08:36.469530+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:08:36.469530+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:47.757854+0000","scrub_duration":0.001636942,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":2051,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110562+0000","last_change":"2024-09-21T10:12:25.718713+0000","last_active":"2024-09-21T10:16:57.110562+0000","last_peered":"2024-09-21T10:16:57.110562+0000","last_clean":"2024-09-21T10:16:57.110562+0000","last_became_active":"2024-09-21T10:12:25.713007+0000","last_became_peered":"2024-09-21T10:12:25.713007+0000","last_unstale":"2024-09-21T10:16:57.110562+0000","last_undegraded":"2024-09-21T10:16:57.110562+0000","last_fullsized":"2024-09-21T10:16:57.110562+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:09.753443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1756,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.632816+0000","last_change":"2024-09-21T10:16:27.321061+0000","last_active":"2024-09-21T10:16:53.632816+0000","last_peered":"2024-09-21T10:16:53.632816+0000","last_clean":"2024-09-21T10:16:53.632816+0000","last_became_active":"2024-09-21T10:16:27.320759+0000","last_became_peered":"2024-09-21T10:16:27.320759+0000","last_unstale":"2024-09-21T10:16:53.632816+0000","last_undegraded":"2024-09-21T10:16:53.632816+0000","last_fullsized":"2024-09-21T10:16:53.632816+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:29:04.861525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2042,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.108698+0000","last_change":"2024-09-21T10:12:25.709467+0000","last_active":"2024-09-21T10:16:57.108698+0000","last_peered":"2024-09-21T10:16:57.108698+0000","last_clean":"2024-09-21T10:16:57.108698+0000","last_became_active":"2024-09-21T10:12:25.706170+0000","last_became_peered":"2024-09-21T10:12:25.706170+0000","last_unstale":"2024-09-21T10:16:57.108698+0000","last_undegraded":"2024-09-21T10:16:57.108698+0000","last_fullsized":"2024-09-21T10:16:57.108698+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:30.335358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"1338'13483","reported_seq":13546,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:02.217569+0000","last_change":"2024-09-21T10:13:28.789659+0000","last_active":"2024-09-21T10:17:02.217569+0000","last_peered":"2024-09-21T10:17:02.217569+0000","last_clean":"2024-09-21T10:17:02.217569+0000","last_became_active":"2024-09-21T10:13:28.789382+0000","last_became_peered":"2024-09-21T10:13:28.789382+0000","last_unstale":"2024-09-21T10:17:02.217569+0000","last_undegraded":"2024-09-21T10:17:02.217569+0000","last_fullsized":"2024-09-21T10:17:02.217569+0000","mapping_epoch":1283,"log_start":"1311'8658","ondisk_log_start":"1311'8658","created":1283,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4825,"log_dups_size":0,"ondisk_log_size":4825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:51.179733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161808384,"num_objects":2469,"num_object_clones":0,"num_object_copies":7407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13483,"num_write_kb":510464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":521,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110016+0000","last_change":"2024-09-21T10:12:50.578581+0000","last_active":"2024-09-21T10:16:57.110016+0000","last_peered":"2024-09-21T10:16:57.110016+0000","last_clean":"2024-09-21T10:16:57.110016+0000","last_became_active":"2024-09-21T10:12:50.578246+0000","last_became_peered":"2024-09-21T10:12:50.578246+0000","last_unstale":"2024-09-21T10:16:57.110016+0000","last_undegraded":"2024-09-21T10:16:57.110016+0000","last_fullsized":"2024-09-21T10:16:57.110016+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.423664+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.423664+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:38.323326+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":2069,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.101680+0000","last_change":"2024-09-21T10:16:27.330519+0000","last_active":"2024-09-21T10:16:57.101680+0000","last_peered":"2024-09-21T10:16:57.101680+0000","last_clean":"2024-09-21T10:16:57.101680+0000","last_became_active":"2024-09-21T10:16:27.317630+0000","last_became_peered":"2024-09-21T10:16:27.317630+0000","last_unstale":"2024-09-21T10:16:57.101680+0000","last_undegraded":"2024-09-21T10:16:57.101680+0000","last_fullsized":"2024-09-21T10:16:57.101680+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:19:53.670529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":105,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631544+0000","last_change":"2024-09-21T10:12:25.704090+0000","last_active":"2024-09-21T10:16:53.631544+0000","last_peered":"2024-09-21T10:16:53.631544+0000","last_clean":"2024-09-21T10:16:53.631544+0000","last_became_active":"2024-09-21T10:12:25.702945+0000","last_became_peered":"2024-09-21T10:12:25.702945+0000","last_unstale":"2024-09-21T10:16:53.631544+0000","last_undegraded":"2024-09-21T10:16:53.631544+0000","last_fullsized":"2024-09-21T10:16:53.631544+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:29.609613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1273,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633425+0000","last_change":"2024-09-21T10:12:50.571672+0000","last_active":"2024-09-21T10:16:53.633425+0000","last_peered":"2024-09-21T10:16:53.633425+0000","last_clean":"2024-09-21T10:16:53.633425+0000","last_became_active":"2024-09-21T10:12:50.571358+0000","last_became_peered":"2024-09-21T10:12:50.571358+0000","last_unstale":"2024-09-21T10:16:53.633425+0000","last_undegraded":"2024-09-21T10:16:53.633425+0000","last_fullsized":"2024-09-21T10:16:53.633425+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:38.440163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1336,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631912+0000","last_change":"2024-09-21T10:16:27.316686+0000","last_active":"2024-09-21T10:16:53.631912+0000","last_peered":"2024-09-21T10:16:53.631912+0000","last_clean":"2024-09-21T10:16:53.631912+0000","last_became_active":"2024-09-21T10:16:27.316186+0000","last_became_peered":"2024-09-21T10:16:27.316186+0000","last_unstale":"2024-09-21T10:16:53.631912+0000","last_undegraded":"2024-09-21T10:16:53.631912+0000","last_fullsized":"2024-09-21T10:16:53.631912+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:45.443605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1366,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631418+0000","last_change":"2024-09-21T10:12:25.713745+0000","last_active":"2024-09-21T10:16:53.631418+0000","last_peered":"2024-09-21T10:16:53.631418+0000","last_clean":"2024-09-21T10:16:53.631418+0000","last_became_active":"2024-09-21T10:12:25.698011+0000","last_became_peered":"2024-09-21T10:12:25.698011+0000","last_unstale":"2024-09-21T10:16:53.631418+0000","last_undegraded":"2024-09-21T10:16:53.631418+0000","last_fullsized":"2024-09-21T10:16:53.631418+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:45.276913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23720,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.630791+0000","last_change":"2024-09-21T10:16:27.338866+0000","last_active":"2024-09-21T10:16:53.630791+0000","last_peered":"2024-09-21T10:16:53.630791+0000","last_clean":"2024-09-21T10:16:53.630791+0000","last_became_active":"2024-09-21T10:16:27.338090+0000","last_became_peered":"2024-09-21T10:16:27.338090+0000","last_unstale":"2024-09-21T10:16:53.630791+0000","last_undegraded":"2024-09-21T10:16:53.630791+0000","last_fullsized":"2024-09-21T10:16:53.630791+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:54.045061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":86,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633388+0000","last_change":"2024-09-21T10:12:25.717602+0000","last_active":"2024-09-21T10:16:53.633388+0000","last_peered":"2024-09-21T10:16:53.633388+0000","last_clean":"2024-09-21T10:16:53.633388+0000","last_became_active":"2024-09-21T10:12:25.705211+0000","last_became_peered":"2024-09-21T10:12:25.705211+0000","last_unstale":"2024-09-21T10:16:53.633388+0000","last_undegraded":"2024-09-21T10:16:53.633388+0000","last_fullsized":"2024-09-21T10:16:53.633388+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:30.822884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"288'100","reported_seq":236,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.145157+0000","last_change":"2024-09-21T10:12:25.698905+0000","last_active":"2024-09-21T10:16:57.145157+0000","last_peered":"2024-09-21T10:16:57.145157+0000","last_clean":"2024-09-21T10:16:57.145157+0000","last_became_active":"2024-09-21T10:12:25.698552+0000","last_became_peered":"2024-09-21T10:12:25.698552+0000","last_unstale":"2024-09-21T10:16:57.145157+0000","last_undegraded":"2024-09-21T10:16:57.145157+0000","last_fullsized":"2024-09-21T10:16:57.145157+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.389965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.389965+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:29:36.307701+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634465+0000","last_change":"2024-09-21T10:16:27.320052+0000","last_active":"2024-09-21T10:16:53.634465+0000","last_peered":"2024-09-21T10:16:53.634465+0000","last_clean":"2024-09-21T10:16:53.634465+0000","last_became_active":"2024-09-21T10:16:27.319716+0000","last_became_peered":"2024-09-21T10:16:27.319716+0000","last_unstale":"2024-09-21T10:16:53.634465+0000","last_undegraded":"2024-09-21T10:16:53.634465+0000","last_fullsized":"2024-09-21T10:16:53.634465+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:15:46.308322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2043,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.108730+0000","last_change":"2024-09-21T10:12:25.707393+0000","last_active":"2024-09-21T10:16:57.108730+0000","last_peered":"2024-09-21T10:16:57.108730+0000","last_clean":"2024-09-21T10:16:57.108730+0000","last_became_active":"2024-09-21T10:12:25.706457+0000","last_became_peered":"2024-09-21T10:12:25.706457+0000","last_unstale":"2024-09-21T10:16:57.108730+0000","last_undegraded":"2024-09-21T10:16:57.108730+0000","last_fullsized":"2024-09-21T10:16:57.108730+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:56.520799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2010,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110435+0000","last_change":"2024-09-21T10:16:07.906476+0000","last_active":"2024-09-21T10:16:57.110435+0000","last_peered":"2024-09-21T10:16:57.110435+0000","last_clean":"2024-09-21T10:16:57.110435+0000","last_became_active":"2024-09-21T10:16:07.906053+0000","last_became_peered":"2024-09-21T10:16:07.906053+0000","last_unstale":"2024-09-21T10:16:57.110435+0000","last_undegraded":"2024-09-21T10:16:57.110435+0000","last_fullsized":"2024-09-21T10:16:57.110435+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:01:57.827302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2085,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.108587+0000","last_change":"2024-09-21T10:16:27.330457+0000","last_active":"2024-09-21T10:16:57.108587+0000","last_peered":"2024-09-21T10:16:57.108587+0000","last_clean":"2024-09-21T10:16:57.108587+0000","last_became_active":"2024-09-21T10:16:27.329527+0000","last_became_peered":"2024-09-21T10:16:27.329527+0000","last_unstale":"2024-09-21T10:16:57.108587+0000","last_undegraded":"2024-09-21T10:16:57.108587+0000","last_fullsized":"2024-09-21T10:16:57.108587+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:16.811107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.11","version":"1338'8162","reported_seq":11995,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:00.827630+0000","last_change":"2024-09-21T10:16:55.901476+0000","last_active":"2024-09-21T10:17:00.827630+0000","last_peered":"2024-09-21T10:17:00.827630+0000","last_clean":"2024-09-21T10:17:00.827630+0000","last_became_active":"2024-09-21T10:16:07.904924+0000","last_became_peered":"2024-09-21T10:16:07.904924+0000","last_unstale":"2024-09-21T10:17:00.827630+0000","last_undegraded":"2024-09-21T10:17:00.827630+0000","last_fullsized":"2024-09-21T10:17:00.827630+0000","mapping_epoch":1325,"log_start":"1289'2054","ondisk_log_start":"1289'2054","created":1292,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":6108,"log_dups_size":0,"ondisk_log_size":6108,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82018304,"num_objects":1252,"num_object_clones":0,"num_object_copies":3756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6584,"num_write_kb":250720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1851,"num_bytes_recovered":121307136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":105,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631754+0000","last_change":"2024-09-21T10:16:27.322614+0000","last_active":"2024-09-21T10:16:53.631754+0000","last_peered":"2024-09-21T10:16:53.631754+0000","last_clean":"2024-09-21T10:16:53.631754+0000","last_became_active":"2024-09-21T10:16:27.321913+0000","last_became_peered":"2024-09-21T10:16:27.321913+0000","last_unstale":"2024-09-21T10:16:53.631754+0000","last_undegraded":"2024-09-21T10:16:53.631754+0000","last_fullsized":"2024-09-21T10:16:53.631754+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:44:39.631658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2057,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.108773+0000","last_change":"2024-09-21T10:16:27.320822+0000","last_active":"2024-09-21T10:16:57.108773+0000","last_peered":"2024-09-21T10:16:57.108773+0000","last_clean":"2024-09-21T10:16:57.108773+0000","last_became_active":"2024-09-21T10:16:27.318888+0000","last_became_peered":"2024-09-21T10:16:27.318888+0000","last_unstale":"2024-09-21T10:16:57.108773+0000","last_undegraded":"2024-09-21T10:16:57.108773+0000","last_fullsized":"2024-09-21T10:16:57.108773+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:50:19.511164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"1338'8141","reported_seq":12104,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:58.674388+0000","last_change":"2024-09-21T10:15:58.105457+0000","last_active":"2024-09-21T10:16:58.674388+0000","last_peered":"2024-09-21T10:16:58.674388+0000","last_clean":"2024-09-21T10:16:58.674388+0000","last_became_active":"2024-09-21T10:14:25.660440+0000","last_became_peered":"2024-09-21T10:14:25.660440+0000","last_unstale":"2024-09-21T10:16:58.674388+0000","last_undegraded":"2024-09-21T10:16:58.674388+0000","last_fullsized":"2024-09-21T10:16:58.674388+0000","mapping_epoch":1302,"log_start":"1291'3140","ondisk_log_start":"1291'3140","created":1292,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":5001,"log_dups_size":0,"ondisk_log_size":5001,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80543744,"num_objects":1229,"num_object_clones":0,"num_object_copies":3687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6551,"num_write_kb":248960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1926,"num_bytes_recovered":126222336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1993,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633788+0000","last_change":"2024-09-21T10:16:07.912578+0000","last_active":"2024-09-21T10:16:53.633788+0000","last_peered":"2024-09-21T10:16:53.633788+0000","last_clean":"2024-09-21T10:16:53.633788+0000","last_became_active":"2024-09-21T10:16:07.912300+0000","last_became_peered":"2024-09-21T10:16:07.912300+0000","last_unstale":"2024-09-21T10:16:53.633788+0000","last_undegraded":"2024-09-21T10:16:53.633788+0000","last_fullsized":"2024-09-21T10:16:53.633788+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:58:19.252218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":760,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.109167+0000","last_change":"2024-09-21T10:16:27.321423+0000","last_active":"2024-09-21T10:16:57.109167+0000","last_peered":"2024-09-21T10:16:57.109167+0000","last_clean":"2024-09-21T10:16:57.109167+0000","last_became_active":"2024-09-21T10:16:27.319009+0000","last_became_peered":"2024-09-21T10:16:27.319009+0000","last_unstale":"2024-09-21T10:16:57.109167+0000","last_undegraded":"2024-09-21T10:16:57.109167+0000","last_fullsized":"2024-09-21T10:16:57.109167+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:32.452405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.13","version":"1338'8532","reported_seq":24806,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:00.862920+0000","last_change":"2024-09-21T10:16:53.680964+0000","last_active":"2024-09-21T10:17:00.862920+0000","last_peered":"2024-09-21T10:17:00.862920+0000","last_clean":"2024-09-21T10:17:00.862920+0000","last_became_active":"2024-09-21T10:16:53.635600+0000","last_became_peered":"2024-09-21T10:16:53.635600+0000","last_unstale":"2024-09-21T10:17:00.862920+0000","last_undegraded":"2024-09-21T10:17:00.862920+0000","last_fullsized":"2024-09-21T10:17:00.862920+0000","mapping_epoch":1336,"log_start":"1290'2678","ondisk_log_start":"1290'2678","created":1292,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":5,"last_scrub":"1300'3938","last_scrub_stamp":"2024-09-21T10:14:22.265794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:22.265794+0000","objects_scrubbed":2294,"log_size":5854,"log_dups_size":0,"ondisk_log_size":5854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:56:39.227571+0000","scrub_duration":688,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80805888,"num_objects":1233,"num_object_clones":0,"num_object_copies":3699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4615,"num_write_kb":113728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6388,"num_bytes_recovered":266797056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2050,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110819+0000","last_change":"2024-09-21T10:16:27.340083+0000","last_active":"2024-09-21T10:16:57.110819+0000","last_peered":"2024-09-21T10:16:57.110819+0000","last_clean":"2024-09-21T10:16:57.110819+0000","last_became_active":"2024-09-21T10:16:27.339405+0000","last_became_peered":"2024-09-21T10:16:27.339405+0000","last_unstale":"2024-09-21T10:16:57.110819+0000","last_undegraded":"2024-09-21T10:16:57.110819+0000","last_fullsized":"2024-09-21T10:16:57.110819+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:11.731090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1315,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.634256+0000","last_change":"2024-09-21T10:16:27.338372+0000","last_active":"2024-09-21T10:16:53.634256+0000","last_peered":"2024-09-21T10:16:53.634256+0000","last_clean":"2024-09-21T10:16:53.634256+0000","last_became_active":"2024-09-21T10:16:27.338029+0000","last_became_peered":"2024-09-21T10:16:27.338029+0000","last_unstale":"2024-09-21T10:16:53.634256+0000","last_undegraded":"2024-09-21T10:16:53.634256+0000","last_fullsized":"2024-09-21T10:16:53.634256+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:52.311869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"1338'8376","reported_seq":16546,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:02.295641+0000","last_change":"2024-09-21T10:16:22.274483+0000","last_active":"2024-09-21T10:17:02.295641+0000","last_peered":"2024-09-21T10:17:02.295641+0000","last_clean":"2024-09-21T10:17:02.295641+0000","last_became_active":"2024-09-21T10:15:27.621305+0000","last_became_peered":"2024-09-21T10:15:27.621305+0000","last_unstale":"2024-09-21T10:17:02.295641+0000","last_undegraded":"2024-09-21T10:17:02.295641+0000","last_fullsized":"2024-09-21T10:17:02.295641+0000","mapping_epoch":1313,"log_start":"1293'3547","ondisk_log_start":"1293'3547","created":1292,"last_epoch_clean":1327,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4829,"log_dups_size":0,"ondisk_log_size":4829,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79069184,"num_objects":1207,"num_object_clones":0,"num_object_copies":3621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6739,"num_write_kb":254240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2013,"num_bytes_recovered":131792896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2076,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.145338+0000","last_change":"2024-09-21T10:16:27.329741+0000","last_active":"2024-09-21T10:16:57.145338+0000","last_peered":"2024-09-21T10:16:57.145338+0000","last_clean":"2024-09-21T10:16:57.145338+0000","last_became_active":"2024-09-21T10:16:27.328855+0000","last_became_peered":"2024-09-21T10:16:27.328855+0000","last_unstale":"2024-09-21T10:16:57.145338+0000","last_undegraded":"2024-09-21T10:16:57.145338+0000","last_fullsized":"2024-09-21T10:16:57.145338+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:27.764838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":44,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.632738+0000","last_change":"2024-09-21T10:16:07.903138+0000","last_active":"2024-09-21T10:16:53.632738+0000","last_peered":"2024-09-21T10:16:53.632738+0000","last_clean":"2024-09-21T10:16:53.632738+0000","last_became_active":"2024-09-21T10:16:07.902844+0000","last_became_peered":"2024-09-21T10:16:07.902844+0000","last_unstale":"2024-09-21T10:16:53.632738+0000","last_undegraded":"2024-09-21T10:16:53.632738+0000","last_fullsized":"2024-09-21T10:16:53.632738+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:39:17.151972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"1338'8149","reported_seq":11442,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:17:00.043351+0000","last_change":"2024-09-21T10:16:33.519619+0000","last_active":"2024-09-21T10:17:00.043351+0000","last_peered":"2024-09-21T10:17:00.043351+0000","last_clean":"2024-09-21T10:17:00.043351+0000","last_became_active":"2024-09-21T10:16:33.501580+0000","last_became_peered":"2024-09-21T10:16:33.501580+0000","last_unstale":"2024-09-21T10:17:00.043351+0000","last_undegraded":"2024-09-21T10:17:00.043351+0000","last_fullsized":"2024-09-21T10:17:00.043351+0000","mapping_epoch":1333,"log_start":"1293'3164","ondisk_log_start":"1293'3164","created":1292,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":5,"last_scrub":"1303'4138","last_scrub_stamp":"2024-09-21T10:14:31.335247+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:14:31.335247+0000","objects_scrubbed":2553,"log_size":4985,"log_dups_size":0,"ondisk_log_size":4985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:28.188051+0000","scrub_duration":943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83820544,"num_objects":1279,"num_object_clones":0,"num_object_copies":3837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4028,"num_write_kb":88128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1555,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1313,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631755+0000","last_change":"2024-09-21T10:13:41.789883+0000","last_active":"2024-09-21T10:16:53.631755+0000","last_peered":"2024-09-21T10:16:53.631755+0000","last_clean":"2024-09-21T10:16:53.631755+0000","last_became_active":"2024-09-21T10:13:41.789620+0000","last_became_peered":"2024-09-21T10:13:41.789620+0000","last_unstale":"2024-09-21T10:16:53.631755+0000","last_undegraded":"2024-09-21T10:16:53.631755+0000","last_fullsized":"2024-09-21T10:16:53.631755+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:36.869540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":289,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631222+0000","last_change":"2024-09-21T10:16:27.324301+0000","last_active":"2024-09-21T10:16:53.631222+0000","last_peered":"2024-09-21T10:16:53.631222+0000","last_clean":"2024-09-21T10:16:53.631222+0000","last_became_active":"2024-09-21T10:16:27.319515+0000","last_became_peered":"2024-09-21T10:16:27.319515+0000","last_unstale":"2024-09-21T10:16:53.631222+0000","last_undegraded":"2024-09-21T10:16:53.631222+0000","last_fullsized":"2024-09-21T10:16:53.631222+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:38.284184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"1338'8125","reported_seq":9425,"reported_epoch":1338,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:17:01.808928+0000","last_change":"2024-09-21T10:16:34.058670+0000","last_active":"2024-09-21T10:17:01.808928+0000","last_peered":"2024-09-21T10:17:01.808928+0000","last_clean":"2024-09-21T10:15:28.617235+0000","last_became_active":"2024-09-21T10:16:31.843224+0000","last_became_peered":"2024-09-21T10:16:31.843224+0000","last_unstale":"2024-09-21T10:17:01.808928+0000","last_undegraded":"2024-09-21T10:16:31.808418+0000","last_fullsized":"2024-09-21T10:16:28.308092+0000","mapping_epoch":1331,"log_start":"1289'1051","ondisk_log_start":"1289'1051","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72777728,"num_objects":1111,"num_object_clones":0,"num_object_copies":3333,"num_objects_missing_on_primary":3302,"num_objects_missing":3302,"num_objects_degraded":4951,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6503,"num_write_kb":243616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":530,"num_bytes_recovered":34668544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,3,4","objects":1136},{"shards":"2,4","objects":-25}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1283,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.633975+0000","last_change":"2024-09-21T10:08:25.430667+0000","last_active":"2024-09-21T10:16:53.633975+0000","last_peered":"2024-09-21T10:16:53.633975+0000","last_clean":"2024-09-21T10:16:53.633975+0000","last_became_active":"2024-09-21T10:08:25.430357+0000","last_became_peered":"2024-09-21T10:08:25.430357+0000","last_unstale":"2024-09-21T10:16:53.633975+0000","last_undegraded":"2024-09-21T10:16:53.633975+0000","last_fullsized":"2024-09-21T10:16:53.633975+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:19.972020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":277,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631567+0000","last_change":"2024-09-21T10:10:13.639231+0000","last_active":"2024-09-21T10:16:53.631567+0000","last_peered":"2024-09-21T10:16:53.631567+0000","last_clean":"2024-09-21T10:16:53.631567+0000","last_became_active":"2024-09-21T10:10:13.622497+0000","last_became_peered":"2024-09-21T10:10:13.622497+0000","last_unstale":"2024-09-21T10:16:53.631567+0000","last_undegraded":"2024-09-21T10:16:53.631567+0000","last_fullsized":"2024-09-21T10:16:53.631567+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:31:46.263003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"1338'8213","reported_seq":8328,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:59.969357+0000","last_change":"2024-09-21T10:15:29.127817+0000","last_active":"2024-09-21T10:16:59.969357+0000","last_peered":"2024-09-21T10:16:59.969357+0000","last_clean":"2024-09-21T10:16:59.969357+0000","last_became_active":"2024-09-21T10:14:54.651071+0000","last_became_peered":"2024-09-21T10:14:54.651071+0000","last_unstale":"2024-09-21T10:16:59.969357+0000","last_undegraded":"2024-09-21T10:16:59.969357+0000","last_fullsized":"2024-09-21T10:16:59.969357+0000","mapping_epoch":1309,"log_start":"1293'3280","ondisk_log_start":"1293'3280","created":1292,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:13:27.730297+0000","objects_scrubbed":0,"log_size":4933,"log_dups_size":0,"ondisk_log_size":4933,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83066880,"num_objects":1268,"num_object_clones":0,"num_object_copies":3804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6574,"num_write_kb":250912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2028,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.109099+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:16:57.109099+0000","last_peered":"2024-09-21T10:16:57.109099+0000","last_clean":"2024-09-21T10:16:57.109099+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:16:57.109099+0000","last_undegraded":"2024-09-21T10:16:57.109099+0000","last_fullsized":"2024-09-21T10:16:57.109099+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":2044,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.110620+0000","last_change":"2024-09-21T10:10:13.651599+0000","last_active":"2024-09-21T10:16:57.110620+0000","last_peered":"2024-09-21T10:16:57.110620+0000","last_clean":"2024-09-21T10:16:57.110620+0000","last_became_active":"2024-09-21T10:10:13.651250+0000","last_became_peered":"2024-09-21T10:10:13.651250+0000","last_unstale":"2024-09-21T10:16:57.110620+0000","last_undegraded":"2024-09-21T10:16:57.110620+0000","last_fullsized":"2024-09-21T10:16:57.110620+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:34.100056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"1338'8263","reported_seq":9418,"reported_epoch":1338,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2024-09-21T10:17:02.351327+0000","last_change":"2024-09-21T10:16:33.703800+0000","last_active":"2024-09-21T10:17:02.351327+0000","last_peered":"2024-09-21T10:17:02.351327+0000","last_clean":"2024-09-21T10:15:31.628723+0000","last_became_active":"2024-09-21T10:16:32.300370+0000","last_became_peered":"2024-09-21T10:16:32.300370+0000","last_unstale":"2024-09-21T10:17:02.351327+0000","last_undegraded":"2024-09-21T10:16:32.205794+0000","last_fullsized":"2024-09-21T10:17:02.351327+0000","mapping_epoch":1331,"log_start":"1293'3409","ondisk_log_start":"1293'3409","created":1292,"last_epoch_clean":1293,"parent":"0.0","parent_split_bits":5,"last_scrub":"1315'6171","last_scrub_stamp":"2024-09-21T10:15:29.169070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:13:27.730297+0000","last_clean_scrub_stamp":"2024-09-21T10:15:29.169070+0000","objects_scrubbed":3301,"log_size":4854,"log_dups_size":0,"ondisk_log_size":4854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:52.694725+0000","scrub_duration":1090,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78053376,"num_objects":1191,"num_object_clones":0,"num_object_copies":3573,"num_objects_missing_on_primary":3412,"num_objects_missing":3412,"num_objects_degraded":1191,"num_objects_misplaced":3412,"num_objects_unfound":0,"num_objects_dirty":1191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2110,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":484,"num_bytes_recovered":31588352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,0],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1182},{"shards":"0,1,4,6","objects":9}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1336,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-21T10:16:53.631244+0000","last_change":"2024-09-21T10:16:27.318860+0000","last_active":"2024-09-21T10:16:53.631244+0000","last_peered":"2024-09-21T10:16:53.631244+0000","last_clean":"2024-09-21T10:16:53.631244+0000","last_became_active":"2024-09-21T10:16:27.317542+0000","last_became_peered":"2024-09-21T10:16:27.317542+0000","last_unstale":"2024-09-21T10:16:53.631244+0000","last_undegraded":"2024-09-21T10:16:53.631244+0000","last_fullsized":"2024-09-21T10:16:53.631244+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:56:58.481255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":47,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-21T10:16:57.102278+0000","last_change":"2024-09-21T10:16:07.914634+0000","last_active":"2024-09-21T10:16:57.102278+0000","last_peered":"2024-09-21T10:16:57.102278+0000","last_clean":"2024-09-21T10:16:57.102278+0000","last_became_active":"2024-09-21T10:16:07.914216+0000","last_became_peered":"2024-09-21T10:16:07.914216+0000","last_unstale":"2024-09-21T10:16:57.102278+0000","last_undegraded":"2024-09-21T10:16:57.102278+0000","last_fullsized":"2024-09-21T10:16:57.102278+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:22:59.593322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":2586476568,"num_objects":39467,"num_object_clones":0,"num_object_copies":118401,"num_objects_missing_on_primary":6714,"num_objects_missing":6714,"num_objects_degraded":6470,"num_objects_misplaced":6849,"num_objects_unfound":0,"num_objects_dirty":39467,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191348,"num_write_kb":6793121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18546,"num_bytes_recovered":961216512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8623566848,"data_stored":8623554632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":145278,"ondisk_log_size":145278,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":24146,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34026,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":724543,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":67,"num_read_kb":170,"num_write":34,"num_write_kb":370,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":724543,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2100,"ondisk_log_size":2100,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398014,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1779428,"kb_used_data":883100,"kb_used_omap":1,"kb_used_meta":896318,"kb_avail":91965724,"statfs":{"total":95995035648,"available":94172901376,"internally_reserved":0,"allocated":904294400,"data_stored":901001066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917829976},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594243,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2172840,"kb_used_data":1268640,"kb_used_omap":1,"kb_used_meta":904190,"kb_avail":91572312,"statfs":{"total":95995035648,"available":93770047488,"internally_reserved":0,"allocated":1299087360,"data_stored":1295790925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":925890904},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790472,"num_pgs":12,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1347980,"kb_used_data":445508,"kb_used_omap":1,"kb_used_meta":902462,"kb_avail":92397172,"statfs":{"total":95995035648,"available":94614704128,"internally_reserved":0,"allocated":456200192,"data_stored":452926477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":924121432},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1224,"seq":5257039970408,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1883560,"kb_used_data":1542876,"kb_used_omap":290,"kb_used_meta":340381,"kb_avail":91861592,"statfs":{"total":95995035648,"available":94066270208,"internally_reserved":0,"allocated":1579905024,"data_stored":1576611628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":297377,"internal_metadata":348550751},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999341,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1952980,"kb_used_data":951436,"kb_used_omap":25,"kb_used_meta":1001510,"kb_avail":91792172,"statfs":{"total":95995035648,"available":93995184128,"internally_reserved":0,"allocated":974270464,"data_stored":970989262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":1025547207},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111284,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2188308,"kb_used_data":1188364,"kb_used_omap":31,"kb_used_meta":999904,"kb_avail":91556844,"statfs":{"total":95995035648,"available":93754208256,"internally_reserved":0,"allocated":1216884736,"data_stored":1213603978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":1023901928},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":2203318223227,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335169,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2502812,"kb_used_data":1594512,"kb_used_omap":31,"kb_used_meta":908256,"kb_avail":91242340,"statfs":{"total":95995035648,"available":93432156160,"internally_reserved":0,"allocated":1632780288,"data_stored":1629494921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":930054240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1617039360,"data_stored":1617035288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":689307648,"data_stored":689307648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1194328064,"data_stored":1194328064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":962134016,"data_stored":962134016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1556090880,"data_stored":1556086808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":441974784,"data_stored":441974784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1285230592,"data_stored":1285226520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":877461504,"data_stored":877461504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:17:07.897 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:17:07.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pg_num 31 2024-09-21T10:17:07.899 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:07.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:07 smithi134 ceph-mon[90085]: pgmap v2118: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 6470/114192 objects degraded (5.666%); 6277/114192 objects misplaced (5.497%); 7.9 MiB/s, 126 objects/s recovering 2024-09-21T10:17:08.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:08.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:08.135 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:08.163 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:08.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:08.265 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:08.299 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:08.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:08.401 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:08.436 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:08.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:08.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.567 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:08.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:08.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:08.696 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:08.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:08.798 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:08.826 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:08.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:17:08.928 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:08.952 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:08.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:09.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:09.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:09.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:09.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:09.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:09.328 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:09.356 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:09.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:09.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:17:09.458 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:09.482 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:09.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:09.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:09.621 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:09.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:09 smithi017 ceph-mon[102913]: pgmap v2119: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 6283/109566 objects degraded (5.734%); 6095/109566 objects misplaced (5.563%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:17:09.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:09 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6283/109566 objects degraded (5.734%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:09.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:09 smithi017 ceph-mon[105151]: pgmap v2119: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 6283/109566 objects degraded (5.734%); 6095/109566 objects misplaced (5.563%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:17:09.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:09 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6283/109566 objects degraded (5.734%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:09.723 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:09.747 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:09.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:09.849 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:09.871 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:09.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:09.906 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:09.972 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:09 smithi134 ceph-mon[90085]: pgmap v2119: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 6283/109566 objects degraded (5.734%); 6095/109566 objects misplaced (5.563%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:17:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:09 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6283/109566 objects degraded (5.734%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:10.001 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:10.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:10.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:10.134 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:10.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:10.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:10.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:10.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:10.396 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:10.399 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:10.431 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:10.457 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:17:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:17:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:17:10.559 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:10.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:10.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:10.626 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:17:10.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:10.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:10.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:10.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:10.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:10 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3267406054' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:10.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:10.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:10.822 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:10 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3267406054' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:10.822 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:10.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:10.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:10.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:10.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:10.987 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:10.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:10 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3267406054' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:10.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:11.089 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:11.118 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:11.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:11.221 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:11.244 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:11.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:11.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:11.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:11.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:11.476 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:11.502 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:11.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:11.604 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:11.634 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:11.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:11.736 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:11.765 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:11.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:11.866 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:11.893 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:11.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:11 smithi017 ceph-mon[102913]: pgmap v2120: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 6283/105648 objects degraded (5.947%); 5678/105648 objects misplaced (5.374%); 8.4 MiB/s, 151 objects/s recovering 2024-09-21T10:17:11.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:11 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:11.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:11 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3267406054' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:11.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:11 smithi017 ceph-mon[102913]: osdmap e1339: 8 total, 8 up, 7 in 2024-09-21T10:17:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:11 smithi017 ceph-mon[105151]: pgmap v2120: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 6283/105648 objects degraded (5.947%); 5678/105648 objects misplaced (5.374%); 8.4 MiB/s, 151 objects/s recovering 2024-09-21T10:17:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:11 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:11 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3267406054' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:11.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:11 smithi017 ceph-mon[105151]: osdmap e1339: 8 total, 8 up, 7 in 2024-09-21T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:11 smithi134 ceph-mon[90085]: pgmap v2120: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 6283/105648 objects degraded (5.947%); 5678/105648 objects misplaced (5.374%); 8.4 MiB/s, 151 objects/s recovering 2024-09-21T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:11 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:11 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3267406054' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:11 smithi134 ceph-mon[90085]: osdmap e1339: 8 total, 8 up, 7 in 2024-09-21T10:17:11.995 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:12.021 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:12.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:12.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:12.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:12.263 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:12.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:12.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:12.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:12.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:12.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:12.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:12.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:12.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:12.686 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:12.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:12.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:12.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:12.925 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:12.955 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:13.057 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:13.088 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:13.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:13.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:13.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:13.329 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:13.359 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:13.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:13.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:13.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:13.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:13.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:13.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:13.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:13.723 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:13.750 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:13.761 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:13 smithi017 ceph-mon[102913]: pgmap v2122: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 6097/100122 objects degraded (6.090%); 5457/100122 objects misplaced (5.450%); 7.3 MiB/s, 138 objects/s recovering 2024-09-21T10:17:13.761 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:13 smithi017 ceph-mon[105151]: pgmap v2122: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 6097/100122 objects degraded (6.090%); 5457/100122 objects misplaced (5.450%); 7.3 MiB/s, 138 objects/s recovering 2024-09-21T10:17:13.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:13.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:13.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:13.986 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:13 smithi134 ceph-mon[90085]: pgmap v2122: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 6097/100122 objects degraded (6.090%); 5457/100122 objects misplaced (5.450%); 7.3 MiB/s, 138 objects/s recovering 2024-09-21T10:17:14.015 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:14.117 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:14.148 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:14.250 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:14.274 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:14.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:14.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:14.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:14.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:14.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:14.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:14.648 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:14.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:14.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:14.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:14.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:14.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:14.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:14.946 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:14.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:14 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6097/100122 objects degraded (6.090%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:14.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:17:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:17:14] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-21T10:17:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:14 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6097/100122 objects degraded (6.090%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:14 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6097/100122 objects degraded (6.090%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:15.048 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:15.081 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:15.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:15.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:15.218 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:15.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:15.320 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.351 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:15.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:15.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:15.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:15.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:15.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:15.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:15.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:15 smithi017 ceph-mon[102913]: pgmap v2123: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 6097/97899 objects degraded (6.228%); 5375/97899 objects misplaced (5.490%); 4.3 MiB/s, 160 objects/s recovering 2024-09-21T10:17:15.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:15 smithi017 ceph-mon[105151]: pgmap v2123: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 6097/97899 objects degraded (6.228%); 5375/97899 objects misplaced (5.490%); 4.3 MiB/s, 160 objects/s recovering 2024-09-21T10:17:15.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:15.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:15.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:15.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:15.904 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:15.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:15 smithi134 ceph-mon[90085]: pgmap v2123: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 6097/97899 objects degraded (6.228%); 5375/97899 objects misplaced (5.490%); 4.3 MiB/s, 160 objects/s recovering 2024-09-21T10:17:16.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:16.039 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:16.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:16.142 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.174 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:16.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:16.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:16.311 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:16.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:16.412 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:16.439 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:16.541 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:16.568 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:16.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:16.645 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pg_num to 31 2024-09-21T10:17:16.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:17:16.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:16.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:16.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:16.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:16.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:16 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4019589058' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-21T10:17:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:16 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:16 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:16.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:16 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4019589058' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-21T10:17:16.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:16 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:16.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:16 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:16.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:16 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4019589058' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-21T10:17:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:16 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:16 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:17.017 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:17.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:17.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:17.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:17.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:17.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:17.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:17.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:17.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:17.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:17.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:17.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:17.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:17.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:17.639 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:17.667 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:17.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:17.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:17.907 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:17.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:17 smithi017 ceph-mon[105151]: pgmap v2124: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 6097/95886 objects degraded (6.359%); 5375/95886 objects misplaced (5.606%); 4.3 MiB/s, 160 objects/s recovering 2024-09-21T10:17:17.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4019589058' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-21T10:17:17.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:17 smithi017 ceph-mon[105151]: osdmap e1340: 8 total, 8 up, 7 in 2024-09-21T10:17:17.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:17 smithi017 ceph-mon[102913]: pgmap v2124: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 6097/95886 objects degraded (6.359%); 5375/95886 objects misplaced (5.606%); 4.3 MiB/s, 160 objects/s recovering 2024-09-21T10:17:17.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4019589058' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-21T10:17:17.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:17 smithi017 ceph-mon[102913]: osdmap e1340: 8 total, 8 up, 7 in 2024-09-21T10:17:17.933 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:17.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:17 smithi134 ceph-mon[90085]: pgmap v2124: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 6097/95886 objects degraded (6.359%); 5375/95886 objects misplaced (5.606%); 4.3 MiB/s, 160 objects/s recovering 2024-09-21T10:17:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4019589058' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-21T10:17:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:17 smithi134 ceph-mon[90085]: osdmap e1340: 8 total, 8 up, 7 in 2024-09-21T10:17:18.035 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:18.059 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:18.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:18.136 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:17:18.136 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57270 9.a 9:5e5a1cc8:::benchmark_data_smithi017_237924_object57269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.423992+0000", 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 473.70747323000001, 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2984869460000001, 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.137 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57270 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.423992+0000", 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.423992+0000", 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.423993+0000", 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4000000000000001e-07 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424017+0000", 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4017000000000001e-05 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424026+0000", 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7399999999999993e-06 2024-09-21T10:17:18.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424032+0000", 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6229999999999999e-06 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424137+0000", 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010459 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424290+0000", 2024-09-21T10:17:18.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153287 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.424441+0000", 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015061100000000001 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.446330+0000", 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021888744000000002 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.446341+0000", 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1245e-05 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.437328+0000", 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99098753299999998 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.437346+0000", 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7173e-05 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.722057+0000", 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.284711774 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.722479+0000", 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042196900000000001 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57273 9.a 9:52bf49fb:::benchmark_data_smithi017_237924_object57272:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-09-21T10:17:18.143 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.428944+0000", 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 473.70252110299998, 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3377784579999998, 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57273 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428944+0000", 2024-09-21T10:17:18.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428944+0000", 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428946+0000", 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.556e-06 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428977+0000", 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0793999999999997e-05 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428990+0000", 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2798e-05 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.428999+0000", 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2830000000000007e-06 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.431002+0000", 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020034300000000001 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.431137+0000", 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134222 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.431270+0000", 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013351199999999999 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.446375+0000", 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015105159999999999 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.446384+0000", 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8429999999999999e-06 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.448028+0000", 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0016439619999999 2024-09-21T10:17:18.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.448042+0000", 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4419999999999999e-05 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.725409+0000", 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2773661299999999 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.729090+0000", 2024-09-21T10:17:18.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003681171 2024-09-21T10:17:18.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.766612+0000", 2024-09-21T10:17:18.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.037522487 2024-09-21T10:17:18.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.766723+0000", 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011069 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57274 9.d 9:b4817d95:::benchmark_data_smithi017_237924_object57273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:17:18.151 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.442406+0000", 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 473.689059988, 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2971451329999999, 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57274 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442406+0000", 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442406+0000", 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442407+0000", 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4789999999999999e-06 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442436+0000", 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9147999999999999e-05 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442447+0000", 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0674e-05 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.442455+0000", 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0949999999999996e-06 2024-09-21T10:17:18.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.445511+0000", 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030560850000000001 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.445522+0000", 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1126e-05 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.437369+0000", 2024-09-21T10:17:18.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.991847005 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.437380+0000", 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1082e-05 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.738280+0000", 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.300899509 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.739551+0000", 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00127093 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57285 9.a 9:5a7c97ce:::benchmark_data_smithi017_237924_object57284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.453285+0000", 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 473.67818022099999, 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3177990209999999, 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.157 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57285 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453285+0000", 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453285+0000", 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453286+0000", 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0729999999999999e-06 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453316+0000", 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.959e-05 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453334+0000", 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7645000000000001e-05 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.453342+0000", 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7350000000000002e-06 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.455020+0000", 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016776269999999999 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.455032+0000", 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1943000000000001e-05 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.448075+0000", 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99304325000000004 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.448081+0000", 2024-09-21T10:17:18.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2790000000000004e-06 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.729139+0000", 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2810577990000001 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.729308+0000", 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000168338 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:17:18.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.729489+0000", 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018096599999999999 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.740163+0000", 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010674865 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.742987+0000", 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028231110000000001 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.766774+0000", 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023787557000000001 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.766827+0000", 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3258999999999999e-05 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.771084+0000", 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042569840000000001 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.164 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:57301 9.a 9:548d621b:::benchmark_data_smithi017_237924_object57300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:24.475566+0000", 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 473.65589918400002, 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2977826700000001, 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.165 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 57301 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475566+0000", 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475566+0000", 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475568+0000", 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1939999999999999e-06 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475591+0000", 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3132000000000001e-05 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475613+0000", 2024-09-21T10:17:18.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2254000000000001e-05 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475623+0000", 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8500000000000006e-06 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475691+0000", 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8668000000000001e-05 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:24.475704+0000", 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2285e-05 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.461721+0000", 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.986017276 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:25.461736+0000", 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.482e-05 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.734589+0000", 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.272852927 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.734767+0000", 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017805100000000001 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:17:18.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.734950+0000", 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018336700000000001 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.740209+0000", 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052588000000000001 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.742993+0000", 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027842850000000001 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.771175+0000", 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.02818128 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.771231+0000", 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6097000000000002e-05 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:26.773349+0000", 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021183840000000001 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.172 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:71121 9.0 9:089477fe:::benchmark_data_smithi017_237924_object71120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1248)", 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:09:45.644059+0000", 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 452.48740657899998, 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0834903279999999, 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 71121 2024-09-21T10:17:18.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644059+0000", 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644059+0000", 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644061+0000", 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2460000000000002e-06 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644124+0000", 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2621999999999998e-05 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644145+0000", 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1228000000000001e-05 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644157+0000", 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1630999999999999e-05 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644222+0000", 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5735999999999995e-05 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:45.644230+0000", 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0260000000000007e-06 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:46.454119+0000", 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.80988898399999998 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:46.454127+0000", 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7500000000000003e-06 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.668601+0000", 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.214473871 2024-09-21T10:17:18.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.668746+0000", 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144979 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.668897+0000", 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000150808 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.684975+0000", 2024-09-21T10:17:18.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016078080000000002 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.685064+0000", 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9204999999999997e-05 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.724053+0000", 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.038988876999999998 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.724115+0000", 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1860999999999996e-05 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:09:47.727549+0000", 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034344240000000002 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173494 9.a 9:50c8d2e5:::benchmark_data_smithi017_237924_object80490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:17:18.180 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.672366+0000", 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 293.45909938699998, 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2677540870000001, 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173494 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672366+0000", 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672366+0000", 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672367+0000", 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.296e-06 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672373+0000", 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3009999999999997e-06 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672384+0000", 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1387e-05 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.672392+0000", 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7950000000000008e-06 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.673331+0000", 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00093947300000000002 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.673451+0000", 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119657 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.673880+0000", 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042885500000000001 2024-09-21T10:17:18.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.681603+0000", 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077231490000000003 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686212+0000", 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046092870000000001 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.711576+0000", 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025363765999999999 2024-09-21T10:17:18.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.711589+0000", 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3047999999999999e-05 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.691955+0000", 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.98036599499999999 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.691968+0000", 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2887999999999999e-05 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.927698+0000", 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.235729887 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.927823+0000", 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124857 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.940061+0000", 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012238657999999999 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.940120+0000", 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8788e-05 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173497 9.a 9:5b90c862:::benchmark_data_smithi017_237924_object80493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:17:18.188 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.682723+0000", 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 293.44874222800001, 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.265087597, 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173497 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682723+0000", 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682723+0000", 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682725+0000", 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5430000000000001e-06 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682731+0000", 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4389999999999997e-06 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682748+0000", 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7028e-05 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682756+0000", 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7300000000000005e-06 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683107+0000", 2024-09-21T10:17:18.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035098600000000002 2024-09-21T10:17:18.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:18.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683251+0000", 2024-09-21T10:17:18.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014426200000000001 2024-09-21T10:17:18.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.683395+0000", 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014337700000000001 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686218+0000", 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028229240000000001 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712051+0000", 2024-09-21T10:17:18.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.025833394999999999 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712062+0000", 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0796e-05 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692006+0000", 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97994381699999999 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692014+0000", 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7839999999999992e-06 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928245+0000", 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.236230741 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928362+0000", 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117108 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947740+0000", 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019377426999999999 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947811+0000", 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1240000000000002e-05 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173499 9.a 9:51167347:::benchmark_data_smithi017_237924_object80495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:17:18.197 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.682799+0000", 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 293.44866687899997, 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2655015469999999, 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173499 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682799+0000", 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682799+0000", 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682799+0000", 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2100000000000002e-07 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682802+0000", 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.407e-06 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682808+0000", 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.404e-06 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682813+0000", 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3109999999999996e-06 2024-09-21T10:17:18.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686247+0000", 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003433859 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686362+0000", 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114931 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:17:18.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.686478+0000", 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116415 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.689225+0000", 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027468509999999998 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712092+0000", 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022866424 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712101+0000", 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9260000000000006e-06 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692045+0000", 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97994389900000001 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692055+0000", 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0183999999999999e-05 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928407+0000", 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2363518550000001 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928493+0000", 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6203000000000001e-05 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.948233+0000", 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.019740119 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.948300+0000", 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7337999999999998e-05 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173506 9.a 9:5e300410:::benchmark_data_smithi017_237924_object80502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.691017+0000", 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 293.440448167, 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2555493759999998, 2024-09-21T10:17:18.205 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173506 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691017+0000", 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691017+0000", 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691019+0000", 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.68e-06 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691024+0000", 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8729999999999998e-06 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691036+0000", 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1798e-05 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.691043+0000", 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6480000000000003e-06 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712526+0000", 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021482858000000001 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.712537+0000", 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0359e-05 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692079+0000", 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97954288099999998 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692089+0000", 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8290000000000003e-06 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928542+0000", 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2364527670000001 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928669+0000", 2024-09-21T10:17:18.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127415 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.928804+0000", 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134895 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933737+0000", 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049323420000000001 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933793+0000", 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6239999999999997e-05 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936412+0000", 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026192839999999999 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946186+0000", 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0097739379999999994 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946225+0000", 2024-09-21T10:17:18.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8436999999999998e-05 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946567+0000", 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034213199999999998 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173517 9.a 9:533fe112:::benchmark_data_smithi017_237924_object80513:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.728151+0000", 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 293.403314137, 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.218795536, 2024-09-21T10:17:18.213 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173517 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728151+0000", 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728151+0000", 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728154+0000", 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2800000000000002e-06 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728160+0000", 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4169999999999997e-06 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728178+0000", 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8159999999999999e-05 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728189+0000", 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0264e-05 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728271+0000", 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2862999999999996e-05 2024-09-21T10:17:18.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.728284+0000", 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2395000000000001e-05 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692113+0000", 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96382926999999996 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692122+0000", 2024-09-21T10:17:18.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6899999999999998e-06 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.931313+0000", 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2391907739999999 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.931439+0000", 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012628199999999999 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.931510+0000", 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0813000000000001e-05 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936463+0000", 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049534410000000003 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936523+0000", 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9979000000000003e-05 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936592+0000", 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8757999999999995e-05 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946624+0000", 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01003248 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946662+0000", 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7984000000000001e-05 2024-09-21T10:17:18.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.946947+0000", 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028468599999999998 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173519 9.a 9:5cd44cac:::benchmark_data_smithi017_237924_object80515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.733984+0000", 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 293.39748116099997, 2024-09-21T10:17:18.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2132821030000001, 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173519 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.733984+0000", 2024-09-21T10:17:18.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.733984+0000", 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.733986+0000", 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.291e-06 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.733990+0000", 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1500000000000001e-06 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.734003+0000", 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2782999999999999e-05 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.734009+0000", 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2840000000000003e-06 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.734077+0000", 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8062e-05 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.734084+0000", 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4170000000000003e-06 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692148+0000", 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.95806323500000001 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.692156+0000", 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8869999999999998e-06 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933829+0000", 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2416723649999999 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.933906+0000", 2024-09-21T10:17:18.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6699999999999994e-05 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5,7", 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.934023+0000", 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117877 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936557+0000", 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002533712 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936585+0000", 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7634999999999999e-05 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.936595+0000", 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8619999999999998e-06 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947020+0000", 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010425462 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947080+0000", 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9521000000000002e-05 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.947266+0000", 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018686000000000001 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173491 9.3 9:c0ca1344:::benchmark_data_smithi017_237924_object80487:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1268)", 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:25.694632+0000", 2024-09-21T10:17:18.229 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 292.43683403, 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2226775640000001, 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173491 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694632+0000", 2024-09-21T10:17:18.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694632+0000", 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694633+0000", 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.516e-06 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694638+0000", 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5619999999999997e-06 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694649+0000", 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1382e-05 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694658+0000", 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4210000000000001e-06 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694720+0000", 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1409999999999996e-05 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.694729+0000", 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3190000000000001e-06 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917233+0000", 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2225039360000001 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917309+0000", 2024-09-21T10:17:18.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6018000000000003e-05 2024-09-21T10:17:18.234 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.234 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.234 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.234 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:174750 9.0 9:0193018d:::benchmark_data_smithi017_237924_object81746:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1270)", 2024-09-21T10:17:18.234 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:28.597666+0000", 2024-09-21T10:17:18.235 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.53379912299999, 2024-09-21T10:17:18.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0207220859999999, 2024-09-21T10:17:18.235 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.235 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.235 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 174750 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597666+0000", 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597666+0000", 2024-09-21T10:17:18.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597668+0000", 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5149999999999999e-06 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597673+0000", 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2650000000000004e-06 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597686+0000", 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2313999999999999e-05 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.597711+0000", 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5914999999999998e-05 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.598218+0000", 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00050644500000000001 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.598227+0000", 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1719999999999996e-06 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.609945+0000", 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.011718382 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.610028+0000", 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2486999999999998e-05 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.610100+0000", 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2378000000000004e-05 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.612698+0000", 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025973670000000002 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.615409+0000", 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027111029999999999 2024-09-21T10:17:18.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618273+0000", 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028642310000000001 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618340+0000", 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6830999999999998e-05 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618389+0000", 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8680999999999997e-05 2024-09-21T10:17:18.241 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:174754 9.0 9:05c8886c:::benchmark_data_smithi017_237924_object81750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1270)", 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:28.602933+0000", 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.52853275500001, 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0185616390000001, 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.242 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 174754 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602933+0000", 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602933+0000", 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602935+0000", 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3190000000000002e-06 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602940+0000", 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2179999999999998e-06 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602958+0000", 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7509000000000001e-05 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.602967+0000", 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9840000000000007e-06 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.603550+0000", 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00058335699999999995 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.603573+0000", 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2855999999999999e-05 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.612748+0000", 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.009174802 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.612874+0000", 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012605099999999999 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.612996+0000", 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012250600000000001 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618448+0000", 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054516620000000003 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618600+0000", 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000152304 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621393+0000", 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002792905 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621447+0000", 2024-09-21T10:17:18.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3891000000000003e-05 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621494+0000", 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7274999999999997e-05 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:174764 9.0 9:02d0e704:::benchmark_data_smithi017_237924_object81760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1270)", 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:28.619871+0000", 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.51159419999999, 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.001702117, 2024-09-21T10:17:18.248 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 174764 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619871+0000", 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619871+0000", 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619873+0000", 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.55e-06 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619877+0000", 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.031e-06 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619886+0000", 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2410000000000001e-06 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619892+0000", 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3219999999999998e-06 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619946+0000", 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3049e-05 2024-09-21T10:17:18.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:28.619956+0000", 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0458999999999999e-05 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.615467+0000", 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99551071499999999 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.615602+0000", 2024-09-21T10:17:18.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013529899999999999 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.615735+0000", 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000132652 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618655+0000", 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029202709999999999 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.618709+0000", 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4308999999999998e-05 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621505+0000", 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027956669999999999 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621541+0000", 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6576999999999999e-05 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:29.621573+0000", 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1974999999999998e-05 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51624 10.12 10:4d635f7b:::benchmark_data_smithi017_268344_object51623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.574502+0000", 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 193.55696359199999, 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0964674430000001, 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.255 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51624 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574502+0000", 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574502+0000", 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574505+0000", 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8320000000000002e-06 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574574+0000", 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8708999999999996e-05 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574592+0000", 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8173e-05 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574601+0000", 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7899999999999994e-06 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652357+0000", 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077755938999999996 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652365+0000", 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9119999999999998e-06 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.670836+0000", 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0184705590000001 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.670969+0000", 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013352899999999999 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51630 10.12 10:4a1f2cd2:::benchmark_data_smithi017_268344_object51629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.574639+0000", 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 193.556826438, 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1024330849999999, 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51630 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574639+0000", 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574639+0000", 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.261 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574640+0000", 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4e-07 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574688+0000", 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8414000000000001e-05 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574698+0000", 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5109999999999992e-06 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.574705+0000", 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9750000000000001e-06 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652386+0000", 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077681106 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652392+0000", 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8440000000000003e-06 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.671339+0000", 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.018947681 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.672973+0000", 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001633633 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.676980+0000", 2024-09-21T10:17:18.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040064209999999996 2024-09-21T10:17:18.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.677072+0000", 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2659999999999997e-05 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51632 10.12 10:48e3c976:::benchmark_data_smithi017_268344_object51631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.575319+0000", 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 193.55614683300001, 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1098050749999999, 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.266 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51632 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575319+0000", 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575319+0000", 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575320+0000", 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.305e-06 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575368+0000", 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7852999999999997e-05 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575383+0000", 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5378000000000001e-05 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.575390+0000", 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1919999999999999e-06 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652413+0000", 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.077022304999999999 2024-09-21T10:17:18.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652419+0000", 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0480000000000004e-06 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673028+0000", 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.020609163 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673203+0000", 2024-09-21T10:17:18.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175508 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.673384+0000", 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000181003 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.677117+0000", 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037329780000000001 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.679278+0000", 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021605589999999998 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.684683+0000", 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0054047239999999996 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.684769+0000", 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6170999999999999e-05 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.685124+0000", 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035488799999999998 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:51634 10.2 10:464e25bd:::benchmark_data_smithi017_268344_object51633:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1292)", 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:14:04.611011+0000", 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 193.52045430199999, 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.053044901, 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:18.273 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51634 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611011+0000", 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:18.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611011+0000", 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611014+0000", 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7949999999999999e-06 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611049+0000", 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4715999999999998e-05 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611075+0000", 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6681000000000001e-05 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.611089+0000", 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3224000000000001e-05 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652442+0000", 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.041353194000000003 2024-09-21T10:17:18.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:04.652448+0000", 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3400000000000003e-06 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.653378+0000", 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0009295229999999 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:18.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.653597+0000", 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000219378 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.654044+0000", 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000446792 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.662105+0000", 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080612300000000008 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.663041+0000", 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00093635699999999997 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.663961+0000", 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00091951100000000003 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.664014+0000", 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2849999999999997e-05 2024-09-21T10:17:18.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:14:05.664056+0000", 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.231e-05 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:18.280 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:17:18.287 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:18.388 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:18.415 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:18.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:18.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:18.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:18.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:18.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:18.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:18.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:18.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:18.836 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:18.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:18 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-21T10:17:18.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:18 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-21T10:17:18.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.971 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:18.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:18 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-21T10:17:19.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:19.102 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:19.204 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:19.234 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:19.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:19.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:19.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:19.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:19.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:19.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:19.611 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:19.643 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:19.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:19.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:19.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:19.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:19.920 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[102913]: pgmap v2126: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 5881/90705 objects degraded (6.484%); 5152/90705 objects misplaced (5.680%); 3.1 MiB/s, 138 objects/s recovering 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 6097/95886 objects degraded (6.359%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[102913]: osdmap e1341: 8 total, 8 up, 7 in 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[105151]: pgmap v2126: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 5881/90705 objects degraded (6.484%); 5152/90705 objects misplaced (5.680%); 3.1 MiB/s, 138 objects/s recovering 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 6097/95886 objects degraded (6.359%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[105151]: osdmap e1341: 8 total, 8 up, 7 in 2024-09-21T10:17:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:19.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:19 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:19 smithi134 ceph-mon[90085]: pgmap v2126: 119 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 113 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 5881/90705 objects degraded (6.484%); 5152/90705 objects misplaced (5.680%); 3.1 MiB/s, 138 objects/s recovering 2024-09-21T10:17:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:19 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 6097/95886 objects degraded (6.359%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:19 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-21T10:17:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:19 smithi134 ceph-mon[90085]: osdmap e1341: 8 total, 8 up, 7 in 2024-09-21T10:17:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:19 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:19 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:20.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:20.055 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:20.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:20.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:20.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:20.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:20.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:20.431 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:20.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.464 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:20.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:17:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:17:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:17:20.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:20.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:20.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:20.700 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:20.728 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:20.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:20.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:20.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:20.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:20 smithi017 ceph-mon[102913]: osdmap e1342: 8 total, 8 up, 7 in 2024-09-21T10:17:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:20 smithi017 ceph-mon[105151]: osdmap e1342: 8 total, 8 up, 7 in 2024-09-21T10:17:20.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:20.991 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:20.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:20 smithi134 ceph-mon[90085]: osdmap e1342: 8 total, 8 up, 7 in 2024-09-21T10:17:21.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:21.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:21.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:21.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:17:21.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.273 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:21.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:21.374 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:21.399 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:21.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:21.501 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:21.527 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:21.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:21.591 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:17:21.591 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:21.628 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:17:21.628 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:21.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:21.665 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:17:21.666 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:21.667 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:21.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:21.696 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:17:21.697 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:21.724 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:17:21.724 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:21.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:17:21.749 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:21.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:21.778 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:17:21.778 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:21.778 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:21 smithi017 ceph-mon[102913]: pgmap v2129: 129 pgs: 2 peering, 7 unknown, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 114 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 5881/87027 objects degraded (6.758%); 5055/87027 objects misplaced (5.809%); 2.2 MiB/s, 170 objects/s recovering 2024-09-21T10:17:21.779 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:21 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2759229846' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:21.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:21 smithi017 ceph-mon[105151]: pgmap v2129: 129 pgs: 2 peering, 7 unknown, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 114 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 5881/87027 objects degraded (6.758%); 5055/87027 objects misplaced (5.809%); 2.2 MiB/s, 170 objects/s recovering 2024-09-21T10:17:21.779 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:21 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2759229846' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:21.807 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:17:21.808 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:21.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:21.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:21.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:21.936 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:21.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:21 smithi134 ceph-mon[90085]: pgmap v2129: 129 pgs: 2 peering, 7 unknown, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 114 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 5881/87027 objects degraded (6.758%); 5055/87027 objects misplaced (5.809%); 2.2 MiB/s, 170 objects/s recovering 2024-09-21T10:17:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:21 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2759229846' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:22.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:22.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:22.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:22.173 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:22.197 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:22.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:22.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:22.325 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:22.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:22.427 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:22.455 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:22.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:22.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:22.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:22.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:22.692 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:22.717 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:22.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:22.818 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:22.845 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:22.918 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:17:22.918 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:17:22.919 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-21T10:17:22.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 110 3.0 2024-09-21T10:17:22.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:22.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:22.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:23.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:23.118 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:23.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:23.220 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:23.248 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:23.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:23.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:23.379 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:23.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:23.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:23.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:23.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:23.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:23.736 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:23.767 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:23.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:23.869 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:23.900 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:23.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:23 smithi017 ceph-mon[102913]: pgmap v2130: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 119 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 888 op/s; 5654/79896 objects degraded (7.077%); 4807/79896 objects misplaced (6.017%); 4.5 MiB/s, 207 objects/s recovering 2024-09-21T10:17:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:23 smithi017 ceph-mon[102913]: osdmap e1343: 8 total, 8 up, 7 in 2024-09-21T10:17:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:23 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5654/79896 objects degraded (7.077%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:23 smithi017 ceph-mon[105151]: pgmap v2130: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 119 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 888 op/s; 5654/79896 objects degraded (7.077%); 4807/79896 objects misplaced (6.017%); 4.5 MiB/s, 207 objects/s recovering 2024-09-21T10:17:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:23 smithi017 ceph-mon[105151]: osdmap e1343: 8 total, 8 up, 7 in 2024-09-21T10:17:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:23 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5654/79896 objects degraded (7.077%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:23 smithi134 ceph-mon[90085]: pgmap v2130: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 unknown, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 119 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 888 op/s; 5654/79896 objects degraded (7.077%); 4807/79896 objects misplaced (6.017%); 4.5 MiB/s, 207 objects/s recovering 2024-09-21T10:17:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:23 smithi134 ceph-mon[90085]: osdmap e1343: 8 total, 8 up, 7 in 2024-09-21T10:17:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:23 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5654/79896 objects degraded (7.077%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:24.002 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:24.031 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:24.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:24.133 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:24.164 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:24.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:24.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:24.300 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:24.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:24.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:24.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:24.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:24.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:24.574 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:24.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:24.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:24.714 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:24.729 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:24 smithi017 ceph-mon[105151]: osdmap e1344: 8 total, 8 up, 7 in 2024-09-21T10:17:24.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:24 smithi017 ceph-mon[102913]: osdmap e1344: 8 total, 8 up, 7 in 2024-09-21T10:17:24.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:17:24.817 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:24.845 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:24.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:24.947 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:24.978 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:24.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:24 smithi134 ceph-mon[90085]: osdmap e1344: 8 total, 8 up, 7 in 2024-09-21T10:17:25.080 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:25.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.117 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:25.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:25.127 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:17:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:17:24] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-21T10:17:25.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:25.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:25.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:25.344 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:25.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:25.399 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:25.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:25.501 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:25.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:25.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:25.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:25.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:25.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:25.722 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:17:25.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:25.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:25.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:25.931 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:25.960 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:25 smithi134 ceph-mon[90085]: pgmap v2133: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 5326/76521 objects degraded (6.960%); 4509/76521 objects misplaced (5.893%); 4.4 MiB/s, 344 objects/s recovering 2024-09-21T10:17:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:25 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2203838215' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:26.062 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:26.100 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:26.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:26.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:25 smithi017 ceph-mon[102913]: pgmap v2133: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 5326/76521 objects degraded (6.960%); 4509/76521 objects misplaced (5.893%); 4.4 MiB/s, 344 objects/s recovering 2024-09-21T10:17:26.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:26.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:25 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2203838215' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:25 smithi017 ceph-mon[105151]: pgmap v2133: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 5326/76521 objects degraded (6.960%); 4509/76521 objects misplaced (5.893%); 4.4 MiB/s, 344 objects/s recovering 2024-09-21T10:17:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:26.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:25 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2203838215' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:26.202 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:26.233 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:26.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:26.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:26.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:26.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:26.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:26.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:26.604 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:26.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.638 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:26.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:26.739 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:26.773 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:26.875 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:26.902 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:26.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:26 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:26 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2203838215' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:26 smithi134 ceph-mon[90085]: osdmap e1345: 8 total, 8 up, 7 in 2024-09-21T10:17:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:26 smithi134 ceph-mon[90085]: 10.11 scrub starts 2024-09-21T10:17:27.004 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:27.030 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:27.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:27.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:27.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:27.174 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:26 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:27.174 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:26 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2203838215' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:27.174 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:26 smithi017 ceph-mon[102913]: osdmap e1345: 8 total, 8 up, 7 in 2024-09-21T10:17:27.175 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:26 smithi017 ceph-mon[102913]: 10.11 scrub starts 2024-09-21T10:17:27.175 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:26 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:27.175 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:26 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2203838215' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:27.175 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:26 smithi017 ceph-mon[105151]: osdmap e1345: 8 total, 8 up, 7 in 2024-09-21T10:17:27.175 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:26 smithi017 ceph-mon[105151]: 10.11 scrub starts 2024-09-21T10:17:27.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:27.284 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:27.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:27.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:27.417 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:27.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:27.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:27.546 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:27.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:27.679 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:27.782 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:27.811 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:27.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:27.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:27.943 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:27 smithi134 ceph-mon[90085]: pgmap v2135: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 5326/73767 objects degraded (7.220%); 4509/73767 objects misplaced (6.112%); 4.3 MiB/s, 202 objects/s recovering 2024-09-21T10:17:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:27 smithi134 ceph-mon[90085]: 1.d scrub starts 2024-09-21T10:17:28.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:28.083 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:28.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:28.130 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:27 smithi017 ceph-mon[102913]: pgmap v2135: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 5326/73767 objects degraded (7.220%); 4509/73767 objects misplaced (6.112%); 4.3 MiB/s, 202 objects/s recovering 2024-09-21T10:17:28.130 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:27 smithi017 ceph-mon[102913]: 1.d scrub starts 2024-09-21T10:17:28.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:27 smithi017 ceph-mon[105151]: pgmap v2135: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 5326/73767 objects degraded (7.220%); 4509/73767 objects misplaced (6.112%); 4.3 MiB/s, 202 objects/s recovering 2024-09-21T10:17:28.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:27 smithi017 ceph-mon[105151]: 1.d scrub starts 2024-09-21T10:17:28.185 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:28.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:28.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:28.325 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:28.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.357 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:28.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:28.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:28.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:28.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.636 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:28.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:28.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:28.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:28.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:28.879 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:28.908 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:28.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:28 smithi134 ceph-mon[90085]: 1.9 scrub starts 2024-09-21T10:17:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:28 smithi134 ceph-mon[90085]: 1.9 scrub ok 2024-09-21T10:17:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:28 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 5326/73767 objects degraded (7.220%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:29.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:29.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:29.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:29.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:28 smithi017 ceph-mon[102913]: 1.9 scrub starts 2024-09-21T10:17:29.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:28 smithi017 ceph-mon[102913]: 1.9 scrub ok 2024-09-21T10:17:29.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:28 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 5326/73767 objects degraded (7.220%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:29.146 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:28 smithi017 ceph-mon[105151]: 1.9 scrub starts 2024-09-21T10:17:29.146 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:28 smithi017 ceph-mon[105151]: 1.9 scrub ok 2024-09-21T10:17:29.147 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:28 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 5326/73767 objects degraded (7.220%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:29.147 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:29.173 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:29.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:29.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:29.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:29.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:29.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:29.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:29.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:29.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:29.593 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:29.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:29.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:29.733 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:29.734 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:29.734 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:29.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:29.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:29.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:29.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:29 smithi134 ceph-mon[90085]: pgmap v2136: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 124 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 645/67185 objects degraded (0.960%); 4776/67185 objects misplaced (7.109%); 5.0 MiB/s, 214 objects/s recovering 2024-09-21T10:17:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:29 smithi134 ceph-mon[90085]: 1.1d scrub starts 2024-09-21T10:17:30.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.008 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:30.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:30.109 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:30.142 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:30.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:29 smithi017 ceph-mon[102913]: pgmap v2136: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 124 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 645/67185 objects degraded (0.960%); 4776/67185 objects misplaced (7.109%); 5.0 MiB/s, 214 objects/s recovering 2024-09-21T10:17:30.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:29 smithi017 ceph-mon[102913]: 1.1d scrub starts 2024-09-21T10:17:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:29 smithi017 ceph-mon[105151]: pgmap v2136: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 124 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 645/67185 objects degraded (0.960%); 4776/67185 objects misplaced (7.109%); 5.0 MiB/s, 214 objects/s recovering 2024-09-21T10:17:30.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:29 smithi017 ceph-mon[105151]: 1.1d scrub starts 2024-09-21T10:17:30.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.273 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:30.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:30.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:30.405 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:17:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:17:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:17:30.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:30.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:30.639 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:30.670 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:30.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:30.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:30.903 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:30.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.935 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:30.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:31.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:31.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.069 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:31.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:31.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:31.198 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:31.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:31.333 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:31.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:31.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:31.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:31.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:31.604 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:31.706 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:31.735 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:31.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:31.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:17:31.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:31.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[102913]: pgmap v2137: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 645/61785 objects degraded (1.044%); 4632/61785 objects misplaced (7.497%); 4.3 MiB/s, 170 objects/s recovering 2024-09-21T10:17:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[105151]: pgmap v2137: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 645/61785 objects degraded (1.044%); 4632/61785 objects misplaced (7.497%); 4.3 MiB/s, 170 objects/s recovering 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:17:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:31.981 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:31 smithi134 ceph-mon[90085]: pgmap v2137: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 645/61785 objects degraded (1.044%); 4632/61785 objects misplaced (7.497%); 4.3 MiB/s, 170 objects/s recovering 2024-09-21T10:17:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:17:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:17:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:32.010 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:32.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:32.112 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:32.138 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:32.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:32.240 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:32.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.266 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:32.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:32.367 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:32.396 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:32.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:32.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:32.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:32.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:32.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:32.678 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:32.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:32.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:32.823 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:32.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:32.925 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:32.955 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:33.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:33.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:33.195 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:33.225 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:33.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:33.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:33.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:33.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:33.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:33.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:33.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:33.603 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:33.632 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:33.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:33.733 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:33.760 INFO:teuthology.orchestra.run.smithi017.stdout:moved 19 / 387 (4.90956%) 2024-09-21T10:17:33.760 INFO:teuthology.orchestra.run.smithi017.stdout:avg 55.2857 2024-09-21T10:17:33.760 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 18.2343 -> 17.8942 (expected baseline 6.88388) 2024-09-21T10:17:33.760 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 12 -> 15 pgs (0.217054 -> 0.271318 * mean) 2024-09-21T10:17:33.760 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.6 with 70 -> 69 pgs (1.26615 -> 1.24806 * mean) 2024-09-21T10:17:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:17:33.761 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2024-09-21T10:17:33.761 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 3 2024-09-21T10:17:33.761 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:17:33.761 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0149 2024-09-21T10:17:33.761 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0164 2024-09-21T10:17:33.761 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 weight 1.0000 -> 0.7929 2024-09-21T10:17:33.761 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 0.2900 -> 0.3520 2024-09-21T10:17:33.761 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 weight 1.0000 -> 0.9074 2024-09-21T10:17:33.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:33.876 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:33.897 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:33.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:33.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:33 smithi017 ceph-mon[102913]: pgmap v2138: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 124 active+clean; 1.1 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 492/52638 objects degraded (0.935%); 4153/52638 objects misplaced (7.890%); 6.0 MiB/s, 186 objects/s recovering 2024-09-21T10:17:33.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:33 smithi017 ceph-mon[102913]: from='client.46247 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:17:33.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:33 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'51966','3':'59465','5':'23066'}"} : dispatch 2024-09-21T10:17:33.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:33 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 492/52638 objects degraded (0.935%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:33.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:33 smithi017 ceph-mon[105151]: pgmap v2138: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 124 active+clean; 1.1 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 492/52638 objects degraded (0.935%); 4153/52638 objects misplaced (7.890%); 6.0 MiB/s, 186 objects/s recovering 2024-09-21T10:17:33.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:33 smithi017 ceph-mon[105151]: from='client.46247 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:17:33.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:33 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'51966','3':'59465','5':'23066'}"} : dispatch 2024-09-21T10:17:33.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:33 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 492/52638 objects degraded (0.935%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:33.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:34.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:34.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:34.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:34.146 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:33 smithi134 ceph-mon[90085]: pgmap v2138: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 124 active+clean; 1.1 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 492/52638 objects degraded (0.935%); 4153/52638 objects misplaced (7.890%); 6.0 MiB/s, 186 objects/s recovering 2024-09-21T10:17:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:33 smithi134 ceph-mon[90085]: from='client.46247 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:17:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:33 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'51966','3':'59465','5':'23066'}"} : dispatch 2024-09-21T10:17:34.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:33 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 492/52638 objects degraded (0.935%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:34.248 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:34.271 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:34.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:34.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:34.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:34.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:34.517 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:34.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:34.561 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-21T10:17:34.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 1000 1.0 2024-09-21T10:17:34.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:34.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:34.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:34.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:34.780 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:34.793 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:17:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:17:34] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.51.0" 2024-09-21T10:17:34.794 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:34 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'51966','3':'59465','5':'23066'}"}]': finished 2024-09-21T10:17:34.794 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:34 smithi017 ceph-mon[102913]: osdmap e1346: 8 total, 8 up, 7 in 2024-09-21T10:17:34.794 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:34 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'51966','3':'59465','5':'23066'}"}]': finished 2024-09-21T10:17:34.794 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:34 smithi017 ceph-mon[105151]: osdmap e1346: 8 total, 8 up, 7 in 2024-09-21T10:17:34.814 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:17:34.814 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:34.834 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:17:34.834 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:34.850 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:17:34.851 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:34.877 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:17:34.878 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:34.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:34.901 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:17:34.901 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:34.910 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:34.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:34.917 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:17:34.917 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:34.934 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:17:34.934 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:34.953 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:17:34.953 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:35.012 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:35.034 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:35.137 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:35.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:35.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:35.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:34 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'51966','3':'59465','5':'23066'}"}]': finished 2024-09-21T10:17:35.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:34 smithi134 ceph-mon[90085]: osdmap e1346: 8 total, 8 up, 7 in 2024-09-21T10:17:35.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:35.284 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:35.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:35.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:35.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:35.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:35.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:35.590 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:35.637 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:35.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:35.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:35.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:35.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:35.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:35.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:35 smithi017 ceph-mon[102913]: pgmap v2140: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1010 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 492/48528 objects degraded (1.014%); 3997/48528 objects misplaced (8.236%); 5.1 MiB/s, 80 objects/s recovering 2024-09-21T10:17:35.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:35 smithi017 ceph-mon[102913]: osdmap e1347: 8 total, 8 up, 7 in 2024-09-21T10:17:35.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:35 smithi017 ceph-mon[102913]: 10.6 starting backfill to osd.5 from (0'0,0'0] MAX to 1345'8975 2024-09-21T10:17:35.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3255895600' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:35 smithi017 ceph-mon[105151]: pgmap v2140: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1010 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 492/48528 objects degraded (1.014%); 3997/48528 objects misplaced (8.236%); 5.1 MiB/s, 80 objects/s recovering 2024-09-21T10:17:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:35 smithi017 ceph-mon[105151]: osdmap e1347: 8 total, 8 up, 7 in 2024-09-21T10:17:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:35 smithi017 ceph-mon[105151]: 10.6 starting backfill to osd.5 from (0'0,0'0] MAX to 1345'8975 2024-09-21T10:17:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3255895600' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:36.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:36.070 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:36.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:36.144 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:35 smithi134 ceph-mon[90085]: pgmap v2140: 129 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 124 active+clean; 1010 MiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s; 492/48528 objects degraded (1.014%); 3997/48528 objects misplaced (8.236%); 5.1 MiB/s, 80 objects/s recovering 2024-09-21T10:17:36.144 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:35 smithi134 ceph-mon[90085]: osdmap e1347: 8 total, 8 up, 7 in 2024-09-21T10:17:36.144 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:35 smithi134 ceph-mon[90085]: 10.6 starting backfill to osd.5 from (0'0,0'0] MAX to 1345'8975 2024-09-21T10:17:36.144 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3255895600' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:36.172 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:36.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.194 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:36.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:36.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:36.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:36.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:36.419 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:36.440 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:36.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:36.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:36.564 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:36.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:36.665 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:36.687 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:36.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:36.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:36.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:36.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:36.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:36.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:37.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:37.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:37.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:37.129 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[102913]: 10.4 starting backfill to osd.2 from (0'0,0'0] MAX to 1345'9061 2024-09-21T10:17:37.129 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[102913]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 1345'14940 2024-09-21T10:17:37.129 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[102913]: osdmap e1348: 8 total, 8 up, 7 in 2024-09-21T10:17:37.129 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[102913]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 1345'9200 2024-09-21T10:17:37.129 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[102913]: 10.15 starting backfill to osd.4 from (0'0,0'0] MAX to 1345'8983 2024-09-21T10:17:37.130 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[102913]: 10.11 starting backfill to osd.2 from (0'0,0'0] MAX to 1345'8930 2024-09-21T10:17:37.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[105151]: 10.4 starting backfill to osd.2 from (0'0,0'0] MAX to 1345'9061 2024-09-21T10:17:37.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[105151]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 1345'14940 2024-09-21T10:17:37.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[105151]: osdmap e1348: 8 total, 8 up, 7 in 2024-09-21T10:17:37.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[105151]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 1345'9200 2024-09-21T10:17:37.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[105151]: 10.15 starting backfill to osd.4 from (0'0,0'0] MAX to 1345'8983 2024-09-21T10:17:37.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:36 smithi017 ceph-mon[105151]: 10.11 starting backfill to osd.2 from (0'0,0'0] MAX to 1345'8930 2024-09-21T10:17:37.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:37.229 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:36 smithi134 ceph-mon[90085]: 10.4 starting backfill to osd.2 from (0'0,0'0] MAX to 1345'9061 2024-09-21T10:17:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:36 smithi134 ceph-mon[90085]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 1345'14940 2024-09-21T10:17:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:36 smithi134 ceph-mon[90085]: osdmap e1348: 8 total, 8 up, 7 in 2024-09-21T10:17:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:36 smithi134 ceph-mon[90085]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 1345'9200 2024-09-21T10:17:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:36 smithi134 ceph-mon[90085]: 10.15 starting backfill to osd.4 from (0'0,0'0] MAX to 1345'8983 2024-09-21T10:17:37.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:36 smithi134 ceph-mon[90085]: 10.11 starting backfill to osd.2 from (0'0,0'0] MAX to 1345'8930 2024-09-21T10:17:37.330 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:37.356 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:37.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:37.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:37.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:37.627 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:37.724 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:37.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.793 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:37.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:37.894 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:37.923 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:38.024 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:38.060 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:38.162 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:38.191 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:38.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:37 smithi017 ceph-mon[102913]: pgmap v2143: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 965 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 856 op/s; 492/46362 objects degraded (1.061%); 4479/46362 objects misplaced (9.661%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T10:17:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:37 smithi017 ceph-mon[102913]: osdmap e1349: 8 total, 8 up, 7 in 2024-09-21T10:17:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:37 smithi017 ceph-mon[105151]: pgmap v2143: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 965 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 856 op/s; 492/46362 objects degraded (1.061%); 4479/46362 objects misplaced (9.661%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T10:17:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:37 smithi017 ceph-mon[105151]: osdmap e1349: 8 total, 8 up, 7 in 2024-09-21T10:17:38.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:37 smithi134 ceph-mon[90085]: pgmap v2143: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 122 active+clean; 965 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 856 op/s; 492/46362 objects degraded (1.061%); 4479/46362 objects misplaced (9.661%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T10:17:38.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:37 smithi134 ceph-mon[90085]: osdmap e1349: 8 total, 8 up, 7 in 2024-09-21T10:17:38.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:38.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:38.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:38.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:38.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:38.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:38.616 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:38.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:38.717 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:38.749 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:38.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:38 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 492/46362 objects degraded (1.061%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:38.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:38 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 492/46362 objects degraded (1.061%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:38.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:38.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:17:38.890 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:38.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:38.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:39.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:39.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:39.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:39.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:38 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 492/46362 objects degraded (1.061%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:39.299 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:39.334 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:39.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:39.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:39.479 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:39.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:39.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:39.594 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:17:39.594 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:239696 10.8 10:1456df7f:::benchmark_data_smithi017_268344_object113765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:17:38.834993+0000", 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0035106489999999998, 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035380390000000002, 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 239696 2024-09-21T10:17:39.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.834993+0000", 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.834993+0000", 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.834994+0000", 2024-09-21T10:17:39.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1379999999999999e-06 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.834999+0000", 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7700000000000001e-06 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.835012+0000", 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3288999999999999e-05 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.835020+0000", 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7500000000000003e-06 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.835113+0000", 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2845999999999998e-05 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.835223+0000", 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010993399999999999 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.835331+0000", 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010848 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46058.0:239697 10.e e1349/1283 10:7d2cf1c0:::benchmark_data_smithi017_268344_object113766:head v 1349'15094, mlcod=1349'15091)", 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:17:38.837075+0000", 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.001429107, 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015363239999999999, 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:39.599 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837075+0000", 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837075+0000", 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837076+0000", 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7000000000000004e-07 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837082+0000", 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1240000000000002e-06 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837094+0000", 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1875999999999999e-05 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837102+0000", 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8289999999999997e-06 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837155+0000", 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2803999999999998e-05 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837175+0000", 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9375e-05 2024-09-21T10:17:39.602 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46058.0:239689 10.14 e1349/1331 ondisk, result = 0)", 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:17:38.837174+0000", 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0013301739999999999, 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014914139999999999, 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:39.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837174+0000", 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837174+0000", 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837174+0000", 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0399999999999996e-07 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:39.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837178+0000", 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6119999999999999e-06 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837186+0000", 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7729999999999999e-06 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837191+0000", 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3600000000000004e-06 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:39.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837851+0000", 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00066009900000000004 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837909+0000", 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8171000000000003e-05 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:239701 10.3 10:c3310a38:::benchmark_data_smithi017_268344_object113770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:17:38.837826+0000", 2024-09-21T10:17:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00067762200000000001, 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00089186199999999999, 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 239701 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837826+0000", 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837826+0000", 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837827+0000", 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.218e-06 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837832+0000", 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9439999999999996e-06 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837844+0000", 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1161e-05 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837851+0000", 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3549999999999999e-06 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837904+0000", 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2812999999999997e-05 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.838022+0000", 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118486 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,5", 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.838144+0000", 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122067 2024-09-21T10:17:39.610 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46058.0:239699 10.f e1349/1283)", 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:17:38.837877+0000", 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00062730100000000003, 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00090282999999999997, 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:39.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837877+0000", 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837877+0000", 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837877+0000", 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1099999999999995e-07 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837882+0000", 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1290000000000004e-06 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837893+0000", 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0149999999999999e-05 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837899+0000", 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1480000000000003e-06 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:39.613 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46058.0:239698 10.a e1349/1347 10:5cf0cd20:::benchmark_data_smithi017_268344_object113767:head v 1349'15102, mlcod=1349'15100)", 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:17:38.837920+0000", 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00058368500000000002, 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000900222, 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837920+0000", 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837920+0000", 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837921+0000", 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9499999999999998e-07 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837924+0000", 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7179999999999998e-06 2024-09-21T10:17:39.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837931+0000", 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5860000000000001e-06 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837936+0000", 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6659999999999999e-06 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.838029+0000", 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3468999999999996e-05 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.838047+0000", 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8328000000000001e-05 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.617 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46058.0:239696 10.8 e1349/1325)", 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:17:38.837966+0000", 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00053765400000000004, 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00090120399999999996, 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837966+0000", 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837966+0000", 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837967+0000", 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0400000000000002e-07 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837969+0000", 2024-09-21T10:17:39.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5040000000000001e-06 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837976+0000", 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7889999999999997e-06 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837981+0000", 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6809999999999997e-06 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:239689 10.14 10:297bb311:::benchmark_data_smithi017_268344_object113758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:17:38.826825+0000", 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.011678367, 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012084805000000001, 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 239689 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.621 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.826825+0000", 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.826825+0000", 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.826827+0000", 2024-09-21T10:17:39.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4339999999999999e-06 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.826831+0000", 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8630000000000001e-06 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.826844+0000", 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3118999999999999e-05 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.826852+0000", 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9829999999999996e-06 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.828831+0000", 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019788269999999998 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.831012+0000", 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021812440000000002 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.832129+0000", 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011166189999999999 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.836988+0000", 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048590710000000004 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837138+0000", 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000150653 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.837965+0000", 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082703900000000001 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:17:38.838353+0000", 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000387931 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.626 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:17:39.627 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-21T10:17:39.627 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 8 2024-09-21T10:17:39.627 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:17:39.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:39.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:39.730 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:39.760 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:39.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:39.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:39.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.895 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:39.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:39.996 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:40.023 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:40.059 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:39 smithi017 ceph-mon[102913]: pgmap v2145: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 120 active+clean; 893 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 422/42921 objects degraded (0.983%); 5223/42921 objects misplaced (12.169%); 7.2 MiB/s, 121 objects/s recovering 2024-09-21T10:17:40.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:39 smithi017 ceph-mon[105151]: pgmap v2145: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 120 active+clean; 893 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 422/42921 objects degraded (0.983%); 5223/42921 objects misplaced (12.169%); 7.2 MiB/s, 121 objects/s recovering 2024-09-21T10:17:40.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:40.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:40.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:40.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:39 smithi134 ceph-mon[90085]: pgmap v2145: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 120 active+clean; 893 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 422/42921 objects degraded (0.983%); 5223/42921 objects misplaced (12.169%); 7.2 MiB/s, 121 objects/s recovering 2024-09-21T10:17:40.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:40.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:40.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:40.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:40.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:40.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:17:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:17:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:17:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:17:40.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:40.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:40.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:40.702 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:40.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.885 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:40.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:40.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:41.017 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:41.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:41.120 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:41.157 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:41.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:41.259 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:41.288 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:41.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:41.391 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:41.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.419 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:41.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:41.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:41.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:41.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:41.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.703 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:41.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:41.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:41.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:41.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:41.944 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:41.973 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:41.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:42.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:42.081 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:41 smithi017 ceph-mon[102913]: pgmap v2146: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 824 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 422/39606 objects degraded (1.065%); 5668/39606 objects misplaced (14.311%); 10 MiB/s, 169 objects/s recovering 2024-09-21T10:17:42.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:41 smithi017 ceph-mon[105151]: pgmap v2146: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 824 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 422/39606 objects degraded (1.065%); 5668/39606 objects misplaced (14.311%); 10 MiB/s, 169 objects/s recovering 2024-09-21T10:17:42.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:42.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:42.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:41 smithi134 ceph-mon[90085]: pgmap v2146: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 824 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 422/39606 objects degraded (1.065%); 5668/39606 objects misplaced (14.311%); 10 MiB/s, 169 objects/s recovering 2024-09-21T10:17:42.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:42.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:42.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:42.406 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:42.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:42.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.543 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:42.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:42.644 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:42.675 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:42.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:42.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:42.819 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:42.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:42.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:42.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.959 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:42.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:43.060 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:43.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.096 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:43.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:43.197 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:43.226 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:43.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:43.328 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:43.359 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:43.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:43.461 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:43.493 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:43.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:43.635 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:43.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:43.737 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:43.775 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:43.873 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:17:43.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:43.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:44.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:44.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:44.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:44.151 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:44.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:44.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:43 smithi017 ceph-mon[105151]: pgmap v2147: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 620 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 249/29808 objects degraded (0.835%); 4642/29808 objects misplaced (15.573%); 9.3 MiB/s, 227 objects/s recovering 2024-09-21T10:17:44.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:43 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 249/29808 objects degraded (0.835%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:44.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:43 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2518195817' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:44.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:43 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:44.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:43 smithi017 ceph-mon[102913]: pgmap v2147: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 620 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 249/29808 objects degraded (0.835%); 4642/29808 objects misplaced (15.573%); 9.3 MiB/s, 227 objects/s recovering 2024-09-21T10:17:44.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:43 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 249/29808 objects degraded (0.835%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:44.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:43 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2518195817' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:44.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:43 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:44.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:43 smithi134 ceph-mon[90085]: pgmap v2147: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 620 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 249/29808 objects degraded (0.835%); 4642/29808 objects misplaced (15.573%); 9.3 MiB/s, 227 objects/s recovering 2024-09-21T10:17:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:43 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 249/29808 objects degraded (0.835%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:43 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2518195817' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:43 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:44.321 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:44.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.364 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:44.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:44.465 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:44.499 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:44.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:44.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:44.741 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:44.778 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:44.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:44.880 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:44.911 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:44.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:45.013 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:45.035 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:44 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:17:45.035 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:44 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:45.035 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:44 smithi017 ceph-mon[102913]: osdmap e1350: 8 total, 8 up, 7 in 2024-09-21T10:17:45.035 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:44 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:17:45.035 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:44 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:45.035 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:44 smithi017 ceph-mon[105151]: osdmap e1350: 8 total, 8 up, 7 in 2024-09-21T10:17:45.036 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:17:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:17:44] "GET /metrics HTTP/1.1" 200 36767 "" "Prometheus/2.51.0" 2024-09-21T10:17:45.044 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:45.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:45.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:45.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:45.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:44 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:17:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:44 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:44 smithi134 ceph-mon[90085]: osdmap e1350: 8 total, 8 up, 7 in 2024-09-21T10:17:45.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:45.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:45.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:45.443 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:45.472 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:45.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:45.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:45.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:45.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:45.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:45.752 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:45.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:45.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:45.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:45.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:45.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:45 smithi017 ceph-mon[102913]: pgmap v2149: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 525 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 878 op/s; 249/25266 objects degraded (0.986%); 4478/25266 objects misplaced (17.723%); 9.1 MiB/s, 221 objects/s recovering 2024-09-21T10:17:45.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:45 smithi017 ceph-mon[105151]: pgmap v2149: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 525 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 878 op/s; 249/25266 objects degraded (0.986%); 4478/25266 objects misplaced (17.723%); 9.1 MiB/s, 221 objects/s recovering 2024-09-21T10:17:45.994 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:46.032 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:46.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:46.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:46.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:46.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:45 smithi134 ceph-mon[90085]: pgmap v2149: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 525 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 878 op/s; 249/25266 objects degraded (0.986%); 4478/25266 objects misplaced (17.723%); 9.1 MiB/s, 221 objects/s recovering 2024-09-21T10:17:46.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:46.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:46.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:46.408 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:46.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:46.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:46.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:46.597 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:46.632 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:46.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:46.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:46.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.772 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:46.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:46.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:17:46.843 INFO:teuthology.orchestra.run.smithi017.stdout:moved 1 / 387 (0.258398%) 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:avg 55.2857 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 17.8942 -> 17.4905 (expected baseline 6.88388) 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 15 -> 16 pgs (0.271318 -> 0.289406 * mean) 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 69 -> 69 pgs (1.24806 -> 1.24806 * mean) 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0123 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.1230 2024-09-21T10:17:46.844 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 0.3520 -> 0.3629 2024-09-21T10:17:46.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:46.906 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:46.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:46 smithi017 ceph-mon[102913]: from='client.55747 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:17:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'5':'23781'}"} : dispatch 2024-09-21T10:17:46.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:46 smithi017 ceph-mon[105151]: from='client.55747 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:17:46.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'5':'23781'}"} : dispatch 2024-09-21T10:17:47.008 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:47.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:47.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:47.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:47.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.190 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:47.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:46 smithi134 ceph-mon[90085]: from='client.55747 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:17:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'5':'23781'}"} : dispatch 2024-09-21T10:17:47.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:47.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:47.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:47.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:47.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:47.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:47.595 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:47.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:47.697 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:47.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.736 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:47.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:47.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:47.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:47.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:48.022 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:48.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:48.124 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:48.154 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:47 smithi017 ceph-mon[102913]: pgmap v2150: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 448 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 249/21546 objects degraded (1.156%); 4126/21546 objects misplaced (19.150%); 7.4 MiB/s, 181 objects/s recovering 2024-09-21T10:17:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:47 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'23781'}"}]': finished 2024-09-21T10:17:48.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:47 smithi017 ceph-mon[102913]: osdmap e1351: 8 total, 8 up, 7 in 2024-09-21T10:17:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:47 smithi017 ceph-mon[105151]: pgmap v2150: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 448 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 249/21546 objects degraded (1.156%); 4126/21546 objects misplaced (19.150%); 7.4 MiB/s, 181 objects/s recovering 2024-09-21T10:17:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:47 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'23781'}"}]': finished 2024-09-21T10:17:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:47 smithi017 ceph-mon[105151]: osdmap e1351: 8 total, 8 up, 7 in 2024-09-21T10:17:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:47 smithi134 ceph-mon[90085]: pgmap v2150: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 120 active+clean; 448 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 249/21546 objects degraded (1.156%); 4126/21546 objects misplaced (19.150%); 7.4 MiB/s, 181 objects/s recovering 2024-09-21T10:17:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:47 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'23781'}"}]': finished 2024-09-21T10:17:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:47 smithi134 ceph-mon[90085]: osdmap e1351: 8 total, 8 up, 7 in 2024-09-21T10:17:48.257 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:48.295 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:48.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:48.397 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:48.409 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:17:48.410 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:17:48.428 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:48.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:48.453 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:17:48.453 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:48.495 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:17:48.496 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:48.528 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:17:48.529 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:48.529 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:48.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:48.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:17:48.564 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:48.593 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:17:48.593 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:48.626 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:17:48.626 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:48.655 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:17:48.655 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:48.666 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:48.695 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:48.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:48.797 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:48.833 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:48.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:48.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:48.977 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:49.081 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:49.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.113 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:49.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:48 smithi017 ceph-mon[102913]: osdmap e1352: 8 total, 8 up, 7 in 2024-09-21T10:17:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:48 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2673879848' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:49.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:48 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 249/21546 objects degraded (1.156%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:48 smithi017 ceph-mon[105151]: osdmap e1352: 8 total, 8 up, 7 in 2024-09-21T10:17:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:48 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2673879848' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:49.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:48 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 249/21546 objects degraded (1.156%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:49.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:49.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:48 smithi134 ceph-mon[90085]: osdmap e1352: 8 total, 8 up, 7 in 2024-09-21T10:17:49.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:48 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2673879848' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:49.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:48 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 249/21546 objects degraded (1.156%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:49.252 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:49.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:49.354 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:49.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:49.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:49.477 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:49.491 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 125928 objects 2024-09-21T10:17:49.491 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :159.915 2024-09-21T10:17:49.499 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:49.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:49.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:49.624 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-09-21T10:17:49.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-09-21T10:17:49.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:49.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:49.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:49.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:49.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:49.876 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:49.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:49.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:49 smithi017 ceph-mon[102913]: pgmap v2153: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 7 active+remapped+backfill_wait, 119 active+clean; 270 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 67/13020 objects degraded (0.515%); 3201/13020 objects misplaced (24.585%); 1.9 MiB/s, 71 objects/s recovering 2024-09-21T10:17:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:49 smithi017 ceph-mon[105151]: pgmap v2153: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 7 active+remapped+backfill_wait, 119 active+clean; 270 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 67/13020 objects degraded (0.515%); 3201/13020 objects misplaced (24.585%); 1.9 MiB/s, 71 objects/s recovering 2024-09-21T10:17:49.978 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:50.000 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:50.101 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:50.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:50.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:49 smithi134 ceph-mon[90085]: pgmap v2153: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 7 active+remapped+backfill_wait, 119 active+clean; 270 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 67/13020 objects degraded (0.515%); 3201/13020 objects misplaced (24.585%); 1.9 MiB/s, 71 objects/s recovering 2024-09-21T10:17:50.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:50.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:50.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:50.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:50.371 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:50.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:50.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:17:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:17:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:17:50.496 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:50.599 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:50.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:50.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:50.724 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:50.748 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:50.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:50.819 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:17:50.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 110 1.0 2024-09-21T10:17:50.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:50.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:50.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:50.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:51.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:51.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:51.116 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:51.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.142 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:51.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:51.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:51.267 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:51.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:51.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:51.396 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:51.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:51.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:51.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:51.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:51.622 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:51.648 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:51.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:51.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:51.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:51.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:51.899 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:51.920 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:52.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:52.025 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:52.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:52.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:51 smithi017 ceph-mon[102913]: pgmap v2154: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 7 active+remapped+backfill_wait, 119 active+clean; 131 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 67/6360 objects degraded (1.053%); 2862/6360 objects misplaced (45.000%); 1.7 MiB/s, 64 objects/s recovering 2024-09-21T10:17:52.135 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:51 smithi017 ceph-mon[105151]: pgmap v2154: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 7 active+remapped+backfill_wait, 119 active+clean; 131 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 67/6360 objects degraded (1.053%); 2862/6360 objects misplaced (45.000%); 1.7 MiB/s, 64 objects/s recovering 2024-09-21T10:17:52.154 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:52.154 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:52.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:52.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:51 smithi134 ceph-mon[90085]: pgmap v2154: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 7 active+remapped+backfill_wait, 119 active+clean; 131 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 67/6360 objects degraded (1.053%); 2862/6360 objects misplaced (45.000%); 1.7 MiB/s, 64 objects/s recovering 2024-09-21T10:17:52.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:52.282 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:52.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:52.384 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:52.392 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:52.410 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:52.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:52.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:52.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:52.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:52.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:52.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:52.762 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:52.788 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:52.889 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:52.915 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:52.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:53.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:53.043 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:53.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:53.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:53.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:53.295 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:53.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:53.422 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:53.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:53.524 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:53.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:53.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:53.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:53.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:53.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:53.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:53.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:53.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:53.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:53.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:53.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:17:54.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:54.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:54.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:54.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:54.178 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:54.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:53 smithi017 ceph-mon[102913]: pgmap v2155: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 119 active+clean; 23 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 1/1152 objects degraded (0.087%); 2320/1152 objects misplaced (201.389%); 1.9 MiB/s, 151 objects/s recovering 2024-09-21T10:17:54.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:53 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 1/1152 objects degraded (0.087%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:54.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:53 smithi017 ceph-mon[105151]: pgmap v2155: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 119 active+clean; 23 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 1/1152 objects degraded (0.087%); 2320/1152 objects misplaced (201.389%); 1.9 MiB/s, 151 objects/s recovering 2024-09-21T10:17:54.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:53 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 1/1152 objects degraded (0.087%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:53 smithi134 ceph-mon[90085]: pgmap v2155: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 119 active+clean; 23 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 1/1152 objects degraded (0.087%); 2320/1152 objects misplaced (201.389%); 1.9 MiB/s, 151 objects/s recovering 2024-09-21T10:17:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:53 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 1/1152 objects degraded (0.087%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:54.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:54.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:54.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:54.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:54.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:54.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:54.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:54.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:54.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:54.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:54.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:54.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:54.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:54.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:54.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:54.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:54.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:55.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:55.044 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:17:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:17:54] "GET /metrics HTTP/1.1" 200 36767 "" "Prometheus/2.51.0" 2024-09-21T10:17:55.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:55.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:55.150 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:55.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.173 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:55.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:55.274 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:55.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:55.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:55.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:55.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:55.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:55.544 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:55.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:55.646 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:55.666 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:55.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:55.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:55.790 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:55.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:55.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:55.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:17:56.020 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:56.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.043 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:56.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:56.144 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:56.164 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:56.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:55 smithi017 ceph-mon[105151]: pgmap v2156: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 119 active+clean; 12 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 1/600 objects degraded (0.167%); 2301/600 objects misplaced (383.500%); 1.9 MiB/s, 151 objects/s recovering 2024-09-21T10:17:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:55 smithi017 ceph-mon[105151]: osdmap e1353: 8 total, 8 up, 7 in 2024-09-21T10:17:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:56.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:56.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:55 smithi017 ceph-mon[102913]: pgmap v2156: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 119 active+clean; 12 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 1/600 objects degraded (0.167%); 2301/600 objects misplaced (383.500%); 1.9 MiB/s, 151 objects/s recovering 2024-09-21T10:17:56.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:55 smithi017 ceph-mon[102913]: osdmap e1353: 8 total, 8 up, 7 in 2024-09-21T10:17:56.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:56.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:55 smithi134 ceph-mon[90085]: pgmap v2156: 129 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 119 active+clean; 12 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 1/600 objects degraded (0.167%); 2301/600 objects misplaced (383.500%); 1.9 MiB/s, 151 objects/s recovering 2024-09-21T10:17:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:55 smithi134 ceph-mon[90085]: osdmap e1353: 8 total, 8 up, 7 in 2024-09-21T10:17:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:17:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:56.266 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:56.288 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:56.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:56.390 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:56.412 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:56.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:56.514 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:56.535 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:56.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:56.636 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:56.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:56.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:56.760 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:56.781 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:56.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:56.912 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:56.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:57.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:57.036 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:57.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:57.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:57.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:57.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:57.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:56 smithi017 ceph-mon[102913]: osdmap e1354: 8 total, 8 up, 7 in 2024-09-21T10:17:57.183 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:56 smithi017 ceph-mon[105151]: osdmap e1354: 8 total, 8 up, 7 in 2024-09-21T10:17:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:56 smithi134 ceph-mon[90085]: osdmap e1354: 8 total, 8 up, 7 in 2024-09-21T10:17:57.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:57.282 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:57.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:57.303 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:57.303 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:57.304 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:57.383 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:57.404 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:57.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:57.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:57.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:57.657 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:57.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:57.759 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:57.781 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:57.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:17:57.893 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:17:57.894 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_5' removed 2024-09-21T10:17:57.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:57.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:17:58.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:58.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:58.153 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:58.174 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:58.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:58.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:57 smithi017 ceph-mon[105151]: pgmap v2159: 129 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 118 active+clean; 12 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 1/600 objects degraded (0.167%); 2301/600 objects misplaced (383.500%); 528 KiB/s, 97 objects/s recovering 2024-09-21T10:17:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:57 smithi017 ceph-mon[105151]: osdmap e1355: 8 total, 8 up, 7 in 2024-09-21T10:17:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3022245967' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:17:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2049017089' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:58.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:57 smithi017 ceph-mon[102913]: pgmap v2159: 129 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 118 active+clean; 12 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 1/600 objects degraded (0.167%); 2301/600 objects misplaced (383.500%); 528 KiB/s, 97 objects/s recovering 2024-09-21T10:17:58.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:57 smithi017 ceph-mon[102913]: osdmap e1355: 8 total, 8 up, 7 in 2024-09-21T10:17:58.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3022245967' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:17:58.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2049017089' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:57 smithi134 ceph-mon[90085]: pgmap v2159: 129 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 118 active+clean; 12 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 1/600 objects degraded (0.167%); 2301/600 objects misplaced (383.500%); 528 KiB/s, 97 objects/s recovering 2024-09-21T10:17:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:57 smithi134 ceph-mon[90085]: osdmap e1355: 8 total, 8 up, 7 in 2024-09-21T10:17:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3022245967' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:17:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2049017089' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:58.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:58.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.312 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:58.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:58.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:58.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:58.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:58.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:58.558 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:58.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:58.660 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:58.681 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:58.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:58.782 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:58.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.804 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:58.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:58.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:58.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:59.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:59.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:59.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:59.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3022245967' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:17:59.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2049017089' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:17:59.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[102913]: osdmap e1356: 8 total, 8 up, 7 in 2024-09-21T10:17:59.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-21T10:17:59.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[102913]: Health check update: Degraded data redundancy: 1/600 objects degraded (0.167%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:59.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:59.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3022245967' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:17:59.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2049017089' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:17:59.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[105151]: osdmap e1356: 8 total, 8 up, 7 in 2024-09-21T10:17:59.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-21T10:17:59.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:58 smithi017 ceph-mon[105151]: Health check update: Degraded data redundancy: 1/600 objects degraded (0.167%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:59.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:59.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:59.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:59.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:58 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:59.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:58 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3022245967' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:17:59.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:58 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2049017089' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:17:59.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:58 smithi134 ceph-mon[90085]: osdmap e1356: 8 total, 8 up, 7 in 2024-09-21T10:17:59.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:58 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-21T10:17:59.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:58 smithi134 ceph-mon[90085]: Health check update: Degraded data redundancy: 1/600 objects degraded (0.167%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-21T10:17:59.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:59.301 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:59.403 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:59.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:59.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:59.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:59.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:59.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:59.673 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:17:59.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:59.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:59.797 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:59.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:59.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:59.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:17:59.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:00.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:00.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:00.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:00.145 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:59 smithi017 ceph-mon[102913]: pgmap v2162: 103 pgs: 103 active+clean; 713 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:00.145 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:59 smithi017 ceph-mon[102913]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/600 objects degraded (0.167%), 2 pgs degraded, 1 pg undersized) 2024-09-21T10:18:00.145 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-21T10:18:00.145 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:17:59 smithi017 ceph-mon[102913]: osdmap e1357: 8 total, 8 up, 7 in 2024-09-21T10:18:00.145 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:59 smithi017 ceph-mon[105151]: pgmap v2162: 103 pgs: 103 active+clean; 713 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:00.145 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:59 smithi017 ceph-mon[105151]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/600 objects degraded (0.167%), 2 pgs degraded, 1 pg undersized) 2024-09-21T10:18:00.145 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-21T10:18:00.145 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:17:59 smithi017 ceph-mon[105151]: osdmap e1357: 8 total, 8 up, 7 in 2024-09-21T10:18:00.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:00.171 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:18:00.171 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:18:00.171 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2024-09-21T10:18:00.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_6 16 2024-09-21T10:18:00.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:00.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:00.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:59 smithi134 ceph-mon[90085]: pgmap v2162: 103 pgs: 103 active+clean; 713 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:00.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:59 smithi134 ceph-mon[90085]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/600 objects degraded (0.167%), 2 pgs degraded, 1 pg undersized) 2024-09-21T10:18:00.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-21T10:18:00.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:17:59 smithi134 ceph-mon[90085]: osdmap e1357: 8 total, 8 up, 7 in 2024-09-21T10:18:00.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:00.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:00.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:00.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:00.425 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:18:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:18:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:18:00.528 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:00.553 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:00.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:00.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:00.680 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:00.781 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:00.805 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:00.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:00.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:00.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:00.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:01.031 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:01.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:01.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:01.156 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:01.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:01.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:01.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:00 smithi017 ceph-mon[102913]: osdmap e1358: 8 total, 8 up, 7 in 2024-09-21T10:18:01.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:00 smithi017 ceph-mon[105151]: osdmap e1358: 8 total, 8 up, 7 in 2024-09-21T10:18:01.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:00 smithi134 ceph-mon[90085]: osdmap e1358: 8 total, 8 up, 7 in 2024-09-21T10:18:01.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:01.304 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:01.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:01.429 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:01.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:01.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:01.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:01.657 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:01.683 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:01.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:01.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:01.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:01.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:01.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:01.932 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:02.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:02.056 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:02.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:02.158 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:02.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.186 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:02.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:01 smithi017 ceph-mon[102913]: pgmap v2165: 103 pgs: 103 active+clean; 726 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 4.0 KiB/s rd, 1 op/s 2024-09-21T10:18:02.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:01 smithi017 ceph-mon[105151]: pgmap v2165: 103 pgs: 103 active+clean; 726 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 4.0 KiB/s rd, 1 op/s 2024-09-21T10:18:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:01 smithi134 ceph-mon[90085]: pgmap v2165: 103 pgs: 103 active+clean; 726 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 4.0 KiB/s rd, 1 op/s 2024-09-21T10:18:02.287 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:02.312 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:02.413 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:02.437 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:02.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:02.538 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:02.551 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:18:02.551 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:18:02.562 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:02.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:02.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:02.789 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:02.811 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:02.912 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:02.937 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:02.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:03.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:03.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:03.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:03.164 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:03.189 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:03.291 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:03.316 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:03.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:03.441 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:03.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:18:03.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:03.573 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:03.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:03.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:03.698 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:03.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:03.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:03.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:18:03.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:03.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:03.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:04.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:04.075 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:04.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:04.089 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:18:04.089 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:18:04.111 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:18:04.111 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:04.118 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:04.134 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:18:04.134 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:04.134 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:03 smithi017 ceph-mon[102913]: pgmap v2166: 103 pgs: 103 active+clean; 739 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 5.8 KiB/s rd, 2 op/s 2024-09-21T10:18:04.135 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:03 smithi017 ceph-mon[105151]: pgmap v2166: 103 pgs: 103 active+clean; 739 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 5.8 KiB/s rd, 2 op/s 2024-09-21T10:18:04.141 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:04.158 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:18:04.158 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:04.174 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:18:04.174 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:04.177 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:04.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:18:04.191 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:18:04.202 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:04.206 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:18:04.206 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:04.222 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:18:04.222 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:04.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:03 smithi134 ceph-mon[90085]: pgmap v2166: 103 pgs: 103 active+clean; 739 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 5.8 KiB/s rd, 2 op/s 2024-09-21T10:18:04.304 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:04.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:04.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:04.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:04.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:04.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:04.556 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:04.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:04.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:04.708 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:04.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:04.832 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:04.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:04.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:04.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:04.993 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:04 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/335692813' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:04.993 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:04 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/335692813' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:04.994 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:18:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:18:04] "GET /metrics HTTP/1.1" 200 36758 "" "Prometheus/2.51.0" 2024-09-21T10:18:05.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:05.069 INFO:teuthology.orchestra.run.smithi017.stdout:moved 17 / 309 (5.50162%) 2024-09-21T10:18:05.069 INFO:teuthology.orchestra.run.smithi017.stdout:avg 44.1429 2024-09-21T10:18:05.069 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 13.4847 -> 13.4316 (expected baseline 6.15116) 2024-09-21T10:18:05.070 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 15 -> 17 pgs (0.339806 -> 0.385113 * mean) 2024-09-21T10:18:05.070 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.3 with 55 -> 48 pgs (1.24595 -> 1.08738 * mean) 2024-09-21T10:18:05.070 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:18:05.070 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2024-09-21T10:18:05.070 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2024-09-21T10:18:05.070 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:18:05.070 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0094 2024-09-21T10:18:05.071 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0103 2024-09-21T10:18:05.071 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 weight 1.0000 -> 0.8371 2024-09-21T10:18:05.071 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 weight 0.9074 -> 0.7789 2024-09-21T10:18:05.071 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 0.3629 -> 0.3249 2024-09-21T10:18:05.071 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2024-09-21T10:18:05.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:05.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:05.183 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:05.209 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:05.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:04 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/335692813' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:05.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:05.334 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:05.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:05.395 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:18:05 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T10:18:05.067+0000 7f3131349640 -1 mgr.server reply reply (3) No such process no change 2024-09-21T10:18:05.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:05.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:05.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:05.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:05.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:05.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:05.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:05.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:05.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:05.831 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:05.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:05.954 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:06.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:06.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:06.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:05 smithi017 ceph-mon[102913]: pgmap v2167: 103 pgs: 103 active+clean; 739 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 5.1 KiB/s rd, 11 KiB/s wr, 3 op/s; 18 KiB/s, 0 objects/s recovering 2024-09-21T10:18:06.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:06.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:05 smithi017 ceph-mon[105151]: pgmap v2167: 103 pgs: 103 active+clean; 739 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 5.1 KiB/s rd, 11 KiB/s wr, 3 op/s; 18 KiB/s, 0 objects/s recovering 2024-09-21T10:18:06.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:06.202 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:06.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:05 smithi134 ceph-mon[90085]: pgmap v2167: 103 pgs: 103 active+clean; 739 KiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 5.1 KiB/s rd, 11 KiB/s wr, 3 op/s; 18 KiB/s, 0 objects/s recovering 2024-09-21T10:18:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:06.294 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-21T10:18:06.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 1000 3.0 2024-09-21T10:18:06.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:06.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:06.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:06.486 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:06.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:06.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:06.611 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:06.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:06.713 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:06.739 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:06.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:06.840 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:06.865 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:06.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:06.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:06.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:07.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:07.114 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:06 smithi017 ceph-mon[102913]: from='client.46289 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:18:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:06 smithi017 ceph-mon[105151]: from='client.46289 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:18:07.215 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:07.241 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:07.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:06 smithi134 ceph-mon[90085]: from='client.46289 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:18:07.343 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:07.368 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:07.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:07.469 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:07.495 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:07.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:07.596 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.623 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:07.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:07.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:07.745 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:07.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:07.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:07.874 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:07.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:07.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:07.997 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:08.099 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:08.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:08.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:08.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:07 smithi017 ceph-mon[102913]: pgmap v2168: 103 pgs: 103 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 4.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 15 KiB/s, 0 objects/s recovering 2024-09-21T10:18:08.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:07 smithi017 ceph-mon[105151]: pgmap v2168: 103 pgs: 103 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 4.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 15 KiB/s, 0 objects/s recovering 2024-09-21T10:18:08.226 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:07 smithi134 ceph-mon[90085]: pgmap v2168: 103 pgs: 103 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 4.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 15 KiB/s, 0 objects/s recovering 2024-09-21T10:18:08.251 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:08.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:08.352 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:08.377 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:08.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:08.479 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:08.503 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:08.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:08.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.628 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:08.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:08.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:08.754 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:08.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:08.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:08.879 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:08.981 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:09.008 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:09.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:09.020 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:18:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:18:09.020 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-21T10:18:09.020 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:18:09.110 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:09.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:09.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:09.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:09.256 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:09.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:09.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:09.481 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:09.507 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:09.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:09.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:09.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:09.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:09.733 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:09.759 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:09.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:09.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:18:09.824 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:09.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:09.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:09.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:09.930 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_6' created 2024-09-21T10:18:09.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:10.012 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:10.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:10.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:10.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:10.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:10.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:18:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:09 smithi017 ceph-mon[102913]: pgmap v2169: 103 pgs: 103 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 3.5 KiB/s rd, 7.7 KiB/s wr, 2 op/s; 13 KiB/s, 0 objects/s recovering 2024-09-21T10:18:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:09 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/710692485' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-21T10:18:10.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:09 smithi017 ceph-mon[105151]: pgmap v2169: 103 pgs: 103 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 3.5 KiB/s rd, 7.7 KiB/s wr, 2 op/s; 13 KiB/s, 0 objects/s recovering 2024-09-21T10:18:10.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:09 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/710692485' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-21T10:18:10.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:09 smithi134 ceph-mon[90085]: pgmap v2169: 103 pgs: 103 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 3.5 KiB/s rd, 7.7 KiB/s wr, 2 op/s; 13 KiB/s, 0 objects/s recovering 2024-09-21T10:18:10.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:09 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/710692485' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-21T10:18:10.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:10.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:10.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:10.363 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:10.389 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:10.491 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:18:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:18:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:18:10.516 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:10.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:10.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:10.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2024-09-21T10:18:10.644 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:10.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:10.746 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:10.771 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:10.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:10.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:10.996 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:11.020 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:11.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:11.121 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:11.144 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:11.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:11.178 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:10 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/710692485' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-21T10:18:11.178 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:10 smithi017 ceph-mon[102913]: osdmap e1359: 8 total, 8 up, 7 in 2024-09-21T10:18:11.178 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:11.178 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:11.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:10 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/710692485' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-21T10:18:11.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:10 smithi017 ceph-mon[105151]: osdmap e1359: 8 total, 8 up, 7 in 2024-09-21T10:18:11.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:11.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:10 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/710692485' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-21T10:18:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:10 smithi134 ceph-mon[90085]: osdmap e1359: 8 total, 8 up, 7 in 2024-09-21T10:18:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:11.245 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:11.268 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:11.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:11.391 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:11.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:11.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:11.515 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:11.616 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:11.671 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:11.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:11.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:11.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:11.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:11.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:11.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:11.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:12.020 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:12.044 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:12.045 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:12.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:12.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:12.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:11 smithi017 ceph-mon[102913]: pgmap v2171: 119 pgs: 6 creating+peering, 10 unknown, 103 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 1.7 KiB/s rd, 3.7 KiB/s wr, 1 op/s; 11 KiB/s, 0 objects/s recovering 2024-09-21T10:18:12.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:11 smithi017 ceph-mon[102913]: osdmap e1360: 8 total, 8 up, 7 in 2024-09-21T10:18:12.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:11 smithi017 ceph-mon[102913]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:18:12.159 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:11 smithi017 ceph-mon[105151]: pgmap v2171: 119 pgs: 6 creating+peering, 10 unknown, 103 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 1.7 KiB/s rd, 3.7 KiB/s wr, 1 op/s; 11 KiB/s, 0 objects/s recovering 2024-09-21T10:18:12.159 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:11 smithi017 ceph-mon[105151]: osdmap e1360: 8 total, 8 up, 7 in 2024-09-21T10:18:12.159 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:11 smithi017 ceph-mon[105151]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:18:12.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:12.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:11 smithi134 ceph-mon[90085]: pgmap v2171: 119 pgs: 6 creating+peering, 10 unknown, 103 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 1.7 KiB/s rd, 3.7 KiB/s wr, 1 op/s; 11 KiB/s, 0 objects/s recovering 2024-09-21T10:18:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:11 smithi134 ceph-mon[90085]: osdmap e1360: 8 total, 8 up, 7 in 2024-09-21T10:18:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:11 smithi134 ceph-mon[90085]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:18:12.272 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:12.297 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:12.400 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:12.425 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:12.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:12.549 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:12.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:12.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:12.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:12.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:12.746 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:12.746 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:12.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:12.801 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:12.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:12.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:12.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:12.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:13.028 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:13.053 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:13.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:13.180 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:13.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:12 smithi017 ceph-mon[102913]: osdmap e1361: 8 total, 8 up, 7 in 2024-09-21T10:18:13.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:12 smithi017 ceph-mon[105151]: osdmap e1361: 8 total, 8 up, 7 in 2024-09-21T10:18:13.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:12 smithi134 ceph-mon[90085]: osdmap e1361: 8 total, 8 up, 7 in 2024-09-21T10:18:13.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:13.307 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:13.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:13.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:13.434 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:13.535 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:13.557 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:13.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:13.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:13.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:13.805 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:13.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:13.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:13.932 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:13.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:13.956 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:18:14.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:14.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:14.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:14.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:14.181 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:13 smithi017 ceph-mon[105151]: pgmap v2174: 119 pgs: 9 creating+peering, 2 unknown, 108 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:13 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2666907340' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:14.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:13 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:14.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:13 smithi017 ceph-mon[102913]: pgmap v2174: 119 pgs: 9 creating+peering, 2 unknown, 108 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:14.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:13 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2666907340' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:14.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:13 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:14.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:13 smithi134 ceph-mon[90085]: pgmap v2174: 119 pgs: 9 creating+peering, 2 unknown, 108 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:14.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:13 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2666907340' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:14.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:13 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:14.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:14.308 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:14.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:14.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:14.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:14.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:14.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:14.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:14.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:14.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:14.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:14.804 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:14.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:14.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:14.931 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:14.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:14.939 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:18:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:18:14] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-21T10:18:15.033 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:15.059 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:15.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:15.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:15.186 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:15.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:14 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:15.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:14 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:15.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:14 smithi017 ceph-mon[102913]: osdmap e1362: 8 total, 8 up, 7 in 2024-09-21T10:18:15.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:14 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:15.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:14 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:15.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:14 smithi017 ceph-mon[105151]: osdmap e1362: 8 total, 8 up, 7 in 2024-09-21T10:18:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:14 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:14 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:14 smithi134 ceph-mon[90085]: osdmap e1362: 8 total, 8 up, 7 in 2024-09-21T10:18:15.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:15.313 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:15.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:15.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:15.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:15.538 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:15.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:15.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:15.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:15.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:15.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:15.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:15.813 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:15.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:15.915 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:15.941 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:15.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:16.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:16.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:16.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:16.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:16.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:16.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:16.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:15 smithi134 ceph-mon[90085]: pgmap v2176: 119 pgs: 9 creating+peering, 110 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:16.290 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:16.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:15 smithi017 ceph-mon[102913]: pgmap v2176: 119 pgs: 9 creating+peering, 110 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:16.309 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:15 smithi017 ceph-mon[105151]: pgmap v2176: 119 pgs: 9 creating+peering, 110 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:16.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.317 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:16.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:16.327 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2024-09-21T10:18:16.327 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2024-09-21T10:18:16.327 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 15.6205 -> 15.6205 (expected baseline 6.61168) 2024-09-21T10:18:16.327 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 16 -> 16 pgs (0.313726 -> 0.313726 * mean) 2024-09-21T10:18:16.327 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.6 with 64 -> 64 pgs (1.2549 -> 1.2549 * mean) 2024-09-21T10:18:16.328 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:18:16.328 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2024-09-21T10:18:16.328 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 3 2024-09-21T10:18:16.328 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:18:16.328 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0093 2024-09-21T10:18:16.328 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0932 2024-09-21T10:18:16.328 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2024-09-21T10:18:16.418 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:16.443 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:16.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:16.545 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:16.570 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:16.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:16.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:16.696 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:16.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:18:16.798 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:16.823 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:16.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:16.925 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:16.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:16.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:17.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:17.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:17.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:17.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:17.195 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:17.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:17.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:17.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:17.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:17.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:17.444 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:17.546 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:17.575 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:17.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:17.677 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:17.702 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:17.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:17.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:17.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:17.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:17.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:17.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:17.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:18.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:18.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:18.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:18.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:18.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:18.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:18.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:17 smithi134 ceph-mon[90085]: from='client.46307 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:18:18.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:17 smithi134 ceph-mon[90085]: pgmap v2177: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:18.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:18.328 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:18.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:18.396 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:17 smithi017 ceph-mon[105151]: from='client.46307 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:18:18.396 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:17 smithi017 ceph-mon[105151]: pgmap v2177: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:18.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:17 smithi017 ceph-mon[102913]: from='client.46307 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:18:18.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:17 smithi017 ceph-mon[102913]: pgmap v2177: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:18.429 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:18.455 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:18.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:18.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:18.579 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:18.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:18.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:18.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.703 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:18.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:18.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:18.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:18.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:18.833 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-21T10:18:18.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 1000 0.05 2024-09-21T10:18:18.929 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:18.955 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:19.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:19.079 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:19.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:19.181 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:19.206 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:19.308 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:19.334 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:19.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:19.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:19.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:19.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:19.560 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:19.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:19.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:19.687 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:19.713 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:19.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:19.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:19.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:19.851 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:18:19.851 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:18:19.851 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:18:19.851 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:18:19.851 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:18:19.939 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:19.965 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:19.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:20.050 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:19 smithi017 ceph-mon[102913]: pgmap v2178: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:20.051 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:19 smithi017 ceph-mon[105151]: pgmap v2178: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:20.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:20.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:20.191 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:20.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:19 smithi134 ceph-mon[90085]: pgmap v2178: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:20.217 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:20.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:20.268 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:18:20.268 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:18:20.277 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:18:20.277 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:20.293 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:18:20.293 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:20.310 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:18:20.310 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:20.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:20.326 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:18:20.327 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:20.340 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:20.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:20.342 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:18:20.343 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:18:20.359 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:18:20.359 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:20.375 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:18:20.375 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:20.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:20.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:20.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:18:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:18:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:18:20.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:20.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:20.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:20.690 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:20.717 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:20.818 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:20.842 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:20.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:20.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:20.971 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_6' 2024-09-21T10:18:21.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:21.092 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:21.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:21.194 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:21.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:20 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/801134965' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:18:21.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:20 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:18:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:20 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/612432788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:21.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:20 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:20 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/801134965' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:18:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:20 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:18:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:20 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/612432788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:21.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:20 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:21.219 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:21.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:20 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/801134965' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:18:21.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:20 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:18:21.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:20 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/612432788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:21.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:20 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:21.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:18:21.321 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:21.346 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:21.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:21.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:21.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:21.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:21.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:21.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:21.699 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:21.724 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:21.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:21.827 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:21.852 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:21.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:21.954 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:21.980 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:21.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:22.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:21 smithi017 ceph-mon[102913]: pgmap v2179: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:22.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:21 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:18:22.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:21 smithi017 ceph-mon[102913]: osdmap e1363: 8 total, 8 up, 7 in 2024-09-21T10:18:22.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:21 smithi017 ceph-mon[105151]: pgmap v2179: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:22.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:21 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:18:22.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:21 smithi017 ceph-mon[105151]: osdmap e1363: 8 total, 8 up, 7 in 2024-09-21T10:18:22.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:22.106 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:22.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:22.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:22.231 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:22.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:22.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:21 smithi134 ceph-mon[90085]: pgmap v2179: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:22.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:21 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:18:22.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:21 smithi134 ceph-mon[90085]: osdmap e1363: 8 total, 8 up, 7 in 2024-09-21T10:18:22.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:22.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:22.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:22.483 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:22.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:22.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:22.607 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:22.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:22.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:22.732 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:22.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:22.788 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:22.788 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:22.788 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:22.834 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:22.860 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:22.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:22.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:22.984 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:22.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:23.040 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2024-09-21T10:18:23.042 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:18:23.042 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:18:23.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:23.091 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2024-09-21T10:18:23.092 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:18:23.092 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_304926 2024-09-21T10:18:23.092 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:18:23.092 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:18:23.120 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:23.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:23 smithi017 ceph-mon[105151]: pgmap v2181: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:23.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:23 smithi017 ceph-mon[105151]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:18:23.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:23 smithi017 ceph-mon[102913]: pgmap v2181: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:23.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:23 smithi017 ceph-mon[102913]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:18:23.222 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:23.256 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:23.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:23.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:23.400 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:23 smithi134 ceph-mon[90085]: pgmap v2181: 119 pgs: 119 active+clean; 739 KiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2024-09-21T10:18:23.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:23 smithi134 ceph-mon[90085]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:18:23.501 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:23.533 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:23.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:23.636 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:23.670 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:23.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:23.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:23.809 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:23.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:23.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:23.943 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:23.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:24.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:24.078 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:24.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:24.092 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3856 3840 239.985 240 0.00386497 0.00415389 2024-09-21T10:18:24.180 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:24.211 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:24.313 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:24.346 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:24.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:24.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:24.488 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:24.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:24.590 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:24.622 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:24.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:24.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:18:24.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:24.793 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:24.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:24.894 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.927 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:24.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:24.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:18:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:18:24] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-21T10:18:25.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:25.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:25.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:25.093 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 7396 7380 230.573 221.25 0.00905901 0.00432669 2024-09-21T10:18:25.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:25.225 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:25.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:25.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:25.367 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:25.469 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:25.504 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:25.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:25.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:25.646 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:25.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:25.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:25.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:25.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:25.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:25.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:25.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:25 smithi017 ceph-mon[102913]: pgmap v2182: 119 pgs: 119 active+clean; 22 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 2.1 MiB/s wr, 34 op/s 2024-09-21T10:18:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:25 smithi017 ceph-mon[105151]: pgmap v2182: 119 pgs: 119 active+clean; 22 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 2.1 MiB/s wr, 34 op/s 2024-09-21T10:18:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:25 smithi134 ceph-mon[90085]: pgmap v2182: 119 pgs: 119 active+clean; 22 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 2.1 MiB/s wr, 34 op/s 2024-09-21T10:18:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:26.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:26.074 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:26.093 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 10691 10675 222.355 205.938 0.00303528 0.00448916 2024-09-21T10:18:26.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:26.215 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:26.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:26.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:26.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:26.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:26.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:26.478 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:26.580 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:26.614 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:26.716 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:26.736 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:26.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:26.839 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:26.870 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:26.972 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:27.004 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:27.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:27.093 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 15 13623 13608 212.591 183.312 0.00869955 0.00469554 2024-09-21T10:18:27.106 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:27.137 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:27.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:27.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:27.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:27.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:27.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:27.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.423 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:27.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:27.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:27.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:27.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:27.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:27.701 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:27.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:27.802 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:27.840 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:27.943 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:27.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:27 smithi017 ceph-mon[105151]: pgmap v2183: 119 pgs: 119 active+clean; 318 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 32 MiB/s wr, 507 op/s 2024-09-21T10:18:27.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:27 smithi017 ceph-mon[102913]: pgmap v2183: 119 pgs: 119 active+clean; 318 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 32 MiB/s wr, 507 op/s 2024-09-21T10:18:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.970 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:27.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:27 smithi134 ceph-mon[90085]: pgmap v2183: 119 pgs: 119 active+clean; 318 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 32 MiB/s wr, 507 op/s 2024-09-21T10:18:28.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:28.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 16595 16579 207.125 185.688 0.00753701 0.00482108 2024-09-21T10:18:28.108 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:28.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:28.210 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:28.230 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:28.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:28.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:28.370 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:28.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:28.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:28.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:28.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:28.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:28.657 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:28.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:28.759 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:28.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:28.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:28.930 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:29.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:29.071 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:29.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 19367 19351 201.461 173.25 0.00605859 0.00495803 2024-09-21T10:18:29.173 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:29.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.207 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:29.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:29.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:29.354 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:29.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:29.456 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:29.487 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:29.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:29.589 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:29.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.622 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:29.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:29.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:29 smithi017 ceph-mon[102913]: pgmap v2184: 119 pgs: 119 active+clean; 590 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-21T10:18:29.685 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:29 smithi017 ceph-mon[105151]: pgmap v2184: 119 pgs: 119 active+clean; 590 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-21T10:18:29.723 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:29.755 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:29.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:29.856 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:29.887 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:29.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:29.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:29 smithi134 ceph-mon[90085]: pgmap v2184: 119 pgs: 119 active+clean; 590 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s 2024-09-21T10:18:30.005 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:30.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.034 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:30.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:30.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 21830 21814 194.672 153.938 0.00458981 0.00512902 2024-09-21T10:18:30.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:30.178 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:30.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:30.280 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:30.311 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:30.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:30.412 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:30.447 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:30.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:18:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:18:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:18:30.549 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:30.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.590 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:30.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:30.646 INFO:teuthology.orchestra.run.smithi017.stdout:moved 3 / 357 (0.840336%) 2024-09-21T10:18:30.646 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2024-09-21T10:18:30.646 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 15.6205 -> 14.4716 (expected baseline 6.61168) 2024-09-21T10:18:30.646 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 16 -> 19 pgs (0.313726 -> 0.372549 * mean) 2024-09-21T10:18:30.646 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.6 with 64 -> 63 pgs (1.2549 -> 1.23529 * mean) 2024-09-21T10:18:30.647 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:18:30.647 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2024-09-21T10:18:30.647 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2024-09-21T10:18:30.647 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:18:30.647 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0121 2024-09-21T10:18:30.647 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.1205 2024-09-21T10:18:30.648 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 0.3629 -> 0.4129 2024-09-21T10:18:30.651 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:18:30.690 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:30.717 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:30.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:30.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:30.843 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:30.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:30.883 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:30 smithi017 ceph-mon[105151]: from='client.36513 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:18:30.883 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'5':'27057'}"} : dispatch 2024-09-21T10:18:30.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:30 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2619594150' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:30.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:30 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:30.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:30 smithi017 ceph-mon[102913]: from='client.36513 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:18:30.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'5':'27057'}"} : dispatch 2024-09-21T10:18:30.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:30 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2619594150' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:30.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:30 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:30.890 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:18:30.890 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:18:30.945 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:30.967 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:30.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:30 smithi134 ceph-mon[90085]: from='client.36513 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:18:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'5':'27057'}"} : dispatch 2024-09-21T10:18:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:30 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2619594150' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:30 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:31.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:31.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.091 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:31.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:31.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 23414 23398 182.716 99 0.0021271 0.00521799 2024-09-21T10:18:31.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:31.214 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:31.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:31.337 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:31.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:31.438 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:31.459 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:31.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:31.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:31.582 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:18:31.584 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:31.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:31.686 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:31.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:31.810 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:31.833 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:31.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:31.854 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:18:31.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[102913]: pgmap v2185: 119 pgs: 119 active+clean; 1.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:18:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'27057'}"}]': finished 2024-09-21T10:18:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:18:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[102913]: osdmap e1364: 8 total, 8 up, 7 in 2024-09-21T10:18:31.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:18:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[105151]: pgmap v2185: 119 pgs: 119 active+clean; 1.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:18:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'27057'}"}]': finished 2024-09-21T10:18:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:18:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[105151]: osdmap e1364: 8 total, 8 up, 7 in 2024-09-21T10:18:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:18:31.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:31.982 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:31 smithi134 ceph-mon[90085]: pgmap v2185: 119 pgs: 119 active+clean; 1.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:18:31.982 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:31 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:31.982 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'5':'27057'}"}]': finished 2024-09-21T10:18:31.982 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:31 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:18:31.982 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:31 smithi134 ceph-mon[90085]: osdmap e1364: 8 total, 8 up, 7 in 2024-09-21T10:18:31.982 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:18:32.059 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:32.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 23414 23398 162.42 0 - 0.00521799 2024-09-21T10:18:32.118 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:32.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:32.219 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:32.241 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:32.343 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:32.365 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:32.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:32.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:32.489 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:32.591 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:32.612 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:32.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:32.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:32.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:32.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:32.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:32.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[102913]: 1.e scrub starts 2024-09-21T10:18:32.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[102913]: 1.e scrub ok 2024-09-21T10:18:32.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[102913]: 1.12 scrub starts 2024-09-21T10:18:32.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[102913]: 1.12 scrub ok 2024-09-21T10:18:32.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[102913]: osdmap e1365: 8 total, 8 up, 7 in 2024-09-21T10:18:32.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[105151]: 1.e scrub starts 2024-09-21T10:18:32.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[105151]: 1.e scrub ok 2024-09-21T10:18:32.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[105151]: 1.12 scrub starts 2024-09-21T10:18:32.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[105151]: 1.12 scrub ok 2024-09-21T10:18:32.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:32 smithi017 ceph-mon[105151]: osdmap e1365: 8 total, 8 up, 7 in 2024-09-21T10:18:32.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:32.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:32.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:32 smithi134 ceph-mon[90085]: 1.e scrub starts 2024-09-21T10:18:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:32 smithi134 ceph-mon[90085]: 1.e scrub ok 2024-09-21T10:18:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:32 smithi134 ceph-mon[90085]: 1.12 scrub starts 2024-09-21T10:18:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:32 smithi134 ceph-mon[90085]: 1.12 scrub ok 2024-09-21T10:18:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:32 smithi134 ceph-mon[90085]: osdmap e1365: 8 total, 8 up, 7 in 2024-09-21T10:18:33.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:33.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 23686 23670 147.882 8.5 0.00420755 0.00675594 2024-09-21T10:18:33.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:33.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:33.226 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.260 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:33.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:33.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:33.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:33.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:33.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:33.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:33.646 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:33.678 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:33.780 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:33.800 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:33 smithi017 ceph-mon[102913]: pgmap v2188: 119 pgs: 1 peering, 118 active+clean; 1.4 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 171 MiB/s wr, 2.74k op/s 2024-09-21T10:18:33.800 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:33 smithi017 ceph-mon[102913]: osdmap e1366: 8 total, 8 up, 7 in 2024-09-21T10:18:33.800 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:33 smithi017 ceph-mon[105151]: pgmap v2188: 119 pgs: 1 peering, 118 active+clean; 1.4 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 171 MiB/s wr, 2.74k op/s 2024-09-21T10:18:33.800 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:33 smithi017 ceph-mon[105151]: osdmap e1366: 8 total, 8 up, 7 in 2024-09-21T10:18:33.811 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:33.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:33.913 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:33.947 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:33.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:33 smithi134 ceph-mon[90085]: pgmap v2188: 119 pgs: 1 peering, 118 active+clean; 1.4 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 171 MiB/s wr, 2.74k op/s 2024-09-21T10:18:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:33 smithi134 ceph-mon[90085]: osdmap e1366: 8 total, 8 up, 7 in 2024-09-21T10:18:34.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:34.085 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:34.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:34.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 25776 25760 146.313 130.625 0.00437674 0.00682865 2024-09-21T10:18:34.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:34.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:34.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:34.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:34.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:34.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:34.463 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:34.497 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:34.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:34.631 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:34.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:34.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3928740853' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:34.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3928740853' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:34.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:34.772 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:34.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:34.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:34.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.911 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:34.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:34.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:18:34.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:18:34.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:18:34.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:18:34.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:18:34.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:18:34.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:18:34.924 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:18:34.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:18:34.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:34.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:34.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:34.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:34.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:18:34.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:34.924 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3928740853' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:35.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:35.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:35.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:35.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 27827 27811 144.802 128.188 0.00933951 0.00690065 2024-09-21T10:18:35.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:35.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:35.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:35.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:18:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:18:34] "GET /metrics HTTP/1.1" 200 36721 "" "Prometheus/2.51.0" 2024-09-21T10:18:35.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:35.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:35.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:35.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:35.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:35.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:35.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:35.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:35.693 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:35.730 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:35.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:35.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:35.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:35.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:35.922 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:35 smithi134 ceph-mon[90085]: pgmap v2190: 119 pgs: 1 peering, 118 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-21T10:18:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:35 smithi017 ceph-mon[102913]: pgmap v2190: 119 pgs: 1 peering, 118 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-21T10:18:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:35 smithi017 ceph-mon[105151]: pgmap v2190: 119 pgs: 1 peering, 118 active+clean; 1.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-21T10:18:35.957 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:35.991 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:35.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:36.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:36.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 29856 29840 143.417 126.812 0.00543143 0.00696662 2024-09-21T10:18:36.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:36.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:36.262 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:36.364 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.405 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:36.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:36.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:18:36.482 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:18:36.482 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:18:36.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:36.482 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:30464 11.a e1366/1365 11:589f37a0:::benchmark_data_smithi017_304926_object30463:head v 1366'1878, mlcod=1363'247)", 2024-09-21T10:18:36.482 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:36.476830+0000", 2024-09-21T10:18:36.482 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.001288862, 2024-09-21T10:18:36.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001317052, 2024-09-21T10:18:36.482 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:36.482 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.476830+0000", 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.476830+0000", 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:36.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.476829+0000", 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.476890+0000", 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1080000000000005e-05 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.476902+0000", 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1402e-05 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:36.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.476908+0000", 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.122e-06 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.477135+0000", 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00022754000000000001 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.477159+0000", 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3790000000000001e-05 2024-09-21T10:18:36.485 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:36.486 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:36.486 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:36.486 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:36.486 INFO:teuthology.orchestra.run.smithi134.stdout: ], 2024-09-21T10:18:36.486 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 1 2024-09-21T10:18:36.486 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:18:36.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:36.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:36.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:36.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:36.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:36.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:36.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:36 smithi134 ceph-mon[90085]: 1.5 scrub starts 2024-09-21T10:18:36.781 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:36.805 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:36.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:36.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:36.945 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:36 smithi017 ceph-mon[102913]: 1.5 scrub starts 2024-09-21T10:18:36.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:36 smithi017 ceph-mon[105151]: 1.5 scrub starts 2024-09-21T10:18:37.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:37.086 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:37.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 31434 31418 140.218 98.625 0.00363155 0.00712335 2024-09-21T10:18:37.189 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:37.229 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:37.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:37.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:37.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.368 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:37.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:37.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:37.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:37.592 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:37.615 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:37.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:37.717 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:37.754 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:37.856 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:37.893 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:37.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:37 smithi017 ceph-mon[102913]: 1.d scrub starts 2024-09-21T10:18:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:37 smithi017 ceph-mon[102913]: 1.d scrub ok 2024-09-21T10:18:37.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:37 smithi017 ceph-mon[102913]: pgmap v2191: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 1288/79263 objects misplaced (1.625%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:18:37.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:37 smithi017 ceph-mon[105151]: 1.d scrub starts 2024-09-21T10:18:37.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:37 smithi017 ceph-mon[105151]: 1.d scrub ok 2024-09-21T10:18:37.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:37 smithi017 ceph-mon[105151]: pgmap v2191: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 1288/79263 objects misplaced (1.625%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:18:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:37 smithi134 ceph-mon[90085]: 1.d scrub starts 2024-09-21T10:18:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:37 smithi134 ceph-mon[90085]: 1.d scrub ok 2024-09-21T10:18:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:37 smithi134 ceph-mon[90085]: pgmap v2191: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 1288/79263 objects misplaced (1.625%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:18:37.995 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:38.028 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:38.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:38.070 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:18:38.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 32998 32982 137.387 97.75 0.0119886 0.007273 2024-09-21T10:18:38.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:38.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.154 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:38.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:38.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:38.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:38.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:38.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:38.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:38.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:38.514 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:18:38.514 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:18:38.515 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-21T10:18:38.515 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:18:38.515 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:18:38.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-21T10:18:38.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:38.563 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:38.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:38.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:38.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:38.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:38.758 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:38 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:18:38.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:38.859 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:38.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:38.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:38 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:18:38.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:38 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:18:38.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:38.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:39.097 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 15 34716 34701 135.515 107.438 0.00379993 0.00737286 2024-09-21T10:18:39.100 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:18:39.101 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:39.153 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:39.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:39.256 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:39.292 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:39.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:39.394 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:39.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:39.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:39.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:39.552 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:39.654 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:39.693 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:39.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:39.827 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:39.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:39.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:39.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:39 smithi017 ceph-mon[102913]: 1.3 scrub starts 2024-09-21T10:18:39.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:39 smithi017 ceph-mon[102913]: pgmap v2192: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s; 1288/86988 objects misplaced (1.481%); 1.2 MiB/s, 19 objects/s recovering 2024-09-21T10:18:39.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:39 smithi017 ceph-mon[105151]: 1.3 scrub starts 2024-09-21T10:18:39.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:39 smithi017 ceph-mon[105151]: pgmap v2192: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s; 1288/86988 objects misplaced (1.481%); 1.2 MiB/s, 19 objects/s recovering 2024-09-21T10:18:39.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.961 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:39.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:39 smithi134 ceph-mon[90085]: 1.3 scrub starts 2024-09-21T10:18:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:39 smithi134 ceph-mon[90085]: pgmap v2192: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s; 1288/86988 objects misplaced (1.481%); 1.2 MiB/s, 19 objects/s recovering 2024-09-21T10:18:40.063 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.089 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:40.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 36288 36272 133.319 98.1875 0.0195073 0.00748742 2024-09-21T10:18:40.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:40.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:40.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:40.315 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.346 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:40.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.448 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:40.481 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:40.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:18:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:18:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:18:40.583 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:40.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:40.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:40.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:40.850 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.873 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:40.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:40 smithi017 ceph-mon[102913]: 1.17 scrub starts 2024-09-21T10:18:40.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:40 smithi017 ceph-mon[105151]: 1.17 scrub starts 2024-09-21T10:18:40.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:40.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:40 smithi134 ceph-mon[90085]: 1.17 scrub starts 2024-09-21T10:18:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:40.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:41.097 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 37869 37853 131.402 98.8125 0.00501259 0.00759527 2024-09-21T10:18:41.101 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:41.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.126 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:41.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:41.227 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:41.262 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:41.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:41.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:41.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:41.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.535 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:41.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:41.636 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:41.670 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:41.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:41.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:41.809 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:41.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:41.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:41.871 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:41.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:41.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:41 smithi017 ceph-mon[102913]: pgmap v2193: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 885/100116 objects misplaced (0.884%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T10:18:41.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:41 smithi017 ceph-mon[105151]: pgmap v2193: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 885/100116 objects misplaced (0.884%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T10:18:41.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:41.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:41.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:41 smithi134 ceph-mon[90085]: pgmap v2193: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 885/100116 objects misplaced (0.884%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T10:18:42.071 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:42.097 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 39366 39350 129.41 93.5625 0.00616148 0.00772173 2024-09-21T10:18:42.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:42.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:42.213 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:42.247 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:42.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:42.382 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:42.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:42.484 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:42.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.515 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:42.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:42.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:42.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.657 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:42.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:42.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:42.796 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:42.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:42.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:42.938 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:42.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:43.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:43.083 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:43.097 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:18:43.096035+0000 min lat: 0.00118998 max lat: 2.30472 avg lat: 0.00781435 2024-09-21T10:18:43.097 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:18:43.097 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 40946 40930 127.877 98.75 0.00819248 0.00781435 2024-09-21T10:18:43.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:43.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:43.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:43.334 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:43.363 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:43.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:43.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:43.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:43.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:43.606 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:43.640 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:43.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:18:43.741 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:43.771 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:43.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:43.873 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:43.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:43.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:43 smithi017 ceph-mon[102913]: 1.1a scrub starts 2024-09-21T10:18:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:43 smithi017 ceph-mon[102913]: pgmap v2194: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 885/111957 objects misplaced (0.790%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T10:18:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:43 smithi017 ceph-mon[105151]: 1.1a scrub starts 2024-09-21T10:18:43.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:43 smithi017 ceph-mon[105151]: pgmap v2194: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 885/111957 objects misplaced (0.790%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T10:18:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:43 smithi134 ceph-mon[90085]: 1.1a scrub starts 2024-09-21T10:18:43.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:43 smithi134 ceph-mon[90085]: pgmap v2194: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s; 885/111957 objects misplaced (0.790%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T10:18:44.009 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:44.045 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:44.097 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 42302 42286 125.823 84.75 0.0149186 0.00794158 2024-09-21T10:18:44.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:44.189 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:44.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:44.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:44.329 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:44.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:44.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:44.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:44.569 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:44.601 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:44.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:44.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:44.742 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:44.844 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:44.884 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:44.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:44.986 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:45.020 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:45.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:45.097 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 44010 43994 124.956 106.75 0.00641336 0.00799833 2024-09-21T10:18:45.122 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:45.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:45.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:18:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:18:44] "GET /metrics HTTP/1.1" 200 36760 "" "Prometheus/2.51.0" 2024-09-21T10:18:45.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:45.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:45.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:45.401 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:45.435 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:45.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:45.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:45.574 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:45.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:45.676 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:45.714 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:45.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:45.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:45.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:45.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:45.866 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:45 smithi134 ceph-mon[90085]: pgmap v2195: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 885/114858 objects misplaced (0.771%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:18:45.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:46.000 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:46.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:46.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:45 smithi017 ceph-mon[102913]: pgmap v2195: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 885/114858 objects misplaced (0.771%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:18:46.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:45 smithi017 ceph-mon[105151]: pgmap v2195: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 885/114858 objects misplaced (0.771%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T10:18:46.100 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 45815 45799 124.413 112.812 0.0133964 0.00803041 2024-09-21T10:18:46.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:46.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:46.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:46.249 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:46.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.294 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:46.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:46.388 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:18:46.388 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:18:46.388 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:18:46.388 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:18:46.388 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:18:46.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:46.428 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:46.530 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:46.568 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:46.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:46.670 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:46.702 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:46.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:46.803 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:46.840 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:46.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:46.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:46.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.982 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:46.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:47.083 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:47.100 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 47520 47504 123.669 106.562 0.00496444 0.0080802 2024-09-21T10:18:47.115 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:47.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:47.148 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:18:47.148 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "" 2024-09-21T10:18:47.148 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:18:47.218 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:47.250 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:47.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:47.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:18:47.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.388 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:47.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:47.489 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:47.532 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:47.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:47.634 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:47.695 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:47.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:47.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:47.835 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:47.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:47.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:47 smithi017 ceph-mon[102913]: 1.1c scrub starts 2024-09-21T10:18:47.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:47 smithi017 ceph-mon[102913]: pgmap v2196: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s; 344/128391 objects misplaced (0.268%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:18:47.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:47 smithi017 ceph-mon[105151]: 1.1c scrub starts 2024-09-21T10:18:47.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:47 smithi017 ceph-mon[105151]: pgmap v2196: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s; 344/128391 objects misplaced (0.268%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:18:47.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:47.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:47.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:47 smithi134 ceph-mon[90085]: 1.1c scrub starts 2024-09-21T10:18:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:47 smithi134 ceph-mon[90085]: pgmap v2196: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s; 344/128391 objects misplaced (0.268%); 5.7 MiB/s, 91 objects/s recovering 2024-09-21T10:18:48.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:48.100 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 49139 49123 122.77 101.188 0.00732313 0.0081395 2024-09-21T10:18:48.129 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:48.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:48.159 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:18:48.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:48.273 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:48.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:48.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:48.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:48.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:48.516 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:48.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.559 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:48.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:48.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:48.681 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:48.744 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:48.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:48.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:48.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:48.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:49.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:49.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:49.100 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 50429 50413 121.149 80.625 0.00293002 0.0081752 2024-09-21T10:18:49.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:49.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:49.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:49.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.263 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:49.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:49.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:49.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:49.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[105151]: 1.1b scrub starts 2024-09-21T10:18:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[105151]: 1.1b scrub ok 2024-09-21T10:18:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[105151]: pgmap v2197: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 344/135846 objects misplaced (0.253%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T10:18:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[105151]: osdmap e1367: 8 total, 8 up, 7 in 2024-09-21T10:18:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2668372991' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:49.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:49.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:49.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:49.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[102913]: 1.1b scrub starts 2024-09-21T10:18:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[102913]: 1.1b scrub ok 2024-09-21T10:18:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[102913]: pgmap v2197: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 344/135846 objects misplaced (0.253%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T10:18:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[102913]: osdmap e1367: 8 total, 8 up, 7 in 2024-09-21T10:18:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2668372991' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:49.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:49 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:49.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:49 smithi134 ceph-mon[90085]: 1.1b scrub starts 2024-09-21T10:18:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:49 smithi134 ceph-mon[90085]: 1.1b scrub ok 2024-09-21T10:18:49.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:49 smithi134 ceph-mon[90085]: pgmap v2197: 119 pgs: 1 active+recovering+undersized+remapped, 118 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 344/135846 objects misplaced (0.253%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T10:18:49.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:49 smithi134 ceph-mon[90085]: osdmap e1367: 8 total, 8 up, 7 in 2024-09-21T10:18:49.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:49 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2668372991' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:49.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:49 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:49.511 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:49.612 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:49.635 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:49.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:49.690 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:18:49.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:49.776 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:49.878 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:49.913 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:49.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:50.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:50.050 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:50.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:50.100 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 50969 50953 117.912 33.75 0.00642379 0.00847557 2024-09-21T10:18:50.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:50.194 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:50.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:50.295 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:50.334 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:50.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:50.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:50.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:50 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:18:50.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:50 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:18:50.444 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:50 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:50.445 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:50 smithi134 ceph-mon[90085]: osdmap e1368: 8 total, 8 up, 7 in 2024-09-21T10:18:50.445 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:18:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:18:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:18:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:50 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:18:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:50 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:18:50.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:50 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:50.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:50 smithi017 ceph-mon[102913]: osdmap e1368: 8 total, 8 up, 7 in 2024-09-21T10:18:50.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:50 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:18:50.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:50 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:18:50.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:50 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:50.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:50 smithi017 ceph-mon[105151]: osdmap e1368: 8 total, 8 up, 7 in 2024-09-21T10:18:50.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:50.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:50.573 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:50.597 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:18:50.597 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:18:50.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:18:50.597 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:18:50.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.597 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:97657 9.6 e1254/1238 9:66ff8220:::benchmark_data_smithi017_237924_object4653:head v 1254'6054, mlcod=1254'6052)", 2024-09-21T10:18:50.597 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:10:33.566008+0000", 2024-09-21T10:18:50.597 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 497.02540830599997, 2024-09-21T10:18:50.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.102021377, 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.566008+0000", 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.566008+0000", 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.566006+0000", 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.566021+0000", 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4943000000000001e-05 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.566049+0000", 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7827e-05 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.566072+0000", 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2643e-05 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.566105+0000", 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.3572000000000003e-05 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.566118+0000", 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3138e-05 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.667999+0000", 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.101880078 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.668030+0000", 2024-09-21T10:18:50.601 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.116e-05 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:97655 9.a e1254/1245 9:5965e1c6:::benchmark_data_smithi017_237924_object4651:head v 1254'6116, mlcod=1242'867)", 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:10:33.568678+0000", 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 497.02273873000001, 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.098614950000000007, 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.602 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.568678+0000", 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.568678+0000", 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.568671+0000", 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999928 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.603 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.568688+0000", 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7382999999999999e-05 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.660594+0000", 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.091906283000000005 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.660605+0000", 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0623e-05 2024-09-21T10:18:50.604 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.660651+0000", 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5994e-05 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.660679+0000", 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7977999999999999e-05 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.667259+0000", 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0065799320000000001 2024-09-21T10:18:50.605 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.667293+0000", 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4093e-05 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:97658 9.6 e1254/1238 9:6e946b82:::benchmark_data_smithi017_237924_object4654:head v 1254'6055, mlcod=1254'6052)", 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:10:33.569937+0000", 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 497.02147978800002, 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.10043326900000001, 2024-09-21T10:18:50.606 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.569937+0000", 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.569937+0000", 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.607 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.569936+0000", 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.569941+0000", 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7899999999999999e-06 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.660599+0000", 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.090658160000000002 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.608 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.660608+0000", 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5059999999999995e-06 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.665343+0000", 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0047355549999999998 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.665361+0000", 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7870999999999999e-05 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.609 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.670348+0000", 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0049864250000000001 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:10:33.670370+0000", 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2532999999999999e-05 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:171056 9.b e1266/1252 9:dbe40d8e:::benchmark_data_smithi017_237924_object78052:head v 1266'10828, mlcod=1247'4555)", 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:21.290642+0000", 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 389.30077447299999, 2024-09-21T10:18:50.610 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.075158142999999997, 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.290642+0000", 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.290642+0000", 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.611 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.290642+0000", 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.290647+0000", 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6969999999999998e-06 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.290655+0000", 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9009999999999994e-06 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.612 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.290659+0000", 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7450000000000001e-06 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.290685+0000", 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.6274999999999998e-05 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.613 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.359212+0000", 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.068526474000000004 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.365779+0000", 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0065670549999999996 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.365800+0000", 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1665e-05 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.614 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:171063 9.a e1266/1245 9:59260471:::benchmark_data_smithi017_237924_object78059:head v 1266'10695, mlcod=1242'867)", 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:21.292803+0000", 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 389.298613176, 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.070752564000000004, 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.292803+0000", 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.615 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.292803+0000", 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.292803+0000", 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.292809+0000", 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.947e-06 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.616 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.359062+0000", 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.066253775000000001 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.359069+0000", 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3060000000000003e-06 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.359110+0000", 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0794999999999998e-05 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.617 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.359130+0000", 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0460000000000001e-05 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.363519+0000", 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0043885979999999996 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.363556+0000", 2024-09-21T10:18:50.618 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7462000000000001e-05 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.45851.0:171067 9.6 e1266/1238 9:6c99f9a2:::benchmark_data_smithi017_237924_object78063:head v 1266'10694, mlcod=1266'10692)", 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:21.294500+0000", 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 389.29691631499998, 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.076674689000000004, 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.619 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.294500+0000", 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.294500+0000", 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.294500+0000", 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.620 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.294507+0000", 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3429999999999998e-06 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.358988+0000", 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.064480671000000003 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.359003+0000", 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5645999999999999e-05 2024-09-21T10:18:50.621 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.359052+0000", 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8325999999999998e-05 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.359074+0000", 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2269e-05 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.371142+0000", 2024-09-21T10:18:50.622 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.012068407999999999 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:21.371175+0000", 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2727e-05 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.46058.0:250070 10.a e1352/1347 10:56b1c2c6:::benchmark_data_smithi017_268344_object124139:head v 1352'15776, mlcod=1352'15774)", 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:17:48.014378+0000", 2024-09-21T10:18:50.623 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 62.577038612999999, 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.017291333999999998, 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.014378+0000", 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.014378+0000", 2024-09-21T10:18:50.624 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.625 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.625 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.625 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.625 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.014377+0000", 2024-09-21T10:18:50.625 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:18:50.625 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.625 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.625 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.014387+0000", 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0034000000000001e-05 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.014407+0000", 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0112e-05 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.014414+0000", 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3130000000000001e-06 2024-09-21T10:18:50.626 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.014471+0000", 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6781999999999999e-05 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.014500+0000", 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9215999999999999e-05 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.031526+0000", 2024-09-21T10:18:50.627 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.017025518999999999 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:17:48.031669+0000", 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000143478 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:31213 11.a e1366/1365 11:55226ecc:::benchmark_data_smithi017_304926_object31212:head v 1366'1931, mlcod=1363'274)", 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:36.981763+0000", 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 13.609653996, 2024-09-21T10:18:50.628 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.027276477, 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.981763+0000", 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.981763+0000", 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.629 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.981762+0000", 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.981904+0000", 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000141977 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.630 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.981916+0000", 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2300000000000001e-05 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.981921+0000", 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1320000000000002e-06 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.988358+0000", 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0064369559999999998 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.631 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.988419+0000", 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0792999999999999e-05 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.009001+0000", 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.020582296999999999 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.009039+0000", 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7684e-05 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.632 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:31217 11.a e1366/1365 11:55f3f002:::benchmark_data_smithi017_304926_object31216:head v 1366'1932, mlcod=1363'274)", 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:36.984927+0000", 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 13.606489536, 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.026352248000000002, 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.984927+0000", 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.633 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.984927+0000", 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.984926+0000", 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.984966+0000", 2024-09-21T10:18:50.634 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9468999999999998e-05 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.984976+0000", 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0421999999999999e-05 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.984982+0000", 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.959e-06 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.635 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.995506+0000", 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010523731999999999 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.995564+0000", 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7689000000000003e-05 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.011204+0000", 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.015640007000000001 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.636 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.011279+0000", 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5780999999999998e-05 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:31220 11.a e1366/1365 11:5ca265b2:::benchmark_data_smithi017_304926_object31219:head v 1366'1933, mlcod=1363'274)", 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:36.984990+0000", 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 13.606426969999999, 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.026342540000000001, 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.637 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.984990+0000", 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.984990+0000", 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.984989+0000", 2024-09-21T10:18:50.638 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:18:50.639 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.639 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.639 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.639 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.985018+0000", 2024-09-21T10:18:50.639 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9113999999999999e-05 2024-09-21T10:18:50.639 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.640 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:50.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:50.640 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.985026+0000", 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2729999999999997e-06 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.985030+0000", 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1069999999999998e-06 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.998015+0000", 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.012984913000000001 2024-09-21T10:18:50.641 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.998077+0000", 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1832e-05 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.011283+0000", 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013206397 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.642 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.011332+0000", 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9249000000000002e-05 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:31230 11.a e1366/1365 11:51091463:::benchmark_data_smithi017_304926_object31229:head v 1366'1934, mlcod=1363'274)", 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:36.987066+0000", 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 13.604351072, 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.024311914, 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.643 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.987066+0000", 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.987066+0000", 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.987063+0000", 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999971 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.644 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.987141+0000", 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8102000000000005e-05 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.987156+0000", 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5108e-05 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:36.987162+0000", 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5780000000000004e-06 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.645 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.000607+0000", 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013445063 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.000662+0000", 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4512999999999998e-05 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.011334+0000", 2024-09-21T10:18:50.646 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010672381999999999 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.011378+0000", 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3179000000000001e-05 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:31713 11.a e1366/1365 11:5088f3f2:::benchmark_data_smithi017_304926_object31712:head v 1366'1970, mlcod=1363'288)", 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:37.245049+0000", 2024-09-21T10:18:50.647 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 13.346367320000001, 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014021829, 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.245049+0000", 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.245049+0000", 2024-09-21T10:18:50.648 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.245049+0000", 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.245076+0000", 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.6519000000000001e-05 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.649 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.245092+0000", 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6358000000000002e-05 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.245098+0000", 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8889999999999999e-06 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.249874+0000", 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0047761720000000004 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.650 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.249966+0000", 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1687999999999994e-05 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.258920+0000", 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0089544900000000007 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.259071+0000", 2024-09-21T10:18:50.651 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015099300000000001 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:32320 11.a e1366/1365 11:50d48974:::benchmark_data_smithi017_304926_object32319:head v 1366'2004, mlcod=1363'356)", 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:37.744106+0000", 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 12.847311037000001, 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013754381, 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.652 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.744106+0000", 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.744106+0000", 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.744105+0000", 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.653 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.744279+0000", 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017427499999999999 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.744299+0000", 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9882999999999999e-05 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.744306+0000", 2024-09-21T10:18:50.654 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5119999999999997e-06 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.744546+0000", 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000240301 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.744569+0000", 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3404000000000002e-05 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.655 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.757821+0000", 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013251959000000001 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:37.757860+0000", 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8838999999999999e-05 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:32899 11.3 e1366/1359 11:cabb73cf:::benchmark_data_smithi017_304926_object32898:head v 1366'1995, mlcod=1366'1992)", 2024-09-21T10:18:50.656 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:38.034024+0000", 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 12.557392613999999, 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.015231285000000001, 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.034024+0000", 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.034024+0000", 2024-09-21T10:18:50.657 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.034023+0000", 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.034050+0000", 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7818e-05 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.658 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.034071+0000", 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0934e-05 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.034079+0000", 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4710000000000001e-06 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.034239+0000", 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00016009999999999999 2024-09-21T10:18:50.659 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.034284+0000", 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5090999999999999e-05 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.049058+0000", 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014774137 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.660 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.049255+0000", 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019724599999999999 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:33137 11.a e1366/1365 11:5ebabc6a:::benchmark_data_smithi017_304926_object33136:head v 1366'2062, mlcod=1363'393)", 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:38.236548+0000", 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 12.354869015, 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.016585669000000001, 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.661 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.236548+0000", 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.236548+0000", 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.236547+0000", 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:18:50.662 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.236747+0000", 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000199953 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.236761+0000", 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4061e-05 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.236767+0000", 2024-09-21T10:18:50.663 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.319e-06 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.238829+0000", 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020617869999999998 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.238884+0000", 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4710000000000003e-05 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.664 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.253093+0000", 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014208721000000001 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.253133+0000", 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0775000000000002e-05 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:33139 11.a e1366/1365 11:58b3c418:::benchmark_data_smithi017_304926_object33138:head v 1366'2063, mlcod=1363'394)", 2024-09-21T10:18:50.665 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:38.239307+0000", 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 12.352110089, 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.01386019, 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.239307+0000", 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.239307+0000", 2024-09-21T10:18:50.666 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.239305+0000", 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.239358+0000", 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2717999999999999e-05 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.239374+0000", 2024-09-21T10:18:50.667 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5690999999999999e-05 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.239380+0000", 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9549999999999997e-06 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.242204+0000", 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002824408 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.242228+0000", 2024-09-21T10:18:50.668 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4070000000000002e-05 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.253140+0000", 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010911311999999999 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:38.253167+0000", 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7158999999999999e-05 2024-09-21T10:18:50.669 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:38670 11.a e1366/1365 11:5e704c2b:::benchmark_data_smithi017_304926_object38669:head v 1366'2416, mlcod=1363'726)", 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:41.620216+0000", 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 8.9712001810000004, 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.048765913000000001, 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.670 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.620216+0000", 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.620216+0000", 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.620216+0000", 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.671 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.620330+0000", 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000114516 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.620342+0000", 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1841000000000001e-05 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.620347+0000", 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0669999999999999e-06 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.672 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.620400+0000", 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2868999999999998e-05 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.620421+0000", 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.09e-05 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.668892+0000", 2024-09-21T10:18:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.048470918000000002 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.668982+0000", 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.0717000000000006e-05 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:38679 11.3 e1366/1359 11:c8c66c7a:::benchmark_data_smithi017_304926_object38678:head v 1366'2347, mlcod=1366'2345)", 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:41.624400+0000", 2024-09-21T10:18:50.674 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 8.9670166449999993, 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.042404675000000003, 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.624400+0000", 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.624400+0000", 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.675 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.624399+0000", 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.624426+0000", 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.7421999999999999e-05 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.662518+0000", 2024-09-21T10:18:50.676 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.038091851000000003 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.662531+0000", 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2785e-05 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.662972+0000", 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00044071099999999999 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.662995+0000", 2024-09-21T10:18:50.677 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.3462e-05 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.666722+0000", 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0037271790000000002 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.666805+0000", 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.2490999999999993e-05 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.678 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:38719 11.a e1366/1365 11:545bb440:::benchmark_data_smithi017_304926_object38718:head v 1366'2417, mlcod=1363'726)", 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:41.644790+0000", 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 8.9466265499999995, 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.024265278000000001, 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.644790+0000", 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.679 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.644790+0000", 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.644788+0000", 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.644837+0000", 2024-09-21T10:18:50.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8148e-05 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.662545+0000", 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.017708719000000001 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.662551+0000", 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1199999999999999e-06 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.663920+0000", 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001368576 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.663945+0000", 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4516e-05 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.668986+0000", 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0050418429999999998 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.683 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.683 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.669055+0000", 2024-09-21T10:18:50.683 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8997000000000004e-05 2024-09-21T10:18:50.683 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.683 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.683 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.683 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.683 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:38743 11.3 e1366/1359 11:ca992ae5:::benchmark_data_smithi017_304926_object38742:head v 1366'2354, mlcod=1366'2345)", 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:18:41.662974+0000", 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 8.9284427280000003, 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014820244, 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.662974+0000", 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.684 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.662974+0000", 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.662974+0000", 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.663022+0000", 2024-09-21T10:18:50.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8532999999999998e-05 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.663028+0000", 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2740000000000004e-06 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.663033+0000", 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3619999999999999e-06 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.686 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.674750+0000", 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011717521999999999 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.674801+0000", 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0880000000000001e-05 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.677713+0000", 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0029120040000000002 2024-09-21T10:18:50.687 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:18:41.677794+0000", 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0932000000000004e-05 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:18:50.688 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:18:50.741 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:50.778 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:50.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:50.915 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:50.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:51.017 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:51.059 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:51.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:51.101 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 52675 52659 117.509 106.625 0.0068431 0.00850541 2024-09-21T10:18:51.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:51.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:51.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:51.299 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:51.336 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:51.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:51.477 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:51 smithi134 ceph-mon[90085]: pgmap v2200: 119 pgs: 119 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:18:51.579 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:51.617 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:51.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:51.645 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:51 smithi017 ceph-mon[102913]: pgmap v2200: 119 pgs: 119 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:18:51.645 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:51 smithi017 ceph-mon[105151]: pgmap v2200: 119 pgs: 119 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:18:51.679 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:18:51.719 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:51.758 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:51.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:51.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:51.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:51.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:52.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:52.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:52.100 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 54205 54189 116.754 95.625 0.0146245 0.00855939 2024-09-21T10:18:52.106 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:18:52.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:52.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:52.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:52.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:52.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.353 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:52.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:52.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:52.490 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:52.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:52.633 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:52.735 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:52.775 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:52.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:52.877 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:52.911 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:52.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:52.976 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:18:52.976 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:18:52.976 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-21T10:18:52.976 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T10:18:52.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.1 2024-09-21T10:18:53.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:53.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:53.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:53.100 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 55855 55839 116.3 103.125 0.00873485 0.00859353 2024-09-21T10:18:53.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:53.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:53.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:53.319 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:18:53 smithi017 systemd[1]: Stopping Ceph osd.1 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:18:53.334 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:53.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:53.436 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:53.470 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:53.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:53.572 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:53.610 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:53.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:53.712 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:53.754 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:53.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:53.767 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:18:53.767 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:18:53.803 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:18:53.803 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:53.838 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:18:53.838 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:53.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:53 smithi017 ceph-mon[102913]: 1.15 scrub starts 2024-09-21T10:18:53.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:53 smithi017 ceph-mon[102913]: 1.15 scrub ok 2024-09-21T10:18:53.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:53 smithi017 ceph-mon[102913]: pgmap v2201: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:18:53.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:53 smithi017 ceph-mon[105151]: 1.15 scrub starts 2024-09-21T10:18:53.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:53 smithi017 ceph-mon[105151]: 1.15 scrub ok 2024-09-21T10:18:53.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:53 smithi017 ceph-mon[105151]: pgmap v2201: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:18:53.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:53.873 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:18:53.874 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:53.890 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:53 smithi134 ceph-mon[90085]: 1.15 scrub starts 2024-09-21T10:18:53.890 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:53 smithi134 ceph-mon[90085]: 1.15 scrub ok 2024-09-21T10:18:53.890 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:53 smithi134 ceph-mon[90085]: pgmap v2201: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 6.9 MiB/s, 110 objects/s recovering 2024-09-21T10:18:53.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:53.907 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:18:53.907 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:53.939 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:18:53.939 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:18:53.970 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:18:53.970 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:53.996 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:54.000 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:18:54.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.035 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:54.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:54.101 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 57433 57417 115.729 98.625 0.0170506 0.00863498 2024-09-21T10:18:54.136 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:54.172 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:54.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:54.274 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:54.311 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:54.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:54.378 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:18:54.379 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:18:54.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:54.451 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:54.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:54.553 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:54.582 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:54.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:54.686 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:54.726 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:54.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:54.827 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:54.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.865 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:54.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:54 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2883176926' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:54.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:18:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:18:54] "GET /metrics HTTP/1.1" 200 36760 "" "Prometheus/2.51.0" 2024-09-21T10:18:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:54 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2883176926' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:54.965 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:54 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2883176926' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:55.003 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:55.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:55.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:18:55.101 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 59042 59026 115.255 100.562 0.00657195 0.00867078 2024-09-21T10:18:55.105 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:55.132 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:55.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:55.202 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:18:55.235 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:55.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.285 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:55.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:55.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:55.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:55.538 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:55.589 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:55.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:55.615 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:18:55.625 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[119509]: 2024-09-21T10:18:55.492+0000 7f03a55c9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:18:55.626 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[119509]: 2024-09-21T10:18:55.492+0000 7f03a55c9640 -1 osd.1 1368 *** Got signal Terminated *** 2024-09-21T10:18:55.626 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[119509]: 2024-09-21T10:18:55.492+0000 7f03a55c9640 -1 osd.1 1368 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:18:55.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:55.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:55.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:55.869 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:55.927 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:55 smithi134 ceph-mon[90085]: pgmap v2202: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 2.7 MiB/s, 42 objects/s recovering 2024-09-21T10:18:55.927 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:55.927 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:55.928 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:55 smithi134 ceph-mon[90085]: osd.1 marked itself down and dead 2024-09-21T10:18:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-mon[102913]: pgmap v2202: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 2.7 MiB/s, 42 objects/s recovering 2024-09-21T10:18:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:55.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-mon[102913]: osd.1 marked itself down and dead 2024-09-21T10:18:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-mon[105151]: pgmap v2202: 119 pgs: 119 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 2.7 MiB/s, 42 objects/s recovering 2024-09-21T10:18:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:55.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:55 smithi017 ceph-mon[105151]: osd.1 marked itself down and dead 2024-09-21T10:18:55.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:56.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.015 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:56.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:56.040 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:56.101 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 60482 60466 114.489 90 0.0132725 0.00872906 2024-09-21T10:18:56.116 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:56.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:56.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:56.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:56.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:56.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:56.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:56.436 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:56.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:56.538 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:56.568 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:56.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:56.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:56.706 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:56.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:56.803 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:56 smithi017 ceph-mon[102913]: 1.1d scrub starts 2024-09-21T10:18:56.803 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:56 smithi017 ceph-mon[102913]: 1.1d scrub ok 2024-09-21T10:18:56.803 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:56.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:56 smithi017 ceph-mon[102913]: osdmap e1369: 8 total, 7 up, 7 in 2024-09-21T10:18:56.804 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:56 smithi017 ceph-mon[105151]: 1.1d scrub starts 2024-09-21T10:18:56.804 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:56 smithi017 ceph-mon[105151]: 1.1d scrub ok 2024-09-21T10:18:56.804 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:56.804 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:56 smithi017 ceph-mon[105151]: osdmap e1369: 8 total, 7 up, 7 in 2024-09-21T10:18:56.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:56.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:56.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:56.966 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:56 smithi134 ceph-mon[90085]: 1.1d scrub starts 2024-09-21T10:18:56.966 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:56 smithi134 ceph-mon[90085]: 1.1d scrub ok 2024-09-21T10:18:56.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:56.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:56 smithi134 ceph-mon[90085]: osdmap e1369: 8 total, 7 up, 7 in 2024-09-21T10:18:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.995 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:56.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:57.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:18:57.096 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:57.102 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 62020 62004 113.944 96.125 0.00490933 0.00877153 2024-09-21T10:18:57.131 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:18:56 smithi017 podman[308471]: 2024-09-21 10:18:56.801136486 +0000 UTC m=+2.242547157 container died 228e704d86811dd16f9d70a40fd4d555b290ff83fde7842e8cfcda4090f4493f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T10:18:57.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:57.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:57.237 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:57.273 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:57.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:57.375 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:57.408 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:57.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:57.510 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:57.540 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:57.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:57.642 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:57.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:57.809 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:18:57.810 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:18:57.819 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:57.869 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:57 smithi017 ceph-mon[102913]: 1.5 scrub starts 2024-09-21T10:18:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:57 smithi017 ceph-mon[102913]: 1.5 scrub ok 2024-09-21T10:18:57.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:57 smithi017 ceph-mon[102913]: pgmap v2204: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:18:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:57 smithi017 ceph-mon[105151]: 1.5 scrub starts 2024-09-21T10:18:57.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:57 smithi017 ceph-mon[105151]: 1.5 scrub ok 2024-09-21T10:18:57.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:57 smithi017 ceph-mon[105151]: pgmap v2204: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:18:57.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:57 smithi134 ceph-mon[90085]: 1.5 scrub starts 2024-09-21T10:18:57.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:57 smithi134 ceph-mon[90085]: 1.5 scrub ok 2024-09-21T10:18:57.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:57 smithi134 ceph-mon[90085]: pgmap v2204: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:18:58.011 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:58.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:58.103 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 63509 63493 113.346 93.0625 0.0132073 0.00881807 2024-09-21T10:18:58.113 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:58.145 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:58.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:58.247 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:58.286 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:58.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:58.388 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:58.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:58.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:58.499 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:18:58 smithi017 podman[308471]: 2024-09-21 10:18:58.244941968 +0000 UTC m=+3.686352639 container cleanup 228e704d86811dd16f9d70a40fd4d555b290ff83fde7842e8cfcda4090f4493f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T10:18:58.500 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:18:58 smithi017 bash[308471]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1 2024-09-21T10:18:58.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:58.552 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:58.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:58.606 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:18:58.654 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:58.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.687 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:58.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:58.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:58.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:58.928 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:58.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:18:58 smithi017 podman[308753]: 2024-09-21 10:18:58.497102258 +0000 UTC m=+1.695164579 container remove 228e704d86811dd16f9d70a40fd4d555b290ff83fde7842e8cfcda4090f4493f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-21T10:18:58.974 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:59.025 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:18:59.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:59.103 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 64885 64869 112.586 86 0.0103983 0.00887523 2024-09-21T10:18:59.112 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:59.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:59.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:59.260 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:18:59.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:59.330 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:59.330 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:59.330 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:59 smithi134 ceph-mon[90085]: pgmap v2205: 119 pgs: 119 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s; 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:18:59.331 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:18:59.331 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:18:59.331 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:18:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:59.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:59.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:59.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:59.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[102913]: pgmap v2205: 119 pgs: 119 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s; 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:18:59.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:18:59.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:18:59.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:59.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:59.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:59.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[105151]: pgmap v2205: 119 pgs: 119 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s; 2.4 MiB/s, 38 objects/s recovering 2024-09-21T10:18:59.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:18:59.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:18:59.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:18:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:18:59.403 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:59.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:59.505 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:59.532 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:59.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:59.634 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:59.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.672 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:59.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:59.773 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:59.807 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:59.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:59.818 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:59.908 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:59.937 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:18:59.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:00.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:00.081 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:00.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:00.103 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 66265 66249 111.874 86.25 0.0228248 0.00893335 2024-09-21T10:19:00.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:00.234 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:00.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:00.336 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:00.372 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:00.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:00.374 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:00 smithi017 ceph-mon[102913]: 1.7 scrub starts 2024-09-21T10:19:00.374 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:00 smithi017 ceph-mon[102913]: 1.7 scrub ok 2024-09-21T10:19:00.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:00 smithi017 ceph-mon[105151]: 1.7 scrub starts 2024-09-21T10:19:00.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:00 smithi017 ceph-mon[105151]: 1.7 scrub ok 2024-09-21T10:19:00.473 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:00 smithi134 ceph-mon[90085]: 1.7 scrub starts 2024-09-21T10:19:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:00 smithi134 ceph-mon[90085]: 1.7 scrub ok 2024-09-21T10:19:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:19:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:19:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:19:00.507 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:00.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:00.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:00.642 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:00.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:00.744 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:00.781 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:00.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:00.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:00.920 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:00.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:01.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:01.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:01.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:01.090 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:19:01.103 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 67672 67656 111.243 87.9375 0.017803 0.00898324 2024-09-21T10:19:01.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:01.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.201 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:01.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:01.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:01.334 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[102913]: 1.17 scrub starts 2024-09-21T10:19:01.334 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[102913]: 1.17 scrub ok 2024-09-21T10:19:01.334 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3929908885' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:01.334 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:01.334 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[102913]: pgmap v2206: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-09-21T10:19:01.335 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[105151]: 1.17 scrub starts 2024-09-21T10:19:01.335 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[105151]: 1.17 scrub ok 2024-09-21T10:19:01.335 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3929908885' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:01.335 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:01.335 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:01 smithi017 ceph-mon[105151]: pgmap v2206: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-09-21T10:19:01.335 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:19:01 smithi017 podman[309241]: 2024-09-21 10:19:00.919880203 +0000 UTC m=+0.029590329 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:19:01.335 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:19:01 smithi017 podman[309241]: 2024-09-21 10:19:01.095514606 +0000 UTC m=+0.205224724 container create 08fa01397b418d3c3b542ff416d77f9fe996023761052ed73c78905321f381f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T10:19:01.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:01.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:01.439 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:01.457 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:01 smithi134 ceph-mon[90085]: 1.17 scrub starts 2024-09-21T10:19:01.457 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:01 smithi134 ceph-mon[90085]: 1.17 scrub ok 2024-09-21T10:19:01.458 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:01 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3929908885' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:01.458 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:01 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:01.458 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:01 smithi134 ceph-mon[90085]: pgmap v2206: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-09-21T10:19:01.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:01.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:01.586 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:01.618 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:19:01.618 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:19:01.632 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:01.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:01.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:01.774 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:01.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:01.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:01.914 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:19:01 smithi017 podman[309241]: 2024-09-21 10:19:01.567743214 +0000 UTC m=+0.677453347 container init 08fa01397b418d3c3b542ff416d77f9fe996023761052ed73c78905321f381f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T10:19:01.914 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:19:01 smithi017 podman[309241]: 2024-09-21 10:19:01.580323574 +0000 UTC m=+0.690033686 container start 08fa01397b418d3c3b542ff416d77f9fe996023761052ed73c78905321f381f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:19:01.914 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:19:01 smithi017 podman[309241]: 2024-09-21 10:19:01.910853271 +0000 UTC m=+1.020563385 container attach 08fa01397b418d3c3b542ff416d77f9fe996023761052ed73c78905321f381f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T10:19:01.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:02.015 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:02.047 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:02.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 69111 69095 110.697 89.9375 0.00749455 0.00902848 2024-09-21T10:19:02.149 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:02.185 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:02.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:02.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:02 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:02.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:02 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:02.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:02 smithi017 ceph-mon[105151]: osdmap e1370: 8 total, 7 up, 7 in 2024-09-21T10:19:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:02 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:02 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:02 smithi017 ceph-mon[102913]: osdmap e1370: 8 total, 7 up, 7 in 2024-09-21T10:19:02.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:02.320 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:02.330 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:02 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:02.330 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:02 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:02.330 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:02 smithi134 ceph-mon[90085]: osdmap e1370: 8 total, 7 up, 7 in 2024-09-21T10:19:02.374 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:19:02.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:02.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:02.559 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:02.606 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:02.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:02.708 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:02.761 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:02.827 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:19:02.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:02.896 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:02.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:02.997 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:03.025 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:03.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:03.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:19:03.102917+0000 min lat: 0.00118998 max lat: 2.30472 avg lat: 0.00908028 2024-09-21T10:19:03.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:19:03.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 70478 70462 110.065 85.4375 0.0126131 0.00908028 2024-09-21T10:19:03.126 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:03.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:03.238 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:03 smithi134 ceph-mon[90085]: pgmap v2208: 119 pgs: 119 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-21T10:19:03.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:03.293 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:03.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:03.395 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:03.433 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:03.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:03.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:03 smithi017 ceph-mon[102913]: pgmap v2208: 119 pgs: 119 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-21T10:19:03.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:03 smithi017 ceph-mon[105151]: pgmap v2208: 119 pgs: 119 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-21T10:19:03.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:03.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:03.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:03.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:03.713 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:03.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:03.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:03.848 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:03.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:03.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:03.996 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:04.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:04.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 71827 71811 109.437 84.3125 0.00830528 0.00913206 2024-09-21T10:19:04.138 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:04.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:04.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:04.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:04.378 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:04.414 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:04.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:04.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:04.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:04.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:04.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:04.687 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:04.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:04.789 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:04.820 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:04.922 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:04.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:19:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:19:04] "GET /metrics HTTP/1.1" 200 36755 "" "Prometheus/2.51.0" 2024-09-21T10:19:04.958 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:04.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:05.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:05.091 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:05.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:05.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 73236 73220 108.928 88.0625 0.0138392 0.00917412 2024-09-21T10:19:05.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:05.235 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:05.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:05.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:05.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:05.478 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:05.509 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:05.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:05.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:05.628 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:19:05.629 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:05.633 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-21T10:19:05.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.656 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:05.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:05.672 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:19:05.673 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:05 smithi017 ceph-mon[102913]: pgmap v2209: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-21T10:19:05.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:05 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1439330829' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:05 smithi017 ceph-mon[105151]: pgmap v2209: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-21T10:19:05.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:05 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1439330829' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:05.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:19:05 smithi017 podman[309241]: 2024-09-21 10:19:05.549020724 +0000 UTC m=+4.658730838 container died 08fa01397b418d3c3b542ff416d77f9fe996023761052ed73c78905321f381f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:19:05.707 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:19:05.707 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:05.738 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:19:05.738 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:05.756 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:05.769 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:19:05.769 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:19:05.785 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:05.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:05.801 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:19:05.801 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:05.839 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:19:05.839 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:05.886 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:05.891 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:05 smithi134 ceph-mon[90085]: pgmap v2209: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-21T10:19:05.891 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:05 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1439330829' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:05.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:19:05.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:05.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:06.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.074 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:06.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:06.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 74493 74477 108.222 78.5625 0.0117129 0.00923477 2024-09-21T10:19:06.174 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:06.199 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:06.301 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:06.335 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:06.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:06.437 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:06.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.475 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:06.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:06.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:06.612 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:06.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:06.714 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:06.746 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:06.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:06.848 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:06.871 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:06.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:06.911 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:19:06.911 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:19:06.911 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.911 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:74383 11.e 11:783268e1:::benchmark_data_smithi017_304926_object74382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1370)", 2024-09-21T10:19:06.911 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:19:06.014040+0000", 2024-09-21T10:19:06.911 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0045963239999999997, 2024-09-21T10:19:06.911 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0046211919999999997, 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 74383 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.014040+0000", 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.912 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.014040+0000", 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.014038+0000", 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.014085+0000", 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6419999999999999e-05 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.913 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.014105+0000", 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0707999999999999e-05 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.014111+0000", 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7760000000000003e-06 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015111+0000", 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00099952599999999993 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.914 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015249+0000", 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013794999999999999 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015385+0000", 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013615599999999999 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.018463+0000", 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0030787520000000001 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.915 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.018471+0000", 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9829999999999996e-06 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:74386 11.d e1370/1359 11:b9822821:::benchmark_data_smithi017_304926_object74385:head v 1370'4596, mlcod=1370'4594)", 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:19:06.015610+0000", 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0030267060000000001, 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0031140019999999998, 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:06.916 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015610+0000", 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015610+0000", 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015609+0000", 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:19:06.917 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015640+0000", 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.1275999999999997e-05 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015654+0000", 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3689999999999999e-05 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015659+0000", 2024-09-21T10:19:06.918 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2020000000000003e-06 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015701+0000", 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1730999999999997e-05 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015719+0000", 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7889000000000001e-05 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.919 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:74387 11.7 11:eff52dc9:::benchmark_data_smithi017_304926_object74386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1370)", 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:19:06.015800+0000", 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0028359309999999999, 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002958544, 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 74387 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:06.920 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015800+0000", 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015800+0000", 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015799+0000", 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:19:06.921 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015845+0000", 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5812000000000001e-05 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015857+0000", 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2235999999999999e-05 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.922 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015862+0000", 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1089999999999997e-06 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.015909+0000", 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6322000000000001e-05 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.016015+0000", 2024-09-21T10:19:06.923 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010653 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,7", 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:19:06.016133+0000", 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000118256 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:06.924 INFO:teuthology.orchestra.run.smithi134.stdout: ], 2024-09-21T10:19:06.925 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 3 2024-09-21T10:19:06.925 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:19:06.973 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:07.004 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:07.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 75718 75702 107.502 76.5625 0.0155483 0.00929679 2024-09-21T10:19:07.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:07.140 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:07.242 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:07.280 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:07.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:07.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:07.418 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:07.520 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:07.562 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:07.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:07.664 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:07.696 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:07.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:07.798 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:07.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:07.938 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:07.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:07 smithi017 ceph-mon[102913]: pgmap v2210: 119 pgs: 119 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-21T10:19:07.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:07 smithi017 ceph-mon[105151]: pgmap v2210: 119 pgs: 119 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-21T10:19:07.973 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:07.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:07 smithi134 ceph-mon[90085]: pgmap v2210: 119 pgs: 119 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-21T10:19:08.075 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:08.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 77019 77003 106.92 81.3125 0.0187323 0.00934786 2024-09-21T10:19:08.115 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:08.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:08.217 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:08.254 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:08.356 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:08.388 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:08.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:08.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:08.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:08.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:08.614 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:08.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.650 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:08.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:08.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:08.787 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:08.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:08.888 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:08.925 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:08.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:09.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:09.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.069 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:09.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:09.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 78208 78192 106.211 74.3125 0.00810396 0.00941006 2024-09-21T10:19:09.170 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:09.209 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:09.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:09.254 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:19:09.313 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:09.370 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:09.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:09.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:09.509 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:09.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:09.611 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:09.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.659 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:09.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:09.748 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:19:09.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:09.798 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:09.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:09.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:09.928 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:09.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:09.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:09 smithi017 ceph-mon[102913]: pgmap v2211: 119 pgs: 119 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-21T10:19:09.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:09 smithi017 ceph-mon[105151]: pgmap v2211: 119 pgs: 119 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-21T10:19:09.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:09 smithi134 ceph-mon[90085]: pgmap v2211: 119 pgs: 119 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-21T10:19:10.029 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:10.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.068 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:10.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:10.104 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 79220 79204 105.297 63.25 0.00350576 0.00948978 2024-09-21T10:19:10.169 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:10.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:10.306 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:10.342 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:10.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:10.482 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:10.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:19:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:19:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:19:10.584 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:10.622 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:10.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:10.725 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:10.766 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:10.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:10.868 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:10.901 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:10.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:10.920 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:19:10.920 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:19:11.003 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:11.049 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:11.105 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 80400 80384 104.64 73.75 0.0331291 0.00954747 2024-09-21T10:19:11.151 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:11.191 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:11.293 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:11.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:11.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:11.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:11.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:11.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:11.498 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:19:11.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:11.499 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:11 smithi017 ceph-mon[102913]: pgmap v2212: 119 pgs: 119 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-21T10:19:11.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:19:11.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:11.499 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:11 smithi017 ceph-mon[105151]: pgmap v2212: 119 pgs: 119 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-21T10:19:11.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.1.service' 2024-09-21T10:19:11.530 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:11.588 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:11.619 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:11.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:11.624 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:19:11.624 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:11.624 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:11 smithi134 ceph-mon[90085]: pgmap v2212: 119 pgs: 119 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-21T10:19:11.665 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:19:11.665 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T10:19:11.721 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:11.752 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:11.853 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:11.886 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:11.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:12.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:12.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:12.105 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 81622 81606 104.063 76.375 0.0105047 0.00960276 2024-09-21T10:19:12.134 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:12.168 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:12.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:12.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:12.312 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:12.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:12.414 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:12.455 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:12.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:12.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:12.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:12.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:12.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:12.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:12.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:12.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:12.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:12.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:12.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:19:12.973 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:13.013 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:13.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:13.105 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 82800 82784 103.454 73.625 0.0348291 0.0096606 2024-09-21T10:19:13.115 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:13.147 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:13.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:13.248 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:13.280 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:13.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:13.382 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.417 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:13.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:13.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:13.556 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:13.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:13.658 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:13.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.696 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:13.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:13.767 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:13.797 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:13.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.833 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:13.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:13.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:13 smithi017 ceph-mon[102913]: pgmap v2213: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:19:13.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:13 smithi017 ceph-mon[105151]: pgmap v2213: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:19:13.934 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:13.958 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:13 smithi134 ceph-mon[90085]: pgmap v2213: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:19:14.056 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:19:14.056 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:19:14.056 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:19:14.056 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:19:14.056 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:19:14.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:14.089 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:14.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:14.105 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 83958 83942 102.845 72.375 0.0177264 0.00971779 2024-09-21T10:19:14.190 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.223 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:14.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:14.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:14.362 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:14.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:14.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:14.502 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:14.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:14.604 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.641 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:14.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:14.650 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:19:14.721 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/679827302' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:14.721 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:14 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:14.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:14.766 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:19:14.778 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:14.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:14.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.917 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:14.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:14.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/679827302' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:14.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:14 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/679827302' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:14 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:14.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:19:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:19:14] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-09-21T10:19:15.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:15.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:15.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:15.105 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 85237 85221 102.404 79.9375 0.00493175 0.00976043 2024-09-21T10:19:15.160 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:15.209 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:19:15.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.215 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:15.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:15.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:15.360 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:15.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:15.462 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:15.498 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:15.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:15.600 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.642 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:15.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:15 smithi017 ceph-mon[102913]: pgmap v2214: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:19:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:15 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:15.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:15 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:15.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:15 smithi017 ceph-mon[102913]: osdmap e1371: 8 total, 7 up, 7 in 2024-09-21T10:19:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:15 smithi017 ceph-mon[105151]: pgmap v2214: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:19:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:15 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:15 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:15.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:15 smithi017 ceph-mon[105151]: osdmap e1371: 8 total, 7 up, 7 in 2024-09-21T10:19:15.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:15.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:15.883 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:15.920 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:15.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:15 smithi134 ceph-mon[90085]: pgmap v2214: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:19:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:15 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:15 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:15 smithi134 ceph-mon[90085]: osdmap e1371: 8 total, 7 up, 7 in 2024-09-21T10:19:16.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:16.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:16.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:16.105 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 86526 86510 101.992 80.5625 0.0103623 0.00979979 2024-09-21T10:19:16.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:16.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:16.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:16.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:16.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:16.445 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:16.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:16.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:16.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.624 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:16.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:16.666 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [1] live_osds: [0, 2, 3, 5, 6, 7, 4] 2024-09-21T10:19:16.666 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:19:16.666 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-09-21T10:19:16.666 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:19:16.666 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:19:16.667 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-09-21T10:19:16.725 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:16.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.768 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:16.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:16.869 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:16.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:16.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:17.007 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:17.042 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:17.106 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 87728 87712 101.494 75.125 0.0226438 0.00984726 2024-09-21T10:19:17.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:17.178 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:17.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:17.280 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:17.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:17.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:17.431 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:57282 9.e 9:7d14f317:::benchmark_data_smithi017_237924_object57281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:09:24.450175+0000", 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 592.98973652300003, 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1387745909999998, 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.448 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 57282 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450175+0000", 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.449 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450175+0000", 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450173+0000", 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450368+0000", 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019504900000000001 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.450 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450404+0000", 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5596000000000003e-05 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450416+0000", 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2003e-05 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450508+0000", 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1644999999999994e-05 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.451 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:24.450524+0000", 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6843000000000001e-05 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:25.429753+0000", 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97922897900000005 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:25.429766+0000", 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2617e-05 2024-09-21T10:19:17.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.570352+0000", 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.140585728 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.571070+0000", 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00071844699999999999 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1", 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.571211+0000", 2024-09-21T10:19:17.453 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000140648 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.573315+0000", 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002104201 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.580792+0000", 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0074773510000000001 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.454 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.587425+0000", 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0066328389999999997 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.587470+0000", 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4362e-05 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:09:26.588950+0000", 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001480386 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.455 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:173513 9.e 9:71c17054:::benchmark_data_smithi017_237924_object80509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:24.724062+0000", 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 412.71584988299998, 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4376691149999998, 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 173513 2024-09-21T10:19:17.456 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724062+0000", 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724062+0000", 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724060+0000", 2024-09-21T10:19:17.457 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724071+0000", 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1567e-05 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724093+0000", 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1767999999999999e-05 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724101+0000", 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.4549999999999995e-06 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724163+0000", 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1303000000000006e-05 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724176+0000", 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2779000000000001e-05 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.459 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689519+0000", 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96534338099999994 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689531+0000", 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1839e-05 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.123518+0000", 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4339874020000001 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.460 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.123669+0000", 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015062300000000001 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1,7", 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.123813+0000", 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000144487 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.134005+0000", 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010191817000000001 2024-09-21T10:19:17.461 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.134077+0000", 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1769999999999999e-05 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.147373+0000", 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013295968999999999 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161450+0000", 2024-09-21T10:19:17.462 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014077265 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161485+0000", 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5074000000000003e-05 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161731+0000", 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000246016 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.463 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62006 10.10 10:0a3d3700:::benchmark_data_smithi017_268344_object62005:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.010013+0000", 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.429899256, 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.063320321, 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62006 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.464 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010013+0000", 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010013+0000", 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010011+0000", 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.465 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010189+0000", 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017731600000000001 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010201+0000", 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2072000000000001e-05 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010207+0000", 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7780000000000004e-06 2024-09-21T10:19:17.466 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010266+0000", 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8933e-05 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010274+0000", 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0260000000000007e-06 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.467 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997736+0000", 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.98746292599999996 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997747+0000", 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0535e-05 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.057043+0000", 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0592958029999999 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.468 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.057156+0000", 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011309 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.057287+0000", 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000131483 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060730+0000", 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0034428140000000002 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.469 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072326+0000", 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011595764999999999 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072522+0000", 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019573 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072558+0000", 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6087999999999997e-05 2024-09-21T10:19:17.470 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073333+0000", 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00077519299999999998 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62045 10.10 10:09934317:::benchmark_data_smithi017_268344_object62044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.028735+0000", 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.41117724499998, 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.045632221, 2024-09-21T10:19:17.471 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62045 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028735+0000", 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028735+0000", 2024-09-21T10:19:17.472 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028734+0000", 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028791+0000", 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7380999999999998e-05 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028802+0000", 2024-09-21T10:19:17.473 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0868e-05 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028807+0000", 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9620000000000001e-06 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028858+0000", 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0646999999999998e-05 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.474 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.475 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028866+0000", 2024-09-21T10:19:17.475 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1000000000000004e-06 2024-09-21T10:19:17.475 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.475 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997771+0000", 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96890569500000001 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997778+0000", 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0460000000000002e-06 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.057972+0000", 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.060194053 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.476 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.058100+0000", 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000128169 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.058264+0000", 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000164649 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.061464+0000", 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0031997639999999999 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.477 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072356+0000", 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010891616 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073623+0000", 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001267114 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073680+0000", 2024-09-21T10:19:17.478 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6700999999999998e-05 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.074367+0000", 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00068730600000000005 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62049 10.10 10:0ccbb6b8:::benchmark_data_smithi017_268344_object62048:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.028883+0000", 2024-09-21T10:19:17.479 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.41102846299998, 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.046507053, 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62049 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028883+0000", 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.480 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028883+0000", 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028883+0000", 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028947+0000", 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4085999999999996e-05 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.481 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028955+0000", 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9500000000000001e-06 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028959+0000", 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3000000000000003e-06 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.029019+0000", 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9722000000000002e-05 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.482 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.029025+0000", 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7610000000000004e-06 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997799+0000", 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96877391599999996 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997805+0000", 2024-09-21T10:19:17.483 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2349999999999996e-06 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.058878+0000", 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.061073696 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.059013+0000", 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013482699999999999 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.059166+0000", 2024-09-21T10:19:17.484 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015248900000000001 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.062458+0000", 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032921339999999999 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072397+0000", 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0099391219999999999 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.074478+0000", 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020814050000000001 2024-09-21T10:19:17.485 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.074526+0000", 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7168e-05 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.075391+0000", 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00086493200000000001 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.486 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62065 10.10 10:0ecadf09:::benchmark_data_smithi017_268344_object62064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.036833+0000", 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.40307858, 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.039298166, 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62065 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.487 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036833+0000", 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036833+0000", 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036832+0000", 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.488 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036959+0000", 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012738300000000001 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036970+0000", 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1007e-05 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036976+0000", 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4369999999999998e-06 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.489 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.037047+0000", 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1067000000000002e-05 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.037056+0000", 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6930000000000002e-06 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997826+0000", 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96076999299999999 2024-09-21T10:19:17.490 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997832+0000", 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9649999999999996e-06 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.059887+0000", 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.062054651 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060031+0000", 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000144122 2024-09-21T10:19:17.491 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060202+0000", 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000170421 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064305+0000", 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0041030090000000003 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072427+0000", 2024-09-21T10:19:17.492 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0081220770000000001 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.075436+0000", 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0030096419999999999 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.075474+0000", 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7456999999999997e-05 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.076132+0000", 2024-09-21T10:19:17.493 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000657735 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62079 10.10 10:08d2baad:::benchmark_data_smithi017_268344_object62078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.042330+0000", 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.39758237699999, 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0347306949999999, 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.494 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62079 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042330+0000", 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042330+0000", 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.495 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042328+0000", 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042375+0000", 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7429999999999998e-05 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042388+0000", 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2347000000000001e-05 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.496 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042393+0000", 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.921e-06 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042453+0000", 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0038999999999999e-05 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042462+0000", 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6180000000000001e-06 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.497 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997855+0000", 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95539260199999998 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997861+0000", 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1670000000000004e-06 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060760+0000", 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0628989769999999 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.498 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060892+0000", 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013164200000000001 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060987+0000", 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5452999999999996e-05 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064317+0000", 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0033294599999999998 2024-09-21T10:19:17.499 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072451+0000", 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0081343519999999992 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.076247+0000", 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0037959030000000002 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.076302+0000", 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5504000000000003e-05 2024-09-21T10:19:17.500 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077060+0000", 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000757794 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62118 10.10 10:0a3d075f:::benchmark_data_smithi017_268344_object62117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.066075+0000", 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.37383643599998, 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0117501670000002, 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.501 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62118 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066075+0000", 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.502 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066075+0000", 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066074+0000", 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066104+0000", 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9998999999999999e-05 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.503 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066114+0000", 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.3149999999999998e-06 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066118+0000", 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5990000000000004e-06 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066173+0000", 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4737999999999997e-05 2024-09-21T10:19:17.504 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066181+0000", 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8490000000000006e-06 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997893+0000", 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.93171239400000005 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997900+0000", 2024-09-21T10:19:17.505 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2829999999999998e-06 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.061493+0000", 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.063593528 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.061625+0000", 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132127 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.506 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.061796+0000", 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017107900000000001 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.065992+0000", 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0041953980000000004 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072475+0000", 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.006483707 2024-09-21T10:19:17.507 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077124+0000", 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0046483269999999998 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077183+0000", 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.897e-05 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.508 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077826+0000", 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00064299799999999999 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62139 10.10 10:0ba8e010:::benchmark_data_smithi017_268344_object62138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.077457+0000", 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.362455377, 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0014778230000001, 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.509 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62139 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077457+0000", 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077457+0000", 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.510 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077456+0000", 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077533+0000", 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.7880000000000007e-05 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077547+0000", 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.401e-05 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.511 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077554+0000", 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2280000000000003e-06 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077619+0000", 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4863000000000006e-05 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077628+0000", 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6630000000000005e-06 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.512 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997923+0000", 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.92029435400000004 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997929+0000", 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1510000000000001e-06 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.062497+0000", 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.064568411 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.513 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.062619+0000", 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012207599999999999 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.062783+0000", 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000164072 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066011+0000", 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032278200000000002 2024-09-21T10:19:17.514 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.515 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:19:17.515 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.515 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.515 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072503+0000", 2024-09-21T10:19:17.515 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0064917580000000003 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.078076+0000", 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0055731849999999996 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.078142+0000", 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5740000000000004e-05 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.516 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.078934+0000", 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00079261100000000003 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62252 10.10 10:0bdce5c9:::benchmark_data_smithi017_268344_object62251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.141789+0000", 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.29812305299998, 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9393023760000001, 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.517 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62252 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141789+0000", 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141789+0000", 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.518 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141788+0000", 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141836+0000", 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7958999999999999e-05 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141851+0000", 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5679e-05 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.519 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141857+0000", 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6130000000000003e-06 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141918+0000", 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0735e-05 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141928+0000", 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0206e-05 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.520 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997949+0000", 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.85602108099999996 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997955+0000", 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0800000000000002e-06 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.063480+0000", 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0655249870000001 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.521 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.063614+0000", 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000133984 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.063768+0000", 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015351300000000001 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067623+0000", 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0038550450000000001 2024-09-21T10:19:17.522 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073393+0000", 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0057700340000000003 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.080053+0000", 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0066600729999999999 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.080109+0000", 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6363000000000002e-05 2024-09-21T10:19:17.523 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.081091+0000", 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00098213399999999996 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62285 10.10 10:0b189abc:::benchmark_data_smithi017_268344_object62284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.159865+0000", 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.28004715399999, 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9218552840000001, 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.524 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62285 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159865+0000", 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159865+0000", 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.525 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159863+0000", 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159921+0000", 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7759e-05 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159935+0000", 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.401e-05 2024-09-21T10:19:17.526 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159941+0000", 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1940000000000003e-06 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159998+0000", 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.715e-05 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.527 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.160005+0000", 2024-09-21T10:19:17.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2089999999999999e-06 2024-09-21T10:19:17.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.528 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.528 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997977+0000", 2024-09-21T10:19:17.528 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.83797192300000001 2024-09-21T10:19:17.528 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.528 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.529 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:17.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:17.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997984+0000", 2024-09-21T10:19:17.530 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1310000000000003e-06 2024-09-21T10:19:17.530 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.530 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.530 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.530 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064350+0000", 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.066366404 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064432+0000", 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1909000000000005e-05 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064526+0000", 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.4266999999999997e-05 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.531 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067633+0000", 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0031064310000000002 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073451+0000", 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0058181439999999999 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.081135+0000", 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0076844399999999998 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.532 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.081172+0000", 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6900000000000002e-05 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.081720+0000", 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00054806099999999997 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.533 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62298 10.10 10:0dc9853e:::benchmark_data_smithi017_268344_object62297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.167360+0000", 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.27255208100001, 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9187170839999999, 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62298 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.534 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167360+0000", 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167360+0000", 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167355+0000", 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999948 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.535 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167433+0000", 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8127000000000006e-05 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167446+0000", 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2996000000000001e-05 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167452+0000", 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9039999999999997e-06 2024-09-21T10:19:17.536 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167496+0000", 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4366999999999997e-05 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167503+0000", 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8900000000000001e-06 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998005+0000", 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.83050180600000001 2024-09-21T10:19:17.537 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998010+0000", 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4929999999999998e-06 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.065213+0000", 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0672026370000001 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.065334+0000", 2024-09-21T10:19:17.538 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012127499999999999 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.065466+0000", 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013228900000000001 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069466+0000", 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0039995300000000003 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.539 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073499+0000", 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0040332170000000004 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.084549+0000", 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011049892 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.084604+0000", 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4500000000000003e-05 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.540 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.086077+0000", 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0014733599999999999 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62326 10.10 10:09ee21e6:::benchmark_data_smithi017_268344_object62325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.184514+0000", 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.25539827400002, 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9027839289999999, 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.541 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62326 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184514+0000", 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184514+0000", 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.542 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184513+0000", 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184562+0000", 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9391000000000002e-05 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184572+0000", 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0069e-05 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.543 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184576+0000", 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4510000000000002e-06 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184618+0000", 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1137000000000003e-05 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184623+0000", 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6400000000000002e-06 2024-09-21T10:19:17.544 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998032+0000", 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.813408833 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998038+0000", 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0700000000000003e-06 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066048+0000", 2024-09-21T10:19:17.545 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0680093959999999 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066180+0000", 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132409 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066359+0000", 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017886399999999999 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.546 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069474+0000", 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0031148870000000002 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073545+0000", 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0040713709999999998 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.086141+0000", 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.012596424 2024-09-21T10:19:17.547 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.086217+0000", 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5721000000000002e-05 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.087298+0000", 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0010804 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.548 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62337 10.10 10:0ed49e86:::benchmark_data_smithi017_268344_object62336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.191812+0000", 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.24810032300002, 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.89657432, 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62337 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.549 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191812+0000", 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191812+0000", 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191811+0000", 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.550 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191863+0000", 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2701999999999998e-05 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191879+0000", 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5159e-05 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191883+0000", 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4700000000000004e-06 2024-09-21T10:19:17.551 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191926+0000", 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2605999999999998e-05 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191932+0000", 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2020000000000001e-06 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998059+0000", 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.80612681600000002 2024-09-21T10:19:17.552 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998064+0000", 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5809999999999996e-06 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066911+0000", 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.068846288 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067004+0000", 2024-09-21T10:19:17.553 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.3441000000000002e-05 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067107+0000", 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000103467 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.071083+0000", 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0039751409999999997 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.554 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073578+0000", 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0024952199999999998 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.087347+0000", 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013768786 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.087406+0000", 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9669000000000001e-05 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.555 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.088386+0000", 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00097964100000000006 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62347 10.10 10:081c52cb:::benchmark_data_smithi017_268344_object62346:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.198848+0000", 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.24106362499998, 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8906097960000001, 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.556 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62347 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198848+0000", 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198848+0000", 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.557 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198848+0000", 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198886+0000", 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8263999999999998e-05 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198892+0000", 2024-09-21T10:19:17.558 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5369999999999997e-06 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198895+0000", 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9189999999999999e-06 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198940+0000", 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5080999999999997e-05 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198945+0000", 2024-09-21T10:19:17.559 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5709999999999997e-06 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998087+0000", 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.79914121199999999 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998092+0000", 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8810000000000001e-06 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.560 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067678+0000", 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0695851270000001 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067813+0000", 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013508699999999999 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067970+0000", 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015738500000000001 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.561 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072284+0000", 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0043139349999999996 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.074428+0000", 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021445000000000001 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.088430+0000", 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014001764 2024-09-21T10:19:17.562 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.088518+0000", 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7372000000000003e-05 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.089458+0000", 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00094053099999999998 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.563 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62353 10.10 10:0b0475b7:::benchmark_data_smithi017_268344_object62352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.203016+0000", 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.23689581899998, 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8887158500000001, 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62353 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.564 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203016+0000", 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203016+0000", 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203015+0000", 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.565 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203066+0000", 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0473000000000003e-05 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203073+0000", 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8900000000000007e-06 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203077+0000", 2024-09-21T10:19:17.566 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.365e-06 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203131+0000", 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4598999999999997e-05 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203141+0000", 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5100000000000004e-06 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998137+0000", 2024-09-21T10:19:17.567 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.79499567100000001 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998146+0000", 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.9789999999999999e-06 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.068505+0000", 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.070359587 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.068634+0000", 2024-09-21T10:19:17.568 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000128956 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.068765+0000", 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013051800000000001 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072361+0000", 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003596821 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.569 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.076196+0000", 2024-09-21T10:19:17.570 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003834414 2024-09-21T10:19:17.570 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.570 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.570 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.570 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.090509+0000", 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014313090000000001 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.090560+0000", 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.151e-05 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.091732+0000", 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001171524 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.571 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62385 10.10 10:0e102290:::benchmark_data_smithi017_268344_object62384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.241726+0000", 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 306.19818607799999, 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.850994201, 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62385 2024-09-21T10:19:17.572 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241726+0000", 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241726+0000", 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.573 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241725+0000", 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241778+0000", 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3606000000000001e-05 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241787+0000", 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1309999999999995e-06 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.574 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241792+0000", 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.126e-06 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241832+0000", 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0929000000000001e-05 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241838+0000", 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.519e-06 2024-09-21T10:19:17.575 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998181+0000", 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.75634329300000003 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998186+0000", 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.498e-06 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069517+0000", 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0713316340000001 2024-09-21T10:19:17.576 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069650+0000", 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132099 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069825+0000", 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000175415 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072367+0000", 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0025424760000000001 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.577 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077894+0000", 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0055264299999999997 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.091783+0000", 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013889225 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.091836+0000", 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3300000000000001e-05 2024-09-21T10:19:17.578 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.092720+0000", 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00088369999999999996 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62487 10.10 10:0b19b1b2:::benchmark_data_smithi017_268344_object62486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.462833+0000", 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 305.977079274, 2024-09-21T10:19:17.579 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.630773469, 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62487 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462833+0000", 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.580 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462833+0000", 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462828+0000", 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999952 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462882+0000", 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4044e-05 2024-09-21T10:19:17.581 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462892+0000", 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8230000000000006e-06 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462897+0000", 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9450000000000001e-06 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462942+0000", 2024-09-21T10:19:17.582 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5859999999999998e-05 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462949+0000", 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7689999999999999e-06 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998215+0000", 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.53526555300000001 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.583 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998222+0000", 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2099999999999996e-06 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.070360+0000", 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.072138456 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.070487+0000", 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012635700000000001 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.584 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.070612+0000", 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012507600000000001 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072371+0000", 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00175961 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077947+0000", 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0055752609999999998 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.585 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.092779+0000", 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014832675999999999 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.092836+0000", 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6518000000000003e-05 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.093606+0000", 2024-09-21T10:19:17.586 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000770216 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:107632 10.d 10:bae075aa:::benchmark_data_smithi017_268344_object107631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1309)", 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:53.648790+0000", 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 263.79112186499998, 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0769366060000001, 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.587 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 107632 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.648790+0000", 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.648790+0000", 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.588 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.648788+0000", 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.650952+0000", 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021638780000000002 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.650970+0000", 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7306e-05 2024-09-21T10:19:17.589 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.650976+0000", 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1099999999999999e-06 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.651039+0000", 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3382999999999999e-05 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.590 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.651048+0000", 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8189999999999998e-06 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.666773+0000", 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0157253959999999 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for missing object", 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.666919+0000", 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000145599 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.591 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.696188+0000", 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029269046 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.696233+0000", 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4929999999999998e-05 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.725635+0000", 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029402333999999999 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.592 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.725727+0000", 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1385999999999995e-05 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:197548 10.2 10:40a3616b:::benchmark_data_smithi017_268344_object71617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:16:26.328429+0000", 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 171.11148255500001, 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.330358935, 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:19:17.593 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 197548 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328429+0000", 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328429+0000", 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328429+0000", 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328432+0000", 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.061e-06 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:19:17.595 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328445+0000", 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3256e-05 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328450+0000", 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5279999999999997e-06 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328484+0000", 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4725999999999999e-05 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.596 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328490+0000", 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6239999999999999e-06 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:27.306205+0000", 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97771489 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:27.306215+0000", 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0118e-05 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.597 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.639779+0000", 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.333564252 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.639912+0000", 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132662 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.640308+0000", 2024-09-21T10:19:17.598 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00039639500000000002 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.652812+0000", 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.012503284 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.657104+0000", 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0042921979999999997 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:19:17.599 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.657983+0000", 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00087909900000000005 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.658020+0000", 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7431999999999997e-05 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.658788+0000", 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00076788599999999996 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.600 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:19:17.601 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:19:17.601 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:19:17.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:17.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:17.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:17.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:17.807 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:17.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:17.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:17.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:17 smithi017 ceph-mon[102913]: pgmap v2216: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-21T10:19:17.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:17 smithi017 ceph-mon[105151]: pgmap v2216: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-21T10:19:17.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.952 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:17.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:17 smithi134 ceph-mon[90085]: pgmap v2216: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-21T10:19:18.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:19:18.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:18.105 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 88897 88881 100.977 73.0625 0.0169036 0.00989799 2024-09-21T10:19:18.113 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:19:18.113 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:18.119 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-21T10:19:18.142 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:18.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:18.160 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:19:18.160 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:18.197 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:19:18.198 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:18.231 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:19:18.231 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:18.244 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:18.266 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:19:18.267 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:19:18.275 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:18.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:18.304 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:19:18.305 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:18.335 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:19:18.335 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:18.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:18.411 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:18.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:18.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:18.547 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:18.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:18.649 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:18.682 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:18.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:18.784 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:18.831 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:18.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:18.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:18 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2738696784' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:18.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:18 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2738696784' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:18.963 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:18.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:18 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2738696784' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:19.065 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:19.096 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:19.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:19.105 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 90086 90070 100.501 74.3125 0.011199 0.0099453 2024-09-21T10:19:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:19:19.173 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:19:19.198 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:19.233 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:19.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:19.247 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:19.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:19.377 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:19.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:19.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:19.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:19.615 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:19.647 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:19.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:19.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:19.775 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:19.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:19.877 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:19.915 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:19.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:19 smithi017 ceph-mon[102913]: pgmap v2217: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:19:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:19 smithi017 ceph-mon[105151]: pgmap v2217: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:19:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:19 smithi134 ceph-mon[90085]: pgmap v2217: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:19:20.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:20.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:20.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:20.107 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 91210 91194 99.9684 70.25 0.013237 0.00999822 2024-09-21T10:19:20.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:20.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:20.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:20.299 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:20.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.322 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:20.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:20.423 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:20.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.450 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:20.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:19:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:19:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:19:20.552 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:20.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.593 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:20.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:20.694 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:20.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.730 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:20.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:20.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:19:20.830 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:20.862 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:20.964 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:20.997 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:20.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:21.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:21.106 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 92226 92210 99.3396 63.5 0.013462 0.0100611 2024-09-21T10:19:21.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:21.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:21.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:19:21.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:21.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:21.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:21.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:21.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:21.493 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:21.566 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:21.600 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:21.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:21.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.741 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:21.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:21.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:21.883 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:21.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:21 smithi017 ceph-mon[102913]: pgmap v2218: 119 pgs: 119 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-21T10:19:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:21 smithi017 ceph-mon[105151]: pgmap v2218: 119 pgs: 119 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-21T10:19:21.979 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:21 smithi134 ceph-mon[90085]: pgmap v2218: 119 pgs: 119 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-21T10:19:21.984 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:22.024 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:22.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:22.107 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 93372 93356 98.8699 71.625 0.0154177 0.0101095 2024-09-21T10:19:22.125 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:22.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:22.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:22.260 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:22.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:22.398 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:22.439 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:22.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:22.538 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:22.541 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:22.572 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:22.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:22.680 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:19:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.716 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:22.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:22.744 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:19:22.744 INFO:teuthology.orchestra.run.smithi134.stdout: "success": "" 2024-09-21T10:19:22.745 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:19:22.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:22.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.855 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:22.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:22 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4123538544' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:22.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:22 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:22 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4123538544' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:22 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:22.956 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:22.987 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:22.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:22 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4123538544' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:22 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:19:23.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:23.107 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:19:23.105949+0000 min lat: 0.00118998 max lat: 2.30472 avg lat: 0.0101449 2024-09-21T10:19:23.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:19:23.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 15 94617 94602 98.5199 77.875 0.0103419 0.0101449 2024-09-21T10:19:23.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.133 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:23.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:23.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:23.278 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:23.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:23.379 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:23.420 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:23.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:23.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:23.567 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:23.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:23.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:23.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:23 smithi017 ceph-mon[102913]: pgmap v2219: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-21T10:19:23.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:23 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:23.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:23 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:19:23.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:23 smithi017 ceph-mon[102913]: osdmap e1372: 8 total, 7 up, 7 in 2024-09-21T10:19:23.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:23 smithi017 ceph-mon[105151]: pgmap v2219: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-21T10:19:23.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:23 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:23.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:23 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:19:23.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:23 smithi017 ceph-mon[105151]: osdmap e1372: 8 total, 7 up, 7 in 2024-09-21T10:19:23.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:23.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:23.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:23.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.851 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:23.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:23.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:23 smithi134 ceph-mon[90085]: pgmap v2219: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-21T10:19:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:23 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:23 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:19:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:23 smithi134 ceph-mon[90085]: osdmap e1372: 8 total, 7 up, 7 in 2024-09-21T10:19:23.995 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:23.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:24.097 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:24.107 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 95879 95863 98.1967 78.8125 0.00779605 0.0101789 2024-09-21T10:19:24.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.128 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:24.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:24.206 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:19:24.206 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:19:24.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:24.261 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:24.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:24.363 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:24.396 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:24.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:24.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:24.538 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:24.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:24.640 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:24.674 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:24.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:24.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:24.817 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:24.838 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:19:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:19:24] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-09-21T10:19:24.919 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:24.955 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:24.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:25.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:25.100 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:25.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:25.107 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 97187 97171 97.9314 81.75 0.00779536 0.0102063 2024-09-21T10:19:25.202 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:25.233 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:25.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:25.335 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:25.380 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:25.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:25.481 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:25.519 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:25.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:25.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:25.657 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:25.759 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:25.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:25.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:25.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:25.931 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:25.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:25 smithi017 ceph-mon[105151]: 1.3 scrub starts 2024-09-21T10:19:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:25 smithi017 ceph-mon[105151]: 1.3 scrub ok 2024-09-21T10:19:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:25 smithi017 ceph-mon[105151]: pgmap v2221: 119 pgs: 119 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:19:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:25 smithi017 ceph-mon[102913]: 1.3 scrub starts 2024-09-21T10:19:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:25 smithi017 ceph-mon[102913]: 1.3 scrub ok 2024-09-21T10:19:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:25 smithi017 ceph-mon[102913]: pgmap v2221: 119 pgs: 119 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:19:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:25 smithi134 ceph-mon[90085]: 1.3 scrub starts 2024-09-21T10:19:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:25 smithi134 ceph-mon[90085]: 1.3 scrub ok 2024-09-21T10:19:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:25 smithi134 ceph-mon[90085]: pgmap v2221: 119 pgs: 119 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:19:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:26.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:19:26.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:26.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 98467 98451 97.6457 80 0.0106328 0.010236 2024-09-21T10:19:26.113 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:26.215 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:26.245 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:26.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:26.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:26.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:26.503 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:26.532 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:26.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:26.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.695 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:26.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:26.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:19:26.755 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:26.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:26.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:26.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:26.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:26 smithi017 ceph-mon[102913]: 1.1a scrub starts 2024-09-21T10:19:26.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:26 smithi017 ceph-mon[102913]: 1.1a scrub ok 2024-09-21T10:19:26.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:26 smithi017 ceph-mon[105151]: 1.1a scrub starts 2024-09-21T10:19:26.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:26 smithi017 ceph-mon[105151]: 1.1a scrub ok 2024-09-21T10:19:26.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:26 smithi134 ceph-mon[90085]: 1.1a scrub starts 2024-09-21T10:19:26.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:26 smithi134 ceph-mon[90085]: 1.1a scrub ok 2024-09-21T10:19:27.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:27.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:27.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 99652 99636 97.2772 74.0625 0.00734324 0.0102745 2024-09-21T10:19:27.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:27.137 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:27.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:27.239 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:27.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.270 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:27.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:27.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:27.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:27.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:27.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:27.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:27.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:27.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:27.691 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:27.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:27.793 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:27.827 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:27.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:27.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:27 smithi017 ceph-mon[102913]: 1.1c scrub starts 2024-09-21T10:19:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:27 smithi017 ceph-mon[102913]: 1.1c scrub ok 2024-09-21T10:19:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:27 smithi017 ceph-mon[102913]: pgmap v2222: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:19:27.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:27 smithi017 ceph-mon[105151]: 1.1c scrub starts 2024-09-21T10:19:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:27 smithi017 ceph-mon[105151]: 1.1c scrub ok 2024-09-21T10:19:27.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:27 smithi017 ceph-mon[105151]: pgmap v2222: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:19:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:27.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:27 smithi134 ceph-mon[90085]: 1.1c scrub starts 2024-09-21T10:19:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:27 smithi134 ceph-mon[90085]: 1.1c scrub ok 2024-09-21T10:19:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:27 smithi134 ceph-mon[90085]: pgmap v2222: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:19:28.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:28.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.104 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:28.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:28.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 100903 100887 96.9835 78.1875 0.00798918 0.010306 2024-09-21T10:19:28.205 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:28.239 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:28.340 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:28.372 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:28.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:28.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:28.552 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:28.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:28.654 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:28.687 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:28.719 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [1] live_osds: [0, 2, 3, 5, 6, 7, 4] 2024-09-21T10:19:28.719 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:19:28.719 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:19:28.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:19:28.789 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:28.828 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:28.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:28.929 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:28.962 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:29.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:29.094 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:29.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:29.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 102117 102101 96.6635 75.875 0.00411542 0.0103357 2024-09-21T10:19:29.197 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:29.231 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:29.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:29.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:29.370 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:29.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:29.472 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:29.510 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:29.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:29.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:29.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:29.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:29.745 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:29.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.778 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:29.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:29.879 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:29.916 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:29.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:29 smithi134 ceph-mon[90085]: pgmap v2223: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:19:30.018 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:30.049 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:30.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:30.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 103292 103276 96.3168 73.4375 0.00404892 0.0103781 2024-09-21T10:19:30.150 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:29 smithi017 ceph-mon[102913]: pgmap v2223: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:19:30.151 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:29 smithi017 ceph-mon[105151]: pgmap v2223: 119 pgs: 119 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:19:30.151 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:30.184 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:30.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:30.330 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:30.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:30.393 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:19:30.393 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:19:30.393 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:30.393 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-21T10:19:30.394 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:30.419 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:19:30.420 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:30.432 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:30.453 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:19:30.454 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:30.471 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:30.486 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:19:30.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:19:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:19:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:19:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:19:30.519 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:19:30.519 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:30.555 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:19:30.555 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:30.573 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:30.606 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:30.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:30.708 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:30.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.749 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:30.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:30.850 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:30.885 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:30.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:30.987 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:30 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2487604012' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:31.022 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:31.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:31.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 104572 104556 96.0768 80 0.00728789 0.0104033 2024-09-21T10:19:31.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:31.161 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:31.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:30 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2487604012' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:31.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:30 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2487604012' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:31.264 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:31.302 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:31.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:31.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:31.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:31.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:31.543 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:31.576 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:31.678 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:31.714 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:31.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:31.815 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.850 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:31.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:31.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:31.987 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:31.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:31 smithi134 ceph-mon[90085]: pgmap v2224: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:19:32.088 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:32.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 105873 105857 95.8627 81.3125 0.00680618 0.0104269 2024-09-21T10:19:32.129 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:32.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:32.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:31 smithi017 ceph-mon[102913]: pgmap v2224: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:19:32.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:31 smithi017 ceph-mon[105151]: pgmap v2224: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:19:32.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:19:32.267 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:19:32.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:32.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:32.398 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:32.433 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:32.534 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:32.563 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:32.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:32.665 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:32.707 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:32.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:32.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:32.822 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:32.834 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:32.855 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:19:32.855 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:19:32.855 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:19:32.855 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:19:32.855 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:19:32.873 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:32.936 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:32.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:32.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:33.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:33.103 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:33.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:33.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 107096 107080 95.5851 76.4375 0.00885407 0.0104569 2024-09-21T10:19:33.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:33.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:33.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:33.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:33.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:33.488 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:33.519 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:33.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:33.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:33.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:33.745 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:33.782 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:33.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:33.843 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:33 smithi017 ceph-mon[102913]: pgmap v2225: 119 pgs: 119 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:19:33.843 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:33 smithi017 ceph-mon[105151]: pgmap v2225: 119 pgs: 119 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:19:33.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:33.926 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:33 smithi134 ceph-mon[90085]: pgmap v2225: 119 pgs: 119 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-21T10:19:34.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:34.065 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:34.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:34.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 108187 108171 95.1992 68.1875 0.0194671 0.0104991 2024-09-21T10:19:34.166 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:34.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:34.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:34.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:19:34.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.355 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:34.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:34.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:34.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.494 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:34.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:34.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:34.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:34.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:34.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:34.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:34.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:34.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:34.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:35.020 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:35.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:19:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:19:34] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.51.0" 2024-09-21T10:19:35.057 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:35.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:35.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 109260 109244 94.8084 67.0625 0.0111031 0.0105428 2024-09-21T10:19:35.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:35.204 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:35.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:35.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:35.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:35.442 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:35.463 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:35.565 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:35.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:35.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:35.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:35.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:35.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:35.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:35.853 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:35.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:35.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:35.983 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:35 smithi134 ceph-mon[90085]: pgmap v2226: 119 pgs: 119 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:19:36.085 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:36.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 110271 110255 94.3752 63.1875 0.0107439 0.010591 2024-09-21T10:19:36.124 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:36.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:35 smithi017 ceph-mon[102913]: pgmap v2226: 119 pgs: 119 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:19:36.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:35 smithi017 ceph-mon[105151]: pgmap v2226: 119 pgs: 119 active+clean; 6.4 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:19:36.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:36.266 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:36.368 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:36.398 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:36.499 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:36.533 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:36.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:36.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:36.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:36.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:36.761 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:36.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:36.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:36.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:36.941 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:36 smithi017 ceph-mon[102913]: 1.16 scrub starts 2024-09-21T10:19:36.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:36 smithi017 ceph-mon[102913]: 1.16 scrub ok 2024-09-21T10:19:36.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:36 smithi017 ceph-mon[105151]: 1.16 scrub starts 2024-09-21T10:19:36.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:36 smithi017 ceph-mon[105151]: 1.16 scrub ok 2024-09-21T10:19:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:36 smithi134 ceph-mon[90085]: 1.16 scrub starts 2024-09-21T10:19:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:36 smithi134 ceph-mon[90085]: 1.16 scrub ok 2024-09-21T10:19:37.043 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:37.065 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:37.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:37.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 111266 111250 93.9402 62.1875 0.00294686 0.0106191 2024-09-21T10:19:37.166 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:37.188 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:37.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:37.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:37.331 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:37.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:37.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:37.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:37.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:37.572 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:37.611 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:37.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:37.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:37.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:37.849 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:37.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.888 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:37.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:37.989 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:37 smithi134 ceph-mon[90085]: pgmap v2227: 119 pgs: 119 active+clean; 6.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:19:38.027 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:38.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 112234 112218 93.4944 60.5 0.0113049 0.0106909 2024-09-21T10:19:38.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:38.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:38.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:37 smithi017 ceph-mon[102913]: pgmap v2227: 119 pgs: 119 active+clean; 6.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:19:38.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:37 smithi017 ceph-mon[105151]: pgmap v2227: 119 pgs: 119 active+clean; 6.6 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:19:38.272 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:38.312 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:38.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:38.414 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:38.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:19:38.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.449 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:38.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:38.550 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:38.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.583 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:38.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:38.684 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:38.717 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:38.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:38.819 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.854 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:38.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:38.886 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:38 smithi017 ceph-mon[102913]: 1.1e scrub starts 2024-09-21T10:19:38.886 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:38 smithi017 ceph-mon[102913]: 1.1e scrub ok 2024-09-21T10:19:38.887 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:38 smithi017 ceph-mon[105151]: 1.1e scrub starts 2024-09-21T10:19:38.887 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:38 smithi017 ceph-mon[105151]: 1.1e scrub ok 2024-09-21T10:19:38.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:38.985 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:38.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:39.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:39.111 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 113367 113351 93.1958 70.8125 0.0105002 0.0107254 2024-09-21T10:19:39.121 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:39.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:39.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:38 smithi134 ceph-mon[90085]: 1.1e scrub starts 2024-09-21T10:19:39.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:38 smithi134 ceph-mon[90085]: 1.1e scrub ok 2024-09-21T10:19:39.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:39.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:39.361 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:39.389 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:39.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:39.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:39.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:39.674 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:39.744 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:39.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:39.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.826 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:39.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:39.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:39.968 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:40.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:39 smithi017 ceph-mon[102913]: pgmap v2228: 119 pgs: 119 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-21T10:19:40.027 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:39 smithi017 ceph-mon[105151]: pgmap v2228: 119 pgs: 119 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-21T10:19:40.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:40.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 114543 114527 92.9399 73.5 0.0106085 0.0107546 2024-09-21T10:19:40.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:40.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:40.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:39 smithi134 ceph-mon[90085]: pgmap v2228: 119 pgs: 119 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-21T10:19:40.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.258 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:40.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:40.359 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:40.387 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:40.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:40.488 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:19:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:19:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:19:40.509 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:40.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:40.611 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:40.646 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:40.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:40.747 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:40.785 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:40.882 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:19:40.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:40.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:40.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:41.028 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:41.064 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:41.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:41.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 115439 115423 92.4664 56 0.0098033 0.0108103 2024-09-21T10:19:41.165 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:41.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:41.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/707002765' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:41.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:40 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:41.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:41.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/707002765' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:41.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:40 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:41.197 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:41.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/707002765' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:40 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:41.300 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:41.329 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:41.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:41.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:41.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:41.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:41.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.618 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:41.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:41.719 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:41.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.752 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:41.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:41.853 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:41.888 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:41.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:41.955 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:19:41.957 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:19:41.962 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:41.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:42.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:42.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:42.052 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[102913]: pgmap v2229: 119 pgs: 119 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:19:42.052 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:19:42.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:19:42.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:42.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[102913]: osdmap e1373: 8 total, 7 up, 7 in 2024-09-21T10:19:42.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[105151]: pgmap v2229: 119 pgs: 119 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:19:42.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:19:42.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:19:42.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:42.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:41 smithi017 ceph-mon[105151]: osdmap e1373: 8 total, 7 up, 7 in 2024-09-21T10:19:42.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 116634 116618 92.2412 74.6875 0.0115226 0.0108365 2024-09-21T10:19:42.137 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:42.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:42.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:41 smithi134 ceph-mon[90085]: pgmap v2229: 119 pgs: 119 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:19:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:41 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:19:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:41 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:19:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:41 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:41 smithi134 ceph-mon[90085]: osdmap e1373: 8 total, 7 up, 7 in 2024-09-21T10:19:42.278 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:42.310 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:42.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:42.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:42.446 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:42.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:42.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:42.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:42.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:42.731 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:42.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:42.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:42.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:42.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:42.943 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":2230,"stamp":"2024-09-21T10:19:40.768301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7377859849,"num_objects":112595,"num_object_clones":0,"num_object_copies":337785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":112595,"num_whiteouts":0,"num_read":19,"num_read_kb":51,"num_write":112643,"num_write_kb":7204622,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1515,"num_bytes_recovered":95453723,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86528,"ondisk_log_size":86528,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":27816432,"kb_used_data":21521144,"kb_used_omap":384,"kb_used_meta":6294847,"kb_avail":628399632,"statfs":{"total":671965249536,"available":643481223168,"internally_reserved":0,"allocated":22037651456,"data_stored":22013445231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":393354,"internal_metadata":6445924214},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,5,8,7,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":748361516,"num_objects":11421,"num_object_clones":0,"num_object_copies":34263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11421,"num_whiteouts":0,"num_read":-14,"num_read_kb":-34,"num_write":11414,"num_write_kb":730934,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":-26836,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.184629"},"pg_stats":[{"pgid":"1.1d","version":"288'100","reported_seq":181,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647122+0000","last_change":"2024-09-21T10:18:55.372468+0000","last_active":"2024-09-21T10:19:14.647122+0000","last_peered":"2024-09-21T10:19:14.647122+0000","last_clean":"2024-09-21T10:19:14.647122+0000","last_became_active":"2024-09-21T10:17:59.913598+0000","last_became_peered":"2024-09-21T10:17:59.913598+0000","last_unstale":"2024-09-21T10:19:14.647122+0000","last_undegraded":"2024-09-21T10:19:14.647122+0000","last_fullsized":"2024-09-21T10:19:14.647122+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:55.372392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:55.372392+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:15:40.399506+0000","scrub_duration":0.0042594110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":129,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.668884+0000","last_change":"2024-09-21T10:17:34.790860+0000","last_active":"2024-09-21T10:19:22.668884+0000","last_peered":"2024-09-21T10:19:22.668884+0000","last_clean":"2024-09-21T10:19:22.668884+0000","last_became_active":"2024-09-21T10:17:34.779226+0000","last_became_peered":"2024-09-21T10:17:34.779226+0000","last_unstale":"2024-09-21T10:19:22.668884+0000","last_undegraded":"2024-09-21T10:19:22.668884+0000","last_fullsized":"2024-09-21T10:19:22.668884+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:45:00.128138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1370,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646462+0000","last_change":"2024-09-21T10:16:27.318860+0000","last_active":"2024-09-21T10:19:14.646462+0000","last_peered":"2024-09-21T10:19:14.646462+0000","last_clean":"2024-09-21T10:19:14.646462+0000","last_became_active":"2024-09-21T10:16:27.317542+0000","last_became_peered":"2024-09-21T10:16:27.317542+0000","last_unstale":"2024-09-21T10:19:14.646462+0000","last_undegraded":"2024-09-21T10:19:14.646462+0000","last_fullsized":"2024-09-21T10:19:14.646462+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:56:58.481255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":2113,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.677630+0000","last_change":"2024-09-21T10:10:13.651599+0000","last_active":"2024-09-21T10:19:22.677630+0000","last_peered":"2024-09-21T10:19:22.677630+0000","last_clean":"2024-09-21T10:19:22.677630+0000","last_became_active":"2024-09-21T10:10:13.651250+0000","last_became_peered":"2024-09-21T10:10:13.651250+0000","last_unstale":"2024-09-21T10:19:22.677630+0000","last_undegraded":"2024-09-21T10:19:22.677630+0000","last_fullsized":"2024-09-21T10:19:22.677630+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:34.100056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2097,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.680483+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:19:22.680483+0000","last_peered":"2024-09-21T10:19:22.680483+0000","last_clean":"2024-09-21T10:19:22.680483+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:19:22.680483+0000","last_undegraded":"2024-09-21T10:19:22.680483+0000","last_fullsized":"2024-09-21T10:19:22.680483+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"288'100","reported_seq":37,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:26.113882+0000","last_change":"2024-09-21T10:19:26.113832+0000","last_active":"2024-09-21T10:19:26.113882+0000","last_peered":"2024-09-21T10:19:26.113882+0000","last_clean":"2024-09-21T10:19:26.113882+0000","last_became_active":"2024-09-21T10:17:59.912636+0000","last_became_peered":"2024-09-21T10:17:59.912636+0000","last_unstale":"2024-09-21T10:19:26.113882+0000","last_undegraded":"2024-09-21T10:19:26.113882+0000","last_fullsized":"2024-09-21T10:19:26.113882+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:26.113747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:26.113747+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:57:42.993691+0000","scrub_duration":0.0042569799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":311,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647246+0000","last_change":"2024-09-21T10:10:13.639231+0000","last_active":"2024-09-21T10:19:14.647246+0000","last_peered":"2024-09-21T10:19:14.647246+0000","last_clean":"2024-09-21T10:19:14.647246+0000","last_became_active":"2024-09-21T10:10:13.622497+0000","last_became_peered":"2024-09-21T10:10:13.622497+0000","last_unstale":"2024-09-21T10:19:14.647246+0000","last_undegraded":"2024-09-21T10:19:14.647246+0000","last_fullsized":"2024-09-21T10:19:14.647246+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:31:46.263003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1317,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647051+0000","last_change":"2024-09-21T10:08:25.430667+0000","last_active":"2024-09-21T10:19:14.647051+0000","last_peered":"2024-09-21T10:19:14.647051+0000","last_clean":"2024-09-21T10:19:14.647051+0000","last_became_active":"2024-09-21T10:08:25.430357+0000","last_became_peered":"2024-09-21T10:08:25.430357+0000","last_unstale":"2024-09-21T10:19:14.647051+0000","last_undegraded":"2024-09-21T10:19:14.647051+0000","last_fullsized":"2024-09-21T10:19:14.647051+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:19.972020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"288'100","reported_seq":396,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:37.200842+0000","last_change":"2024-09-21T10:19:37.200842+0000","last_active":"2024-09-21T10:19:37.200842+0000","last_peered":"2024-09-21T10:19:37.200842+0000","last_clean":"2024-09-21T10:19:37.200842+0000","last_became_active":"2024-09-21T10:17:59.914846+0000","last_became_peered":"2024-09-21T10:17:59.914846+0000","last_unstale":"2024-09-21T10:19:37.200842+0000","last_undegraded":"2024-09-21T10:19:37.200842+0000","last_fullsized":"2024-09-21T10:19:37.200842+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:37.200788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:37.200788+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:17:48.733459+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":323,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646959+0000","last_change":"2024-09-21T10:16:27.324301+0000","last_active":"2024-09-21T10:19:14.646959+0000","last_peered":"2024-09-21T10:19:14.646959+0000","last_clean":"2024-09-21T10:19:14.646959+0000","last_became_active":"2024-09-21T10:16:27.319515+0000","last_became_peered":"2024-09-21T10:16:27.319515+0000","last_unstale":"2024-09-21T10:19:14.646959+0000","last_undegraded":"2024-09-21T10:19:14.646959+0000","last_fullsized":"2024-09-21T10:19:14.646959+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:38.284184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1347,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646552+0000","last_change":"2024-09-21T10:13:41.789883+0000","last_active":"2024-09-21T10:19:14.646552+0000","last_peered":"2024-09-21T10:19:14.646552+0000","last_clean":"2024-09-21T10:19:14.646552+0000","last_became_active":"2024-09-21T10:13:41.789620+0000","last_became_peered":"2024-09-21T10:13:41.789620+0000","last_unstale":"2024-09-21T10:19:14.646552+0000","last_undegraded":"2024-09-21T10:19:14.646552+0000","last_fullsized":"2024-09-21T10:19:14.646552+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:36.869540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":88,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647061+0000","last_change":"2024-09-21T10:17:34.770306+0000","last_active":"2024-09-21T10:19:14.647061+0000","last_peered":"2024-09-21T10:19:14.647061+0000","last_clean":"2024-09-21T10:19:14.647061+0000","last_became_active":"2024-09-21T10:17:34.769690+0000","last_became_peered":"2024-09-21T10:17:34.769690+0000","last_unstale":"2024-09-21T10:19:14.647061+0000","last_undegraded":"2024-09-21T10:19:14.647061+0000","last_fullsized":"2024-09-21T10:19:14.647061+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:00:49.468067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2152,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.669170+0000","last_change":"2024-09-21T10:17:34.794267+0000","last_active":"2024-09-21T10:19:22.669170+0000","last_peered":"2024-09-21T10:19:22.669170+0000","last_clean":"2024-09-21T10:19:22.669170+0000","last_became_active":"2024-09-21T10:17:34.792230+0000","last_became_peered":"2024-09-21T10:17:34.792230+0000","last_unstale":"2024-09-21T10:19:22.669170+0000","last_undegraded":"2024-09-21T10:19:22.669170+0000","last_fullsized":"2024-09-21T10:19:22.669170+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:37.529702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"288'100","reported_seq":1276,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.679721+0000","last_change":"2024-09-21T10:17:59.912179+0000","last_active":"2024-09-21T10:19:22.679721+0000","last_peered":"2024-09-21T10:19:22.679721+0000","last_clean":"2024-09-21T10:19:22.679721+0000","last_became_active":"2024-09-21T10:17:59.911742+0000","last_became_peered":"2024-09-21T10:17:59.911742+0000","last_unstale":"2024-09-21T10:19:22.679721+0000","last_undegraded":"2024-09-21T10:19:22.679721+0000","last_fullsized":"2024-09-21T10:19:22.679721+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:15.663228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:15.663228+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:20:22.144298+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1349,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646988+0000","last_change":"2024-09-21T10:16:27.338372+0000","last_active":"2024-09-21T10:19:14.646988+0000","last_peered":"2024-09-21T10:19:14.646988+0000","last_clean":"2024-09-21T10:19:14.646988+0000","last_became_active":"2024-09-21T10:16:27.338029+0000","last_became_peered":"2024-09-21T10:16:27.338029+0000","last_unstale":"2024-09-21T10:19:14.646988+0000","last_undegraded":"2024-09-21T10:19:14.646988+0000","last_fullsized":"2024-09-21T10:19:14.646988+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:52.311869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2119,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.677919+0000","last_change":"2024-09-21T10:16:27.340083+0000","last_active":"2024-09-21T10:19:22.677919+0000","last_peered":"2024-09-21T10:19:22.677919+0000","last_clean":"2024-09-21T10:19:22.677919+0000","last_became_active":"2024-09-21T10:16:27.339405+0000","last_became_peered":"2024-09-21T10:16:27.339405+0000","last_unstale":"2024-09-21T10:19:22.677919+0000","last_undegraded":"2024-09-21T10:19:22.677919+0000","last_fullsized":"2024-09-21T10:19:22.677919+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:11.731090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"288'100","reported_seq":576,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.669368+0000","last_change":"2024-09-21T10:17:59.915389+0000","last_active":"2024-09-21T10:19:22.669368+0000","last_peered":"2024-09-21T10:19:22.669368+0000","last_clean":"2024-09-21T10:19:22.669368+0000","last_became_active":"2024-09-21T10:17:59.909290+0000","last_became_peered":"2024-09-21T10:17:59.909290+0000","last_unstale":"2024-09-21T10:19:22.669368+0000","last_undegraded":"2024-09-21T10:19:22.669368+0000","last_fullsized":"2024-09-21T10:19:22.669368+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:20:12.193740+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":829,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.680215+0000","last_change":"2024-09-21T10:16:27.321423+0000","last_active":"2024-09-21T10:19:22.680215+0000","last_peered":"2024-09-21T10:19:22.680215+0000","last_clean":"2024-09-21T10:19:22.680215+0000","last_became_active":"2024-09-21T10:16:27.319009+0000","last_became_peered":"2024-09-21T10:16:27.319009+0000","last_unstale":"2024-09-21T10:19:22.680215+0000","last_undegraded":"2024-09-21T10:19:22.680215+0000","last_fullsized":"2024-09-21T10:19:22.680215+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:32.452405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2036,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650213+0000","last_change":"2024-09-21T10:17:34.778067+0000","last_active":"2024-09-21T10:19:14.650213+0000","last_peered":"2024-09-21T10:19:14.650213+0000","last_clean":"2024-09-21T10:19:14.650213+0000","last_became_active":"2024-09-21T10:17:34.777744+0000","last_became_peered":"2024-09-21T10:17:34.777744+0000","last_unstale":"2024-09-21T10:19:14.650213+0000","last_undegraded":"2024-09-21T10:19:14.650213+0000","last_fullsized":"2024-09-21T10:19:14.650213+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:15:54.571208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1b","version":"288'100","reported_seq":34,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646953+0000","last_change":"2024-09-21T10:18:48.373065+0000","last_active":"2024-09-21T10:19:14.646953+0000","last_peered":"2024-09-21T10:19:14.646953+0000","last_clean":"2024-09-21T10:19:14.646953+0000","last_became_active":"2024-09-21T10:17:59.910945+0000","last_became_peered":"2024-09-21T10:17:59.910945+0000","last_unstale":"2024-09-21T10:19:14.646953+0000","last_undegraded":"2024-09-21T10:19:14.646953+0000","last_fullsized":"2024-09-21T10:19:14.646953+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:48.372952+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:18:48.372952+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:15:04.784278+0000","scrub_duration":0.024424405,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2133,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.679774+0000","last_change":"2024-09-21T10:17:34.777326+0000","last_active":"2024-09-21T10:19:22.679774+0000","last_peered":"2024-09-21T10:19:22.679774+0000","last_clean":"2024-09-21T10:19:22.679774+0000","last_became_active":"2024-09-21T10:17:34.776889+0000","last_became_peered":"2024-09-21T10:17:34.776889+0000","last_unstale":"2024-09-21T10:19:22.679774+0000","last_undegraded":"2024-09-21T10:19:22.679774+0000","last_fullsized":"2024-09-21T10:19:22.679774+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:44:01.933785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":139,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.653677+0000","last_change":"2024-09-21T10:16:27.322614+0000","last_active":"2024-09-21T10:19:14.653677+0000","last_peered":"2024-09-21T10:19:14.653677+0000","last_clean":"2024-09-21T10:19:14.653677+0000","last_became_active":"2024-09-21T10:16:27.321913+0000","last_became_peered":"2024-09-21T10:16:27.321913+0000","last_unstale":"2024-09-21T10:19:14.653677+0000","last_undegraded":"2024-09-21T10:19:14.653677+0000","last_fullsized":"2024-09-21T10:19:14.653677+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:44:39.631658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"288'100","reported_seq":37,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:25.436198+0000","last_change":"2024-09-21T10:19:25.436153+0000","last_active":"2024-09-21T10:19:25.436198+0000","last_peered":"2024-09-21T10:19:25.436198+0000","last_clean":"2024-09-21T10:19:25.436198+0000","last_became_active":"2024-09-21T10:17:59.912015+0000","last_became_peered":"2024-09-21T10:17:59.912015+0000","last_unstale":"2024-09-21T10:19:25.436198+0000","last_undegraded":"2024-09-21T10:19:25.436198+0000","last_fullsized":"2024-09-21T10:19:25.436198+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:25.436066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:25.436066+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:38:18.381058+0000","scrub_duration":0.0032626159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2155,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.678929+0000","last_change":"2024-09-21T10:16:27.330457+0000","last_active":"2024-09-21T10:19:22.678929+0000","last_peered":"2024-09-21T10:19:22.678929+0000","last_clean":"2024-09-21T10:19:22.678929+0000","last_became_active":"2024-09-21T10:16:27.329527+0000","last_became_peered":"2024-09-21T10:16:27.329527+0000","last_unstale":"2024-09-21T10:19:22.678929+0000","last_undegraded":"2024-09-21T10:19:22.678929+0000","last_fullsized":"2024-09-21T10:19:22.678929+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:16.811107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2079,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.677918+0000","last_change":"2024-09-21T10:16:07.906476+0000","last_active":"2024-09-21T10:19:22.677918+0000","last_peered":"2024-09-21T10:19:22.677918+0000","last_clean":"2024-09-21T10:19:22.677918+0000","last_became_active":"2024-09-21T10:16:07.906053+0000","last_became_peered":"2024-09-21T10:16:07.906053+0000","last_unstale":"2024-09-21T10:19:22.677918+0000","last_undegraded":"2024-09-21T10:19:22.677918+0000","last_fullsized":"2024-09-21T10:19:22.677918+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:01:57.827302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"288'100","reported_seq":34,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650378+0000","last_change":"2024-09-21T10:18:52.067685+0000","last_active":"2024-09-21T10:19:14.650378+0000","last_peered":"2024-09-21T10:19:14.650378+0000","last_clean":"2024-09-21T10:19:14.650378+0000","last_became_active":"2024-09-21T10:17:59.910073+0000","last_became_peered":"2024-09-21T10:17:59.910073+0000","last_unstale":"2024-09-21T10:19:14.650378+0000","last_undegraded":"2024-09-21T10:19:14.650378+0000","last_fullsized":"2024-09-21T10:19:14.650378+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:52.067480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:52.067480+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:51.001507+0000","scrub_duration":0.0041194669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2113,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.679920+0000","last_change":"2024-09-21T10:12:25.707393+0000","last_active":"2024-09-21T10:19:22.679920+0000","last_peered":"2024-09-21T10:19:22.679920+0000","last_clean":"2024-09-21T10:19:22.679920+0000","last_became_active":"2024-09-21T10:12:25.706457+0000","last_became_peered":"2024-09-21T10:12:25.706457+0000","last_unstale":"2024-09-21T10:19:22.679920+0000","last_undegraded":"2024-09-21T10:19:22.679920+0000","last_fullsized":"2024-09-21T10:19:22.679920+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:56.520799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":133,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650297+0000","last_change":"2024-09-21T10:16:27.320052+0000","last_active":"2024-09-21T10:19:14.650297+0000","last_peered":"2024-09-21T10:19:14.650297+0000","last_clean":"2024-09-21T10:19:14.650297+0000","last_became_active":"2024-09-21T10:16:27.319716+0000","last_became_peered":"2024-09-21T10:16:27.319716+0000","last_unstale":"2024-09-21T10:19:14.650297+0000","last_undegraded":"2024-09-21T10:19:14.650297+0000","last_fullsized":"2024-09-21T10:19:14.650297+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:15:46.308322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"288'100","reported_seq":308,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.668735+0000","last_change":"2024-09-21T10:12:25.698905+0000","last_active":"2024-09-21T10:19:22.668735+0000","last_peered":"2024-09-21T10:19:22.668735+0000","last_clean":"2024-09-21T10:19:22.668735+0000","last_became_active":"2024-09-21T10:12:25.698552+0000","last_became_peered":"2024-09-21T10:12:25.698552+0000","last_unstale":"2024-09-21T10:19:22.668735+0000","last_undegraded":"2024-09-21T10:19:22.668735+0000","last_fullsized":"2024-09-21T10:19:22.668735+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.389965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.389965+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:17:52.573175+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":119,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650300+0000","last_change":"2024-09-21T10:12:25.717602+0000","last_active":"2024-09-21T10:19:14.650300+0000","last_peered":"2024-09-21T10:19:14.650300+0000","last_clean":"2024-09-21T10:19:14.650300+0000","last_became_active":"2024-09-21T10:12:25.705211+0000","last_became_peered":"2024-09-21T10:12:25.705211+0000","last_unstale":"2024-09-21T10:19:14.650300+0000","last_undegraded":"2024-09-21T10:19:14.650300+0000","last_fullsized":"2024-09-21T10:19:14.650300+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:30.822884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":102,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.652801+0000","last_change":"2024-09-21T10:16:27.338866+0000","last_active":"2024-09-21T10:19:14.652801+0000","last_peered":"2024-09-21T10:19:14.652801+0000","last_clean":"2024-09-21T10:19:14.652801+0000","last_became_active":"2024-09-21T10:16:27.338090+0000","last_became_peered":"2024-09-21T10:16:27.338090+0000","last_unstale":"2024-09-21T10:19:14.652801+0000","last_undegraded":"2024-09-21T10:19:14.652801+0000","last_fullsized":"2024-09-21T10:19:14.652801+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:54.045061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"288'100","reported_seq":119,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649712+0000","last_change":"2024-09-21T10:19:00.110407+0000","last_active":"2024-09-21T10:19:14.649712+0000","last_peered":"2024-09-21T10:19:14.649712+0000","last_clean":"2024-09-21T10:19:14.649712+0000","last_became_active":"2024-09-21T10:17:59.912144+0000","last_became_peered":"2024-09-21T10:17:59.912144+0000","last_unstale":"2024-09-21T10:19:14.649712+0000","last_undegraded":"2024-09-21T10:19:14.649712+0000","last_fullsized":"2024-09-21T10:19:14.649712+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:00.110351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:00.110351+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:06:32.995133+0000","scrub_duration":0.005953931,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1400,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647052+0000","last_change":"2024-09-21T10:12:25.713745+0000","last_active":"2024-09-21T10:19:14.647052+0000","last_peered":"2024-09-21T10:19:14.647052+0000","last_clean":"2024-09-21T10:19:14.647052+0000","last_became_active":"2024-09-21T10:12:25.698011+0000","last_became_peered":"2024-09-21T10:12:25.698011+0000","last_unstale":"2024-09-21T10:19:14.647052+0000","last_undegraded":"2024-09-21T10:19:14.647052+0000","last_fullsized":"2024-09-21T10:19:14.647052+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:45.276913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23720,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1370,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646672+0000","last_change":"2024-09-21T10:16:27.316686+0000","last_active":"2024-09-21T10:19:14.646672+0000","last_peered":"2024-09-21T10:19:14.646672+0000","last_clean":"2024-09-21T10:19:14.646672+0000","last_became_active":"2024-09-21T10:16:27.316186+0000","last_became_peered":"2024-09-21T10:16:27.316186+0000","last_unstale":"2024-09-21T10:19:14.646672+0000","last_undegraded":"2024-09-21T10:19:14.646672+0000","last_fullsized":"2024-09-21T10:19:14.646672+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:45.443605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"288'100","reported_seq":475,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:35.203265+0000","last_change":"2024-09-21T10:19:35.203265+0000","last_active":"2024-09-21T10:19:35.203265+0000","last_peered":"2024-09-21T10:19:35.203265+0000","last_clean":"2024-09-21T10:19:35.203265+0000","last_became_active":"2024-09-21T10:17:59.912488+0000","last_became_peered":"2024-09-21T10:17:59.912488+0000","last_unstale":"2024-09-21T10:19:35.203265+0000","last_undegraded":"2024-09-21T10:19:35.203265+0000","last_fullsized":"2024-09-21T10:19:35.203265+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:35.203193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:35.203193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:49:07.827686+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"1372'6971","reported_seq":6992,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:38.781926+0000","last_change":"2024-09-21T10:18:10.935476+0000","last_active":"2024-09-21T10:19:38.781926+0000","last_peered":"2024-09-21T10:19:38.781926+0000","last_clean":"2024-09-21T10:19:38.781926+0000","last_became_active":"2024-09-21T10:18:10.935178+0000","last_became_peered":"2024-09-21T10:18:10.935178+0000","last_unstale":"2024-09-21T10:19:38.781926+0000","last_undegraded":"2024-09-21T10:19:38.781926+0000","last_fullsized":"2024-09-21T10:19:38.781926+0000","mapping_epoch":1359,"log_start":"1366'1900","ondisk_log_start":"1366'1900","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5071,"log_dups_size":0,"ondisk_log_size":5071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:02:54.677232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":456851456,"num_objects":6971,"num_object_clones":0,"num_object_copies":20913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6971,"num_write_kb":446144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1317,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646485+0000","last_change":"2024-09-21T10:17:34.767553+0000","last_active":"2024-09-21T10:19:14.646485+0000","last_peered":"2024-09-21T10:19:14.646485+0000","last_clean":"2024-09-21T10:19:14.646485+0000","last_became_active":"2024-09-21T10:17:34.767246+0000","last_became_peered":"2024-09-21T10:17:34.767246+0000","last_unstale":"2024-09-21T10:19:14.646485+0000","last_undegraded":"2024-09-21T10:19:14.646485+0000","last_fullsized":"2024-09-21T10:19:14.646485+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:52.295023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":139,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.653442+0000","last_change":"2024-09-21T10:12:25.704090+0000","last_active":"2024-09-21T10:19:14.653442+0000","last_peered":"2024-09-21T10:19:14.653442+0000","last_clean":"2024-09-21T10:19:14.653442+0000","last_became_active":"2024-09-21T10:12:25.702945+0000","last_became_peered":"2024-09-21T10:12:25.702945+0000","last_unstale":"2024-09-21T10:19:14.653442+0000","last_undegraded":"2024-09-21T10:19:14.653442+0000","last_fullsized":"2024-09-21T10:19:14.653442+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:29.609613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":2138,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.668577+0000","last_change":"2024-09-21T10:16:27.330519+0000","last_active":"2024-09-21T10:19:22.668577+0000","last_peered":"2024-09-21T10:19:22.668577+0000","last_clean":"2024-09-21T10:19:22.668577+0000","last_became_active":"2024-09-21T10:16:27.317630+0000","last_became_peered":"2024-09-21T10:16:27.317630+0000","last_unstale":"2024-09-21T10:19:22.668577+0000","last_undegraded":"2024-09-21T10:19:22.668577+0000","last_fullsized":"2024-09-21T10:19:22.668577+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:19:53.670529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":593,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.676119+0000","last_change":"2024-09-21T10:12:50.578581+0000","last_active":"2024-09-21T10:19:22.676119+0000","last_peered":"2024-09-21T10:19:22.676119+0000","last_clean":"2024-09-21T10:19:22.676119+0000","last_became_active":"2024-09-21T10:12:50.578246+0000","last_became_peered":"2024-09-21T10:12:50.578246+0000","last_unstale":"2024-09-21T10:19:22.676119+0000","last_undegraded":"2024-09-21T10:19:22.676119+0000","last_fullsized":"2024-09-21T10:19:22.676119+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.423664+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.423664+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:43.948827+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1372'6988","reported_seq":7008,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:35.949481+0000","last_change":"2024-09-21T10:18:10.941491+0000","last_active":"2024-09-21T10:19:35.949481+0000","last_peered":"2024-09-21T10:19:35.949481+0000","last_clean":"2024-09-21T10:19:35.949481+0000","last_became_active":"2024-09-21T10:18:10.941206+0000","last_became_peered":"2024-09-21T10:18:10.941206+0000","last_unstale":"2024-09-21T10:19:35.949481+0000","last_undegraded":"2024-09-21T10:19:35.949481+0000","last_fullsized":"2024-09-21T10:19:35.949481+0000","mapping_epoch":1359,"log_start":"1366'1900","ondisk_log_start":"1366'1900","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5088,"log_dups_size":0,"ondisk_log_size":5088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:46.198535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":457965568,"num_objects":6988,"num_object_clones":0,"num_object_copies":20964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6988,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2118,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.679706+0000","last_change":"2024-09-21T10:17:34.771415+0000","last_active":"2024-09-21T10:19:22.679706+0000","last_peered":"2024-09-21T10:19:22.679706+0000","last_clean":"2024-09-21T10:19:22.679706+0000","last_became_active":"2024-09-21T10:17:34.771087+0000","last_became_peered":"2024-09-21T10:17:34.771087+0000","last_unstale":"2024-09-21T10:19:22.679706+0000","last_undegraded":"2024-09-21T10:19:22.679706+0000","last_fullsized":"2024-09-21T10:19:22.679706+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:07.509252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1790,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647531+0000","last_change":"2024-09-21T10:16:27.321061+0000","last_active":"2024-09-21T10:19:14.647531+0000","last_peered":"2024-09-21T10:19:14.647531+0000","last_clean":"2024-09-21T10:19:14.647531+0000","last_became_active":"2024-09-21T10:16:27.320759+0000","last_became_peered":"2024-09-21T10:16:27.320759+0000","last_unstale":"2024-09-21T10:19:14.647531+0000","last_undegraded":"2024-09-21T10:19:14.647531+0000","last_fullsized":"2024-09-21T10:19:14.647531+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:29:04.861525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":2121,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.678220+0000","last_change":"2024-09-21T10:12:25.718713+0000","last_active":"2024-09-21T10:19:22.678220+0000","last_peered":"2024-09-21T10:19:22.678220+0000","last_clean":"2024-09-21T10:19:22.678220+0000","last_became_active":"2024-09-21T10:12:25.713007+0000","last_became_peered":"2024-09-21T10:12:25.713007+0000","last_unstale":"2024-09-21T10:19:22.678220+0000","last_undegraded":"2024-09-21T10:19:22.678220+0000","last_fullsized":"2024-09-21T10:19:22.678220+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:09.753443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":295,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647262+0000","last_change":"2024-09-21T10:18:56.346749+0000","last_active":"2024-09-21T10:19:14.647262+0000","last_peered":"2024-09-21T10:19:14.647262+0000","last_clean":"2024-09-21T10:19:14.647262+0000","last_became_active":"2024-09-21T10:17:19.685397+0000","last_became_peered":"2024-09-21T10:17:19.685397+0000","last_unstale":"2024-09-21T10:19:14.647262+0000","last_undegraded":"2024-09-21T10:19:14.647262+0000","last_fullsized":"2024-09-21T10:19:14.647262+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:56.346587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:56.346587+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:24.457314+0000","scrub_duration":0.0039726459999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1372'6905","reported_seq":6942,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:36.946320+0000","last_change":"2024-09-21T10:18:10.938262+0000","last_active":"2024-09-21T10:19:36.946320+0000","last_peered":"2024-09-21T10:19:36.946320+0000","last_clean":"2024-09-21T10:19:36.946320+0000","last_became_active":"2024-09-21T10:18:10.937958+0000","last_became_peered":"2024-09-21T10:18:10.937958+0000","last_unstale":"2024-09-21T10:19:36.946320+0000","last_undegraded":"2024-09-21T10:19:36.946320+0000","last_fullsized":"2024-09-21T10:19:36.946320+0000","mapping_epoch":1359,"log_start":"1363'600","ondisk_log_start":"1363'600","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:31.071784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":452526080,"num_objects":6905,"num_object_clones":0,"num_object_copies":20715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6905,"num_write_kb":441920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2071,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649692+0000","last_change":"2024-09-21T10:16:07.913449+0000","last_active":"2024-09-21T10:19:14.649692+0000","last_peered":"2024-09-21T10:19:14.649692+0000","last_clean":"2024-09-21T10:19:14.649692+0000","last_became_active":"2024-09-21T10:16:07.913117+0000","last_became_peered":"2024-09-21T10:16:07.913117+0000","last_unstale":"2024-09-21T10:19:14.649692+0000","last_undegraded":"2024-09-21T10:19:14.649692+0000","last_fullsized":"2024-09-21T10:19:14.649692+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:24.442444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":113,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650392+0000","last_change":"2024-09-21T10:12:25.709819+0000","last_active":"2024-09-21T10:19:14.650392+0000","last_peered":"2024-09-21T10:19:14.650392+0000","last_clean":"2024-09-21T10:19:14.650392+0000","last_became_active":"2024-09-21T10:12:25.709548+0000","last_became_peered":"2024-09-21T10:12:25.709548+0000","last_unstale":"2024-09-21T10:19:14.650392+0000","last_undegraded":"2024-09-21T10:19:14.650392+0000","last_fullsized":"2024-09-21T10:19:14.650392+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:35.986614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":754,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.668947+0000","last_change":"2024-09-21T10:16:27.329301+0000","last_active":"2024-09-21T10:19:22.668947+0000","last_peered":"2024-09-21T10:19:22.668947+0000","last_clean":"2024-09-21T10:19:22.668947+0000","last_became_active":"2024-09-21T10:16:27.328337+0000","last_became_peered":"2024-09-21T10:16:27.328337+0000","last_unstale":"2024-09-21T10:19:22.668947+0000","last_undegraded":"2024-09-21T10:19:22.668947+0000","last_fullsized":"2024-09-21T10:19:22.668947+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:40.669541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":462,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.677315+0000","last_change":"2024-09-21T10:17:19.741388+0000","last_active":"2024-09-21T10:19:22.677315+0000","last_peered":"2024-09-21T10:19:22.677315+0000","last_clean":"2024-09-21T10:19:22.677315+0000","last_became_active":"2024-09-21T10:17:19.741030+0000","last_became_peered":"2024-09-21T10:17:19.741030+0000","last_unstale":"2024-09-21T10:19:22.677315+0000","last_undegraded":"2024-09-21T10:19:22.677315+0000","last_fullsized":"2024-09-21T10:19:22.677315+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:10:37.256819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:10:37.256819+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:20:21.114371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":6,"num_read_kb":17,"num_write":3,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":13417,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"1372'6894","reported_seq":6931,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:36.946068+0000","last_change":"2024-09-21T10:18:10.938914+0000","last_active":"2024-09-21T10:19:36.946068+0000","last_peered":"2024-09-21T10:19:36.946068+0000","last_clean":"2024-09-21T10:19:36.946068+0000","last_became_active":"2024-09-21T10:18:10.938570+0000","last_became_peered":"2024-09-21T10:18:10.938570+0000","last_unstale":"2024-09-21T10:19:36.946068+0000","last_undegraded":"2024-09-21T10:19:36.946068+0000","last_fullsized":"2024-09-21T10:19:36.946068+0000","mapping_epoch":1359,"log_start":"1363'600","ondisk_log_start":"1363'600","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":6294,"log_dups_size":0,"ondisk_log_size":6294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:14.496321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":451805184,"num_objects":6894,"num_object_clones":0,"num_object_copies":20682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6894,"num_write_kb":441216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":578,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646907+0000","last_change":"2024-09-21T10:16:27.337427+0000","last_active":"2024-09-21T10:19:14.646907+0000","last_peered":"2024-09-21T10:19:14.646907+0000","last_clean":"2024-09-21T10:19:14.646907+0000","last_became_active":"2024-09-21T10:16:27.336986+0000","last_became_peered":"2024-09-21T10:16:27.336986+0000","last_unstale":"2024-09-21T10:19:14.646907+0000","last_undegraded":"2024-09-21T10:19:14.646907+0000","last_fullsized":"2024-09-21T10:19:14.646907+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:51:38.804787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":132,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650331+0000","last_change":"2024-09-21T10:16:27.328517+0000","last_active":"2024-09-21T10:19:14.650331+0000","last_peered":"2024-09-21T10:19:14.650331+0000","last_clean":"2024-09-21T10:19:14.650331+0000","last_became_active":"2024-09-21T10:16:27.328203+0000","last_became_peered":"2024-09-21T10:16:27.328203+0000","last_unstale":"2024-09-21T10:19:14.650331+0000","last_undegraded":"2024-09-21T10:19:14.650331+0000","last_fullsized":"2024-09-21T10:19:14.650331+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:54.497430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1118,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.678303+0000","last_change":"2024-09-21T10:18:31.648581+0000","last_active":"2024-09-21T10:19:22.678303+0000","last_peered":"2024-09-21T10:19:22.678303+0000","last_clean":"2024-09-21T10:19:22.678303+0000","last_became_active":"2024-09-21T10:18:31.648253+0000","last_became_peered":"2024-09-21T10:18:31.648253+0000","last_unstale":"2024-09-21T10:19:22.678303+0000","last_undegraded":"2024-09-21T10:19:22.678303+0000","last_fullsized":"2024-09-21T10:19:22.678303+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:48.168055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":109,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650856+0000","last_change":"2024-09-21T10:18:59.120831+0000","last_active":"2024-09-21T10:19:14.650856+0000","last_peered":"2024-09-21T10:19:14.650856+0000","last_clean":"2024-09-21T10:19:14.650856+0000","last_became_active":"2024-09-21T10:17:34.778770+0000","last_became_peered":"2024-09-21T10:17:34.778770+0000","last_unstale":"2024-09-21T10:19:14.650856+0000","last_undegraded":"2024-09-21T10:19:14.650856+0000","last_fullsized":"2024-09-21T10:19:14.650856+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:59.120678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:59.120678+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:53.877284+0000","scrub_duration":0.0034448999999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.b","version":"1372'7149","reported_seq":7186,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.239651+0000","last_change":"2024-09-21T10:18:10.939692+0000","last_active":"2024-09-21T10:19:40.239651+0000","last_peered":"2024-09-21T10:19:40.239651+0000","last_clean":"2024-09-21T10:19:40.239651+0000","last_became_active":"2024-09-21T10:18:10.939184+0000","last_became_peered":"2024-09-21T10:18:10.939184+0000","last_unstale":"2024-09-21T10:19:40.239651+0000","last_undegraded":"2024-09-21T10:19:40.239651+0000","last_fullsized":"2024-09-21T10:19:40.239651+0000","mapping_epoch":1359,"log_start":"1366'2300","ondisk_log_start":"1366'2300","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4849,"log_dups_size":0,"ondisk_log_size":4849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:33:41.090367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":468516864,"num_objects":7149,"num_object_clones":0,"num_object_copies":21447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7149,"num_write_kb":457536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":828,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.669149+0000","last_change":"2024-09-21T10:16:27.315307+0000","last_active":"2024-09-21T10:19:22.669149+0000","last_peered":"2024-09-21T10:19:22.669149+0000","last_clean":"2024-09-21T10:19:22.669149+0000","last_became_active":"2024-09-21T10:16:27.314999+0000","last_became_peered":"2024-09-21T10:16:27.314999+0000","last_unstale":"2024-09-21T10:19:22.669149+0000","last_undegraded":"2024-09-21T10:19:22.669149+0000","last_fullsized":"2024-09-21T10:19:22.669149+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:34.047746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1315,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646984+0000","last_change":"2024-09-21T10:10:13.686243+0000","last_active":"2024-09-21T10:19:14.646984+0000","last_peered":"2024-09-21T10:19:14.646984+0000","last_clean":"2024-09-21T10:19:14.646984+0000","last_became_active":"2024-09-21T10:10:13.685894+0000","last_became_peered":"2024-09-21T10:10:13.685894+0000","last_unstale":"2024-09-21T10:19:14.646984+0000","last_undegraded":"2024-09-21T10:19:14.646984+0000","last_fullsized":"2024-09-21T10:19:14.646984+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:12.394732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1116,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649523+0000","last_change":"2024-09-21T10:16:27.343418+0000","last_active":"2024-09-21T10:19:14.649523+0000","last_peered":"2024-09-21T10:19:14.649523+0000","last_clean":"2024-09-21T10:19:14.649523+0000","last_became_active":"2024-09-21T10:16:27.339592+0000","last_became_peered":"2024-09-21T10:16:27.339592+0000","last_unstale":"2024-09-21T10:19:14.649523+0000","last_undegraded":"2024-09-21T10:19:14.649523+0000","last_fullsized":"2024-09-21T10:19:14.649523+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:42.895218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":757,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646675+0000","last_change":"2024-09-21T10:16:07.911971+0000","last_active":"2024-09-21T10:19:14.646675+0000","last_peered":"2024-09-21T10:19:14.646675+0000","last_clean":"2024-09-21T10:19:14.646675+0000","last_became_active":"2024-09-21T10:16:07.911538+0000","last_became_peered":"2024-09-21T10:16:07.911538+0000","last_unstale":"2024-09-21T10:19:14.646675+0000","last_undegraded":"2024-09-21T10:19:14.646675+0000","last_fullsized":"2024-09-21T10:19:14.646675+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:19.901507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:19.901507+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:29.782749+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.8","version":"1372'7228","reported_seq":7262,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.232068+0000","last_change":"2024-09-21T10:18:10.940811+0000","last_active":"2024-09-21T10:19:40.232068+0000","last_peered":"2024-09-21T10:19:40.232068+0000","last_clean":"2024-09-21T10:19:40.232068+0000","last_became_active":"2024-09-21T10:18:10.940121+0000","last_became_peered":"2024-09-21T10:18:10.940121+0000","last_unstale":"2024-09-21T10:19:40.232068+0000","last_undegraded":"2024-09-21T10:19:40.232068+0000","last_fullsized":"2024-09-21T10:19:40.232068+0000","mapping_epoch":1359,"log_start":"1366'2300","ondisk_log_start":"1366'2300","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4928,"log_dups_size":0,"ondisk_log_size":4928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:46.766106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":473694208,"num_objects":7228,"num_object_clones":0,"num_object_copies":21684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7228,"num_write_kb":462592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649883+0000","last_change":"2024-09-21T10:18:31.646956+0000","last_active":"2024-09-21T10:19:14.649883+0000","last_peered":"2024-09-21T10:19:14.649883+0000","last_clean":"2024-09-21T10:19:14.649883+0000","last_became_active":"2024-09-21T10:18:31.646598+0000","last_became_peered":"2024-09-21T10:18:31.646598+0000","last_unstale":"2024-09-21T10:19:14.649883+0000","last_undegraded":"2024-09-21T10:19:14.649883+0000","last_fullsized":"2024-09-21T10:19:14.649883+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:44:28.469526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.652613+0000","last_change":"2024-09-21T10:16:27.322540+0000","last_active":"2024-09-21T10:19:14.652613+0000","last_peered":"2024-09-21T10:19:14.652613+0000","last_clean":"2024-09-21T10:19:14.652613+0000","last_became_active":"2024-09-21T10:16:27.322003+0000","last_became_peered":"2024-09-21T10:16:27.322003+0000","last_unstale":"2024-09-21T10:19:14.652613+0000","last_undegraded":"2024-09-21T10:19:14.652613+0000","last_fullsized":"2024-09-21T10:19:14.652613+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:13.361229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2148,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.668284+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:19:22.668284+0000","last_peered":"2024-09-21T10:19:22.668284+0000","last_clean":"2024-09-21T10:19:22.668284+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:19:22.668284+0000","last_undegraded":"2024-09-21T10:19:22.668284+0000","last_fullsized":"2024-09-21T10:19:22.668284+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":123,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649895+0000","last_change":"2024-09-21T10:13:17.734264+0000","last_active":"2024-09-21T10:19:14.649895+0000","last_peered":"2024-09-21T10:19:14.649895+0000","last_clean":"2024-09-21T10:19:14.649895+0000","last_became_active":"2024-09-21T10:12:25.707372+0000","last_became_peered":"2024-09-21T10:12:25.707372+0000","last_unstale":"2024-09-21T10:19:14.649895+0000","last_undegraded":"2024-09-21T10:19:14.649895+0000","last_fullsized":"2024-09-21T10:19:14.649895+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:17.734169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:17.734169+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:59.937743+0000","scrub_duration":0.0017947760000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"1372'7120","reported_seq":7157,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.213681+0000","last_change":"2024-09-21T10:18:10.940549+0000","last_active":"2024-09-21T10:19:40.213681+0000","last_peered":"2024-09-21T10:19:40.213681+0000","last_clean":"2024-09-21T10:19:40.213681+0000","last_became_active":"2024-09-21T10:18:10.940263+0000","last_became_peered":"2024-09-21T10:18:10.940263+0000","last_unstale":"2024-09-21T10:19:40.213681+0000","last_undegraded":"2024-09-21T10:19:40.213681+0000","last_fullsized":"2024-09-21T10:19:40.213681+0000","mapping_epoch":1359,"log_start":"1366'2200","ondisk_log_start":"1366'2200","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4920,"log_dups_size":0,"ondisk_log_size":4920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:13:25.244015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":466616320,"num_objects":7120,"num_object_clones":0,"num_object_copies":21360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7120,"num_write_kb":455680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2020,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.668790+0000","last_change":"2024-09-21T10:17:34.779713+0000","last_active":"2024-09-21T10:19:22.668790+0000","last_peered":"2024-09-21T10:19:22.668790+0000","last_clean":"2024-09-21T10:19:22.668790+0000","last_became_active":"2024-09-21T10:17:34.779351+0000","last_became_peered":"2024-09-21T10:17:34.779351+0000","last_unstale":"2024-09-21T10:19:22.668790+0000","last_undegraded":"2024-09-21T10:19:22.668790+0000","last_fullsized":"2024-09-21T10:19:22.668790+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:42.916450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1934,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646630+0000","last_change":"2024-09-21T10:16:07.907738+0000","last_active":"2024-09-21T10:19:14.646630+0000","last_peered":"2024-09-21T10:19:14.646630+0000","last_clean":"2024-09-21T10:19:14.646630+0000","last_became_active":"2024-09-21T10:16:07.907415+0000","last_became_peered":"2024-09-21T10:16:07.907415+0000","last_unstale":"2024-09-21T10:19:14.646630+0000","last_undegraded":"2024-09-21T10:19:14.646630+0000","last_fullsized":"2024-09-21T10:19:14.646630+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:44.971174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":123,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649718+0000","last_change":"2024-09-21T10:16:27.320147+0000","last_active":"2024-09-21T10:19:14.649718+0000","last_peered":"2024-09-21T10:19:14.649718+0000","last_clean":"2024-09-21T10:19:14.649718+0000","last_became_active":"2024-09-21T10:16:27.319802+0000","last_became_peered":"2024-09-21T10:16:27.319802+0000","last_unstale":"2024-09-21T10:19:14.649718+0000","last_undegraded":"2024-09-21T10:19:14.649718+0000","last_fullsized":"2024-09-21T10:19:14.649718+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:31.946542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":78,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:24.475021+0000","last_change":"2024-09-21T10:19:24.474938+0000","last_active":"2024-09-21T10:19:24.475021+0000","last_peered":"2024-09-21T10:19:24.475021+0000","last_clean":"2024-09-21T10:19:24.475021+0000","last_became_active":"2024-09-21T10:17:34.769498+0000","last_became_peered":"2024-09-21T10:17:34.769498+0000","last_unstale":"2024-09-21T10:19:24.475021+0000","last_undegraded":"2024-09-21T10:19:24.475021+0000","last_fullsized":"2024-09-21T10:19:24.475021+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:24.474841+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:19:24.474841+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:22:34.128086+0000","scrub_duration":0.0093671499999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.a","version":"1372'7186","reported_seq":10125,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.226545+0000","last_change":"2024-09-21T10:18:49.696065+0000","last_active":"2024-09-21T10:19:40.226545+0000","last_peered":"2024-09-21T10:19:40.226545+0000","last_clean":"2024-09-21T10:19:40.226545+0000","last_became_active":"2024-09-21T10:18:49.691461+0000","last_became_peered":"2024-09-21T10:18:49.691461+0000","last_unstale":"2024-09-21T10:19:40.226545+0000","last_undegraded":"2024-09-21T10:19:40.226545+0000","last_fullsized":"2024-09-21T10:19:40.226545+0000","mapping_epoch":1367,"log_start":"1366'2300","ondisk_log_start":"1366'2300","created":1359,"last_epoch_clean":1368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4886,"log_dups_size":0,"ondisk_log_size":4886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:45.812654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":470941696,"num_objects":7186,"num_object_clones":0,"num_object_copies":21558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7186,"num_write_kb":459904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":231,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647477+0000","last_change":"2024-09-21T10:14:54.650325+0000","last_active":"2024-09-21T10:19:14.647477+0000","last_peered":"2024-09-21T10:19:14.647477+0000","last_clean":"2024-09-21T10:19:14.647477+0000","last_became_active":"2024-09-21T10:14:54.650048+0000","last_became_peered":"2024-09-21T10:14:54.650048+0000","last_unstale":"2024-09-21T10:19:14.647477+0000","last_undegraded":"2024-09-21T10:19:14.647477+0000","last_fullsized":"2024-09-21T10:19:14.647477+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:47.899727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2107,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.676227+0000","last_change":"2024-09-21T10:12:50.577072+0000","last_active":"2024-09-21T10:19:22.676227+0000","last_peered":"2024-09-21T10:19:22.676227+0000","last_clean":"2024-09-21T10:19:22.676227+0000","last_became_active":"2024-09-21T10:12:50.576731+0000","last_became_peered":"2024-09-21T10:12:50.576731+0000","last_unstale":"2024-09-21T10:19:22.676227+0000","last_undegraded":"2024-09-21T10:19:22.676227+0000","last_fullsized":"2024-09-21T10:19:22.676227+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:56:06.937599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":619,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.668266+0000","last_change":"2024-09-21T10:17:54.399619+0000","last_active":"2024-09-21T10:19:22.668266+0000","last_peered":"2024-09-21T10:19:22.668266+0000","last_clean":"2024-09-21T10:19:22.668266+0000","last_became_active":"2024-09-21T10:17:47.870309+0000","last_became_peered":"2024-09-21T10:17:47.870309+0000","last_unstale":"2024-09-21T10:19:22.668266+0000","last_undegraded":"2024-09-21T10:19:22.668266+0000","last_fullsized":"2024-09-21T10:19:22.668266+0000","mapping_epoch":1351,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:30.694562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":123,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649644+0000","last_change":"2024-09-21T10:13:20.647735+0000","last_active":"2024-09-21T10:19:14.649644+0000","last_peered":"2024-09-21T10:19:14.649644+0000","last_clean":"2024-09-21T10:19:14.649644+0000","last_became_active":"2024-09-21T10:12:25.708657+0000","last_became_peered":"2024-09-21T10:12:25.708657+0000","last_unstale":"2024-09-21T10:19:14.649644+0000","last_undegraded":"2024-09-21T10:19:14.649644+0000","last_fullsized":"2024-09-21T10:19:14.649644+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:20.647659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:20.647659+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:09.770331+0000","scrub_duration":0.0016005629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.0","version":"1372'7182","reported_seq":7219,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.239340+0000","last_change":"2024-09-21T10:18:10.942106+0000","last_active":"2024-09-21T10:19:40.239340+0000","last_peered":"2024-09-21T10:19:40.239340+0000","last_clean":"2024-09-21T10:19:40.239340+0000","last_became_active":"2024-09-21T10:18:10.940520+0000","last_became_peered":"2024-09-21T10:18:10.940520+0000","last_unstale":"2024-09-21T10:19:40.239340+0000","last_undegraded":"2024-09-21T10:19:40.239340+0000","last_fullsized":"2024-09-21T10:19:40.239340+0000","mapping_epoch":1359,"log_start":"1366'2300","ondisk_log_start":"1366'2300","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:57.174328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":470679552,"num_objects":7182,"num_object_clones":0,"num_object_copies":21546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7182,"num_write_kb":459648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":829,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.678012+0000","last_change":"2024-09-21T10:16:27.333765+0000","last_active":"2024-09-21T10:19:22.678012+0000","last_peered":"2024-09-21T10:19:22.678012+0000","last_clean":"2024-09-21T10:19:22.678012+0000","last_became_active":"2024-09-21T10:16:27.331548+0000","last_became_peered":"2024-09-21T10:16:27.331548+0000","last_unstale":"2024-09-21T10:19:22.678012+0000","last_undegraded":"2024-09-21T10:19:22.678012+0000","last_fullsized":"2024-09-21T10:19:22.678012+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:28:55.065363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2136,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.679878+0000","last_change":"2024-09-21T10:16:27.330267+0000","last_active":"2024-09-21T10:19:22.679878+0000","last_peered":"2024-09-21T10:19:22.679878+0000","last_clean":"2024-09-21T10:19:22.679878+0000","last_became_active":"2024-09-21T10:16:27.329320+0000","last_became_peered":"2024-09-21T10:16:27.329320+0000","last_unstale":"2024-09-21T10:19:22.679878+0000","last_undegraded":"2024-09-21T10:19:22.679878+0000","last_fullsized":"2024-09-21T10:19:22.679878+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:37.062508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":566,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.668325+0000","last_change":"2024-09-21T10:17:19.687362+0000","last_active":"2024-09-21T10:19:22.668325+0000","last_peered":"2024-09-21T10:19:22.668325+0000","last_clean":"2024-09-21T10:19:22.668325+0000","last_became_active":"2024-09-21T10:17:19.687049+0000","last_became_peered":"2024-09-21T10:17:19.687049+0000","last_unstale":"2024-09-21T10:19:22.668325+0000","last_undegraded":"2024-09-21T10:19:22.668325+0000","last_fullsized":"2024-09-21T10:19:22.668325+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:24.686892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:24.686892+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:20:07.195905+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"1372'6868","reported_seq":6888,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:37.482101+0000","last_change":"2024-09-21T10:18:10.943078+0000","last_active":"2024-09-21T10:19:37.482101+0000","last_peered":"2024-09-21T10:19:37.482101+0000","last_clean":"2024-09-21T10:19:37.482101+0000","last_became_active":"2024-09-21T10:18:10.942780+0000","last_became_peered":"2024-09-21T10:18:10.942780+0000","last_unstale":"2024-09-21T10:19:37.482101+0000","last_undegraded":"2024-09-21T10:19:37.482101+0000","last_fullsized":"2024-09-21T10:19:37.482101+0000","mapping_epoch":1359,"log_start":"1366'2100","ondisk_log_start":"1366'2100","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4768,"log_dups_size":0,"ondisk_log_size":4768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:38:22.004424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":450101248,"num_objects":6868,"num_object_clones":0,"num_object_copies":20604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6868,"num_write_kb":439552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1370,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647603+0000","last_change":"2024-09-21T10:16:27.321771+0000","last_active":"2024-09-21T10:19:14.647603+0000","last_peered":"2024-09-21T10:19:14.647603+0000","last_clean":"2024-09-21T10:19:14.647603+0000","last_became_active":"2024-09-21T10:16:27.321423+0000","last_became_peered":"2024-09-21T10:16:27.321423+0000","last_unstale":"2024-09-21T10:19:14.647603+0000","last_undegraded":"2024-09-21T10:19:14.647603+0000","last_fullsized":"2024-09-21T10:19:14.647603+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:04.652246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1328,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646407+0000","last_change":"2024-09-21T10:12:25.697464+0000","last_active":"2024-09-21T10:19:14.646407+0000","last_peered":"2024-09-21T10:19:14.646407+0000","last_clean":"2024-09-21T10:19:14.646407+0000","last_became_active":"2024-09-21T10:12:25.697050+0000","last_became_peered":"2024-09-21T10:12:25.697050+0000","last_unstale":"2024-09-21T10:19:14.646407+0000","last_undegraded":"2024-09-21T10:19:14.646407+0000","last_fullsized":"2024-09-21T10:19:14.646407+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:50.687008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"288'100","reported_seq":193,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.668090+0000","last_change":"2024-09-21T10:17:19.681714+0000","last_active":"2024-09-21T10:19:22.668090+0000","last_peered":"2024-09-21T10:19:22.668090+0000","last_clean":"2024-09-21T10:19:22.668090+0000","last_became_active":"2024-09-21T10:17:19.681360+0000","last_became_peered":"2024-09-21T10:17:19.681360+0000","last_unstale":"2024-09-21T10:19:22.668090+0000","last_undegraded":"2024-09-21T10:19:22.668090+0000","last_fullsized":"2024-09-21T10:19:22.668090+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:30.346035+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:30.346035+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:20:04.181441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":4,"num_read_kb":11,"num_write":2,"num_write_kb":25,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":8945,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.2","version":"1372'7240","reported_seq":7277,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.215709+0000","last_change":"2024-09-21T10:18:10.937252+0000","last_active":"2024-09-21T10:19:40.215709+0000","last_peered":"2024-09-21T10:19:40.215709+0000","last_clean":"2024-09-21T10:19:40.215709+0000","last_became_active":"2024-09-21T10:18:10.936946+0000","last_became_peered":"2024-09-21T10:18:10.936946+0000","last_unstale":"2024-09-21T10:19:40.215709+0000","last_undegraded":"2024-09-21T10:19:40.215709+0000","last_fullsized":"2024-09-21T10:19:40.215709+0000","mapping_epoch":1359,"log_start":"1366'2400","ondisk_log_start":"1366'2400","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4840,"log_dups_size":0,"ondisk_log_size":4840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:49:44.594699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":474480640,"num_objects":7240,"num_object_clones":0,"num_object_copies":21720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7240,"num_write_kb":463360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1347,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647575+0000","last_change":"2024-09-21T10:16:27.320140+0000","last_active":"2024-09-21T10:19:14.647575+0000","last_peered":"2024-09-21T10:19:14.647575+0000","last_clean":"2024-09-21T10:19:14.647575+0000","last_became_active":"2024-09-21T10:16:27.319787+0000","last_became_peered":"2024-09-21T10:16:27.319787+0000","last_unstale":"2024-09-21T10:19:14.647575+0000","last_undegraded":"2024-09-21T10:19:14.647575+0000","last_fullsized":"2024-09-21T10:19:14.647575+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:11.703898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":566,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646823+0000","last_change":"2024-09-21T10:16:27.318938+0000","last_active":"2024-09-21T10:19:14.646823+0000","last_peered":"2024-09-21T10:19:14.646823+0000","last_clean":"2024-09-21T10:19:14.646823+0000","last_became_active":"2024-09-21T10:16:27.318623+0000","last_became_peered":"2024-09-21T10:16:27.318623+0000","last_unstale":"2024-09-21T10:19:14.646823+0000","last_undegraded":"2024-09-21T10:19:14.646823+0000","last_fullsized":"2024-09-21T10:19:14.646823+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:21.912596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":565,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.667761+0000","last_change":"2024-09-21T10:17:19.688581+0000","last_active":"2024-09-21T10:19:22.667761+0000","last_peered":"2024-09-21T10:19:22.667761+0000","last_clean":"2024-09-21T10:19:22.667761+0000","last_became_active":"2024-09-21T10:17:19.687219+0000","last_became_peered":"2024-09-21T10:17:19.687219+0000","last_unstale":"2024-09-21T10:19:22.667761+0000","last_undegraded":"2024-09-21T10:19:22.667761+0000","last_fullsized":"2024-09-21T10:19:22.667761+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:20:09.189549+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":131344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.3","version":"1372'6911","reported_seq":6932,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:37.476236+0000","last_change":"2024-09-21T10:18:10.946676+0000","last_active":"2024-09-21T10:19:37.476236+0000","last_peered":"2024-09-21T10:19:37.476236+0000","last_clean":"2024-09-21T10:19:37.476236+0000","last_became_active":"2024-09-21T10:18:10.946364+0000","last_became_peered":"2024-09-21T10:18:10.946364+0000","last_unstale":"2024-09-21T10:19:37.476236+0000","last_undegraded":"2024-09-21T10:19:37.476236+0000","last_fullsized":"2024-09-21T10:19:37.476236+0000","mapping_epoch":1359,"log_start":"1366'2100","ondisk_log_start":"1366'2100","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4811,"log_dups_size":0,"ondisk_log_size":4811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:54.908078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":452919296,"num_objects":6911,"num_object_clones":0,"num_object_copies":20733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6911,"num_write_kb":442304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1314,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647181+0000","last_change":"2024-09-21T10:10:13.661328+0000","last_active":"2024-09-21T10:19:14.647181+0000","last_peered":"2024-09-21T10:19:14.647181+0000","last_clean":"2024-09-21T10:19:14.647181+0000","last_became_active":"2024-09-21T10:10:13.661028+0000","last_became_peered":"2024-09-21T10:10:13.661028+0000","last_unstale":"2024-09-21T10:19:14.647181+0000","last_undegraded":"2024-09-21T10:19:14.647181+0000","last_fullsized":"2024-09-21T10:19:14.647181+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:36.965214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":2109,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.679534+0000","last_change":"2024-09-21T10:08:25.436371+0000","last_active":"2024-09-21T10:19:22.679534+0000","last_peered":"2024-09-21T10:19:22.679534+0000","last_clean":"2024-09-21T10:19:22.679534+0000","last_became_active":"2024-09-21T10:08:25.435921+0000","last_became_peered":"2024-09-21T10:08:25.435921+0000","last_unstale":"2024-09-21T10:19:22.679534+0000","last_undegraded":"2024-09-21T10:19:22.679534+0000","last_fullsized":"2024-09-21T10:19:22.679534+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:27.612386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":143,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649971+0000","last_change":"2024-09-21T10:17:27.870379+0000","last_active":"2024-09-21T10:19:14.649971+0000","last_peered":"2024-09-21T10:19:14.649971+0000","last_clean":"2024-09-21T10:19:14.649971+0000","last_became_active":"2024-09-21T10:17:19.686196+0000","last_became_peered":"2024-09-21T10:17:19.686196+0000","last_unstale":"2024-09-21T10:19:14.649971+0000","last_undegraded":"2024-09-21T10:19:14.649971+0000","last_fullsized":"2024-09-21T10:19:14.649971+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:17:27.870303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:17:27.870303+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:22.680785+0000","scrub_duration":0.001791443,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"1372'6961","reported_seq":6982,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:35.950110+0000","last_change":"2024-09-21T10:18:10.940665+0000","last_active":"2024-09-21T10:19:35.950110+0000","last_peered":"2024-09-21T10:19:35.950110+0000","last_clean":"2024-09-21T10:19:35.950110+0000","last_became_active":"2024-09-21T10:18:10.940315+0000","last_became_peered":"2024-09-21T10:18:10.940315+0000","last_unstale":"2024-09-21T10:19:35.950110+0000","last_undegraded":"2024-09-21T10:19:35.950110+0000","last_fullsized":"2024-09-21T10:19:35.950110+0000","mapping_epoch":1359,"log_start":"1366'1800","ondisk_log_start":"1366'1800","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5161,"log_dups_size":0,"ondisk_log_size":5161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:08:47.081178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":456196096,"num_objects":6961,"num_object_clones":0,"num_object_copies":20883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6961,"num_write_kb":445504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1971,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646849+0000","last_change":"2024-09-21T10:16:27.325294+0000","last_active":"2024-09-21T10:19:14.646849+0000","last_peered":"2024-09-21T10:19:14.646849+0000","last_clean":"2024-09-21T10:19:14.646849+0000","last_became_active":"2024-09-21T10:16:27.324974+0000","last_became_peered":"2024-09-21T10:16:27.324974+0000","last_unstale":"2024-09-21T10:19:14.646849+0000","last_undegraded":"2024-09-21T10:19:14.646849+0000","last_fullsized":"2024-09-21T10:19:14.646849+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:58.600136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1115,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.676557+0000","last_change":"2024-09-21T10:10:13.678563+0000","last_active":"2024-09-21T10:19:22.676557+0000","last_peered":"2024-09-21T10:19:22.676557+0000","last_clean":"2024-09-21T10:19:22.676557+0000","last_became_active":"2024-09-21T10:10:13.676594+0000","last_became_peered":"2024-09-21T10:10:13.676594+0000","last_unstale":"2024-09-21T10:19:22.676557+0000","last_undegraded":"2024-09-21T10:19:22.676557+0000","last_fullsized":"2024-09-21T10:19:22.676557+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:34.275762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"288'100","reported_seq":198,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647377+0000","last_change":"2024-09-21T10:18:31.396074+0000","last_active":"2024-09-21T10:19:14.647377+0000","last_peered":"2024-09-21T10:19:14.647377+0000","last_clean":"2024-09-21T10:19:14.647377+0000","last_became_active":"2024-09-21T10:17:19.681584+0000","last_became_peered":"2024-09-21T10:17:19.681584+0000","last_unstale":"2024-09-21T10:19:14.647377+0000","last_undegraded":"2024-09-21T10:19:14.647377+0000","last_fullsized":"2024-09-21T10:19:14.647377+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:31.396016+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:31.396016+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:22.138287+0000","scrub_duration":0.001381925,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1372'6876","reported_seq":6897,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:35.938749+0000","last_change":"2024-09-21T10:18:10.942944+0000","last_active":"2024-09-21T10:19:35.938749+0000","last_peered":"2024-09-21T10:19:35.938749+0000","last_clean":"2024-09-21T10:19:35.938749+0000","last_became_active":"2024-09-21T10:18:10.942635+0000","last_became_peered":"2024-09-21T10:18:10.942635+0000","last_unstale":"2024-09-21T10:19:35.938749+0000","last_undegraded":"2024-09-21T10:19:35.938749+0000","last_fullsized":"2024-09-21T10:19:35.938749+0000","mapping_epoch":1359,"log_start":"1366'1700","ondisk_log_start":"1366'1700","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5176,"log_dups_size":0,"ondisk_log_size":5176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:11:10.686262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":450625536,"num_objects":6876,"num_object_clones":0,"num_object_copies":20628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6876,"num_write_kb":440064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":149,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.652938+0000","last_change":"2024-09-21T10:16:27.315202+0000","last_active":"2024-09-21T10:19:14.652938+0000","last_peered":"2024-09-21T10:19:14.652938+0000","last_clean":"2024-09-21T10:19:14.652938+0000","last_became_active":"2024-09-21T10:16:27.314827+0000","last_became_peered":"2024-09-21T10:16:27.314827+0000","last_unstale":"2024-09-21T10:19:14.652938+0000","last_undegraded":"2024-09-21T10:19:14.652938+0000","last_fullsized":"2024-09-21T10:19:14.652938+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:31.494018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":131,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650222+0000","last_change":"2024-09-21T10:12:50.623002+0000","last_active":"2024-09-21T10:19:14.650222+0000","last_peered":"2024-09-21T10:19:14.650222+0000","last_clean":"2024-09-21T10:19:14.650222+0000","last_became_active":"2024-09-21T10:12:50.588355+0000","last_became_peered":"2024-09-21T10:12:50.588355+0000","last_unstale":"2024-09-21T10:19:14.650222+0000","last_undegraded":"2024-09-21T10:19:14.650222+0000","last_fullsized":"2024-09-21T10:19:14.650222+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:22.564459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"288'100","reported_seq":232,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.647238+0000","last_change":"2024-09-21T10:14:54.647582+0000","last_active":"2024-09-21T10:19:14.647238+0000","last_peered":"2024-09-21T10:19:14.647238+0000","last_clean":"2024-09-21T10:19:14.647238+0000","last_became_active":"2024-09-21T10:14:54.647213+0000","last_became_peered":"2024-09-21T10:14:54.647213+0000","last_unstale":"2024-09-21T10:19:14.647238+0000","last_undegraded":"2024-09-21T10:19:14.647238+0000","last_fullsized":"2024-09-21T10:19:14.647238+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:23.746751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:23.746751+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:04.424278+0000","scrub_duration":0.0021986229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"1372'6997","reported_seq":7031,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:36.949793+0000","last_change":"2024-09-21T10:18:10.935134+0000","last_active":"2024-09-21T10:19:36.949793+0000","last_peered":"2024-09-21T10:19:36.949793+0000","last_clean":"2024-09-21T10:19:36.949793+0000","last_became_active":"2024-09-21T10:18:10.934822+0000","last_became_peered":"2024-09-21T10:18:10.934822+0000","last_unstale":"2024-09-21T10:19:36.949793+0000","last_undegraded":"2024-09-21T10:19:36.949793+0000","last_fullsized":"2024-09-21T10:19:36.949793+0000","mapping_epoch":1359,"log_start":"1363'700","ondisk_log_start":"1363'700","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":6297,"log_dups_size":0,"ondisk_log_size":6297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:04:47.127224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458555392,"num_objects":6997,"num_object_clones":0,"num_object_copies":20991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6997,"num_write_kb":447808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1347,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.646751+0000","last_change":"2024-09-21T10:16:27.321387+0000","last_active":"2024-09-21T10:19:14.646751+0000","last_peered":"2024-09-21T10:19:14.646751+0000","last_clean":"2024-09-21T10:19:14.646751+0000","last_became_active":"2024-09-21T10:16:27.321042+0000","last_became_peered":"2024-09-21T10:16:27.321042+0000","last_unstale":"2024-09-21T10:19:14.646751+0000","last_undegraded":"2024-09-21T10:19:14.646751+0000","last_fullsized":"2024-09-21T10:19:14.646751+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:31.231738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":145,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650225+0000","last_change":"2024-09-21T10:17:34.781189+0000","last_active":"2024-09-21T10:19:14.650225+0000","last_peered":"2024-09-21T10:19:14.650225+0000","last_clean":"2024-09-21T10:19:14.650225+0000","last_became_active":"2024-09-21T10:17:34.777855+0000","last_became_peered":"2024-09-21T10:17:34.777855+0000","last_unstale":"2024-09-21T10:19:14.650225+0000","last_undegraded":"2024-09-21T10:19:14.650225+0000","last_fullsized":"2024-09-21T10:19:14.650225+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:42:01.190570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"288'100","reported_seq":587,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.676238+0000","last_change":"2024-09-21T10:17:19.681300+0000","last_active":"2024-09-21T10:19:22.676238+0000","last_peered":"2024-09-21T10:19:22.676238+0000","last_clean":"2024-09-21T10:19:22.676238+0000","last_became_active":"2024-09-21T10:17:19.680813+0000","last_became_peered":"2024-09-21T10:17:19.680813+0000","last_unstale":"2024-09-21T10:19:22.676238+0000","last_undegraded":"2024-09-21T10:19:22.676238+0000","last_fullsized":"2024-09-21T10:19:22.676238+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:27.395700+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:27.395700+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:20:07.185589+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1372'7094","reported_seq":7115,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:38.731779+0000","last_change":"2024-09-21T10:18:10.936688+0000","last_active":"2024-09-21T10:19:38.731779+0000","last_peered":"2024-09-21T10:19:38.731779+0000","last_clean":"2024-09-21T10:19:38.731779+0000","last_became_active":"2024-09-21T10:18:10.936333+0000","last_became_peered":"2024-09-21T10:18:10.936333+0000","last_unstale":"2024-09-21T10:19:38.731779+0000","last_undegraded":"2024-09-21T10:19:38.731779+0000","last_fullsized":"2024-09-21T10:19:38.731779+0000","mapping_epoch":1359,"log_start":"1366'2000","ondisk_log_start":"1366'2000","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5094,"log_dups_size":0,"ondisk_log_size":5094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:14:28.913228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":464912384,"num_objects":7094,"num_object_clones":0,"num_object_copies":21282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7094,"num_write_kb":454016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":123,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650049+0000","last_change":"2024-09-21T10:12:50.573976+0000","last_active":"2024-09-21T10:19:14.650049+0000","last_peered":"2024-09-21T10:19:14.650049+0000","last_clean":"2024-09-21T10:19:14.650049+0000","last_became_active":"2024-09-21T10:12:50.573657+0000","last_became_peered":"2024-09-21T10:12:50.573657+0000","last_unstale":"2024-09-21T10:19:14.650049+0000","last_undegraded":"2024-09-21T10:19:14.650049+0000","last_fullsized":"2024-09-21T10:19:14.650049+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:44:44.245943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":117,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650950+0000","last_change":"2024-09-21T10:12:25.764496+0000","last_active":"2024-09-21T10:19:14.650950+0000","last_peered":"2024-09-21T10:19:14.650950+0000","last_clean":"2024-09-21T10:19:14.650950+0000","last_became_active":"2024-09-21T10:12:25.707263+0000","last_became_peered":"2024-09-21T10:12:25.707263+0000","last_unstale":"2024-09-21T10:19:14.650950+0000","last_undegraded":"2024-09-21T10:19:14.650950+0000","last_fullsized":"2024-09-21T10:19:14.650950+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:45.728577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"288'100","reported_seq":156,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650463+0000","last_change":"2024-09-21T10:18:36.066497+0000","last_active":"2024-09-21T10:19:14.650463+0000","last_peered":"2024-09-21T10:19:14.650463+0000","last_clean":"2024-09-21T10:19:14.650463+0000","last_became_active":"2024-09-21T10:17:19.683947+0000","last_became_peered":"2024-09-21T10:17:19.683947+0000","last_unstale":"2024-09-21T10:19:14.650463+0000","last_undegraded":"2024-09-21T10:19:14.650463+0000","last_fullsized":"2024-09-21T10:19:14.650463+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:36.066408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:36.066408+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:43.382614+0000","scrub_duration":0.006208028,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":179,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.652745+0000","last_change":"2024-09-21T10:17:34.792847+0000","last_active":"2024-09-21T10:19:14.652745+0000","last_peered":"2024-09-21T10:19:14.652745+0000","last_clean":"2024-09-21T10:19:14.652745+0000","last_became_active":"2024-09-21T10:17:34.780504+0000","last_became_peered":"2024-09-21T10:17:34.780504+0000","last_unstale":"2024-09-21T10:19:14.652745+0000","last_undegraded":"2024-09-21T10:19:14.652745+0000","last_fullsized":"2024-09-21T10:19:14.652745+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:36.142424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2122,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.677802+0000","last_change":"2024-09-21T10:16:27.324409+0000","last_active":"2024-09-21T10:19:22.677802+0000","last_peered":"2024-09-21T10:19:22.677802+0000","last_clean":"2024-09-21T10:19:22.677802+0000","last_became_active":"2024-09-21T10:16:27.321554+0000","last_became_peered":"2024-09-21T10:16:27.321554+0000","last_unstale":"2024-09-21T10:19:22.677802+0000","last_undegraded":"2024-09-21T10:19:22.677802+0000","last_fullsized":"2024-09-21T10:19:22.677802+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:56.209301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"288'100","reported_seq":112,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.652619+0000","last_change":"2024-09-21T10:18:31.567928+0000","last_active":"2024-09-21T10:19:14.652619+0000","last_peered":"2024-09-21T10:19:14.652619+0000","last_clean":"2024-09-21T10:19:14.652619+0000","last_became_active":"2024-09-21T10:16:27.337115+0000","last_became_peered":"2024-09-21T10:16:27.337115+0000","last_unstale":"2024-09-21T10:19:14.652619+0000","last_undegraded":"2024-09-21T10:19:14.652619+0000","last_fullsized":"2024-09-21T10:19:14.652619+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:31.567804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:31.567804+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:24.827181+0000","scrub_duration":0.001700088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":113,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649831+0000","last_change":"2024-09-21T10:12:25.700080+0000","last_active":"2024-09-21T10:19:14.649831+0000","last_peered":"2024-09-21T10:19:14.649831+0000","last_clean":"2024-09-21T10:19:14.649831+0000","last_became_active":"2024-09-21T10:12:25.699762+0000","last_became_peered":"2024-09-21T10:12:25.699762+0000","last_unstale":"2024-09-21T10:19:14.649831+0000","last_undegraded":"2024-09-21T10:19:14.649831+0000","last_fullsized":"2024-09-21T10:19:14.649831+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:41.509175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2072,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.677545+0000","last_change":"2024-09-21T10:16:07.912706+0000","last_active":"2024-09-21T10:19:22.677545+0000","last_peered":"2024-09-21T10:19:22.677545+0000","last_clean":"2024-09-21T10:19:22.677545+0000","last_became_active":"2024-09-21T10:16:07.912247+0000","last_became_peered":"2024-09-21T10:16:07.912247+0000","last_unstale":"2024-09-21T10:19:22.677545+0000","last_undegraded":"2024-09-21T10:19:22.677545+0000","last_fullsized":"2024-09-21T10:19:22.677545+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:52.397464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"288'100","reported_seq":112,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.678846+0000","last_change":"2024-09-21T10:16:27.331860+0000","last_active":"2024-09-21T10:19:22.678846+0000","last_peered":"2024-09-21T10:19:22.678846+0000","last_clean":"2024-09-21T10:19:22.678846+0000","last_became_active":"2024-09-21T10:16:27.331320+0000","last_became_peered":"2024-09-21T10:16:27.331320+0000","last_unstale":"2024-09-21T10:19:22.678846+0000","last_undegraded":"2024-09-21T10:19:22.678846+0000","last_fullsized":"2024-09-21T10:19:22.678846+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:16.405582+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:16.405582+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:20:10.134995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":23,"num_write":5,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2022,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649859+0000","last_change":"2024-09-21T10:16:07.907197+0000","last_active":"2024-09-21T10:19:14.649859+0000","last_peered":"2024-09-21T10:19:14.649859+0000","last_clean":"2024-09-21T10:19:14.649859+0000","last_became_active":"2024-09-21T10:16:07.906770+0000","last_became_peered":"2024-09-21T10:16:07.906770+0000","last_unstale":"2024-09-21T10:19:14.649859+0000","last_undegraded":"2024-09-21T10:19:14.649859+0000","last_fullsized":"2024-09-21T10:19:14.649859+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:55:53.815656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.650466+0000","last_change":"2024-09-21T10:12:50.584513+0000","last_active":"2024-09-21T10:19:14.650466+0000","last_peered":"2024-09-21T10:19:14.650466+0000","last_clean":"2024-09-21T10:19:14.650466+0000","last_became_active":"2024-09-21T10:12:50.584107+0000","last_became_peered":"2024-09-21T10:12:50.584107+0000","last_unstale":"2024-09-21T10:19:14.650466+0000","last_undegraded":"2024-09-21T10:19:14.650466+0000","last_fullsized":"2024-09-21T10:19:14.650466+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:23.047403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"288'100","reported_seq":133,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649917+0000","last_change":"2024-09-21T10:13:22.620513+0000","last_active":"2024-09-21T10:19:14.649917+0000","last_peered":"2024-09-21T10:19:14.649917+0000","last_clean":"2024-09-21T10:19:14.649917+0000","last_became_active":"2024-09-21T10:12:50.581542+0000","last_became_peered":"2024-09-21T10:12:50.581542+0000","last_unstale":"2024-09-21T10:19:14.649917+0000","last_undegraded":"2024-09-21T10:19:14.649917+0000","last_fullsized":"2024-09-21T10:19:14.649917+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:22.620446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:22.620446+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:13.111755+0000","scrub_duration":0.0014078199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":119,"reported_epoch":1371,"state":"active+clean","last_fresh":"2024-09-21T10:19:14.649996+0000","last_change":"2024-09-21T10:12:25.720808+0000","last_active":"2024-09-21T10:19:14.649996+0000","last_peered":"2024-09-21T10:19:14.649996+0000","last_clean":"2024-09-21T10:19:14.649996+0000","last_became_active":"2024-09-21T10:12:25.706699+0000","last_became_peered":"2024-09-21T10:12:25.706699+0000","last_unstale":"2024-09-21T10:19:14.649996+0000","last_undegraded":"2024-09-21T10:19:14.649996+0000","last_fullsized":"2024-09-21T10:19:14.649996+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:29.044701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2123,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.676445+0000","last_change":"2024-09-21T10:16:27.318532+0000","last_active":"2024-09-21T10:19:22.676445+0000","last_peered":"2024-09-21T10:19:22.676445+0000","last_clean":"2024-09-21T10:19:22.676445+0000","last_became_active":"2024-09-21T10:16:27.317049+0000","last_became_peered":"2024-09-21T10:16:27.317049+0000","last_unstale":"2024-09-21T10:19:22.676445+0000","last_undegraded":"2024-09-21T10:19:22.676445+0000","last_fullsized":"2024-09-21T10:19:22.676445+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:43.431430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"288'100","reported_seq":727,"reported_epoch":1372,"state":"active+clean","last_fresh":"2024-09-21T10:19:22.680327+0000","last_change":"2024-09-21T10:16:27.324371+0000","last_active":"2024-09-21T10:19:22.680327+0000","last_peered":"2024-09-21T10:19:22.680327+0000","last_clean":"2024-09-21T10:19:22.680327+0000","last_became_active":"2024-09-21T10:16:27.323938+0000","last_became_peered":"2024-09-21T10:16:27.323938+0000","last_unstale":"2024-09-21T10:19:22.680327+0000","last_undegraded":"2024-09-21T10:19:22.680327+0000","last_fullsized":"2024-09-21T10:19:22.680327+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.888350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.888350+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:16.271510+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":7377387520,"num_objects":112570,"num_object_clones":0,"num_object_copies":337710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":112570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":112570,"num_write_kb":7204480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":22040739840,"data_stored":22040739840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83370,"ondisk_log_size":83370,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":24716,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34026,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":466780,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":19,"num_read_kb":51,"num_write":10,"num_write_kb":111,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":729468,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398045,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4409384,"kb_used_data":3513056,"kb_used_omap":1,"kb_used_meta":896318,"kb_avail":89335768,"statfs":{"total":95995035648,"available":91479826432,"internally_reserved":0,"allocated":3597369344,"data_stored":3593898996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917829976},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594274,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4889056,"kb_used_data":3997336,"kb_used_omap":1,"kb_used_meta":891710,"kb_avail":88856096,"statfs":{"total":95995035648,"available":90988642304,"internally_reserved":0,"allocated":4093272064,"data_stored":4089809791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913111384},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790503,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1774860,"kb_used_data":880708,"kb_used_omap":1,"kb_used_meta":894142,"kb_avail":91970292,"statfs":{"total":95995035648,"available":94177579008,"internally_reserved":0,"allocated":901844992,"data_stored":898397815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915601752},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1224,"seq":5257039970440,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3042788,"kb_used_data":2706268,"kb_used_omap":290,"kb_used_meta":336221,"kb_avail":90702364,"statfs":{"total":95995035648,"available":92879220736,"internally_reserved":0,"allocated":2771218432,"data_stored":2767752120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":297377,"internal_metadata":344290911},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999373,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4826704,"kb_used_data":3676616,"kb_used_omap":25,"kb_used_meta":1150054,"kb_avail":88918448,"statfs":{"total":95995035648,"available":91052490752,"internally_reserved":0,"allocated":3764854784,"data_stored":3761403672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":1177656263},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111315,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4265748,"kb_used_data":3141964,"kb_used_omap":31,"kb_used_meta":1123744,"kb_avail":89479404,"statfs":{"total":95995035648,"available":91626909696,"internally_reserved":0,"allocated":3217371136,"data_stored":3213916916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":1150714088},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":2203318223249,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335201,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4607892,"kb_used_data":3605196,"kb_used_omap":31,"kb_used_meta":1002656,"kb_avail":89137260,"statfs":{"total":95995035648,"available":91276554240,"internally_reserved":0,"allocated":3691720704,"data_stored":3688265921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":1026719840},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3690201088,"data_stored":3690201088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3206479872,"data_stored":3206479872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3757834240,"data_stored":3757834240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2769813504,"data_stored":2769813504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":898629632,"data_stored":898629632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4107272192,"data_stored":4107272192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3610509312,"data_stored":3610509312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:19:42.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 31 2024-09-21T10:19:42.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:43.010 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:43.038 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:42 smithi017 ceph-mon[105151]: from='client.46385 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:19:43.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:42 smithi017 ceph-mon[102913]: from='client.46385 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:19:43.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2024-09-21T10:19:43.108503+0000 min lat: 0.00118998 max lat: 2.30472 avg lat: 0.0108567 2024-09-21T10:19:43.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:19:43.110 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 117889 117873 92.0686 78.4375 0.00766803 0.0108567 2024-09-21T10:19:43.112 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:43.147 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:43.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:42 smithi134 ceph-mon[90085]: from='client.46385 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:19:43.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:43.302 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:43.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:43.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:43.441 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:43.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:43.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:43.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.590 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:43.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:43.691 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:43.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.730 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:43.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:43.830 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:43.865 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:43.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:43.967 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:44.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:44.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:44.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:43 smithi017 ceph-mon[102913]: pgmap v2231: 119 pgs: 119 active+clean; 7.0 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:19:44.061 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:43 smithi017 ceph-mon[105151]: pgmap v2231: 119 pgs: 119 active+clean; 7.0 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:19:44.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:44.114 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 119138 119122 91.8937 78.0625 0.0129218 0.0108775 2024-09-21T10:19:44.153 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:44.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:43 smithi134 ceph-mon[90085]: pgmap v2231: 119 pgs: 119 active+clean; 7.0 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:19:44.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:44.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:44.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:44.397 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:44.430 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:44.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:44.531 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:44.563 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:44.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:44.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:44.697 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:44.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:44.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:44.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:44.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:44.882 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:19:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:19:44] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.51.0" 2024-09-21T10:19:44.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:44.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.992 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:44.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:45.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:45.111 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 15 120428 120413 91.757 80.6875 0.0245854 0.0108936 2024-09-21T10:19:45.132 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:45.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:45.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:45.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:45.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:45.383 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:45.417 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:45.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:19:45.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:45.598 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:45.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:45.700 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:45.745 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:45.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:45.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.894 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:45.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:45.995 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:46.024 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:46.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:46.111 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 121599 121583 91.5324 73.125 0.010605 0.0109202 2024-09-21T10:19:46.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:46.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:46.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:46.172 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:45 smithi017 ceph-mon[102913]: pgmap v2232: 119 pgs: 119 active+clean; 7.1 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 69 MiB/s wr, 1.09k op/s 2024-09-21T10:19:46.172 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:45 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/610001990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:46.172 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:45 smithi017 ceph-mon[105151]: pgmap v2232: 119 pgs: 119 active+clean; 7.1 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 69 MiB/s wr, 1.09k op/s 2024-09-21T10:19:46.172 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:45 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/610001990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:45 smithi134 ceph-mon[90085]: pgmap v2232: 119 pgs: 119 active+clean; 7.1 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 69 MiB/s wr, 1.09k op/s 2024-09-21T10:19:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:45 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/610001990' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:46.267 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:46.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:46.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:46.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.442 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:46.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:46.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:46.578 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:46.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:46.680 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:46.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:46.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:46.793 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:19:46.793 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:19:46.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:46.849 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:46.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:46.951 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:46.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.989 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:46.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:47.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:47.111 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 122886 122870 91.4003 80.4375 0.012746 0.0109364 2024-09-21T10:19:47.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.125 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:47.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:47.226 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:47.264 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:47.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:47.361 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:47.362 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:47.362 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:19:47.362 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:47.362 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:47.365 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:47.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.404 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:47.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:47.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:47.540 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:47.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:47.582 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:47.643 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:47.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:47.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:47.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.816 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:47.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:47.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:47.949 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:47.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:47 smithi017 ceph-mon[102913]: pgmap v2233: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-21T10:19:47.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:47 smithi017 ceph-mon[105151]: pgmap v2233: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-21T10:19:48.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:48.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:48.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 124179 124163 91.2756 80.8125 0.0103194 0.0109505 2024-09-21T10:19:48.113 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:47 smithi134 ceph-mon[90085]: pgmap v2233: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-21T10:19:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:19:48.144 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:19:48.191 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:48.226 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:48.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:48.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:48.369 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:48.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:48.472 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:48.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:48.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:48.660 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:48.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:48.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:48.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:48.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:48.903 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:48.943 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:48.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:19:49.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:49.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.084 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:49.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:49.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 125426 125410 91.1205 77.9375 0.00660851 0.0109688 2024-09-21T10:19:49.185 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.224 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:49.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:49.325 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:49.357 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:49.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:49.459 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:49.470 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:49.494 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:49.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:49.596 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:49.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.636 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:49.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:49.737 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:49.768 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:49.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:49.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:49 smithi017 ceph-mon[105151]: pgmap v2234: 119 pgs: 119 active+clean; 7.4 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:19:49.894 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:49 smithi017 ceph-mon[102913]: pgmap v2234: 119 pgs: 119 active+clean; 7.4 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:19:49.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:49.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:50.010 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:50.045 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:50.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 126676 126660 90.9711 78.125 0.00644827 0.010988 2024-09-21T10:19:50.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:19:50.147 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.192 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:50.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:50.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:49 smithi134 ceph-mon[90085]: pgmap v2234: 119 pgs: 119 active+clean; 7.4 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:19:50.293 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:50.324 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:50.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:50.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:50.469 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:50.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:19:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:19:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:19:50.571 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:50.607 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:50.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:50.709 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:50.748 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:50.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:50.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:50.887 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:50.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:50.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:51.026 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:51.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:51.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 127927 127911 90.8257 78.1875 0.0143627 0.0110052 2024-09-21T10:19:51.128 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:51.160 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:51.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:51.262 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:51.294 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:51.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:51.430 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:51.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:51.568 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:51.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:51.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:51.713 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:51.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:51.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:51.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:51.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:51.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:51 smithi017 ceph-mon[102913]: pgmap v2235: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:19:51.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:51 smithi017 ceph-mon[105151]: pgmap v2235: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:19:51.956 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:51.990 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:51.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:52.091 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:52.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 129170 129154 90.678 77.6875 0.0159656 0.0110231 2024-09-21T10:19:52.125 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:52.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:52.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:51 smithi134 ceph-mon[90085]: pgmap v2235: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:19:52.277 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:52.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:52.294 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:52.379 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:52.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.410 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:52.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:52.419 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:52.511 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:52.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:52.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:52.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:52.681 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:52.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:52.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:52.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:52.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:52.830 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:19:52.831 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 31 2024-09-21T10:19:52.854 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:52 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/300918567' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:52.854 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:52 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:52.854 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:52 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1492386021' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-21T10:19:52.854 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:52 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-21T10:19:52.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:52 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/300918567' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:52.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:52 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:52.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:52 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1492386021' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-21T10:19:52.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:52 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-21T10:19:52.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:52.957 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:52.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:53.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:53.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:53.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:53.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0059 2024-09-21T10:19:53.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 130204 2024-09-21T10:19:53.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2024-09-21T10:19:53.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 90.4135 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 37.0007 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 240 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1446 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 592.012 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3840 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0110562 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0282482 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 2.30472 2024-09-21T10:19:53.113 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00118998 2024-09-21T10:19:53.116 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:19:53.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:53.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:53.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:52 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/300918567' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:52 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:52 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1492386021' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-21T10:19:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:52 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-21T10:19:53.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:53.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:53.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:53.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:53.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:53.617 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:53.645 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:53.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:53.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:53.785 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:53.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:53.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:53.925 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:53.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:54.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:54.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.063 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:54.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:54.150 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[102913]: pgmap v2236: 119 pgs: 119 active+clean; 7.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-21T10:19:54.150 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:54.150 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:54.150 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-21T10:19:54.151 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[102913]: osdmap e1374: 8 total, 7 up, 7 in 2024-09-21T10:19:54.151 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[105151]: pgmap v2236: 119 pgs: 119 active+clean; 7.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-21T10:19:54.151 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:54.151 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:54.151 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-21T10:19:54.151 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:53 smithi017 ceph-mon[105151]: osdmap e1374: 8 total, 7 up, 7 in 2024-09-21T10:19:54.163 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:54.196 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:54.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:53 smithi134 ceph-mon[90085]: pgmap v2236: 119 pgs: 119 active+clean; 7.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-21T10:19:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:53 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:53 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:53 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-21T10:19:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:53 smithi134 ceph-mon[90085]: osdmap e1374: 8 total, 7 up, 7 in 2024-09-21T10:19:54.298 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:54.332 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:54.433 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:54.469 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:54.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:54.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:54.608 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:54.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:54.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:54.748 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:54.850 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:54.886 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:54.915 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:19:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:19:54] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.51.0" 2024-09-21T10:19:54.988 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:55.022 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:55.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:55.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:55.158 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:55.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:55.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:55.296 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:55.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:55.397 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:55.435 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:55.536 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:55.571 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:55.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:55.673 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:55.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.708 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:55.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:55.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:55.847 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:55.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:55.949 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:55.981 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:55.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:56.083 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:56.105 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:56.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:56.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:55 smithi017 ceph-mon[102913]: pgmap v2238: 119 pgs: 119 active+clean; 7.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-21T10:19:56.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:56.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:55 smithi017 ceph-mon[105151]: pgmap v2238: 119 pgs: 119 active+clean; 7.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-21T10:19:56.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:56.206 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:55 smithi134 ceph-mon[90085]: pgmap v2238: 119 pgs: 119 active+clean; 7.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-21T10:19:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.245 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:56.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:56.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:56.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.382 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:56.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:56.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:56.525 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:56.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:56.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:56.663 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:56.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:56.765 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:56.797 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:56.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:56.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:56.933 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:56.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:57.035 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:57.069 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:57.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:57.171 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:57.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:57.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:57.361 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:57.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:57.463 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:57.497 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:57.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:57.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:57.631 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:57.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:57.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:57.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:57.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:57.870 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:19:57.870 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:19:57.870 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2024-09-21T10:19:57.870 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:19:57.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:57.927 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:58.029 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:58.063 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:58.165 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:58.196 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:58.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:57 smithi017 ceph-mon[102913]: pgmap v2239: 119 pgs: 119 active+clean; 7.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 66 MiB/s wr, 1.25k op/s 2024-09-21T10:19:58.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2965621322' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:57 smithi017 ceph-mon[105151]: pgmap v2239: 119 pgs: 119 active+clean; 7.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 66 MiB/s wr, 1.25k op/s 2024-09-21T10:19:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2965621322' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:57 smithi134 ceph-mon[90085]: pgmap v2239: 119 pgs: 119 active+clean; 7.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 66 MiB/s wr, 1.25k op/s 2024-09-21T10:19:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2965621322' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:58.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:58.332 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:58.333 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:19:58.334 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:58.334 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:58.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:58.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:58.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:58.473 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:58.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:58.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:58.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.613 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:58.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:58.714 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.751 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:58.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:58.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:58.889 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:58.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:58.991 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.033 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:59.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:59.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:59.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:19:59.180 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:59.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:59.281 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:59.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.333 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:59.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:59.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:59.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.491 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:59.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:59.592 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:59.632 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:19:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:59.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:59.778 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:59.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:59.831 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:19:59.832 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:19:59.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:59.943 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:19:59.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:59.996 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:00.045 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:00.074 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:59 smithi134 ceph-mon[90085]: pgmap v2240: 119 pgs: 119 active+clean; 7.7 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 55 MiB/s wr, 1.26k op/s 2024-09-21T10:20:00.074 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:19:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:20:00.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.080 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:00.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:00.144 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:59 smithi017 ceph-mon[102913]: pgmap v2240: 119 pgs: 119 active+clean; 7.7 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 55 MiB/s wr, 1.26k op/s 2024-09-21T10:20:00.144 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:19:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:20:00.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:59 smithi017 ceph-mon[105151]: pgmap v2240: 119 pgs: 119 active+clean; 7.7 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 55 MiB/s wr, 1.26k op/s 2024-09-21T10:20:00.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:19:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:20:00.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:00.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:00.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:20:00.324 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.366 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:00.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:00.373 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [1] live_osds: [0, 2, 3, 5, 6, 7, 4] 2024-09-21T10:20:00.373 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:20:00.373 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_6 2024-09-21T10:20:00.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:20:00.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:20:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:20:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:20:00.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.526 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:00.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:00.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:00.671 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:00.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:00.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:00.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:00.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:00.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:00.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:01.060 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.108 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:01.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:01.178 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:00 smithi017 ceph-mon[102913]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-21T10:20:01.178 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:00 smithi017 ceph-mon[102913]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-21T10:20:01.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:00 smithi017 ceph-mon[105151]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-21T10:20:01.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:00 smithi017 ceph-mon[105151]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-21T10:20:01.209 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:01.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:00 smithi134 ceph-mon[90085]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-21T10:20:01.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:00 smithi134 ceph-mon[90085]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-21T10:20:01.248 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:01.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:01.350 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:01.380 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:01.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:01.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:01.520 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:01.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:01.622 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:01.685 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:01.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:01.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:01.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.828 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:01.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:01.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:20:01.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:01.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:02.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:02.109 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:02.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:01 smithi017 ceph-mon[102913]: pgmap v2241: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 1.29k op/s 2024-09-21T10:20:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:01 smithi017 ceph-mon[105151]: pgmap v2241: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 1.29k op/s 2024-09-21T10:20:02.211 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:02.246 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:01 smithi134 ceph-mon[90085]: pgmap v2241: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 1.29k op/s 2024-09-21T10:20:02.251 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:02.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:02.353 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:02.395 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:02.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:02.535 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:02.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:02.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:02.664 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:02.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:02.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:02.799 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:02.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:02.928 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:02.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:03.029 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:03.066 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:03.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:03.168 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:03.212 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:03.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:03.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:03.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.350 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:03.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:03.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:03.484 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:03.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:03.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:03.620 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:03.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:03.721 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:03.755 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:03.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:03.857 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:03.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.891 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:03.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:03.992 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:04.034 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:04.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:04.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:04.168 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:04.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:03 smithi017 ceph-mon[102913]: pgmap v2242: 119 pgs: 119 active+clean; 7.3 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 1.28k op/s 2024-09-21T10:20:04.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:03 smithi017 ceph-mon[105151]: pgmap v2242: 119 pgs: 119 active+clean; 7.3 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 1.28k op/s 2024-09-21T10:20:04.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:03 smithi134 ceph-mon[90085]: pgmap v2242: 119 pgs: 119 active+clean; 7.3 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 1.28k op/s 2024-09-21T10:20:04.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:04.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:04.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:04.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:04.439 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:04.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:04.541 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:04.582 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:04.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:04.683 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:04.720 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:04.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:04.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:04.852 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:04.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:04.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:04.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:05.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:05.132 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:05.200 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:20:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:20:04] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.51.0" 2024-09-21T10:20:05.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:05.268 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:05.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:05.369 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:05.371 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:05.400 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:05.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:05.502 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:05.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.542 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:05.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:05.643 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:05.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:05.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:05.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:05.801 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:05.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:05.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:05.942 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:05.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:05.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:05.947 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:05.987 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:20:05.987 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:20:05.987 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:20:05.987 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:20:05.987 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:20:06.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:06.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:06.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:05 smithi017 ceph-mon[102913]: pgmap v2243: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 86 B/s rd, 14 MiB/s wr, 1.15k op/s 2024-09-21T10:20:06.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:06.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:06.110 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:05 smithi017 ceph-mon[105151]: pgmap v2243: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 86 B/s rd, 14 MiB/s wr, 1.15k op/s 2024-09-21T10:20:06.110 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:06.110 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:06.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:06.221 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:06.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:05 smithi134 ceph-mon[90085]: pgmap v2243: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 86 B/s rd, 14 MiB/s wr, 1.15k op/s 2024-09-21T10:20:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:06.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:06.356 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:06.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:06.458 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:06.493 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:06.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:06.595 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:06.623 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:06.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:06.724 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:06.761 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:06.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:06.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:06.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:07.000 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:07.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:07.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:07.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:07.175 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:07.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:07.277 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.315 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:07.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:07.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:20:07.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:07.487 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:07.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:07.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.629 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:07.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:07.730 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:07.762 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:07.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:07.864 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:07.900 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:08.002 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:08.037 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:08.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:08.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:08.176 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:08.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:07 smithi134 ceph-mon[90085]: pgmap v2244: 119 pgs: 119 active+clean; 7.1 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 85 B/s rd, 10 MiB/s wr, 1.36k op/s 2024-09-21T10:20:08.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:08.314 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:08.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:08.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:07 smithi017 ceph-mon[102913]: pgmap v2244: 119 pgs: 119 active+clean; 7.1 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 85 B/s rd, 10 MiB/s wr, 1.36k op/s 2024-09-21T10:20:08.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:07 smithi017 ceph-mon[105151]: pgmap v2244: 119 pgs: 119 active+clean; 7.1 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 85 B/s rd, 10 MiB/s wr, 1.36k op/s 2024-09-21T10:20:08.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.454 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:08.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:08.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:08.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.591 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:08.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:08.692 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:08.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.731 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:08.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:08.832 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:08.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.874 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:08.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:08.975 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:09.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:09.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:09.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:09.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:09.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:09.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:09.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:09.387 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.423 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:09.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:09.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:09.559 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:09.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:09.661 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:09.693 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:09.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:09.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:09.830 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:09.932 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:09.976 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:10.078 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:10.119 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:10.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:09 smithi017 ceph-mon[102913]: pgmap v2245: 119 pgs: 119 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 85 B/s rd, 1.3 MiB/s wr, 1.23k op/s 2024-09-21T10:20:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:09 smithi017 ceph-mon[105151]: pgmap v2245: 119 pgs: 119 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 85 B/s rd, 1.3 MiB/s wr, 1.23k op/s 2024-09-21T10:20:10.220 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:10.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:09 smithi134 ceph-mon[90085]: pgmap v2245: 119 pgs: 119 active+clean; 6.9 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 85 B/s rd, 1.3 MiB/s wr, 1.23k op/s 2024-09-21T10:20:10.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:20:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:20:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:20:10.258 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:10.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:10.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:10.394 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:10.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:10.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:10.532 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:10.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:10.634 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:10.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:10.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:10.744 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:20:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a[77414]: logger=cleanup t=2024-09-21T10:20:10.297967447Z level=info msg="Completed cleanup jobs" duration=12.433206ms 2024-09-21T10:20:10.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:10.801 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:10.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:10.903 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:10.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.937 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:10.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:11.037 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:11.070 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:11.172 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:11.202 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:11.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:11.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:11.357 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:11.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:11.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:11.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:11.459 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:11.491 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:11.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:11.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:11.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:11.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:11.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:11.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:11.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:11.870 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:11.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.904 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:11.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:12.005 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:12.039 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:12.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:12.175 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:11 smithi017 ceph-mon[102913]: pgmap v2246: 119 pgs: 119 active+clean; 6.7 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s 2024-09-21T10:20:12.175 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:11 smithi017 ceph-mon[105151]: pgmap v2246: 119 pgs: 119 active+clean; 6.7 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s 2024-09-21T10:20:12.188 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:12.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:11 smithi134 ceph-mon[90085]: pgmap v2246: 119 pgs: 119 active+clean; 6.7 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s 2024-09-21T10:20:12.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:12.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.315 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:12.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:12.416 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:12.438 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:12.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:12.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:12.561 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:12.662 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:12.690 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:12.792 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:12.821 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:12.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:12.923 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:12.958 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:12.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:13.062 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:13.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.095 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:13.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:13.196 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:13.233 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:13.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:13.376 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:13.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:13.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:13.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:13.612 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:13.643 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:13.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:13.744 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:13.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.780 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:13.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:13.881 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:13.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.910 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:13.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:14.011 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.047 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:14.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:14.138 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:13 smithi017 ceph-mon[102913]: pgmap v2247: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s 2024-09-21T10:20:14.138 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:13 smithi017 ceph-mon[105151]: pgmap v2247: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s 2024-09-21T10:20:14.148 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:14.173 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:14.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:14.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:13 smithi134 ceph-mon[90085]: pgmap v2247: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s 2024-09-21T10:20:14.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:14.289 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:20:14.289 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:20:14.289 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:20:14.289 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:20:14.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:14.289 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-21T10:20:14.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:14.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:14.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:14.297 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:20:14.297 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:14.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:14.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:14.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:20:14.334 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:14.366 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:20:14.367 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:14.418 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:14.446 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:14.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:14.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:14.583 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:20:14.585 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:20:14.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:14.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:14.687 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.721 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:14.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:14.772 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:14.822 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:14.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.858 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:14.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:14.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:14.997 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:20:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:20:14] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.51.0" 2024-09-21T10:20:14.999 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:15.100 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:15.134 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:15.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:15.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:15.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:15.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:15.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3209701321' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:15.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3209701321' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:15.382 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:15.413 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:15.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3209701321' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:15.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:15.558 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:15.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:15.660 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:15.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:15.713 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:20:15.714 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:20:15.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:15.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.831 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:15.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:15.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:15.970 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:16.071 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:16.106 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:16.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:16.143 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":2247,"stamp":"2024-09-21T10:20:12.600913+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7031305505,"num_objects":107308,"num_object_clones":0,"num_object_copies":321924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107308,"num_whiteouts":0,"num_read":20,"num_read_kb":52,"num_write":153200,"num_write_kb":8333199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1515,"num_bytes_recovered":95453723,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86985,"ondisk_log_size":86985,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":27554724,"kb_used_data":20850092,"kb_used_omap":384,"kb_used_meta":6703999,"kb_avail":628661340,"statfs":{"total":671965249536,"available":643749212160,"internally_reserved":0,"allocated":21350494208,"data_stored":21326220985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":393354,"internal_metadata":6864895862},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,0,9,11,10,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1033437184,"num_objects":-15769,"num_object_clones":0,"num_object_copies":-47307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-15769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15769,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004549"},"pg_stats":[{"pgid":"1.1d","version":"288'100","reported_seq":183,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772491+0000","last_change":"2024-09-21T10:18:55.372468+0000","last_active":"2024-09-21T10:19:40.772491+0000","last_peered":"2024-09-21T10:19:40.772491+0000","last_clean":"2024-09-21T10:19:40.772491+0000","last_became_active":"2024-09-21T10:17:59.913598+0000","last_became_peered":"2024-09-21T10:17:59.913598+0000","last_unstale":"2024-09-21T10:19:40.772491+0000","last_undegraded":"2024-09-21T10:19:40.772491+0000","last_fullsized":"2024-09-21T10:19:40.772491+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:55.372392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:55.372392+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:15:40.399506+0000","scrub_duration":0.0042594110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":133,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.825837+0000","last_change":"2024-09-21T10:17:34.790860+0000","last_active":"2024-09-21T10:19:52.825837+0000","last_peered":"2024-09-21T10:19:52.825837+0000","last_clean":"2024-09-21T10:19:52.825837+0000","last_became_active":"2024-09-21T10:17:34.779226+0000","last_became_peered":"2024-09-21T10:17:34.779226+0000","last_unstale":"2024-09-21T10:19:52.825837+0000","last_undegraded":"2024-09-21T10:19:52.825837+0000","last_fullsized":"2024-09-21T10:19:52.825837+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:45:00.128138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1372,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.770639+0000","last_change":"2024-09-21T10:16:27.318860+0000","last_active":"2024-09-21T10:19:40.770639+0000","last_peered":"2024-09-21T10:19:40.770639+0000","last_clean":"2024-09-21T10:19:40.770639+0000","last_became_active":"2024-09-21T10:16:27.317542+0000","last_became_peered":"2024-09-21T10:16:27.317542+0000","last_unstale":"2024-09-21T10:19:40.770639+0000","last_undegraded":"2024-09-21T10:19:40.770639+0000","last_fullsized":"2024-09-21T10:19:40.770639+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:56:58.481255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":2117,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843543+0000","last_change":"2024-09-21T10:10:13.651599+0000","last_active":"2024-09-21T10:19:52.843543+0000","last_peered":"2024-09-21T10:19:52.843543+0000","last_clean":"2024-09-21T10:19:52.843543+0000","last_became_active":"2024-09-21T10:10:13.651250+0000","last_became_peered":"2024-09-21T10:10:13.651250+0000","last_unstale":"2024-09-21T10:19:52.843543+0000","last_undegraded":"2024-09-21T10:19:52.843543+0000","last_fullsized":"2024-09-21T10:19:52.843543+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:34.100056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2101,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.840951+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:19:52.840951+0000","last_peered":"2024-09-21T10:19:52.840951+0000","last_clean":"2024-09-21T10:19:52.840951+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:19:52.840951+0000","last_undegraded":"2024-09-21T10:19:52.840951+0000","last_fullsized":"2024-09-21T10:19:52.840951+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"288'100","reported_seq":39,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775764+0000","last_change":"2024-09-21T10:19:26.113832+0000","last_active":"2024-09-21T10:19:40.775764+0000","last_peered":"2024-09-21T10:19:40.775764+0000","last_clean":"2024-09-21T10:19:40.775764+0000","last_became_active":"2024-09-21T10:17:59.912636+0000","last_became_peered":"2024-09-21T10:17:59.912636+0000","last_unstale":"2024-09-21T10:19:40.775764+0000","last_undegraded":"2024-09-21T10:19:40.775764+0000","last_fullsized":"2024-09-21T10:19:40.775764+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:26.113747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:26.113747+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:57:42.993691+0000","scrub_duration":0.0042569799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":313,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771094+0000","last_change":"2024-09-21T10:10:13.639231+0000","last_active":"2024-09-21T10:19:40.771094+0000","last_peered":"2024-09-21T10:19:40.771094+0000","last_clean":"2024-09-21T10:19:40.771094+0000","last_became_active":"2024-09-21T10:10:13.622497+0000","last_became_peered":"2024-09-21T10:10:13.622497+0000","last_unstale":"2024-09-21T10:19:40.771094+0000","last_undegraded":"2024-09-21T10:19:40.771094+0000","last_fullsized":"2024-09-21T10:19:40.771094+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:31:46.263003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1319,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771091+0000","last_change":"2024-09-21T10:08:25.430667+0000","last_active":"2024-09-21T10:19:40.771091+0000","last_peered":"2024-09-21T10:19:40.771091+0000","last_clean":"2024-09-21T10:19:40.771091+0000","last_became_active":"2024-09-21T10:08:25.430357+0000","last_became_peered":"2024-09-21T10:08:25.430357+0000","last_unstale":"2024-09-21T10:19:40.771091+0000","last_undegraded":"2024-09-21T10:19:40.771091+0000","last_fullsized":"2024-09-21T10:19:40.771091+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:19.972020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"288'100","reported_seq":402,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843462+0000","last_change":"2024-09-21T10:19:37.200842+0000","last_active":"2024-09-21T10:19:52.843462+0000","last_peered":"2024-09-21T10:19:52.843462+0000","last_clean":"2024-09-21T10:19:52.843462+0000","last_became_active":"2024-09-21T10:17:59.914846+0000","last_became_peered":"2024-09-21T10:17:59.914846+0000","last_unstale":"2024-09-21T10:19:52.843462+0000","last_undegraded":"2024-09-21T10:19:52.843462+0000","last_fullsized":"2024-09-21T10:19:52.843462+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:37.200788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:37.200788+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:26:30.373346+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":325,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.770808+0000","last_change":"2024-09-21T10:16:27.324301+0000","last_active":"2024-09-21T10:19:40.770808+0000","last_peered":"2024-09-21T10:19:40.770808+0000","last_clean":"2024-09-21T10:19:40.770808+0000","last_became_active":"2024-09-21T10:16:27.319515+0000","last_became_peered":"2024-09-21T10:16:27.319515+0000","last_unstale":"2024-09-21T10:19:40.770808+0000","last_undegraded":"2024-09-21T10:19:40.770808+0000","last_fullsized":"2024-09-21T10:19:40.770808+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:38.284184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1349,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771858+0000","last_change":"2024-09-21T10:13:41.789883+0000","last_active":"2024-09-21T10:19:40.771858+0000","last_peered":"2024-09-21T10:19:40.771858+0000","last_clean":"2024-09-21T10:19:40.771858+0000","last_became_active":"2024-09-21T10:13:41.789620+0000","last_became_peered":"2024-09-21T10:13:41.789620+0000","last_unstale":"2024-09-21T10:19:40.771858+0000","last_undegraded":"2024-09-21T10:19:40.771858+0000","last_fullsized":"2024-09-21T10:19:40.771858+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:36.869540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":90,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772434+0000","last_change":"2024-09-21T10:17:34.770306+0000","last_active":"2024-09-21T10:19:40.772434+0000","last_peered":"2024-09-21T10:19:40.772434+0000","last_clean":"2024-09-21T10:19:40.772434+0000","last_became_active":"2024-09-21T10:17:34.769690+0000","last_became_peered":"2024-09-21T10:17:34.769690+0000","last_unstale":"2024-09-21T10:19:40.772434+0000","last_undegraded":"2024-09-21T10:19:40.772434+0000","last_fullsized":"2024-09-21T10:19:40.772434+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:00:49.468067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2156,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.825453+0000","last_change":"2024-09-21T10:17:34.794267+0000","last_active":"2024-09-21T10:19:52.825453+0000","last_peered":"2024-09-21T10:19:52.825453+0000","last_clean":"2024-09-21T10:19:52.825453+0000","last_became_active":"2024-09-21T10:17:34.792230+0000","last_became_peered":"2024-09-21T10:17:34.792230+0000","last_unstale":"2024-09-21T10:19:52.825453+0000","last_undegraded":"2024-09-21T10:19:52.825453+0000","last_fullsized":"2024-09-21T10:19:52.825453+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:37.529702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"288'100","reported_seq":1281,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.840082+0000","last_change":"2024-09-21T10:17:59.912179+0000","last_active":"2024-09-21T10:19:52.840082+0000","last_peered":"2024-09-21T10:19:52.840082+0000","last_clean":"2024-09-21T10:19:52.840082+0000","last_became_active":"2024-09-21T10:17:59.911742+0000","last_became_peered":"2024-09-21T10:17:59.911742+0000","last_unstale":"2024-09-21T10:19:52.840082+0000","last_undegraded":"2024-09-21T10:19:52.840082+0000","last_fullsized":"2024-09-21T10:19:52.840082+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:15.663228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:15.663228+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1351,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771382+0000","last_change":"2024-09-21T10:16:27.338372+0000","last_active":"2024-09-21T10:19:40.771382+0000","last_peered":"2024-09-21T10:19:40.771382+0000","last_clean":"2024-09-21T10:19:40.771382+0000","last_became_active":"2024-09-21T10:16:27.338029+0000","last_became_peered":"2024-09-21T10:16:27.338029+0000","last_unstale":"2024-09-21T10:19:40.771382+0000","last_undegraded":"2024-09-21T10:19:40.771382+0000","last_fullsized":"2024-09-21T10:19:40.771382+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:52.311869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2123,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.838732+0000","last_change":"2024-09-21T10:16:27.340083+0000","last_active":"2024-09-21T10:19:52.838732+0000","last_peered":"2024-09-21T10:19:52.838732+0000","last_clean":"2024-09-21T10:19:52.838732+0000","last_became_active":"2024-09-21T10:16:27.339405+0000","last_became_peered":"2024-09-21T10:16:27.339405+0000","last_unstale":"2024-09-21T10:19:52.838732+0000","last_undegraded":"2024-09-21T10:19:52.838732+0000","last_fullsized":"2024-09-21T10:19:52.838732+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:45:11.731090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"288'100","reported_seq":582,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.824955+0000","last_change":"2024-09-21T10:17:59.915389+0000","last_active":"2024-09-21T10:19:52.824955+0000","last_peered":"2024-09-21T10:19:52.824955+0000","last_clean":"2024-09-21T10:19:52.824955+0000","last_became_active":"2024-09-21T10:17:59.909290+0000","last_became_peered":"2024-09-21T10:17:59.909290+0000","last_unstale":"2024-09-21T10:19:52.824955+0000","last_undegraded":"2024-09-21T10:19:52.824955+0000","last_fullsized":"2024-09-21T10:19:52.824955+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":833,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.840601+0000","last_change":"2024-09-21T10:16:27.321423+0000","last_active":"2024-09-21T10:19:52.840601+0000","last_peered":"2024-09-21T10:19:52.840601+0000","last_clean":"2024-09-21T10:19:52.840601+0000","last_became_active":"2024-09-21T10:16:27.319009+0000","last_became_peered":"2024-09-21T10:16:27.319009+0000","last_unstale":"2024-09-21T10:19:52.840601+0000","last_undegraded":"2024-09-21T10:19:52.840601+0000","last_fullsized":"2024-09-21T10:19:52.840601+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:32.452405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2038,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775494+0000","last_change":"2024-09-21T10:17:34.778067+0000","last_active":"2024-09-21T10:19:40.775494+0000","last_peered":"2024-09-21T10:19:40.775494+0000","last_clean":"2024-09-21T10:19:40.775494+0000","last_became_active":"2024-09-21T10:17:34.777744+0000","last_became_peered":"2024-09-21T10:17:34.777744+0000","last_unstale":"2024-09-21T10:19:40.775494+0000","last_undegraded":"2024-09-21T10:19:40.775494+0000","last_fullsized":"2024-09-21T10:19:40.775494+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:15:54.571208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1b","version":"288'100","reported_seq":36,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772240+0000","last_change":"2024-09-21T10:18:48.373065+0000","last_active":"2024-09-21T10:19:40.772240+0000","last_peered":"2024-09-21T10:19:40.772240+0000","last_clean":"2024-09-21T10:19:40.772240+0000","last_became_active":"2024-09-21T10:17:59.910945+0000","last_became_peered":"2024-09-21T10:17:59.910945+0000","last_unstale":"2024-09-21T10:19:40.772240+0000","last_undegraded":"2024-09-21T10:19:40.772240+0000","last_fullsized":"2024-09-21T10:19:40.772240+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:48.372952+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:18:48.372952+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:15:04.784278+0000","scrub_duration":0.024424405,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2137,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.840583+0000","last_change":"2024-09-21T10:17:34.777326+0000","last_active":"2024-09-21T10:19:52.840583+0000","last_peered":"2024-09-21T10:19:52.840583+0000","last_clean":"2024-09-21T10:19:52.840583+0000","last_became_active":"2024-09-21T10:17:34.776889+0000","last_became_peered":"2024-09-21T10:17:34.776889+0000","last_unstale":"2024-09-21T10:19:52.840583+0000","last_undegraded":"2024-09-21T10:19:52.840583+0000","last_fullsized":"2024-09-21T10:19:52.840583+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:44:01.933785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":141,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.774085+0000","last_change":"2024-09-21T10:16:27.322614+0000","last_active":"2024-09-21T10:19:40.774085+0000","last_peered":"2024-09-21T10:19:40.774085+0000","last_clean":"2024-09-21T10:19:40.774085+0000","last_became_active":"2024-09-21T10:16:27.321913+0000","last_became_peered":"2024-09-21T10:16:27.321913+0000","last_unstale":"2024-09-21T10:19:40.774085+0000","last_undegraded":"2024-09-21T10:19:40.774085+0000","last_fullsized":"2024-09-21T10:19:40.774085+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:44:39.631658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"288'100","reported_seq":39,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.770953+0000","last_change":"2024-09-21T10:19:25.436153+0000","last_active":"2024-09-21T10:19:40.770953+0000","last_peered":"2024-09-21T10:19:40.770953+0000","last_clean":"2024-09-21T10:19:40.770953+0000","last_became_active":"2024-09-21T10:17:59.912015+0000","last_became_peered":"2024-09-21T10:17:59.912015+0000","last_unstale":"2024-09-21T10:19:40.770953+0000","last_undegraded":"2024-09-21T10:19:40.770953+0000","last_fullsized":"2024-09-21T10:19:40.770953+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:25.436066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:25.436066+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:38:18.381058+0000","scrub_duration":0.0032626159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2159,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.839611+0000","last_change":"2024-09-21T10:16:27.330457+0000","last_active":"2024-09-21T10:19:52.839611+0000","last_peered":"2024-09-21T10:19:52.839611+0000","last_clean":"2024-09-21T10:19:52.839611+0000","last_became_active":"2024-09-21T10:16:27.329527+0000","last_became_peered":"2024-09-21T10:16:27.329527+0000","last_unstale":"2024-09-21T10:19:52.839611+0000","last_undegraded":"2024-09-21T10:19:52.839611+0000","last_fullsized":"2024-09-21T10:19:52.839611+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:16.811107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2083,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843226+0000","last_change":"2024-09-21T10:16:07.906476+0000","last_active":"2024-09-21T10:19:52.843226+0000","last_peered":"2024-09-21T10:19:52.843226+0000","last_clean":"2024-09-21T10:19:52.843226+0000","last_became_active":"2024-09-21T10:16:07.906053+0000","last_became_peered":"2024-09-21T10:16:07.906053+0000","last_unstale":"2024-09-21T10:19:52.843226+0000","last_undegraded":"2024-09-21T10:19:52.843226+0000","last_fullsized":"2024-09-21T10:19:52.843226+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:01:57.827302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"288'100","reported_seq":36,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.776651+0000","last_change":"2024-09-21T10:18:52.067685+0000","last_active":"2024-09-21T10:19:40.776651+0000","last_peered":"2024-09-21T10:19:40.776651+0000","last_clean":"2024-09-21T10:19:40.776651+0000","last_became_active":"2024-09-21T10:17:59.910073+0000","last_became_peered":"2024-09-21T10:17:59.910073+0000","last_unstale":"2024-09-21T10:19:40.776651+0000","last_undegraded":"2024-09-21T10:19:40.776651+0000","last_fullsized":"2024-09-21T10:19:40.776651+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:52.067480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:52.067480+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:51.001507+0000","scrub_duration":0.0041194669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2117,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.840295+0000","last_change":"2024-09-21T10:12:25.707393+0000","last_active":"2024-09-21T10:19:52.840295+0000","last_peered":"2024-09-21T10:19:52.840295+0000","last_clean":"2024-09-21T10:19:52.840295+0000","last_became_active":"2024-09-21T10:12:25.706457+0000","last_became_peered":"2024-09-21T10:12:25.706457+0000","last_unstale":"2024-09-21T10:19:52.840295+0000","last_undegraded":"2024-09-21T10:19:52.840295+0000","last_fullsized":"2024-09-21T10:19:52.840295+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:56.520799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":135,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775565+0000","last_change":"2024-09-21T10:16:27.320052+0000","last_active":"2024-09-21T10:19:40.775565+0000","last_peered":"2024-09-21T10:19:40.775565+0000","last_clean":"2024-09-21T10:19:40.775565+0000","last_became_active":"2024-09-21T10:16:27.319716+0000","last_became_peered":"2024-09-21T10:16:27.319716+0000","last_unstale":"2024-09-21T10:19:40.775565+0000","last_undegraded":"2024-09-21T10:19:40.775565+0000","last_fullsized":"2024-09-21T10:19:40.775565+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:15:46.308322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"288'100","reported_seq":313,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.825131+0000","last_change":"2024-09-21T10:12:25.698905+0000","last_active":"2024-09-21T10:19:52.825131+0000","last_peered":"2024-09-21T10:19:52.825131+0000","last_clean":"2024-09-21T10:19:52.825131+0000","last_became_active":"2024-09-21T10:12:25.698552+0000","last_became_peered":"2024-09-21T10:12:25.698552+0000","last_unstale":"2024-09-21T10:19:52.825131+0000","last_undegraded":"2024-09-21T10:19:52.825131+0000","last_fullsized":"2024-09-21T10:19:52.825131+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.389965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.389965+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:27.529104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":121,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.776603+0000","last_change":"2024-09-21T10:12:25.717602+0000","last_active":"2024-09-21T10:19:40.776603+0000","last_peered":"2024-09-21T10:19:40.776603+0000","last_clean":"2024-09-21T10:19:40.776603+0000","last_became_active":"2024-09-21T10:12:25.705211+0000","last_became_peered":"2024-09-21T10:12:25.705211+0000","last_unstale":"2024-09-21T10:19:40.776603+0000","last_undegraded":"2024-09-21T10:19:40.776603+0000","last_fullsized":"2024-09-21T10:19:40.776603+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:30.822884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":104,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772090+0000","last_change":"2024-09-21T10:16:27.338866+0000","last_active":"2024-09-21T10:19:40.772090+0000","last_peered":"2024-09-21T10:19:40.772090+0000","last_clean":"2024-09-21T10:19:40.772090+0000","last_became_active":"2024-09-21T10:16:27.338090+0000","last_became_peered":"2024-09-21T10:16:27.338090+0000","last_unstale":"2024-09-21T10:19:40.772090+0000","last_undegraded":"2024-09-21T10:19:40.772090+0000","last_fullsized":"2024-09-21T10:19:40.772090+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:02:54.045061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"288'100","reported_seq":121,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775688+0000","last_change":"2024-09-21T10:19:00.110407+0000","last_active":"2024-09-21T10:19:40.775688+0000","last_peered":"2024-09-21T10:19:40.775688+0000","last_clean":"2024-09-21T10:19:40.775688+0000","last_became_active":"2024-09-21T10:17:59.912144+0000","last_became_peered":"2024-09-21T10:17:59.912144+0000","last_unstale":"2024-09-21T10:19:40.775688+0000","last_undegraded":"2024-09-21T10:19:40.775688+0000","last_fullsized":"2024-09-21T10:19:40.775688+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:00.110351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:00.110351+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:06:32.995133+0000","scrub_duration":0.005953931,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1402,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.770929+0000","last_change":"2024-09-21T10:12:25.713745+0000","last_active":"2024-09-21T10:19:40.770929+0000","last_peered":"2024-09-21T10:19:40.770929+0000","last_clean":"2024-09-21T10:19:40.770929+0000","last_became_active":"2024-09-21T10:12:25.698011+0000","last_became_peered":"2024-09-21T10:12:25.698011+0000","last_unstale":"2024-09-21T10:19:40.770929+0000","last_undegraded":"2024-09-21T10:19:40.770929+0000","last_fullsized":"2024-09-21T10:19:40.770929+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:45.276913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23720,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1372,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771988+0000","last_change":"2024-09-21T10:16:27.316686+0000","last_active":"2024-09-21T10:19:40.771988+0000","last_peered":"2024-09-21T10:19:40.771988+0000","last_clean":"2024-09-21T10:19:40.771988+0000","last_became_active":"2024-09-21T10:16:27.316186+0000","last_became_peered":"2024-09-21T10:16:27.316186+0000","last_unstale":"2024-09-21T10:19:40.771988+0000","last_undegraded":"2024-09-21T10:19:40.771988+0000","last_fullsized":"2024-09-21T10:19:40.771988+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:45.443605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"288'100","reported_seq":481,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.842761+0000","last_change":"2024-09-21T10:19:35.203265+0000","last_active":"2024-09-21T10:19:52.842761+0000","last_peered":"2024-09-21T10:19:52.842761+0000","last_clean":"2024-09-21T10:19:52.842761+0000","last_became_active":"2024-09-21T10:17:59.912488+0000","last_became_peered":"2024-09-21T10:17:59.912488+0000","last_unstale":"2024-09-21T10:19:52.842761+0000","last_undegraded":"2024-09-21T10:19:52.842761+0000","last_fullsized":"2024-09-21T10:19:52.842761+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:35.203193+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:35.203193+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:05.928718+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"1374'9307","reported_seq":9330,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.800583+0000","last_change":"2024-09-21T10:18:10.935476+0000","last_active":"2024-09-21T10:20:08.800583+0000","last_peered":"2024-09-21T10:20:08.800583+0000","last_clean":"2024-09-21T10:20:08.800583+0000","last_became_active":"2024-09-21T10:18:10.935178+0000","last_became_peered":"2024-09-21T10:18:10.935178+0000","last_unstale":"2024-09-21T10:20:08.800583+0000","last_undegraded":"2024-09-21T10:20:08.800583+0000","last_fullsized":"2024-09-21T10:20:08.800583+0000","mapping_epoch":1359,"log_start":"1370'4300","ondisk_log_start":"1370'4300","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5007,"log_dups_size":0,"ondisk_log_size":5007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:02:54.677232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448069632,"num_objects":6837,"num_object_clones":0,"num_object_copies":20511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9307,"num_write_kb":516608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1319,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771466+0000","last_change":"2024-09-21T10:17:34.767553+0000","last_active":"2024-09-21T10:19:40.771466+0000","last_peered":"2024-09-21T10:19:40.771466+0000","last_clean":"2024-09-21T10:19:40.771466+0000","last_became_active":"2024-09-21T10:17:34.767246+0000","last_became_peered":"2024-09-21T10:17:34.767246+0000","last_unstale":"2024-09-21T10:19:40.771466+0000","last_undegraded":"2024-09-21T10:19:40.771466+0000","last_fullsized":"2024-09-21T10:19:40.771466+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:52.295023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":141,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.774016+0000","last_change":"2024-09-21T10:12:25.704090+0000","last_active":"2024-09-21T10:19:40.774016+0000","last_peered":"2024-09-21T10:19:40.774016+0000","last_clean":"2024-09-21T10:19:40.774016+0000","last_became_active":"2024-09-21T10:12:25.702945+0000","last_became_peered":"2024-09-21T10:12:25.702945+0000","last_unstale":"2024-09-21T10:19:40.774016+0000","last_undegraded":"2024-09-21T10:19:40.774016+0000","last_fullsized":"2024-09-21T10:19:40.774016+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:29.609613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":2142,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.826365+0000","last_change":"2024-09-21T10:16:27.330519+0000","last_active":"2024-09-21T10:19:52.826365+0000","last_peered":"2024-09-21T10:19:52.826365+0000","last_clean":"2024-09-21T10:19:52.826365+0000","last_became_active":"2024-09-21T10:16:27.317630+0000","last_became_peered":"2024-09-21T10:16:27.317630+0000","last_unstale":"2024-09-21T10:19:52.826365+0000","last_undegraded":"2024-09-21T10:19:52.826365+0000","last_fullsized":"2024-09-21T10:19:52.826365+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:19:53.670529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":598,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843660+0000","last_change":"2024-09-21T10:12:50.578581+0000","last_active":"2024-09-21T10:19:52.843660+0000","last_peered":"2024-09-21T10:19:52.843660+0000","last_clean":"2024-09-21T10:19:52.843660+0000","last_became_active":"2024-09-21T10:12:50.578246+0000","last_became_peered":"2024-09-21T10:12:50.578246+0000","last_unstale":"2024-09-21T10:19:52.843660+0000","last_undegraded":"2024-09-21T10:19:52.843660+0000","last_fullsized":"2024-09-21T10:19:52.843660+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.423664+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.423664+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:57:16.476526+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1374'9778","reported_seq":9801,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:10.972543+0000","last_change":"2024-09-21T10:18:10.941491+0000","last_active":"2024-09-21T10:20:10.972543+0000","last_peered":"2024-09-21T10:20:10.972543+0000","last_clean":"2024-09-21T10:20:10.972543+0000","last_became_active":"2024-09-21T10:18:10.941206+0000","last_became_peered":"2024-09-21T10:18:10.941206+0000","last_unstale":"2024-09-21T10:20:10.972543+0000","last_undegraded":"2024-09-21T10:20:10.972543+0000","last_fullsized":"2024-09-21T10:20:10.972543+0000","mapping_epoch":1359,"log_start":"1370'4600","ondisk_log_start":"1370'4600","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5178,"log_dups_size":0,"ondisk_log_size":5178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:46.198535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":447086592,"num_objects":6822,"num_object_clones":0,"num_object_copies":20466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9778,"num_write_kb":531200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2122,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.840498+0000","last_change":"2024-09-21T10:17:34.771415+0000","last_active":"2024-09-21T10:19:52.840498+0000","last_peered":"2024-09-21T10:19:52.840498+0000","last_clean":"2024-09-21T10:19:52.840498+0000","last_became_active":"2024-09-21T10:17:34.771087+0000","last_became_peered":"2024-09-21T10:17:34.771087+0000","last_unstale":"2024-09-21T10:19:52.840498+0000","last_undegraded":"2024-09-21T10:19:52.840498+0000","last_fullsized":"2024-09-21T10:19:52.840498+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:07.509252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1792,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772551+0000","last_change":"2024-09-21T10:16:27.321061+0000","last_active":"2024-09-21T10:19:40.772551+0000","last_peered":"2024-09-21T10:19:40.772551+0000","last_clean":"2024-09-21T10:19:40.772551+0000","last_became_active":"2024-09-21T10:16:27.320759+0000","last_became_peered":"2024-09-21T10:16:27.320759+0000","last_unstale":"2024-09-21T10:19:40.772551+0000","last_undegraded":"2024-09-21T10:19:40.772551+0000","last_fullsized":"2024-09-21T10:19:40.772551+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:29:04.861525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":2125,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843418+0000","last_change":"2024-09-21T10:12:25.718713+0000","last_active":"2024-09-21T10:19:52.843418+0000","last_peered":"2024-09-21T10:19:52.843418+0000","last_clean":"2024-09-21T10:19:52.843418+0000","last_became_active":"2024-09-21T10:12:25.713007+0000","last_became_peered":"2024-09-21T10:12:25.713007+0000","last_unstale":"2024-09-21T10:19:52.843418+0000","last_undegraded":"2024-09-21T10:19:52.843418+0000","last_fullsized":"2024-09-21T10:19:52.843418+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:09.753443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":297,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771223+0000","last_change":"2024-09-21T10:18:56.346749+0000","last_active":"2024-09-21T10:19:40.771223+0000","last_peered":"2024-09-21T10:19:40.771223+0000","last_clean":"2024-09-21T10:19:40.771223+0000","last_became_active":"2024-09-21T10:17:19.685397+0000","last_became_peered":"2024-09-21T10:17:19.685397+0000","last_unstale":"2024-09-21T10:19:40.771223+0000","last_undegraded":"2024-09-21T10:19:40.771223+0000","last_fullsized":"2024-09-21T10:19:40.771223+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:56.346587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:56.346587+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:24.457314+0000","scrub_duration":0.0039726459999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1374'9671","reported_seq":9712,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:12.248960+0000","last_change":"2024-09-21T10:18:10.938262+0000","last_active":"2024-09-21T10:20:12.248960+0000","last_peered":"2024-09-21T10:20:12.248960+0000","last_clean":"2024-09-21T10:20:12.248960+0000","last_became_active":"2024-09-21T10:18:10.937958+0000","last_became_peered":"2024-09-21T10:18:10.937958+0000","last_unstale":"2024-09-21T10:20:12.248960+0000","last_undegraded":"2024-09-21T10:20:12.248960+0000","last_fullsized":"2024-09-21T10:20:12.248960+0000","mapping_epoch":1359,"log_start":"1368'3200","ondisk_log_start":"1368'3200","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":6471,"log_dups_size":0,"ondisk_log_size":6471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:30:31.071784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431947776,"num_objects":6591,"num_object_clones":0,"num_object_copies":19773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9671,"num_write_kb":520384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2073,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.774824+0000","last_change":"2024-09-21T10:16:07.913449+0000","last_active":"2024-09-21T10:19:40.774824+0000","last_peered":"2024-09-21T10:19:40.774824+0000","last_clean":"2024-09-21T10:19:40.774824+0000","last_became_active":"2024-09-21T10:16:07.913117+0000","last_became_peered":"2024-09-21T10:16:07.913117+0000","last_unstale":"2024-09-21T10:19:40.774824+0000","last_undegraded":"2024-09-21T10:19:40.774824+0000","last_fullsized":"2024-09-21T10:19:40.774824+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:24.442444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":115,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.776678+0000","last_change":"2024-09-21T10:12:25.709819+0000","last_active":"2024-09-21T10:19:40.776678+0000","last_peered":"2024-09-21T10:19:40.776678+0000","last_clean":"2024-09-21T10:19:40.776678+0000","last_became_active":"2024-09-21T10:12:25.709548+0000","last_became_peered":"2024-09-21T10:12:25.709548+0000","last_unstale":"2024-09-21T10:19:40.776678+0000","last_undegraded":"2024-09-21T10:19:40.776678+0000","last_fullsized":"2024-09-21T10:19:40.776678+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:35.986614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":758,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.825351+0000","last_change":"2024-09-21T10:16:27.329301+0000","last_active":"2024-09-21T10:19:52.825351+0000","last_peered":"2024-09-21T10:19:52.825351+0000","last_clean":"2024-09-21T10:19:52.825351+0000","last_became_active":"2024-09-21T10:16:27.328337+0000","last_became_peered":"2024-09-21T10:16:27.328337+0000","last_unstale":"2024-09-21T10:19:52.825351+0000","last_undegraded":"2024-09-21T10:19:52.825351+0000","last_fullsized":"2024-09-21T10:19:52.825351+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:40.669541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":467,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843263+0000","last_change":"2024-09-21T10:17:19.741388+0000","last_active":"2024-09-21T10:19:52.843263+0000","last_peered":"2024-09-21T10:19:52.843263+0000","last_clean":"2024-09-21T10:19:52.843263+0000","last_became_active":"2024-09-21T10:17:19.741030+0000","last_became_peered":"2024-09-21T10:17:19.741030+0000","last_unstale":"2024-09-21T10:19:52.843263+0000","last_undegraded":"2024-09-21T10:19:52.843263+0000","last_fullsized":"2024-09-21T10:19:52.843263+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:10:37.256819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:10:37.256819+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":6,"num_read_kb":17,"num_write":3,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":13417,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"1374'9558","reported_seq":9600,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:12.265434+0000","last_change":"2024-09-21T10:18:10.938914+0000","last_active":"2024-09-21T10:20:12.265434+0000","last_peered":"2024-09-21T10:20:12.265434+0000","last_clean":"2024-09-21T10:20:12.265434+0000","last_became_active":"2024-09-21T10:18:10.938570+0000","last_became_peered":"2024-09-21T10:18:10.938570+0000","last_unstale":"2024-09-21T10:20:12.265434+0000","last_undegraded":"2024-09-21T10:20:12.265434+0000","last_fullsized":"2024-09-21T10:20:12.265434+0000","mapping_epoch":1359,"log_start":"1367'3100","ondisk_log_start":"1367'3100","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":6458,"log_dups_size":0,"ondisk_log_size":6458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:14.496321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427229208,"num_objects":6520,"num_object_clones":0,"num_object_copies":19560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6520,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9558,"num_write_kb":514433,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":580,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771300+0000","last_change":"2024-09-21T10:16:27.337427+0000","last_active":"2024-09-21T10:19:40.771300+0000","last_peered":"2024-09-21T10:19:40.771300+0000","last_clean":"2024-09-21T10:19:40.771300+0000","last_became_active":"2024-09-21T10:16:27.336986+0000","last_became_peered":"2024-09-21T10:16:27.336986+0000","last_unstale":"2024-09-21T10:19:40.771300+0000","last_undegraded":"2024-09-21T10:19:40.771300+0000","last_fullsized":"2024-09-21T10:19:40.771300+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:51:38.804787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":134,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775888+0000","last_change":"2024-09-21T10:16:27.328517+0000","last_active":"2024-09-21T10:19:40.775888+0000","last_peered":"2024-09-21T10:19:40.775888+0000","last_clean":"2024-09-21T10:19:40.775888+0000","last_became_active":"2024-09-21T10:16:27.328203+0000","last_became_peered":"2024-09-21T10:16:27.328203+0000","last_unstale":"2024-09-21T10:19:40.775888+0000","last_undegraded":"2024-09-21T10:19:40.775888+0000","last_fullsized":"2024-09-21T10:19:40.775888+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:54.497430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1122,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.838930+0000","last_change":"2024-09-21T10:18:31.648581+0000","last_active":"2024-09-21T10:19:52.838930+0000","last_peered":"2024-09-21T10:19:52.838930+0000","last_clean":"2024-09-21T10:19:52.838930+0000","last_became_active":"2024-09-21T10:18:31.648253+0000","last_became_peered":"2024-09-21T10:18:31.648253+0000","last_unstale":"2024-09-21T10:19:52.838930+0000","last_undegraded":"2024-09-21T10:19:52.838930+0000","last_fullsized":"2024-09-21T10:19:52.838930+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:48.168055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":111,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.776401+0000","last_change":"2024-09-21T10:18:59.120831+0000","last_active":"2024-09-21T10:19:40.776401+0000","last_peered":"2024-09-21T10:19:40.776401+0000","last_clean":"2024-09-21T10:19:40.776401+0000","last_became_active":"2024-09-21T10:17:34.778770+0000","last_became_peered":"2024-09-21T10:17:34.778770+0000","last_unstale":"2024-09-21T10:19:40.776401+0000","last_undegraded":"2024-09-21T10:19:40.776401+0000","last_fullsized":"2024-09-21T10:19:40.776401+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:59.120678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:59.120678+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:53.877284+0000","scrub_duration":0.0034448999999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.b","version":"1374'9449","reported_seq":9489,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:10.275909+0000","last_change":"2024-09-21T10:18:10.939692+0000","last_active":"2024-09-21T10:20:10.275909+0000","last_peered":"2024-09-21T10:20:10.275909+0000","last_clean":"2024-09-21T10:20:10.275909+0000","last_became_active":"2024-09-21T10:18:10.939184+0000","last_became_peered":"2024-09-21T10:18:10.939184+0000","last_unstale":"2024-09-21T10:20:10.275909+0000","last_undegraded":"2024-09-21T10:20:10.275909+0000","last_fullsized":"2024-09-21T10:20:10.275909+0000","mapping_epoch":1359,"log_start":"1370'4600","ondisk_log_start":"1370'4600","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4849,"log_dups_size":0,"ondisk_log_size":4849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:33:41.090367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":440205312,"num_objects":6717,"num_object_clones":0,"num_object_copies":20151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9449,"num_write_kb":517312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":832,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.825545+0000","last_change":"2024-09-21T10:16:27.315307+0000","last_active":"2024-09-21T10:19:52.825545+0000","last_peered":"2024-09-21T10:19:52.825545+0000","last_clean":"2024-09-21T10:19:52.825545+0000","last_became_active":"2024-09-21T10:16:27.314999+0000","last_became_peered":"2024-09-21T10:16:27.314999+0000","last_unstale":"2024-09-21T10:19:52.825545+0000","last_undegraded":"2024-09-21T10:19:52.825545+0000","last_fullsized":"2024-09-21T10:19:52.825545+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:34.047746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1317,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772309+0000","last_change":"2024-09-21T10:10:13.686243+0000","last_active":"2024-09-21T10:19:40.772309+0000","last_peered":"2024-09-21T10:19:40.772309+0000","last_clean":"2024-09-21T10:19:40.772309+0000","last_became_active":"2024-09-21T10:10:13.685894+0000","last_became_peered":"2024-09-21T10:10:13.685894+0000","last_unstale":"2024-09-21T10:19:40.772309+0000","last_undegraded":"2024-09-21T10:19:40.772309+0000","last_fullsized":"2024-09-21T10:19:40.772309+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:12.394732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1118,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.777000+0000","last_change":"2024-09-21T10:16:27.343418+0000","last_active":"2024-09-21T10:19:40.777000+0000","last_peered":"2024-09-21T10:19:40.777000+0000","last_clean":"2024-09-21T10:19:40.777000+0000","last_became_active":"2024-09-21T10:16:27.339592+0000","last_became_peered":"2024-09-21T10:16:27.339592+0000","last_unstale":"2024-09-21T10:19:40.777000+0000","last_undegraded":"2024-09-21T10:19:40.777000+0000","last_fullsized":"2024-09-21T10:19:40.777000+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:42.895218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":759,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771546+0000","last_change":"2024-09-21T10:16:07.911971+0000","last_active":"2024-09-21T10:19:40.771546+0000","last_peered":"2024-09-21T10:19:40.771546+0000","last_clean":"2024-09-21T10:19:40.771546+0000","last_became_active":"2024-09-21T10:16:07.911538+0000","last_became_peered":"2024-09-21T10:16:07.911538+0000","last_unstale":"2024-09-21T10:19:40.771546+0000","last_undegraded":"2024-09-21T10:19:40.771546+0000","last_fullsized":"2024-09-21T10:19:40.771546+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:19.901507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:19.901507+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:22:29.782749+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.8","version":"1374'9576","reported_seq":9617,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:10.274114+0000","last_change":"2024-09-21T10:18:10.940811+0000","last_active":"2024-09-21T10:20:10.274114+0000","last_peered":"2024-09-21T10:20:10.274114+0000","last_clean":"2024-09-21T10:20:10.274114+0000","last_became_active":"2024-09-21T10:18:10.940121+0000","last_became_peered":"2024-09-21T10:18:10.940121+0000","last_unstale":"2024-09-21T10:20:10.274114+0000","last_undegraded":"2024-09-21T10:20:10.274114+0000","last_fullsized":"2024-09-21T10:20:10.274114+0000","mapping_epoch":1359,"log_start":"1370'4700","ondisk_log_start":"1370'4700","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4876,"log_dups_size":0,"ondisk_log_size":4876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:46.766106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":440139776,"num_objects":6716,"num_object_clones":0,"num_object_copies":20148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9576,"num_write_kb":521344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":153,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775808+0000","last_change":"2024-09-21T10:18:31.646956+0000","last_active":"2024-09-21T10:19:40.775808+0000","last_peered":"2024-09-21T10:19:40.775808+0000","last_clean":"2024-09-21T10:19:40.775808+0000","last_became_active":"2024-09-21T10:18:31.646598+0000","last_became_peered":"2024-09-21T10:18:31.646598+0000","last_unstale":"2024-09-21T10:19:40.775808+0000","last_undegraded":"2024-09-21T10:19:40.775808+0000","last_fullsized":"2024-09-21T10:19:40.775808+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:44:28.469526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":104,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771744+0000","last_change":"2024-09-21T10:16:27.322540+0000","last_active":"2024-09-21T10:19:40.771744+0000","last_peered":"2024-09-21T10:19:40.771744+0000","last_clean":"2024-09-21T10:19:40.771744+0000","last_became_active":"2024-09-21T10:16:27.322003+0000","last_became_peered":"2024-09-21T10:16:27.322003+0000","last_unstale":"2024-09-21T10:19:40.771744+0000","last_undegraded":"2024-09-21T10:19:40.771744+0000","last_fullsized":"2024-09-21T10:19:40.771744+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:13.361229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2152,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.824652+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:19:52.824652+0000","last_peered":"2024-09-21T10:19:52.824652+0000","last_clean":"2024-09-21T10:19:52.824652+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:19:52.824652+0000","last_undegraded":"2024-09-21T10:19:52.824652+0000","last_fullsized":"2024-09-21T10:19:52.824652+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":125,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775245+0000","last_change":"2024-09-21T10:13:17.734264+0000","last_active":"2024-09-21T10:19:40.775245+0000","last_peered":"2024-09-21T10:19:40.775245+0000","last_clean":"2024-09-21T10:19:40.775245+0000","last_became_active":"2024-09-21T10:12:25.707372+0000","last_became_peered":"2024-09-21T10:12:25.707372+0000","last_unstale":"2024-09-21T10:19:40.775245+0000","last_undegraded":"2024-09-21T10:19:40.775245+0000","last_fullsized":"2024-09-21T10:19:40.775245+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:17.734169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:17.734169+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:59.937743+0000","scrub_duration":0.0017947760000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"1374'9509","reported_seq":9549,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:10.296481+0000","last_change":"2024-09-21T10:18:10.940549+0000","last_active":"2024-09-21T10:20:10.296481+0000","last_peered":"2024-09-21T10:20:10.296481+0000","last_clean":"2024-09-21T10:20:10.296481+0000","last_became_active":"2024-09-21T10:18:10.940263+0000","last_became_peered":"2024-09-21T10:18:10.940263+0000","last_unstale":"2024-09-21T10:20:10.296481+0000","last_undegraded":"2024-09-21T10:20:10.296481+0000","last_fullsized":"2024-09-21T10:20:10.296481+0000","mapping_epoch":1359,"log_start":"1370'4600","ondisk_log_start":"1370'4600","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4909,"log_dups_size":0,"ondisk_log_size":4909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:13:25.244015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438239232,"num_objects":6687,"num_object_clones":0,"num_object_copies":20061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9509,"num_write_kb":518272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2024,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.826518+0000","last_change":"2024-09-21T10:17:34.779713+0000","last_active":"2024-09-21T10:19:52.826518+0000","last_peered":"2024-09-21T10:19:52.826518+0000","last_clean":"2024-09-21T10:19:52.826518+0000","last_became_active":"2024-09-21T10:17:34.779351+0000","last_became_peered":"2024-09-21T10:17:34.779351+0000","last_unstale":"2024-09-21T10:19:52.826518+0000","last_undegraded":"2024-09-21T10:19:52.826518+0000","last_fullsized":"2024-09-21T10:19:52.826518+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:02:42.916450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1936,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771740+0000","last_change":"2024-09-21T10:16:07.907738+0000","last_active":"2024-09-21T10:19:40.771740+0000","last_peered":"2024-09-21T10:19:40.771740+0000","last_clean":"2024-09-21T10:19:40.771740+0000","last_became_active":"2024-09-21T10:16:07.907415+0000","last_became_peered":"2024-09-21T10:16:07.907415+0000","last_unstale":"2024-09-21T10:19:40.771740+0000","last_undegraded":"2024-09-21T10:19:40.771740+0000","last_fullsized":"2024-09-21T10:19:40.771740+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:13:44.971174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":125,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.777161+0000","last_change":"2024-09-21T10:16:27.320147+0000","last_active":"2024-09-21T10:19:40.777161+0000","last_peered":"2024-09-21T10:19:40.777161+0000","last_clean":"2024-09-21T10:19:40.777161+0000","last_became_active":"2024-09-21T10:16:27.319802+0000","last_became_peered":"2024-09-21T10:16:27.319802+0000","last_unstale":"2024-09-21T10:19:40.777161+0000","last_undegraded":"2024-09-21T10:19:40.777161+0000","last_fullsized":"2024-09-21T10:19:40.777161+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:12:31.946542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":80,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772060+0000","last_change":"2024-09-21T10:19:24.474938+0000","last_active":"2024-09-21T10:19:40.772060+0000","last_peered":"2024-09-21T10:19:40.772060+0000","last_clean":"2024-09-21T10:19:40.772060+0000","last_became_active":"2024-09-21T10:17:34.769498+0000","last_became_peered":"2024-09-21T10:17:34.769498+0000","last_unstale":"2024-09-21T10:19:40.772060+0000","last_undegraded":"2024-09-21T10:19:40.772060+0000","last_fullsized":"2024-09-21T10:19:40.772060+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:24.474841+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:19:24.474841+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:22:34.128086+0000","scrub_duration":0.0093671499999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.a","version":"1374'9515","reported_seq":12458,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:10.244879+0000","last_change":"2024-09-21T10:18:49.696065+0000","last_active":"2024-09-21T10:20:10.244879+0000","last_peered":"2024-09-21T10:20:10.244879+0000","last_clean":"2024-09-21T10:20:10.244879+0000","last_became_active":"2024-09-21T10:18:49.691461+0000","last_became_peered":"2024-09-21T10:18:49.691461+0000","last_unstale":"2024-09-21T10:20:10.244879+0000","last_undegraded":"2024-09-21T10:20:10.244879+0000","last_fullsized":"2024-09-21T10:20:10.244879+0000","mapping_epoch":1367,"log_start":"1370'4600","ondisk_log_start":"1370'4600","created":1359,"last_epoch_clean":1368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4915,"log_dups_size":0,"ondisk_log_size":4915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:45.812654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442302464,"num_objects":6749,"num_object_clones":0,"num_object_copies":20247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9515,"num_write_kb":520448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":233,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771458+0000","last_change":"2024-09-21T10:14:54.650325+0000","last_active":"2024-09-21T10:19:40.771458+0000","last_peered":"2024-09-21T10:19:40.771458+0000","last_clean":"2024-09-21T10:19:40.771458+0000","last_became_active":"2024-09-21T10:14:54.650048+0000","last_became_peered":"2024-09-21T10:14:54.650048+0000","last_unstale":"2024-09-21T10:19:40.771458+0000","last_undegraded":"2024-09-21T10:19:40.771458+0000","last_fullsized":"2024-09-21T10:19:40.771458+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:47.899727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2112,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.839046+0000","last_change":"2024-09-21T10:12:50.577072+0000","last_active":"2024-09-21T10:19:52.839046+0000","last_peered":"2024-09-21T10:19:52.839046+0000","last_clean":"2024-09-21T10:19:52.839046+0000","last_became_active":"2024-09-21T10:12:50.576731+0000","last_became_peered":"2024-09-21T10:12:50.576731+0000","last_unstale":"2024-09-21T10:19:52.839046+0000","last_undegraded":"2024-09-21T10:19:52.839046+0000","last_fullsized":"2024-09-21T10:19:52.839046+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:56:06.937599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":623,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.825191+0000","last_change":"2024-09-21T10:17:54.399619+0000","last_active":"2024-09-21T10:19:52.825191+0000","last_peered":"2024-09-21T10:19:52.825191+0000","last_clean":"2024-09-21T10:19:52.825191+0000","last_became_active":"2024-09-21T10:17:47.870309+0000","last_became_peered":"2024-09-21T10:17:47.870309+0000","last_unstale":"2024-09-21T10:19:52.825191+0000","last_undegraded":"2024-09-21T10:19:52.825191+0000","last_fullsized":"2024-09-21T10:19:52.825191+0000","mapping_epoch":1351,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:30.694562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":125,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.777094+0000","last_change":"2024-09-21T10:13:20.647735+0000","last_active":"2024-09-21T10:19:40.777094+0000","last_peered":"2024-09-21T10:19:40.777094+0000","last_clean":"2024-09-21T10:19:40.777094+0000","last_became_active":"2024-09-21T10:12:25.708657+0000","last_became_peered":"2024-09-21T10:12:25.708657+0000","last_unstale":"2024-09-21T10:19:40.777094+0000","last_undegraded":"2024-09-21T10:19:40.777094+0000","last_fullsized":"2024-09-21T10:19:40.777094+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:20.647659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:20.647659+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:09.770331+0000","scrub_duration":0.0016005629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.0","version":"1374'9522","reported_seq":9564,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:10.293179+0000","last_change":"2024-09-21T10:18:10.942106+0000","last_active":"2024-09-21T10:20:10.293179+0000","last_peered":"2024-09-21T10:20:10.293179+0000","last_clean":"2024-09-21T10:20:10.293179+0000","last_became_active":"2024-09-21T10:18:10.940520+0000","last_became_peered":"2024-09-21T10:18:10.940520+0000","last_unstale":"2024-09-21T10:20:10.293179+0000","last_undegraded":"2024-09-21T10:20:10.293179+0000","last_fullsized":"2024-09-21T10:20:10.293179+0000","mapping_epoch":1359,"log_start":"1370'4600","ondisk_log_start":"1370'4600","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4922,"log_dups_size":0,"ondisk_log_size":4922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:24:57.174328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439222272,"num_objects":6702,"num_object_clones":0,"num_object_copies":20106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9522,"num_write_kb":519168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":833,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843324+0000","last_change":"2024-09-21T10:16:27.333765+0000","last_active":"2024-09-21T10:19:52.843324+0000","last_peered":"2024-09-21T10:19:52.843324+0000","last_clean":"2024-09-21T10:19:52.843324+0000","last_became_active":"2024-09-21T10:16:27.331548+0000","last_became_peered":"2024-09-21T10:16:27.331548+0000","last_unstale":"2024-09-21T10:19:52.843324+0000","last_undegraded":"2024-09-21T10:19:52.843324+0000","last_fullsized":"2024-09-21T10:19:52.843324+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:28:55.065363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2140,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.840799+0000","last_change":"2024-09-21T10:16:27.330267+0000","last_active":"2024-09-21T10:19:52.840799+0000","last_peered":"2024-09-21T10:19:52.840799+0000","last_clean":"2024-09-21T10:19:52.840799+0000","last_became_active":"2024-09-21T10:16:27.329320+0000","last_became_peered":"2024-09-21T10:16:27.329320+0000","last_unstale":"2024-09-21T10:19:52.840799+0000","last_undegraded":"2024-09-21T10:19:52.840799+0000","last_fullsized":"2024-09-21T10:19:52.840799+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:37.062508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":571,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.825457+0000","last_change":"2024-09-21T10:17:19.687362+0000","last_active":"2024-09-21T10:19:52.825457+0000","last_peered":"2024-09-21T10:19:52.825457+0000","last_clean":"2024-09-21T10:19:52.825457+0000","last_became_active":"2024-09-21T10:17:19.687049+0000","last_became_peered":"2024-09-21T10:17:19.687049+0000","last_unstale":"2024-09-21T10:19:52.825457+0000","last_undegraded":"2024-09-21T10:19:52.825457+0000","last_fullsized":"2024-09-21T10:19:52.825457+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:05:24.686892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:05:24.686892+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"1374'9548","reported_seq":9571,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:12.157365+0000","last_change":"2024-09-21T10:18:10.943078+0000","last_active":"2024-09-21T10:20:12.157365+0000","last_peered":"2024-09-21T10:20:12.157365+0000","last_clean":"2024-09-21T10:20:12.157365+0000","last_became_active":"2024-09-21T10:18:10.942780+0000","last_became_peered":"2024-09-21T10:18:10.942780+0000","last_unstale":"2024-09-21T10:20:12.157365+0000","last_undegraded":"2024-09-21T10:20:12.157365+0000","last_fullsized":"2024-09-21T10:20:12.157365+0000","mapping_epoch":1359,"log_start":"1370'4700","ondisk_log_start":"1370'4700","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:38:22.004424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427819008,"num_objects":6528,"num_object_clones":0,"num_object_copies":19584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9548,"num_write_kb":514432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1372,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772651+0000","last_change":"2024-09-21T10:16:27.321771+0000","last_active":"2024-09-21T10:19:40.772651+0000","last_peered":"2024-09-21T10:19:40.772651+0000","last_clean":"2024-09-21T10:19:40.772651+0000","last_became_active":"2024-09-21T10:16:27.321423+0000","last_became_peered":"2024-09-21T10:16:27.321423+0000","last_unstale":"2024-09-21T10:19:40.772651+0000","last_undegraded":"2024-09-21T10:19:40.772651+0000","last_fullsized":"2024-09-21T10:19:40.772651+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:04.652246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1330,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.770730+0000","last_change":"2024-09-21T10:12:25.697464+0000","last_active":"2024-09-21T10:19:40.770730+0000","last_peered":"2024-09-21T10:19:40.770730+0000","last_clean":"2024-09-21T10:19:40.770730+0000","last_became_active":"2024-09-21T10:12:25.697050+0000","last_became_peered":"2024-09-21T10:12:25.697050+0000","last_unstale":"2024-09-21T10:19:40.770730+0000","last_undegraded":"2024-09-21T10:19:40.770730+0000","last_fullsized":"2024-09-21T10:19:40.770730+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:50.687008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"288'100","reported_seq":198,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.824979+0000","last_change":"2024-09-21T10:17:19.681714+0000","last_active":"2024-09-21T10:19:52.824979+0000","last_peered":"2024-09-21T10:19:52.824979+0000","last_clean":"2024-09-21T10:19:52.824979+0000","last_became_active":"2024-09-21T10:17:19.681360+0000","last_became_peered":"2024-09-21T10:17:19.681360+0000","last_unstale":"2024-09-21T10:19:52.824979+0000","last_undegraded":"2024-09-21T10:19:52.824979+0000","last_fullsized":"2024-09-21T10:19:52.824979+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:30.346035+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:30.346035+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":4,"num_read_kb":11,"num_write":2,"num_write_kb":25,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":8945,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.2","version":"1374'9653","reported_seq":9693,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:10.284541+0000","last_change":"2024-09-21T10:18:10.937252+0000","last_active":"2024-09-21T10:20:10.284541+0000","last_peered":"2024-09-21T10:20:10.284541+0000","last_clean":"2024-09-21T10:20:10.284541+0000","last_became_active":"2024-09-21T10:18:10.936946+0000","last_became_peered":"2024-09-21T10:18:10.936946+0000","last_unstale":"2024-09-21T10:20:10.284541+0000","last_undegraded":"2024-09-21T10:20:10.284541+0000","last_fullsized":"2024-09-21T10:20:10.284541+0000","mapping_epoch":1359,"log_start":"1370'4800","ondisk_log_start":"1370'4800","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4853,"log_dups_size":0,"ondisk_log_size":4853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:49:44.594699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":447414272,"num_objects":6827,"num_object_clones":0,"num_object_copies":20481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9653,"num_write_kb":527360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1349,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771593+0000","last_change":"2024-09-21T10:16:27.320140+0000","last_active":"2024-09-21T10:19:40.771593+0000","last_peered":"2024-09-21T10:19:40.771593+0000","last_clean":"2024-09-21T10:19:40.771593+0000","last_became_active":"2024-09-21T10:16:27.319787+0000","last_became_peered":"2024-09-21T10:16:27.319787+0000","last_unstale":"2024-09-21T10:19:40.771593+0000","last_undegraded":"2024-09-21T10:19:40.771593+0000","last_fullsized":"2024-09-21T10:19:40.771593+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:11.703898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":568,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771224+0000","last_change":"2024-09-21T10:16:27.318938+0000","last_active":"2024-09-21T10:19:40.771224+0000","last_peered":"2024-09-21T10:19:40.771224+0000","last_clean":"2024-09-21T10:19:40.771224+0000","last_became_active":"2024-09-21T10:16:27.318623+0000","last_became_peered":"2024-09-21T10:16:27.318623+0000","last_unstale":"2024-09-21T10:19:40.771224+0000","last_undegraded":"2024-09-21T10:19:40.771224+0000","last_fullsized":"2024-09-21T10:19:40.771224+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:21.912596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":570,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.825238+0000","last_change":"2024-09-21T10:17:19.688581+0000","last_active":"2024-09-21T10:19:52.825238+0000","last_peered":"2024-09-21T10:19:52.825238+0000","last_clean":"2024-09-21T10:19:52.825238+0000","last_became_active":"2024-09-21T10:17:19.687219+0000","last_became_peered":"2024-09-21T10:17:19.687219+0000","last_unstale":"2024-09-21T10:19:52.825238+0000","last_undegraded":"2024-09-21T10:19:52.825238+0000","last_fullsized":"2024-09-21T10:19:52.825238+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:00:15.524724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:00:15.524724+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":131344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.3","version":"1374'9567","reported_seq":9590,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:12.257307+0000","last_change":"2024-09-21T10:18:10.946676+0000","last_active":"2024-09-21T10:20:12.257307+0000","last_peered":"2024-09-21T10:20:12.257307+0000","last_clean":"2024-09-21T10:20:12.257307+0000","last_became_active":"2024-09-21T10:18:10.946364+0000","last_became_peered":"2024-09-21T10:18:10.946364+0000","last_unstale":"2024-09-21T10:20:12.257307+0000","last_undegraded":"2024-09-21T10:20:12.257307+0000","last_fullsized":"2024-09-21T10:20:12.257307+0000","mapping_epoch":1359,"log_start":"1370'4800","ondisk_log_start":"1370'4800","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":4767,"log_dups_size":0,"ondisk_log_size":4767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:54.908078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431423488,"num_objects":6583,"num_object_clones":0,"num_object_copies":19749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9567,"num_write_kb":516800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1316,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771002+0000","last_change":"2024-09-21T10:10:13.661328+0000","last_active":"2024-09-21T10:19:40.771002+0000","last_peered":"2024-09-21T10:19:40.771002+0000","last_clean":"2024-09-21T10:19:40.771002+0000","last_became_active":"2024-09-21T10:10:13.661028+0000","last_became_peered":"2024-09-21T10:10:13.661028+0000","last_unstale":"2024-09-21T10:19:40.771002+0000","last_undegraded":"2024-09-21T10:19:40.771002+0000","last_fullsized":"2024-09-21T10:19:40.771002+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:36.965214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":2113,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.840010+0000","last_change":"2024-09-21T10:08:25.436371+0000","last_active":"2024-09-21T10:19:52.840010+0000","last_peered":"2024-09-21T10:19:52.840010+0000","last_clean":"2024-09-21T10:19:52.840010+0000","last_became_active":"2024-09-21T10:08:25.435921+0000","last_became_peered":"2024-09-21T10:08:25.435921+0000","last_unstale":"2024-09-21T10:19:52.840010+0000","last_undegraded":"2024-09-21T10:19:52.840010+0000","last_fullsized":"2024-09-21T10:19:52.840010+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:27.612386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":145,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775036+0000","last_change":"2024-09-21T10:17:27.870379+0000","last_active":"2024-09-21T10:19:40.775036+0000","last_peered":"2024-09-21T10:19:40.775036+0000","last_clean":"2024-09-21T10:19:40.775036+0000","last_became_active":"2024-09-21T10:17:19.686196+0000","last_became_peered":"2024-09-21T10:17:19.686196+0000","last_unstale":"2024-09-21T10:19:40.775036+0000","last_undegraded":"2024-09-21T10:19:40.775036+0000","last_fullsized":"2024-09-21T10:19:40.775036+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:17:27.870303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:17:27.870303+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:22.680785+0000","scrub_duration":0.001791443,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"1374'9669","reported_seq":9692,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:10.989809+0000","last_change":"2024-09-21T10:18:10.940665+0000","last_active":"2024-09-21T10:20:10.989809+0000","last_peered":"2024-09-21T10:20:10.989809+0000","last_clean":"2024-09-21T10:20:10.989809+0000","last_became_active":"2024-09-21T10:18:10.940315+0000","last_became_peered":"2024-09-21T10:18:10.940315+0000","last_unstale":"2024-09-21T10:20:10.989809+0000","last_undegraded":"2024-09-21T10:20:10.989809+0000","last_fullsized":"2024-09-21T10:20:10.989809+0000","mapping_epoch":1359,"log_start":"1370'4500","ondisk_log_start":"1370'4500","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5169,"log_dups_size":0,"ondisk_log_size":5169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:08:47.081178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445186048,"num_objects":6793,"num_object_clones":0,"num_object_copies":20379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9669,"num_write_kb":526784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1973,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772124+0000","last_change":"2024-09-21T10:16:27.325294+0000","last_active":"2024-09-21T10:19:40.772124+0000","last_peered":"2024-09-21T10:19:40.772124+0000","last_clean":"2024-09-21T10:19:40.772124+0000","last_became_active":"2024-09-21T10:16:27.324974+0000","last_became_peered":"2024-09-21T10:16:27.324974+0000","last_unstale":"2024-09-21T10:19:40.772124+0000","last_undegraded":"2024-09-21T10:19:40.772124+0000","last_fullsized":"2024-09-21T10:19:40.772124+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:58.600136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1119,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.844167+0000","last_change":"2024-09-21T10:10:13.678563+0000","last_active":"2024-09-21T10:19:52.844167+0000","last_peered":"2024-09-21T10:19:52.844167+0000","last_clean":"2024-09-21T10:19:52.844167+0000","last_became_active":"2024-09-21T10:10:13.676594+0000","last_became_peered":"2024-09-21T10:10:13.676594+0000","last_unstale":"2024-09-21T10:19:52.844167+0000","last_undegraded":"2024-09-21T10:19:52.844167+0000","last_fullsized":"2024-09-21T10:19:52.844167+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:34.275762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"288'100","reported_seq":200,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772334+0000","last_change":"2024-09-21T10:18:31.396074+0000","last_active":"2024-09-21T10:19:40.772334+0000","last_peered":"2024-09-21T10:19:40.772334+0000","last_clean":"2024-09-21T10:19:40.772334+0000","last_became_active":"2024-09-21T10:17:19.681584+0000","last_became_peered":"2024-09-21T10:17:19.681584+0000","last_unstale":"2024-09-21T10:19:40.772334+0000","last_undegraded":"2024-09-21T10:19:40.772334+0000","last_fullsized":"2024-09-21T10:19:40.772334+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:31.396016+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:31.396016+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:22.138287+0000","scrub_duration":0.001381925,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1374'9588","reported_seq":9611,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:10.988587+0000","last_change":"2024-09-21T10:18:10.942944+0000","last_active":"2024-09-21T10:20:10.988587+0000","last_peered":"2024-09-21T10:20:10.988587+0000","last_clean":"2024-09-21T10:20:10.988587+0000","last_became_active":"2024-09-21T10:18:10.942635+0000","last_became_peered":"2024-09-21T10:18:10.942635+0000","last_unstale":"2024-09-21T10:20:10.988587+0000","last_undegraded":"2024-09-21T10:20:10.988587+0000","last_fullsized":"2024-09-21T10:20:10.988587+0000","mapping_epoch":1359,"log_start":"1370'4500","ondisk_log_start":"1370'4500","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5088,"log_dups_size":0,"ondisk_log_size":5088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:11:10.686262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439877632,"num_objects":6712,"num_object_clones":0,"num_object_copies":20136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9588,"num_write_kb":521600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":151,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.772045+0000","last_change":"2024-09-21T10:16:27.315202+0000","last_active":"2024-09-21T10:19:40.772045+0000","last_peered":"2024-09-21T10:19:40.772045+0000","last_clean":"2024-09-21T10:19:40.772045+0000","last_became_active":"2024-09-21T10:16:27.314827+0000","last_became_peered":"2024-09-21T10:16:27.314827+0000","last_unstale":"2024-09-21T10:19:40.772045+0000","last_undegraded":"2024-09-21T10:19:40.772045+0000","last_fullsized":"2024-09-21T10:19:40.772045+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:13:31.494018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":133,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.776501+0000","last_change":"2024-09-21T10:12:50.623002+0000","last_active":"2024-09-21T10:19:40.776501+0000","last_peered":"2024-09-21T10:19:40.776501+0000","last_clean":"2024-09-21T10:19:40.776501+0000","last_became_active":"2024-09-21T10:12:50.588355+0000","last_became_peered":"2024-09-21T10:12:50.588355+0000","last_unstale":"2024-09-21T10:19:40.776501+0000","last_undegraded":"2024-09-21T10:19:40.776501+0000","last_fullsized":"2024-09-21T10:19:40.776501+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:04:22.564459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"288'100","reported_seq":234,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.774358+0000","last_change":"2024-09-21T10:14:54.647582+0000","last_active":"2024-09-21T10:19:40.774358+0000","last_peered":"2024-09-21T10:19:40.774358+0000","last_clean":"2024-09-21T10:19:40.774358+0000","last_became_active":"2024-09-21T10:14:54.647213+0000","last_became_peered":"2024-09-21T10:14:54.647213+0000","last_unstale":"2024-09-21T10:19:40.774358+0000","last_undegraded":"2024-09-21T10:19:40.774358+0000","last_fullsized":"2024-09-21T10:19:40.774358+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1310,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:23.746751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:23.746751+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:04.424278+0000","scrub_duration":0.0021986229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.6","version":"1374'9759","reported_seq":9800,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:12.233095+0000","last_change":"2024-09-21T10:18:10.935134+0000","last_active":"2024-09-21T10:20:12.233095+0000","last_peered":"2024-09-21T10:20:12.233095+0000","last_clean":"2024-09-21T10:20:12.233095+0000","last_became_active":"2024-09-21T10:18:10.934822+0000","last_became_peered":"2024-09-21T10:18:10.934822+0000","last_unstale":"2024-09-21T10:20:12.233095+0000","last_undegraded":"2024-09-21T10:20:12.233095+0000","last_fullsized":"2024-09-21T10:20:12.233095+0000","mapping_epoch":1359,"log_start":"1368'3300","ondisk_log_start":"1368'3300","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":6459,"log_dups_size":0,"ondisk_log_size":6459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:04:47.127224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433389568,"num_objects":6613,"num_object_clones":0,"num_object_copies":19839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9759,"num_write_kb":523904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1349,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771667+0000","last_change":"2024-09-21T10:16:27.321387+0000","last_active":"2024-09-21T10:19:40.771667+0000","last_peered":"2024-09-21T10:19:40.771667+0000","last_clean":"2024-09-21T10:19:40.771667+0000","last_became_active":"2024-09-21T10:16:27.321042+0000","last_became_peered":"2024-09-21T10:16:27.321042+0000","last_unstale":"2024-09-21T10:19:40.771667+0000","last_undegraded":"2024-09-21T10:19:40.771667+0000","last_fullsized":"2024-09-21T10:19:40.771667+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:31.231738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":147,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775533+0000","last_change":"2024-09-21T10:17:34.781189+0000","last_active":"2024-09-21T10:19:40.775533+0000","last_peered":"2024-09-21T10:19:40.775533+0000","last_clean":"2024-09-21T10:19:40.775533+0000","last_became_active":"2024-09-21T10:17:34.777855+0000","last_became_peered":"2024-09-21T10:17:34.777855+0000","last_unstale":"2024-09-21T10:19:40.775533+0000","last_undegraded":"2024-09-21T10:19:40.775533+0000","last_fullsized":"2024-09-21T10:19:40.775533+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:42:01.190570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"288'100","reported_seq":593,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843852+0000","last_change":"2024-09-21T10:17:19.681300+0000","last_active":"2024-09-21T10:19:52.843852+0000","last_peered":"2024-09-21T10:19:52.843852+0000","last_clean":"2024-09-21T10:19:52.843852+0000","last_became_active":"2024-09-21T10:17:19.680813+0000","last_became_peered":"2024-09-21T10:17:19.680813+0000","last_unstale":"2024-09-21T10:19:52.843852+0000","last_undegraded":"2024-09-21T10:19:52.843852+0000","last_fullsized":"2024-09-21T10:19:52.843852+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:27.395700+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:27.395700+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":163840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1374'9458","reported_seq":9481,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.819676+0000","last_change":"2024-09-21T10:18:10.936688+0000","last_active":"2024-09-21T10:20:08.819676+0000","last_peered":"2024-09-21T10:20:08.819676+0000","last_clean":"2024-09-21T10:20:08.819676+0000","last_became_active":"2024-09-21T10:18:10.936333+0000","last_became_peered":"2024-09-21T10:18:10.936333+0000","last_unstale":"2024-09-21T10:20:08.819676+0000","last_undegraded":"2024-09-21T10:20:08.819676+0000","last_fullsized":"2024-09-21T10:20:08.819676+0000","mapping_epoch":1359,"log_start":"1370'4400","ondisk_log_start":"1370'4400","created":1359,"last_epoch_clean":1360,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:18:09.922022+0000","last_clean_scrub_stamp":"2024-09-21T10:18:09.922022+0000","objects_scrubbed":0,"log_size":5058,"log_dups_size":0,"ondisk_log_size":5058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:14:28.913228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":451280896,"num_objects":6886,"num_object_clones":0,"num_object_copies":20658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9458,"num_write_kb":523008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":125,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.777289+0000","last_change":"2024-09-21T10:12:50.573976+0000","last_active":"2024-09-21T10:19:40.777289+0000","last_peered":"2024-09-21T10:19:40.777289+0000","last_clean":"2024-09-21T10:19:40.777289+0000","last_became_active":"2024-09-21T10:12:50.573657+0000","last_became_peered":"2024-09-21T10:12:50.573657+0000","last_unstale":"2024-09-21T10:19:40.777289+0000","last_undegraded":"2024-09-21T10:19:40.777289+0000","last_fullsized":"2024-09-21T10:19:40.777289+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:44:44.245943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":119,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.776541+0000","last_change":"2024-09-21T10:12:25.764496+0000","last_active":"2024-09-21T10:19:40.776541+0000","last_peered":"2024-09-21T10:19:40.776541+0000","last_clean":"2024-09-21T10:19:40.776541+0000","last_became_active":"2024-09-21T10:12:25.707263+0000","last_became_peered":"2024-09-21T10:12:25.707263+0000","last_unstale":"2024-09-21T10:19:40.776541+0000","last_undegraded":"2024-09-21T10:19:40.776541+0000","last_fullsized":"2024-09-21T10:19:40.776541+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:45.728577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"288'100","reported_seq":158,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.776754+0000","last_change":"2024-09-21T10:18:36.066497+0000","last_active":"2024-09-21T10:19:40.776754+0000","last_peered":"2024-09-21T10:19:40.776754+0000","last_clean":"2024-09-21T10:19:40.776754+0000","last_became_active":"2024-09-21T10:17:19.683947+0000","last_became_peered":"2024-09-21T10:17:19.683947+0000","last_unstale":"2024-09-21T10:19:40.776754+0000","last_undegraded":"2024-09-21T10:19:40.776754+0000","last_fullsized":"2024-09-21T10:19:40.776754+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:36.066408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:36.066408+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:41:43.382614+0000","scrub_duration":0.006208028,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":181,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771956+0000","last_change":"2024-09-21T10:17:34.792847+0000","last_active":"2024-09-21T10:19:40.771956+0000","last_peered":"2024-09-21T10:19:40.771956+0000","last_clean":"2024-09-21T10:19:40.771956+0000","last_became_active":"2024-09-21T10:17:34.780504+0000","last_became_peered":"2024-09-21T10:17:34.780504+0000","last_unstale":"2024-09-21T10:19:40.771956+0000","last_undegraded":"2024-09-21T10:19:40.771956+0000","last_fullsized":"2024-09-21T10:19:40.771956+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:36.142424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2126,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843122+0000","last_change":"2024-09-21T10:16:27.324409+0000","last_active":"2024-09-21T10:19:52.843122+0000","last_peered":"2024-09-21T10:19:52.843122+0000","last_clean":"2024-09-21T10:19:52.843122+0000","last_became_active":"2024-09-21T10:16:27.321554+0000","last_became_peered":"2024-09-21T10:16:27.321554+0000","last_unstale":"2024-09-21T10:19:52.843122+0000","last_undegraded":"2024-09-21T10:19:52.843122+0000","last_fullsized":"2024-09-21T10:19:52.843122+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:56.209301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"288'100","reported_seq":114,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.771809+0000","last_change":"2024-09-21T10:18:31.567928+0000","last_active":"2024-09-21T10:19:40.771809+0000","last_peered":"2024-09-21T10:19:40.771809+0000","last_clean":"2024-09-21T10:19:40.771809+0000","last_became_active":"2024-09-21T10:16:27.337115+0000","last_became_peered":"2024-09-21T10:16:27.337115+0000","last_unstale":"2024-09-21T10:19:40.771809+0000","last_undegraded":"2024-09-21T10:19:40.771809+0000","last_fullsized":"2024-09-21T10:19:40.771809+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:31.567804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:31.567804+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:24.827181+0000","scrub_duration":0.001700088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":115,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.774921+0000","last_change":"2024-09-21T10:12:25.700080+0000","last_active":"2024-09-21T10:19:40.774921+0000","last_peered":"2024-09-21T10:19:40.774921+0000","last_clean":"2024-09-21T10:19:40.774921+0000","last_became_active":"2024-09-21T10:12:25.699762+0000","last_became_peered":"2024-09-21T10:12:25.699762+0000","last_unstale":"2024-09-21T10:19:40.774921+0000","last_undegraded":"2024-09-21T10:19:40.774921+0000","last_fullsized":"2024-09-21T10:19:40.774921+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:41.509175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2076,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.843418+0000","last_change":"2024-09-21T10:16:07.912706+0000","last_active":"2024-09-21T10:19:52.843418+0000","last_peered":"2024-09-21T10:19:52.843418+0000","last_clean":"2024-09-21T10:19:52.843418+0000","last_became_active":"2024-09-21T10:16:07.912247+0000","last_became_peered":"2024-09-21T10:16:07.912247+0000","last_unstale":"2024-09-21T10:19:52.843418+0000","last_undegraded":"2024-09-21T10:19:52.843418+0000","last_fullsized":"2024-09-21T10:19:52.843418+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:52.397464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"288'100","reported_seq":117,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.839789+0000","last_change":"2024-09-21T10:16:27.331860+0000","last_active":"2024-09-21T10:19:52.839789+0000","last_peered":"2024-09-21T10:19:52.839789+0000","last_clean":"2024-09-21T10:19:52.839789+0000","last_became_active":"2024-09-21T10:16:27.331320+0000","last_became_peered":"2024-09-21T10:16:27.331320+0000","last_unstale":"2024-09-21T10:19:52.839789+0000","last_undegraded":"2024-09-21T10:19:52.839789+0000","last_fullsized":"2024-09-21T10:19:52.839789+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:16.405582+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:13:16.405582+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":23,"num_write":5,"num_write_kb":49,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":17890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2024,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775353+0000","last_change":"2024-09-21T10:16:07.907197+0000","last_active":"2024-09-21T10:19:40.775353+0000","last_peered":"2024-09-21T10:19:40.775353+0000","last_clean":"2024-09-21T10:19:40.775353+0000","last_became_active":"2024-09-21T10:16:07.906770+0000","last_became_peered":"2024-09-21T10:16:07.906770+0000","last_unstale":"2024-09-21T10:19:40.775353+0000","last_undegraded":"2024-09-21T10:19:40.775353+0000","last_fullsized":"2024-09-21T10:19:40.775353+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:55:53.815656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":125,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.776818+0000","last_change":"2024-09-21T10:12:50.584513+0000","last_active":"2024-09-21T10:19:40.776818+0000","last_peered":"2024-09-21T10:19:40.776818+0000","last_clean":"2024-09-21T10:19:40.776818+0000","last_became_active":"2024-09-21T10:12:50.584107+0000","last_became_peered":"2024-09-21T10:12:50.584107+0000","last_unstale":"2024-09-21T10:19:40.776818+0000","last_undegraded":"2024-09-21T10:19:40.776818+0000","last_fullsized":"2024-09-21T10:19:40.776818+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:23.047403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"288'100","reported_seq":135,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.777236+0000","last_change":"2024-09-21T10:13:22.620513+0000","last_active":"2024-09-21T10:19:40.777236+0000","last_peered":"2024-09-21T10:19:40.777236+0000","last_clean":"2024-09-21T10:19:40.777236+0000","last_became_active":"2024-09-21T10:12:50.581542+0000","last_became_peered":"2024-09-21T10:12:50.581542+0000","last_unstale":"2024-09-21T10:19:40.777236+0000","last_undegraded":"2024-09-21T10:19:40.777236+0000","last_fullsized":"2024-09-21T10:19:40.777236+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:22.620446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:22.620446+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:13.111755+0000","scrub_duration":0.0014078199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":121,"reported_epoch":1373,"state":"active+clean","last_fresh":"2024-09-21T10:19:40.775048+0000","last_change":"2024-09-21T10:12:25.720808+0000","last_active":"2024-09-21T10:19:40.775048+0000","last_peered":"2024-09-21T10:19:40.775048+0000","last_clean":"2024-09-21T10:19:40.775048+0000","last_became_active":"2024-09-21T10:12:25.706699+0000","last_became_peered":"2024-09-21T10:12:25.706699+0000","last_unstale":"2024-09-21T10:19:40.775048+0000","last_undegraded":"2024-09-21T10:19:40.775048+0000","last_fullsized":"2024-09-21T10:19:40.775048+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:29.044701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2127,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.844037+0000","last_change":"2024-09-21T10:16:27.318532+0000","last_active":"2024-09-21T10:19:52.844037+0000","last_peered":"2024-09-21T10:19:52.844037+0000","last_clean":"2024-09-21T10:19:52.844037+0000","last_became_active":"2024-09-21T10:16:27.317049+0000","last_became_peered":"2024-09-21T10:16:27.317049+0000","last_unstale":"2024-09-21T10:19:52.844037+0000","last_undegraded":"2024-09-21T10:19:52.844037+0000","last_fullsized":"2024-09-21T10:19:52.844037+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:43.431430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"288'100","reported_seq":732,"reported_epoch":1374,"state":"active+clean","last_fresh":"2024-09-21T10:19:52.840794+0000","last_change":"2024-09-21T10:16:27.324371+0000","last_active":"2024-09-21T10:19:52.840794+0000","last_peered":"2024-09-21T10:19:52.840794+0000","last_clean":"2024-09-21T10:19:52.840794+0000","last_became_active":"2024-09-21T10:16:27.323938+0000","last_became_peered":"2024-09-21T10:16:27.323938+0000","last_unstale":"2024-09-21T10:19:52.840794+0000","last_undegraded":"2024-09-21T10:19:52.840794+0000","last_fullsized":"2024-09-21T10:19:52.840794+0000","mapping_epoch":1330,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1331,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.888350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.888350+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:46.898808+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":7030833176,"num_objects":107283,"num_object_clones":0,"num_object_copies":321849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107283,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153127,"num_write_kb":8333057,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21157523456,"data_stored":21157511240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83827,"ondisk_log_size":83827,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":24716,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34026,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":466780,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":19,"num_read_kb":51,"num_write":10,"num_write_kb":111,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":729468,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398052,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4373504,"kb_used_data":3481272,"kb_used_omap":1,"kb_used_meta":892158,"kb_avail":89371648,"statfs":{"total":95995035648,"available":91516567552,"internally_reserved":0,"allocated":3564822528,"data_stored":3561344354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913570136},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594281,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4708028,"kb_used_data":3812148,"kb_used_omap":1,"kb_used_meta":895870,"kb_avail":89037124,"statfs":{"total":95995035648,"available":91174014976,"internally_reserved":0,"allocated":3903639552,"data_stored":3900165381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917371224},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790510,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1777892,"kb_used_data":875420,"kb_used_omap":1,"kb_used_meta":902462,"kb_avail":91967260,"statfs":{"total":95995035648,"available":94174474240,"internally_reserved":0,"allocated":896430080,"data_stored":892975077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":924121432},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1224,"seq":5257039970446,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3275008,"kb_used_data":2645944,"kb_used_omap":290,"kb_used_meta":628765,"kb_avail":90470144,"statfs":{"total":95995035648,"available":92641427456,"internally_reserved":0,"allocated":2709446656,"data_stored":2705968446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":297377,"internal_metadata":643855967},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999379,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4629608,"kb_used_data":3461472,"kb_used_omap":25,"kb_used_meta":1168102,"kb_avail":89115544,"statfs":{"total":95995035648,"available":91254317056,"internally_reserved":0,"allocated":3544547328,"data_stored":3541088390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":1196137415},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111322,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4148528,"kb_used_data":3006568,"kb_used_omap":31,"kb_used_meta":1141792,"kb_avail":89596624,"statfs":{"total":95995035648,"available":91746942976,"internally_reserved":0,"allocated":3078725632,"data_stored":3075259514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":1169195240},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":2203318223249,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335207,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4642156,"kb_used_data":3567268,"kb_used_omap":31,"kb_used_meta":1074848,"kb_avail":89102996,"statfs":{"total":95995035648,"available":91241467904,"internally_reserved":0,"allocated":3652882432,"data_stored":3649419823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":1100644448},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6255,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3618963456,"data_stored":3618963456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3038973952,"data_stored":3038969880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3528196096,"data_stored":3528196096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2695892992,"data_stored":2695888920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":877395968,"data_stored":877395968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3876786176,"data_stored":3876782104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3521314816,"data_stored":3521314816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:20:16.148 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:20:16.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_6 pg_num 26 2024-09-21T10:20:16.208 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.238 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:16.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:16.273 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:15 smithi017 ceph-mon[102913]: from='client.55855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:16.273 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:15 smithi017 ceph-mon[102913]: pgmap v2248: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-21T10:20:16.273 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:15 smithi017 ceph-mon[105151]: from='client.55855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:16.273 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:15 smithi017 ceph-mon[105151]: pgmap v2248: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-21T10:20:16.339 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:16.375 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:16.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:16.477 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:15 smithi134 ceph-mon[90085]: from='client.55855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:15 smithi134 ceph-mon[90085]: pgmap v2248: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-21T10:20:16.498 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:16.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:16.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:16.623 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:16.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:16.746 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:16.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:16.848 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:16.869 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:16.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:16.971 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:17.002 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:17.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:17.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:17.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:17.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:17.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:17.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:17.439 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:17.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:17.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:17.597 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:17.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:17.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:17.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:20:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.741 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:17.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:17.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:17.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:17.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:17.995 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:18.011 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:20:18.026 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:18.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:18.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:18.167 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:18.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:18 smithi017 ceph-mon[102913]: pgmap v2249: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s 2024-09-21T10:20:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:18 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3006010509' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:18 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:18 smithi017 ceph-mon[105151]: pgmap v2249: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s 2024-09-21T10:20:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:18 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3006010509' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:18.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:18 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:18.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:18.284 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:18 smithi134 ceph-mon[90085]: pgmap v2249: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s 2024-09-21T10:20:18.284 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:18 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3006010509' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:18.284 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:18 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:18.299 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:18.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:18.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:18.437 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:18.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:18.539 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:18.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.585 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:18.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:18.686 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:18.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:18.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:18.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:18.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:18.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:18.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:19.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.118 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:19.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:19.219 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:19.254 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:19.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:19.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:19.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.381 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:19.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:19.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:19 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:19.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:19 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:19.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:19 smithi017 ceph-mon[102913]: osdmap e1375: 8 total, 7 up, 7 in 2024-09-21T10:20:19.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:19 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:19.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:19 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:19.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:19 smithi017 ceph-mon[105151]: osdmap e1375: 8 total, 7 up, 7 in 2024-09-21T10:20:19.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:19 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:19 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:19.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:19 smithi134 ceph-mon[90085]: osdmap e1375: 8 total, 7 up, 7 in 2024-09-21T10:20:19.519 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:19.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:19.648 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:19.750 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:19.782 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:19.884 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:19.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:20.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:20.058 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:20.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:20.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.182 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:20.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:20.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:20.312 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:20.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:20.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:20.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.448 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:20.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:20.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:20 smithi017 ceph-mon[102913]: pgmap v2251: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:20:20.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:20 smithi017 ceph-mon[105151]: pgmap v2251: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:20:20.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:20 smithi134 ceph-mon[90085]: pgmap v2251: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:20:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:20:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:20:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:20:20.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:20.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.582 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:20.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:20.683 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:20.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:20.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:20.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:20.849 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:20.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:20.951 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:20.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.984 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:20.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:21.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:21.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.121 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:21.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:21.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:21.246 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:21.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:21.347 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:21.379 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:21.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:21.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:21 smithi017 ceph-mon[102913]: pgmap v2252: 119 pgs: 119 active+clean; 6.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s 2024-09-21T10:20:21.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:21 smithi017 ceph-mon[105151]: pgmap v2252: 119 pgs: 119 active+clean; 6.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s 2024-09-21T10:20:21.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:21.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:21 smithi134 ceph-mon[90085]: pgmap v2252: 119 pgs: 119 active+clean; 6.0 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s 2024-09-21T10:20:21.517 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:21.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:21.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:21.654 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:21.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:21.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:21.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:21.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:21.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.948 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:21.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:22.049 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:22.077 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:22.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:22.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:22.219 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:22.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:22.321 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:22.350 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:22.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:22.452 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:22.490 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:22.592 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:22.622 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:22.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:22.768 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:22.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:22.810 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:22.870 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:22.910 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:22.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:23.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:23.046 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:23.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:23.148 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:23.176 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:23.278 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:23.309 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:23.411 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:23.452 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:23.522 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:23.522 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:23.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:23.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:23.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:23.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:23.723 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:23.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:23.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:23.860 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:23.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:23.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:23 smithi017 ceph-mon[102913]: pgmap v2253: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-21T10:20:23.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:23 smithi017 ceph-mon[105151]: pgmap v2253: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-21T10:20:23.962 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:23 smithi134 ceph-mon[90085]: pgmap v2253: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-21T10:20:23.996 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:23.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:24.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:24.139 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:24.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:24.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:24.280 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:24.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:24.382 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:24.418 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:24.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:24.555 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:24.657 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:24.685 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:24.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:24.787 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:24.819 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:24.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:24.921 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:24.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:20:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:20:24] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.51.0" 2024-09-21T10:20:24.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.960 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:24.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:25.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:25.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:25.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:25.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:20:25.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:25.275 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:25.275 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:25.377 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:25.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.406 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:25.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:25.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:25.551 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:25.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:25.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:25.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.689 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:25.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:25.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:25.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:25.927 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:25 smithi017 ceph-mon[102913]: pgmap v2254: 119 pgs: 119 active+clean; 5.8 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-21T10:20:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:25 smithi017 ceph-mon[105151]: pgmap v2254: 119 pgs: 119 active+clean; 5.8 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-21T10:20:25.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:25.959 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:25.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:20:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:25 smithi134 ceph-mon[90085]: pgmap v2254: 119 pgs: 119 active+clean; 5.8 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.15k op/s 2024-09-21T10:20:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:26.061 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:26.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.093 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:26.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:26.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:26.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:26.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:26.334 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:26.364 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:26.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:26.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:26.510 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:26.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:26.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:26.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.650 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:26.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:26.751 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.787 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:26.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:26.888 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:26.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:26.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:27.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:27.066 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:27.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:27.167 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:27.196 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:27.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:27.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:27.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:27.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:27.443 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:27.475 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:27.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:27.576 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:27.601 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:27.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:27.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:27.740 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:27.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:27.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.872 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:27.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:27.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:27 smithi017 ceph-mon[102913]: pgmap v2255: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-21T10:20:27.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:27 smithi017 ceph-mon[105151]: pgmap v2255: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-21T10:20:27.973 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:27 smithi134 ceph-mon[90085]: pgmap v2255: 119 pgs: 119 active+clean; 5.6 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-21T10:20:28.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:28.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.169 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:28.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:28.270 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:28.300 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:28.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:28.402 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:28.437 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:28.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:28.539 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:28.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.571 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:28.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:28.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:28.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:28.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:28.850 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:28.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:28.915 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 11 pg_num to 26 2024-09-21T10:20:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:28 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1761169872' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:20:28.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:28 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:20:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:28 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1761169872' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:20:28.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:28 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:20:28.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:28.988 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:28.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:28 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1761169872' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:20:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:28 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:20:29.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.127 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:29.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:29.228 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:29.261 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:29.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:29.267 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:29.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:29.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:29.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:29.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:29.537 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:29.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:29.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:29.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:29.774 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:29.804 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:29.906 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:29.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:29 smithi017 ceph-mon[102913]: pgmap v2256: 119 pgs: 119 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-21T10:20:29.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:29 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:20:29.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:29 smithi017 ceph-mon[102913]: osdmap e1376: 8 total, 7 up, 7 in 2024-09-21T10:20:29.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:29 smithi017 ceph-mon[105151]: pgmap v2256: 119 pgs: 119 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-21T10:20:29.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:29 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:20:29.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:29 smithi017 ceph-mon[105151]: osdmap e1376: 8 total, 7 up, 7 in 2024-09-21T10:20:29.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.941 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:29.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:29 smithi134 ceph-mon[90085]: pgmap v2256: 119 pgs: 119 active+clean; 5.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-21T10:20:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:29 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:20:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:29 smithi134 ceph-mon[90085]: osdmap e1376: 8 total, 7 up, 7 in 2024-09-21T10:20:30.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:30.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:30.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:30.177 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:30.209 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:30.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:30.311 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:30.346 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:30.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:30.448 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:30.479 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:20:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:20:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:20:30.580 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:30.615 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:30.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:30.717 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:30.758 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:30.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:30.796 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:30.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:30.910 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:30.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:30.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:20:30.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:20:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:20:31.012 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:31.033 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:31.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:31.135 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:31.156 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:31.257 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:31.278 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:31.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:31.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:31.402 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:31.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:31.504 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:31.525 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:31.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:31.649 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:31.750 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:31.782 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:31.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:31.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:31.914 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[102913]: pgmap v2258: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s 2024-09-21T10:20:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:20:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[102913]: osdmap e1377: 8 total, 7 up, 7 in 2024-09-21T10:20:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:31.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[105151]: pgmap v2258: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s 2024-09-21T10:20:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:20:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[105151]: osdmap e1377: 8 total, 7 up, 7 in 2024-09-21T10:20:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:31.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:31 smithi134 ceph-mon[90085]: pgmap v2258: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.27k op/s 2024-09-21T10:20:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:20:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:31 smithi134 ceph-mon[90085]: osdmap e1377: 8 total, 7 up, 7 in 2024-09-21T10:20:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:32.015 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:32.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:32.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:32.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.193 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:32.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:32.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:32.341 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:32.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:32.442 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:32.473 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:32.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:32.574 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:32.609 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:32.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:32.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:32.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.759 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:32.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:32.860 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:32.897 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:32 smithi017 ceph-mon[102913]: osdmap e1378: 8 total, 7 up, 7 in 2024-09-21T10:20:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:20:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:20:32.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:32 smithi017 ceph-mon[105151]: osdmap e1378: 8 total, 7 up, 7 in 2024-09-21T10:20:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:20:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:20:32.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:32 smithi134 ceph-mon[90085]: osdmap e1378: 8 total, 7 up, 7 in 2024-09-21T10:20:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:20:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:20:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:32.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:33.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:33.140 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:33.172 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:33.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:33.274 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:33.310 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:33.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:33.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:33.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:33.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:33.551 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:33.585 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:33.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:33.687 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:33.720 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:33.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:33.802 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:33 smithi017 ceph-mon[102913]: pgmap v2261: 129 pgs: 2 peering, 7 unknown, 120 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail 2024-09-21T10:20:33.802 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:33 smithi017 ceph-mon[105151]: pgmap v2261: 129 pgs: 2 peering, 7 unknown, 120 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail 2024-09-21T10:20:33.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:33.863 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:33.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:33.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:33 smithi134 ceph-mon[90085]: pgmap v2261: 129 pgs: 2 peering, 7 unknown, 120 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail 2024-09-21T10:20:34.004 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:34.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:34.105 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:34.137 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:34.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:34.238 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:34.239 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:20:34.239 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:34.240 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:34.272 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:34.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:34.373 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:34.406 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:34.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:34.508 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:34.543 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:34.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:34.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:34.680 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:34.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:34.781 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:34.816 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:34.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:34.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:34.929 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3888639659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:34.930 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3888639659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:34.930 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:20:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:20:34] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.51.0" 2024-09-21T10:20:34.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:34.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3888639659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:35.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:35.098 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:35.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:35.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:35.233 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:35.335 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:35.371 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:35.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:35.389 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:20:35.390 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:20:35.421 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [1] live_osds: [0, 2, 3, 5, 6, 7, 4] 2024-09-21T10:20:35.421 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:20:35.421 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2024-09-21T10:20:35.421 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2024-09-21T10:20:35.422 DEBUG:teuthology.orchestra.run.smithi017:osd.1> sudo journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.1.service 2024-09-21T10:20:35.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl start ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.1 2024-09-21T10:20:35.473 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:35.506 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:35.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:35.608 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:35.623 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:35 smithi017 systemd[1]: Starting Ceph osd.1 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:20:35.638 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:35.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:35.777 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:35.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:35.879 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:35.914 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:35.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:35.936 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:35 smithi017 ceph-mon[102913]: pgmap v2262: 129 pgs: 2 peering, 6 unknown, 121 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail 2024-09-21T10:20:35.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:35 smithi017 ceph-mon[105151]: pgmap v2262: 129 pgs: 2 peering, 6 unknown, 121 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail 2024-09-21T10:20:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:35 smithi134 ceph-mon[90085]: pgmap v2262: 129 pgs: 2 peering, 6 unknown, 121 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail 2024-09-21T10:20:36.016 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:36.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:36.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:36.103 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:20:36.103 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:184312 11.0 11:037f7ae2:::benchmark_data_smithi017_304926_object54105:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1378)", 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:35.649474+0000", 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0070929610000000001, 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071229190000000001, 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 184312 2024-09-21T10:20:36.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.649474+0000", 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.649474+0000", 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:20:36.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.649475+0000", 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.156e-06 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.649479+0000", 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0729999999999998e-06 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.649494+0000", 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4882000000000001e-05 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.649501+0000", 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9319999999999999e-06 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.649560+0000", 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9395999999999998e-05 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.649695+0000", 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135048 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.649991+0000", 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029558999999999998 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654312+0000", 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043211710000000004 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654584+0000", 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027222699999999999 2024-09-21T10:20:36.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654737+0000", 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015334100000000001 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.36507.0:184312 11.0 e1378/1377 ondisk, result = 0)", 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:35.654627+0000", 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0019392350000000001, 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020444690000000001, 2024-09-21T10:20:36.109 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654627+0000", 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654627+0000", 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654628+0000", 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6599999999999996e-07 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654635+0000", 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1029999999999996e-06 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654645+0000", 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.397e-06 2024-09-21T10:20:36.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654651+0000", 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0440000000000001e-06 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654725+0000", 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3949000000000006e-05 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.654731+0000", 2024-09-21T10:20:36.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4119999999999998e-06 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:184321 11.a 11:5dc6530e:::benchmark_data_smithi017_304926_object54114:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1378)", 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:35.655032+0000", 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0015342820000000001, 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001670319, 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:20:36.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 184321 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655032+0000", 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655032+0000", 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655033+0000", 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.181e-06 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655037+0000", 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9210000000000002e-06 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655047+0000", 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8430000000000005e-06 2024-09-21T10:20:36.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655054+0000", 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0389999999999999e-06 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655105+0000", 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1131000000000002e-05 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655320+0000", 2024-09-21T10:20:36.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000214098 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655432+0000", 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011198500000000001 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:184323 11.9 11:926f11d7:::benchmark_data_smithi017_304926_object54116:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1378)", 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:35.655200+0000", 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0013662030000000001, 2024-09-21T10:20:36.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015392509999999999, 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 184323 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655200+0000", 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655200+0000", 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655201+0000", 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.16e-07 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655206+0000", 2024-09-21T10:20:36.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.245e-06 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655215+0000", 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8530000000000004e-06 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655222+0000", 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5819999999999998e-06 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655277+0000", 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4987000000000001e-05 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655383+0000", 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106199 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,6", 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655839+0000", 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00045562599999999999 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:20:36.121 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.36507.0:184322 11.5 e1378/1377 11:a79f771a:::benchmark_data_smithi017_304926_object54115:head v 1378'11319, mlcod=1378'11317)", 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:35.655839+0000", 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00072751599999999999, 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00093626899999999999, 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655839+0000", 2024-09-21T10:20:36.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655839+0000", 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655840+0000", 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9699999999999996e-07 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:20:36.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655845+0000", 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9860000000000002e-06 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655864+0000", 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9644999999999999e-05 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655871+0000", 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4819999999999999e-06 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.655989+0000", 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011771 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.656015+0000", 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6489e-05 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.125 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:184326 11.a 11.96f2002a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1378)", 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:35.656179+0000", 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00038730999999999998, 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00062247300000000002, 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 184326 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.656179+0000", 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.656179+0000", 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.656180+0000", 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4799999999999997e-07 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.656185+0000", 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.301e-06 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.656195+0000", 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0273e-05 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:35.656202+0000", 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0360000000000001e-06 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.128 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:20:36.129 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.129 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:20:36.129 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2024-09-21T10:20:36.129 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 6 2024-09-21T10:20:36.129 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:20:36.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:36.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.187 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:36.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:36.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:36.321 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:36.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:36.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:36.460 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:36.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:36.610 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:36.712 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.746 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:36.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:36.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:36.875 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:36.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:36.977 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:36 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:20:37.000 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:37.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:37.102 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:37.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:37.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:37.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:36 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:20:37.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:36 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:20:37.228 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:37.249 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:37.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:37.374 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:37.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:37.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:20:37.475 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:37.496 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:37.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:37.598 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:37.620 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:37.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:37.721 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:37.744 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:37.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:37.845 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:37.866 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:37.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:37.968 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:37.989 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:37.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:37 smithi134 ceph-mon[90085]: pgmap v2263: 129 pgs: 129 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:20:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:37 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:20:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:37 smithi134 ceph-mon[90085]: osdmap e1379: 8 total, 7 up, 7 in 2024-09-21T10:20:38.090 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:38.111 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:38.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:38.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:37 smithi017 ceph-mon[102913]: pgmap v2263: 129 pgs: 129 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:20:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:37 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:20:38.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:37 smithi017 ceph-mon[102913]: osdmap e1379: 8 total, 7 up, 7 in 2024-09-21T10:20:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:37 smithi017 ceph-mon[105151]: pgmap v2263: 129 pgs: 129 active+clean; 4.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:20:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:37 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:20:38.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:37 smithi017 ceph-mon[105151]: osdmap e1379: 8 total, 7 up, 7 in 2024-09-21T10:20:38.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:38.240 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:38.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:38.342 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:38.363 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:38.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:38.464 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:38.486 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:38.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:38.587 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:38.609 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:38.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:38.710 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:38.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.741 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:38.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:38.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:38.878 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:38.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:38 smithi134 ceph-mon[90085]: osdmap e1380: 8 total, 7 up, 7 in 2024-09-21T10:20:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:38 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:20:39.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.013 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:39.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:39.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:39.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:39.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:38 smithi017 ceph-mon[102913]: osdmap e1380: 8 total, 7 up, 7 in 2024-09-21T10:20:39.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:38 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:20:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:38 smithi017 ceph-mon[105151]: osdmap e1380: 8 total, 7 up, 7 in 2024-09-21T10:20:39.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:38 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:20:39.249 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:39.269 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:39.371 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:39.392 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:39.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:39.494 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:39.515 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:39.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:39.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:39.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:39.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:39.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:39.764 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:39.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:39.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:39.892 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:39.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:39 smithi134 ceph-mon[90085]: pgmap v2266: 129 pgs: 129 active+clean; 4.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 74 MiB/s wr, 2.93k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:20:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:39 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:20:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:39 smithi134 ceph-mon[90085]: osdmap e1381: 8 total, 7 up, 7 in 2024-09-21T10:20:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:39 smithi134 ceph-mon[90085]: 11.10 starting backfill to osd.0 from (0'0,0'0] MAX to 1378'11364 2024-09-21T10:20:39.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:39 smithi134 ceph-mon[90085]: 11.10 starting backfill to osd.2 from (0'0,0'0] MAX to 1378'11364 2024-09-21T10:20:39.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:40.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:40.123 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:40.146 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[102913]: pgmap v2266: 129 pgs: 129 active+clean; 4.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 74 MiB/s wr, 2.93k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:20:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:20:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[102913]: osdmap e1381: 8 total, 7 up, 7 in 2024-09-21T10:20:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[102913]: 11.10 starting backfill to osd.0 from (0'0,0'0] MAX to 1378'11364 2024-09-21T10:20:40.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[102913]: 11.10 starting backfill to osd.2 from (0'0,0'0] MAX to 1378'11364 2024-09-21T10:20:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[105151]: pgmap v2266: 129 pgs: 129 active+clean; 4.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 74 MiB/s wr, 2.93k op/s; 0 B/s, 0 objects/s recovering 2024-09-21T10:20:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:20:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[105151]: osdmap e1381: 8 total, 7 up, 7 in 2024-09-21T10:20:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[105151]: 11.10 starting backfill to osd.0 from (0'0,0'0] MAX to 1378'11364 2024-09-21T10:20:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:39 smithi017 ceph-mon[105151]: 11.10 starting backfill to osd.2 from (0'0,0'0] MAX to 1378'11364 2024-09-21T10:20:40.248 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.271 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:40.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:40.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:40.398 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:40.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:20:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:20:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:20:40.499 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:40.521 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:40.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:40.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:40.649 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:40.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:40.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:20:40.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:40.779 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:40.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:40.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:40.944 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:40.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:41.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:41.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:41.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:40 smithi017 ceph-mon[102913]: osdmap e1382: 8 total, 7 up, 7 in 2024-09-21T10:20:41.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:41.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:20:41.098 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:40 smithi017 ceph-mon[105151]: osdmap e1382: 8 total, 7 up, 7 in 2024-09-21T10:20:41.098 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:41.098 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:20:41.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:41.213 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:41.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:40 smithi134 ceph-mon[90085]: osdmap e1382: 8 total, 7 up, 7 in 2024-09-21T10:20:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:20:41.314 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:41.336 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:41.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:41.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:41.466 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:41.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:41.599 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:41.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:41.701 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:41.730 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:41.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:41.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:41.861 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:41.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:41.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:42.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:42.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[102913]: pgmap v2269: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 2998/221301 objects misplaced (1.355%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:20:42.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:20:42.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[102913]: 11.11 starting backfill to osd.2 from (0'0,0'0] MAX to 1379'11256 2024-09-21T10:20:42.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[102913]: osdmap e1383: 8 total, 7 up, 7 in 2024-09-21T10:20:42.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[102913]: 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 1379'11256 2024-09-21T10:20:42.103 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:42.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[105151]: pgmap v2269: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 2998/221301 objects misplaced (1.355%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:20:42.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:20:42.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[105151]: 11.11 starting backfill to osd.2 from (0'0,0'0] MAX to 1379'11256 2024-09-21T10:20:42.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[105151]: osdmap e1383: 8 total, 7 up, 7 in 2024-09-21T10:20:42.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[105151]: 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 1379'11256 2024-09-21T10:20:42.104 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:42.119 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:42.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:41 smithi134 ceph-mon[90085]: pgmap v2269: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 2998/221301 objects misplaced (1.355%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:20:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:20:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:41 smithi134 ceph-mon[90085]: 11.11 starting backfill to osd.2 from (0'0,0'0] MAX to 1379'11256 2024-09-21T10:20:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:41 smithi134 ceph-mon[90085]: osdmap e1383: 8 total, 7 up, 7 in 2024-09-21T10:20:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:41 smithi134 ceph-mon[90085]: 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 1379'11256 2024-09-21T10:20:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:42.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:42.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:42.349 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:42.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:42.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:42.476 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:42.501 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:42.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:42.509 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:42.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:42.631 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:42.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:42.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:42.759 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:20:42.762 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:42.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:42.864 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:42.891 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:42.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:42 smithi017 ceph-mon[102913]: osdmap e1384: 8 total, 7 up, 7 in 2024-09-21T10:20:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:42 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/246680954' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:42.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:20:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:42 smithi017 ceph-mon[105151]: osdmap e1384: 8 total, 7 up, 7 in 2024-09-21T10:20:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:42 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/246680954' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:42.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:20:42.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:43.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:43.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:43.159 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:42 smithi134 ceph-mon[90085]: osdmap e1384: 8 total, 7 up, 7 in 2024-09-21T10:20:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:42 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/246680954' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:43.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:20:43.260 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:43.281 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:43.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:43.383 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:43.405 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:43.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:43.506 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:43.528 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:43.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:43.630 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:43.651 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:43.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:43.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:43.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:43.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:43.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:43.908 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:43.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[102913]: pgmap v2272: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 2998/218877 objects misplaced (1.370%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:20:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/246680954' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:43.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[102913]: osdmap e1385: 8 total, 7 up, 7 in 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[102913]: 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 1381'11545 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[102913]: osdmap e1386: 8 total, 7 up, 7 in 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[105151]: pgmap v2272: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 2998/218877 objects misplaced (1.370%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/246680954' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[105151]: osdmap e1385: 8 total, 7 up, 7 in 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[105151]: 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 1381'11545 2024-09-21T10:20:43.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:43 smithi017 ceph-mon[105151]: osdmap e1386: 8 total, 7 up, 7 in 2024-09-21T10:20:44.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:44.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:44.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:44.163 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:44.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:43 smithi134 ceph-mon[90085]: pgmap v2272: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 2998/218877 objects misplaced (1.370%); 12 MiB/s, 188 objects/s recovering 2024-09-21T10:20:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:43 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:43 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/246680954' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:43 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:20:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:43 smithi134 ceph-mon[90085]: osdmap e1385: 8 total, 7 up, 7 in 2024-09-21T10:20:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:43 smithi134 ceph-mon[90085]: 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 1381'11545 2024-09-21T10:20:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:43 smithi134 ceph-mon[90085]: osdmap e1386: 8 total, 7 up, 7 in 2024-09-21T10:20:44.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:44.268 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:20:44.268 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:44.268 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-21T10:20:44.292 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:44.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:44.293 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:20:44.293 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:44.314 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:20:44.314 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:44.332 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:20:44.332 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:44.351 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:20:44.351 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:44.368 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:20:44.369 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:44.390 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:20:44.391 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:44.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:44.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:44.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:44.523 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:44.544 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:44.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:44.645 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:44.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:44.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:44.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:44.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:44.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:44.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3329007554' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:44.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:44 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:20:44.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:44 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:20:44.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:44 smithi017 ceph-mon[102913]: osdmap e1387: 8 total, 7 up, 7 in 2024-09-21T10:20:44.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3329007554' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:44.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:44 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:20:44.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:44 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:20:44.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:44 smithi017 ceph-mon[105151]: osdmap e1387: 8 total, 7 up, 7 in 2024-09-21T10:20:44.827 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:20:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:20:44] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.51.0" 2024-09-21T10:20:44.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:44.932 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:44.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:45.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:45.067 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:45.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:45.117 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:44 smithi017 podman[320094]: 2024-09-21 10:20:44.729921775 +0000 UTC m=+0.201437937 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:20:45.117 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:44 smithi017 podman[320094]: 2024-09-21 10:20:44.858853056 +0000 UTC m=+0.330369210 container create 7b28ce96182840e2d51571297edacbbb26c243dffdea5272fe28266878e15ca9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.schema-version=1.0) 2024-09-21T10:20:45.169 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:45.218 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:45.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3329007554' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:44 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:20:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:44 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:20:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:44 smithi134 ceph-mon[90085]: osdmap e1387: 8 total, 7 up, 7 in 2024-09-21T10:20:45.320 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:45.344 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:45.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:45.422 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:45 smithi017 podman[320094]: 2024-09-21 10:20:45.262772651 +0000 UTC m=+0.734288818 container init 7b28ce96182840e2d51571297edacbbb26c243dffdea5272fe28266878e15ca9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T10:20:45.423 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:45 smithi017 podman[320094]: 2024-09-21 10:20:45.269445303 +0000 UTC m=+0.740961457 container start 7b28ce96182840e2d51571297edacbbb26c243dffdea5272fe28266878e15ca9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b) 2024-09-21T10:20:45.423 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:45 smithi017 podman[320094]: 2024-09-21 10:20:45.42095626 +0000 UTC m=+0.892472433 container attach 7b28ce96182840e2d51571297edacbbb26c243dffdea5272fe28266878e15ca9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T10:20:45.446 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:45.470 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:45.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:45.572 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:45.597 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:45.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:45.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:45.729 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:45.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:45.858 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:45.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:45.959 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:45.983 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:45.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:46.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:20:46.038 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:45 smithi017 ceph-mon[102913]: pgmap v2275: 129 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 126 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 9676/218163 objects misplaced (4.435%); 192 KiB/s, 2 objects/s recovering 2024-09-21T10:20:46.038 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:45 smithi017 ceph-mon[102913]: 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 1383'11246 2024-09-21T10:20:46.038 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:45 smithi017 ceph-mon[102913]: 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 1383'11246 2024-09-21T10:20:46.038 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:45 smithi017 ceph-mon[102913]: osdmap e1388: 8 total, 7 up, 7 in 2024-09-21T10:20:46.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:45 smithi017 ceph-mon[105151]: pgmap v2275: 129 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 126 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 9676/218163 objects misplaced (4.435%); 192 KiB/s, 2 objects/s recovering 2024-09-21T10:20:46.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:45 smithi017 ceph-mon[105151]: 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 1383'11246 2024-09-21T10:20:46.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:45 smithi017 ceph-mon[105151]: 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 1383'11246 2024-09-21T10:20:46.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:45 smithi017 ceph-mon[105151]: osdmap e1388: 8 total, 7 up, 7 in 2024-09-21T10:20:46.039 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:20:46.040 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:20:46.085 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:46.108 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:46.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:46.210 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:46.233 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:46.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:45 smithi134 ceph-mon[90085]: pgmap v2275: 129 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 126 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 9676/218163 objects misplaced (4.435%); 192 KiB/s, 2 objects/s recovering 2024-09-21T10:20:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:45 smithi134 ceph-mon[90085]: 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 1383'11246 2024-09-21T10:20:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:45 smithi134 ceph-mon[90085]: 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 1383'11246 2024-09-21T10:20:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:45 smithi134 ceph-mon[90085]: osdmap e1388: 8 total, 7 up, 7 in 2024-09-21T10:20:46.335 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:46.359 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:46.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:46.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:46.484 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:46.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:46.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:46.609 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:46.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:46.711 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:46.738 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:46.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:46.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:46.881 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:46.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:47.021 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:47.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:47.122 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:47.163 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:47.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:47.318 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:47.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:47.347 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:47.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:47.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:47.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:47.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:47.605 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:47.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:47.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:47.747 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:47.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:47.817 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:47 smithi017 ceph-mon[102913]: pgmap v2278: 129 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 125 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 13264/216462 objects misplaced (6.128%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:20:47.817 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:47 smithi017 ceph-mon[102913]: osdmap e1389: 8 total, 7 up, 7 in 2024-09-21T10:20:47.817 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:47 smithi017 ceph-mon[102913]: 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 1385'11551 2024-09-21T10:20:47.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:47 smithi017 ceph-mon[105151]: pgmap v2278: 129 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 125 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 13264/216462 objects misplaced (6.128%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:20:47.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:47 smithi017 ceph-mon[105151]: osdmap e1389: 8 total, 7 up, 7 in 2024-09-21T10:20:47.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:47 smithi017 ceph-mon[105151]: 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 1385'11551 2024-09-21T10:20:47.818 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:47 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[320159]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:20:47.818 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:47 smithi017 bash[320094]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:20:47.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:47.886 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:47.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:47 smithi134 ceph-mon[90085]: pgmap v2278: 129 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 125 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 201 op/s; 13264/216462 objects misplaced (6.128%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:20:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:47 smithi134 ceph-mon[90085]: osdmap e1389: 8 total, 7 up, 7 in 2024-09-21T10:20:47.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:47 smithi134 ceph-mon[90085]: 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 1385'11551 2024-09-21T10:20:48.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:48.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:48.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:20:48.106 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:47 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[320159]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:20:48.106 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:47 smithi017 bash[320094]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:20:48.130 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:48.164 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:48.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:48.311 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:48.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:48.413 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:48.446 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:48.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:48.548 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:48.586 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:48.688 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:48.724 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:48.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:48.868 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:48.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:48.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:48.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:48 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:20:48.998 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:49.099 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:49.123 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:49.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:49.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:48 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:20:49.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:48 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:20:49.225 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:49.248 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:49.350 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:49.374 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:49.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:49.383 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:49.384 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:49.476 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:49.499 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:49.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:49.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:49.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:49.727 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:49.754 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:49.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:49.855 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:49.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.878 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:49.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:49.979 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:49 smithi134 ceph-mon[90085]: pgmap v2280: 129 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 125 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 7999/214179 objects misplaced (3.735%); 54 MiB/s, 856 objects/s recovering 2024-09-21T10:20:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:49 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:20:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:49 smithi134 ceph-mon[90085]: osdmap e1390: 8 total, 7 up, 7 in 2024-09-21T10:20:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:49 smithi134 ceph-mon[90085]: 11.3 scrub starts 2024-09-21T10:20:50.007 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:50.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:50.108 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:50.135 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:50.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:49 smithi017 ceph-mon[102913]: pgmap v2280: 129 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 125 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 7999/214179 objects misplaced (3.735%); 54 MiB/s, 856 objects/s recovering 2024-09-21T10:20:50.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:49 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:20:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:49 smithi017 ceph-mon[102913]: osdmap e1390: 8 total, 7 up, 7 in 2024-09-21T10:20:50.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:49 smithi017 ceph-mon[102913]: 11.3 scrub starts 2024-09-21T10:20:50.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:49 smithi017 ceph-mon[105151]: pgmap v2280: 129 pgs: 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 125 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 7999/214179 objects misplaced (3.735%); 54 MiB/s, 856 objects/s recovering 2024-09-21T10:20:50.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:49 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:20:50.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:49 smithi017 ceph-mon[105151]: osdmap e1390: 8 total, 7 up, 7 in 2024-09-21T10:20:50.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:49 smithi017 ceph-mon[105151]: 11.3 scrub starts 2024-09-21T10:20:50.236 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:50.260 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:50.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:50.362 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:50.386 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:50.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:20:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:20:50.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:20:50.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:50.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:50.616 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:50.640 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:50.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:50.741 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:50.768 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:50.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:50.869 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:50.898 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:50.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:50 smithi134 ceph-mon[90085]: 11.2 scrub starts 2024-09-21T10:20:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:50 smithi134 ceph-mon[90085]: 11.2 scrub ok 2024-09-21T10:20:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:50 smithi134 ceph-mon[90085]: osdmap e1391: 8 total, 7 up, 7 in 2024-09-21T10:20:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:50 smithi134 ceph-mon[90085]: 11.1 scrub starts 2024-09-21T10:20:50.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:50 smithi134 ceph-mon[90085]: 11.1 scrub ok 2024-09-21T10:20:51.000 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:51.037 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:51.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:51.139 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:51.173 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:51.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:51.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[102913]: 11.2 scrub starts 2024-09-21T10:20:51.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[102913]: 11.2 scrub ok 2024-09-21T10:20:51.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[102913]: osdmap e1391: 8 total, 7 up, 7 in 2024-09-21T10:20:51.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[102913]: 11.1 scrub starts 2024-09-21T10:20:51.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[102913]: 11.1 scrub ok 2024-09-21T10:20:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[105151]: 11.2 scrub starts 2024-09-21T10:20:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[105151]: 11.2 scrub ok 2024-09-21T10:20:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[105151]: osdmap e1391: 8 total, 7 up, 7 in 2024-09-21T10:20:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[105151]: 11.1 scrub starts 2024-09-21T10:20:51.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-mon[105151]: 11.1 scrub ok 2024-09-21T10:20:51.202 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:50 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[320159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:20:51.202 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:50 smithi017 bash[320094]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:20:51.275 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:51.308 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:51.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:51.451 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:51.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:51.553 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:51.588 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:51.689 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:51.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[320159]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T10:20:51.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 bash[320094]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T10:20:51.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[320159]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T10:20:51.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 bash[320094]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T10:20:51.701 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[320159]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:20:51.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 bash[320094]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:20:51.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 bash[320094]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T10:20:51.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[320159]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T10:20:51.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[320159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:20:51.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 bash[320094]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:20:51.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate[320159]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T10:20:51.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 bash[320094]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T10:20:51.702 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 conmon[320159]: conmon 7b28ce96182840e2d515 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-7b28ce96182840e2d51571297edacbbb26c243dffdea5272fe28266878e15ca9.scope/memory.events 2024-09-21T10:20:51.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.725 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:51.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:51.826 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:51.853 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:51.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:51.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:51.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.986 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:51.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:51.998 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[102913]: pgmap v2283: 129 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 124 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 7999/209607 objects misplaced (3.816%); 54 MiB/s, 857 objects/s recovering 2024-09-21T10:20:51.999 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[102913]: 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 1389'11522 2024-09-21T10:20:51.999 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[102913]: osdmap e1392: 8 total, 7 up, 7 in 2024-09-21T10:20:51.999 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[102913]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 1389'11522 2024-09-21T10:20:51.999 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:51.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[105151]: pgmap v2283: 129 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 124 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 7999/209607 objects misplaced (3.816%); 54 MiB/s, 857 objects/s recovering 2024-09-21T10:20:51.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[105151]: 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 1389'11522 2024-09-21T10:20:51.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[105151]: osdmap e1392: 8 total, 7 up, 7 in 2024-09-21T10:20:51.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[105151]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 1389'11522 2024-09-21T10:20:51.999 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:51 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:52.087 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:52.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:52.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:51 smithi134 ceph-mon[90085]: pgmap v2283: 129 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 124 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 7999/209607 objects misplaced (3.816%); 54 MiB/s, 857 objects/s recovering 2024-09-21T10:20:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:51 smithi134 ceph-mon[90085]: 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 1389'11522 2024-09-21T10:20:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:51 smithi134 ceph-mon[90085]: osdmap e1392: 8 total, 7 up, 7 in 2024-09-21T10:20:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:51 smithi134 ceph-mon[90085]: 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 1389'11522 2024-09-21T10:20:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:51 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:20:52.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:52.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:52.275 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:20:51 smithi017 podman[320094]: 2024-09-21 10:20:51.997345864 +0000 UTC m=+7.468862027 container died 7b28ce96182840e2d51571297edacbbb26c243dffdea5272fe28266878e15ca9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T10:20:52.363 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:52.386 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:52.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:52.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:52.514 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:52.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:52.615 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:52.638 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:52.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:52.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:52.764 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:52.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:52.866 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:52.892 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:52.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:52.993 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:53.031 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:53.132 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:53.165 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:53.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:52 smithi017 ceph-mon[105151]: 11.13 scrub starts 2024-09-21T10:20:53.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:52 smithi017 ceph-mon[105151]: osdmap e1393: 8 total, 7 up, 7 in 2024-09-21T10:20:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:52 smithi017 ceph-mon[102913]: 11.13 scrub starts 2024-09-21T10:20:53.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:52 smithi017 ceph-mon[102913]: osdmap e1393: 8 total, 7 up, 7 in 2024-09-21T10:20:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:52 smithi134 ceph-mon[90085]: 11.13 scrub starts 2024-09-21T10:20:53.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:52 smithi134 ceph-mon[90085]: osdmap e1393: 8 total, 7 up, 7 in 2024-09-21T10:20:53.267 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:53.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:53.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:53.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:53.447 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:53.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:53.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:53.691 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:53.725 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:53.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:53.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:20:53.806 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:53.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:53.871 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:53.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:53.974 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:54.001 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:54.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:53 smithi017 ceph-mon[102913]: pgmap v2286: 129 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 125 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 4142/203724 objects misplaced (2.033%); 31 MiB/s, 491 objects/s recovering 2024-09-21T10:20:54.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:53 smithi017 ceph-mon[102913]: osdmap e1394: 8 total, 7 up, 7 in 2024-09-21T10:20:54.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:53 smithi017 ceph-mon[102913]: osdmap e1395: 8 total, 7 up, 7 in 2024-09-21T10:20:54.011 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:53 smithi017 ceph-mon[105151]: pgmap v2286: 129 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 125 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 4142/203724 objects misplaced (2.033%); 31 MiB/s, 491 objects/s recovering 2024-09-21T10:20:54.011 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:53 smithi017 ceph-mon[105151]: osdmap e1394: 8 total, 7 up, 7 in 2024-09-21T10:20:54.011 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:53 smithi017 ceph-mon[105151]: osdmap e1395: 8 total, 7 up, 7 in 2024-09-21T10:20:54.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:54.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:54.237 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:53 smithi134 ceph-mon[90085]: pgmap v2286: 129 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 125 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 4142/203724 objects misplaced (2.033%); 31 MiB/s, 491 objects/s recovering 2024-09-21T10:20:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:53 smithi134 ceph-mon[90085]: osdmap e1394: 8 total, 7 up, 7 in 2024-09-21T10:20:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:53 smithi134 ceph-mon[90085]: osdmap e1395: 8 total, 7 up, 7 in 2024-09-21T10:20:54.264 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:54.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:54.365 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:54.389 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:54.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:54.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:54.516 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:54.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:54.618 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:54.639 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:54.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:54.741 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:54.768 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:54.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:54.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:54.874 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:20:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:20:54] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.51.0" 2024-09-21T10:20:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.911 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:54.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:55.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:55.053 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:55.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:55.155 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:55.186 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:54 smithi017 ceph-mon[102913]: 11.10 scrub starts 2024-09-21T10:20:55.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:54 smithi017 ceph-mon[102913]: osdmap e1396: 8 total, 7 up, 7 in 2024-09-21T10:20:55.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:54 smithi017 ceph-mon[105151]: 11.10 scrub starts 2024-09-21T10:20:55.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:54 smithi017 ceph-mon[105151]: osdmap e1396: 8 total, 7 up, 7 in 2024-09-21T10:20:55.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:54 smithi134 ceph-mon[90085]: 11.10 scrub starts 2024-09-21T10:20:55.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:54 smithi134 ceph-mon[90085]: osdmap e1396: 8 total, 7 up, 7 in 2024-09-21T10:20:55.288 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:55.325 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:55.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:55.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:55.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:55.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:55.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:55.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:55.695 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:55.733 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:55.835 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:55.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.872 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:55.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:55.973 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:55.994 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:55.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:56.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:56.128 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:56.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[105151]: 11.10 scrub ok 2024-09-21T10:20:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[105151]: pgmap v2289: 129 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+remapped+backfilling, 126 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1843/201279 objects misplaced (0.916%); 65 MiB/s, 1.03k objects/s recovering 2024-09-21T10:20:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[105151]: 11.3 scrub starts 2024-09-21T10:20:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:56.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[105151]: 11.4 scrub starts 2024-09-21T10:20:56.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[102913]: 11.10 scrub ok 2024-09-21T10:20:56.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[102913]: pgmap v2289: 129 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+remapped+backfilling, 126 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1843/201279 objects misplaced (0.916%); 65 MiB/s, 1.03k objects/s recovering 2024-09-21T10:20:56.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[102913]: 11.3 scrub starts 2024-09-21T10:20:56.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:56.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:55 smithi017 ceph-mon[102913]: 11.4 scrub starts 2024-09-21T10:20:56.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:55 smithi134 ceph-mon[90085]: 11.10 scrub ok 2024-09-21T10:20:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:55 smithi134 ceph-mon[90085]: pgmap v2289: 129 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+remapped+backfilling, 126 active+clean; 4.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1843/201279 objects misplaced (0.916%); 65 MiB/s, 1.03k objects/s recovering 2024-09-21T10:20:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:55 smithi134 ceph-mon[90085]: 11.3 scrub starts 2024-09-21T10:20:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:55 smithi134 ceph-mon[90085]: 11.4 scrub starts 2024-09-21T10:20:56.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.265 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:56.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:56.365 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:56.398 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:56.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:56.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:56.531 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:56.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:56.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:56.669 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:56.771 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:56.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:56.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:56.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:56.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:57.031 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:57.063 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:57.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:57.164 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:57.192 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:57.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:56 smithi017 ceph-mon[102913]: 11.3 scrub ok 2024-09-21T10:20:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:20:57.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:56 smithi017 ceph-mon[102913]: 11.4 scrub ok 2024-09-21T10:20:57.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:56 smithi017 ceph-mon[105151]: 11.3 scrub ok 2024-09-21T10:20:57.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:20:57.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:56 smithi017 ceph-mon[105151]: 11.4 scrub ok 2024-09-21T10:20:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:56 smithi134 ceph-mon[90085]: 11.3 scrub ok 2024-09-21T10:20:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:20:57.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:56 smithi134 ceph-mon[90085]: 11.4 scrub ok 2024-09-21T10:20:57.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:57.324 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:57.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:57.426 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:57.446 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:57.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:57.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:57.571 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:57.673 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:57.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:57.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:57.820 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:57.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:57.921 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:57.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:57.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:58.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:58.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:58.174 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:58.195 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:57 smithi017 ceph-mon[102913]: pgmap v2291: 129 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 126 active+clean; 3.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 5813/193440 objects misplaced (3.005%); 32 MiB/s, 515 objects/s recovering 2024-09-21T10:20:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:20:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:57 smithi017 ceph-mon[102913]: osdmap e1397: 8 total, 7 up, 7 in 2024-09-21T10:20:58.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:57 smithi017 ceph-mon[102913]: 11.5 scrub starts 2024-09-21T10:20:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:57 smithi017 ceph-mon[105151]: pgmap v2291: 129 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 126 active+clean; 3.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 5813/193440 objects misplaced (3.005%); 32 MiB/s, 515 objects/s recovering 2024-09-21T10:20:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:20:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:57 smithi017 ceph-mon[105151]: osdmap e1397: 8 total, 7 up, 7 in 2024-09-21T10:20:58.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:57 smithi017 ceph-mon[105151]: 11.5 scrub starts 2024-09-21T10:20:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:57 smithi134 ceph-mon[90085]: pgmap v2291: 129 pgs: 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 1 active+remapped+backfilling, 126 active+clean; 3.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 5813/193440 objects misplaced (3.005%); 32 MiB/s, 515 objects/s recovering 2024-09-21T10:20:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:20:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:57 smithi134 ceph-mon[90085]: osdmap e1397: 8 total, 7 up, 7 in 2024-09-21T10:20:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:57 smithi134 ceph-mon[90085]: 11.5 scrub starts 2024-09-21T10:20:58.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:58.322 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:58.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:58.424 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:58.446 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:58.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:58.547 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.570 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:58.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:58.671 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:58.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.694 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:58.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:58.794 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:58.816 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:58.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:58.955 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:58.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:59.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:59.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:59.180 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:58 smithi017 ceph-mon[102913]: 11.5 scrub ok 2024-09-21T10:20:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:58 smithi017 ceph-mon[102913]: osdmap e1398: 8 total, 7 up, 7 in 2024-09-21T10:20:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:58 smithi017 ceph-mon[102913]: osdmap e1399: 8 total, 7 up, 7 in 2024-09-21T10:20:59.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:58 smithi017 ceph-mon[105151]: 11.5 scrub ok 2024-09-21T10:20:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:58 smithi017 ceph-mon[105151]: osdmap e1398: 8 total, 7 up, 7 in 2024-09-21T10:20:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:58 smithi017 ceph-mon[105151]: osdmap e1399: 8 total, 7 up, 7 in 2024-09-21T10:20:59.207 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:59.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:59.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:58 smithi134 ceph-mon[90085]: 11.5 scrub ok 2024-09-21T10:20:59.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:58 smithi134 ceph-mon[90085]: osdmap e1398: 8 total, 7 up, 7 in 2024-09-21T10:20:59.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:58 smithi134 ceph-mon[90085]: osdmap e1399: 8 total, 7 up, 7 in 2024-09-21T10:20:59.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:59.342 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:59.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:59.465 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:59.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:59.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:59.595 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:20:59.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:59.697 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:59.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.718 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:59.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:59.819 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:59.854 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:59.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:59.956 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:59.987 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:20:59.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:00.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:00.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:00.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:59 smithi017 ceph-mon[102913]: pgmap v2294: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 125 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4389/187833 objects misplaced (2.337%) 2024-09-21T10:21:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:59 smithi017 ceph-mon[102913]: 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 1396'11877 2024-09-21T10:21:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:59 smithi017 ceph-mon[102913]: 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 1396'11877 2024-09-21T10:21:00.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:20:59 smithi017 ceph-mon[102913]: osdmap e1400: 8 total, 7 up, 7 in 2024-09-21T10:21:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:59 smithi017 ceph-mon[105151]: pgmap v2294: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 125 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4389/187833 objects misplaced (2.337%) 2024-09-21T10:21:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:59 smithi017 ceph-mon[105151]: 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 1396'11877 2024-09-21T10:21:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:59 smithi017 ceph-mon[105151]: 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 1396'11877 2024-09-21T10:21:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:20:59 smithi017 ceph-mon[105151]: osdmap e1400: 8 total, 7 up, 7 in 2024-09-21T10:21:00.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:59 smithi134 ceph-mon[90085]: pgmap v2294: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+clean+scrubbing, 125 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4389/187833 objects misplaced (2.337%) 2024-09-21T10:21:00.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:59 smithi134 ceph-mon[90085]: 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 1396'11877 2024-09-21T10:21:00.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:59 smithi134 ceph-mon[90085]: 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 1396'11877 2024-09-21T10:21:00.213 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:20:59 smithi134 ceph-mon[90085]: osdmap e1400: 8 total, 7 up, 7 in 2024-09-21T10:21:00.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:00.251 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:00.352 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:00.379 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:00.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:00.481 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:21:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:21:00.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:21:00.511 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:00.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:00.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:00.652 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:00.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:00.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:00 smithi017 podman[320801]: 2024-09-21 10:21:00.401986725 +0000 UTC m=+8.928146037 container remove 7b28ce96182840e2d51571297edacbbb26c243dffdea5272fe28266878e15ca9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T10:21:00.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:00.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:00.892 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:00.920 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:00.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:01.022 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:01.050 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:01.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:01.152 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:01.182 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:01.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:01.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:01.319 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:01.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:01.420 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:01.454 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:01.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:01.556 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:01.591 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:01.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:01.733 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:01.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:01.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:01.867 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:01.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:01.969 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:02.002 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:02.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:02.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:02.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:02.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:01 smithi017 ceph-mon[102913]: pgmap v2297: 129 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+clean+scrubbing, 125 active+clean; 3.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2142/181629 objects misplaced (1.179%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:21:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:01 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:02.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:01 smithi017 ceph-mon[105151]: pgmap v2297: 129 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+clean+scrubbing, 125 active+clean; 3.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2142/181629 objects misplaced (1.179%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:21:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:01 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:02.228 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:01 smithi134 ceph-mon[90085]: pgmap v2297: 129 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+clean+scrubbing, 125 active+clean; 3.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2142/181629 objects misplaced (1.179%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T10:21:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:01 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:02.257 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:02.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:02.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:02.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:02.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:02.482 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:02.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.519 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:02.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:02.620 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:02.656 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:02.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:02.795 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:02.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:02.896 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:02.924 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:02.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:03.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:03.062 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:03.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:03.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:03.202 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:03.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:03.303 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:03.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:03.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:03.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:03.503 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:03.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:03.605 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:03.638 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:03.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:03.740 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:03.775 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:03.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:03.877 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:03.908 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:03.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:04.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:04.035 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:04.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:04.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:04.177 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:04.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:04.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:03 smithi017 ceph-mon[102913]: pgmap v2298: 129 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 126 active+clean; 3.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2142/176829 objects misplaced (1.211%); 2.2 MiB/s, 34 objects/s recovering 2024-09-21T10:21:04.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:03 smithi017 ceph-mon[105151]: pgmap v2298: 129 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 126 active+clean; 3.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2142/176829 objects misplaced (1.211%); 2.2 MiB/s, 34 objects/s recovering 2024-09-21T10:21:04.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:03 smithi134 ceph-mon[90085]: pgmap v2298: 129 pgs: 1 active+remapped+backfilling, 1 peering, 1 remapped+peering, 126 active+clean; 3.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 2142/176829 objects misplaced (1.211%); 2.2 MiB/s, 34 objects/s recovering 2024-09-21T10:21:04.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:04.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:04.407 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.441 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:04.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:04.542 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:04.574 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:04.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:04.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:04.715 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:04.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:04.853 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:04.903 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:21:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:21:04] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-09-21T10:21:04.955 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:04.984 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:05.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:05.126 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:05.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:04 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-21T10:21:05.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:04 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-21T10:21:05.228 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:05.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:04 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-21T10:21:05.250 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:05.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:05.351 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:05.373 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:05.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:05.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:05.607 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:05.629 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:05.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:05.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:05.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:05.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:05.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:05.884 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:05.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:05.908 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:21:05.986 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:06.008 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:06.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:05 smithi017 ceph-mon[102913]: pgmap v2299: 129 pgs: 2 active+remapped+backfilling, 127 active+clean; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4126/172209 objects misplaced (2.396%) 2024-09-21T10:21:06.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-21T10:21:06.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:05 smithi017 ceph-mon[102913]: osdmap e1401: 8 total, 7 up, 7 in 2024-09-21T10:21:06.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:05 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/902461572' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:06.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:05 smithi017 ceph-mon[105151]: pgmap v2299: 129 pgs: 2 active+remapped+backfilling, 127 active+clean; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4126/172209 objects misplaced (2.396%) 2024-09-21T10:21:06.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-21T10:21:06.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:05 smithi017 ceph-mon[105151]: osdmap e1401: 8 total, 7 up, 7 in 2024-09-21T10:21:06.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:05 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/902461572' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:06.110 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:06.133 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:06.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:06.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:05 smithi134 ceph-mon[90085]: pgmap v2299: 129 pgs: 2 active+remapped+backfilling, 127 active+clean; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4126/172209 objects misplaced (2.396%) 2024-09-21T10:21:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-21T10:21:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:05 smithi134 ceph-mon[90085]: osdmap e1401: 8 total, 7 up, 7 in 2024-09-21T10:21:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:05 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/902461572' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:06.259 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:06.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:06.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:06.386 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:06.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:06.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:06.513 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:06.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:06.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.638 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:06.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:06.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:06.760 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:06.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:06.883 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:06.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:06.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:07.010 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:07.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:07.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:07.147 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:07.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:06 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:21:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:06 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:21:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:06 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/902461572' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:06 smithi017 ceph-mon[102913]: osdmap e1402: 8 total, 7 up, 7 in 2024-09-21T10:21:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:06 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:21:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:06 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:21:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:06 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/902461572' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:06 smithi017 ceph-mon[105151]: osdmap e1402: 8 total, 7 up, 7 in 2024-09-21T10:21:07.223 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:21:07.223 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:07.223 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2024-09-21T10:21:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:06 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:21:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:06 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:21:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:06 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/902461572' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:06 smithi134 ceph-mon[90085]: osdmap e1402: 8 total, 7 up, 7 in 2024-09-21T10:21:07.244 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:21:07.245 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:07.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:07.263 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:21:07.263 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:07.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:07.281 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:21:07.281 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:07.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:21:07.300 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:07.316 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:21:07.316 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:07.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:21:07.334 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:07.375 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:07.400 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:07.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:07.502 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:07.525 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:07.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:07.649 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:07.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:07.773 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:07.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:07.875 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:07.898 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:08.000 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:08.032 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:08.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:08.133 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:08.168 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:08.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:08.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[102913]: pgmap v2302: 129 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 126 active+clean; 3.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2802/162294 objects misplaced (1.726%); 19 MiB/s, 311 objects/s recovering 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[102913]: osdmap e1403: 8 total, 7 up, 7 in 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[102913]: 11.17 starting backfill to osd.5 from (0'0,0'0] MAX to 1400'12133 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[102913]: 11.7 scrub starts 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[102913]: 11.17 starting backfill to osd.6 from (0'0,0'0] MAX to 1400'12133 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3825304244' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[102913]: 11.7 scrub ok 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[105151]: pgmap v2302: 129 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 126 active+clean; 3.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2802/162294 objects misplaced (1.726%); 19 MiB/s, 311 objects/s recovering 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[105151]: osdmap e1403: 8 total, 7 up, 7 in 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[105151]: 11.17 starting backfill to osd.5 from (0'0,0'0] MAX to 1400'12133 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[105151]: 11.7 scrub starts 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[105151]: 11.17 starting backfill to osd.6 from (0'0,0'0] MAX to 1400'12133 2024-09-21T10:21:08.188 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3825304244' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:08.189 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:07 smithi017 ceph-mon[105151]: 11.7 scrub ok 2024-09-21T10:21:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:07 smithi134 ceph-mon[90085]: pgmap v2302: 129 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 126 active+clean; 3.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2802/162294 objects misplaced (1.726%); 19 MiB/s, 311 objects/s recovering 2024-09-21T10:21:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:07 smithi134 ceph-mon[90085]: osdmap e1403: 8 total, 7 up, 7 in 2024-09-21T10:21:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:07 smithi134 ceph-mon[90085]: 11.17 starting backfill to osd.5 from (0'0,0'0] MAX to 1400'12133 2024-09-21T10:21:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:07 smithi134 ceph-mon[90085]: 11.7 scrub starts 2024-09-21T10:21:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:07 smithi134 ceph-mon[90085]: 11.17 starting backfill to osd.6 from (0'0,0'0] MAX to 1400'12133 2024-09-21T10:21:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:07 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3825304244' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:07 smithi134 ceph-mon[90085]: 11.7 scrub ok 2024-09-21T10:21:08.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:08.308 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:08.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:08.409 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:08.444 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:08.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:08.546 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:08.564 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:08 smithi017 podman[321938]: 2024-09-21 10:21:08.166577215 +0000 UTC m=+2.060195594 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:21:08.564 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:08 smithi017 podman[321938]: 2024-09-21 10:21:08.322083269 +0000 UTC m=+2.215701642 container create d5f81ff1b44049ee3d44701860af266987c3b1046776f34aa11549b17cfe1fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T10:21:08.582 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:08.684 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:08.722 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:08.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:08.824 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:08.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:08.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:08.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:08.991 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:08.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:09.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:09.131 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:09.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:08 smithi017 ceph-mon[102913]: osdmap e1404: 8 total, 7 up, 7 in 2024-09-21T10:21:09.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:08 smithi017 ceph-mon[102913]: 11.13 scrub starts 2024-09-21T10:21:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:08 smithi017 ceph-mon[105151]: osdmap e1404: 8 total, 7 up, 7 in 2024-09-21T10:21:09.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:08 smithi017 ceph-mon[105151]: 11.13 scrub starts 2024-09-21T10:21:09.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:09.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:08 smithi134 ceph-mon[90085]: osdmap e1404: 8 total, 7 up, 7 in 2024-09-21T10:21:09.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:08 smithi134 ceph-mon[90085]: 11.13 scrub starts 2024-09-21T10:21:09.270 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:09.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:09.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:09.408 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:09.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:09.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:09.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.550 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:09.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:09.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:09.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:21:09.685 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:09.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:09.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:09.824 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:09.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:09.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:09.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:10.058 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:10.093 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:10.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:10.195 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:10.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:09 smithi017 ceph-mon[102913]: pgmap v2305: 129 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 126 active+clean; 3.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 2802/158832 objects misplaced (1.764%); 9.2 MiB/s, 146 objects/s recovering 2024-09-21T10:21:10.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:09 smithi017 ceph-mon[102913]: 11.14 scrub starts 2024-09-21T10:21:10.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:09 smithi017 ceph-mon[105151]: pgmap v2305: 129 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 126 active+clean; 3.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 2802/158832 objects misplaced (1.764%); 9.2 MiB/s, 146 objects/s recovering 2024-09-21T10:21:10.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:09 smithi017 ceph-mon[105151]: 11.14 scrub starts 2024-09-21T10:21:10.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:09 smithi134 ceph-mon[90085]: pgmap v2305: 129 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 126 active+clean; 3.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 2802/158832 objects misplaced (1.764%); 9.2 MiB/s, 146 objects/s recovering 2024-09-21T10:21:10.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:09 smithi134 ceph-mon[90085]: 11.14 scrub starts 2024-09-21T10:21:10.224 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:10.325 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:10.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.359 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:10.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:10.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:10.483 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:21:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:21:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:21:10.585 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:10.626 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:10.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:10.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:10.761 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:10.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:10.863 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:10.903 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:11.005 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:11.041 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:11.142 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.178 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:11.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:11.279 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:11.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.317 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:11.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:11.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:11.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:11.498 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:21:11.498 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:21:11.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173493 9.1 9:838aea2c:::benchmark_data_smithi017_237924_object80489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.675265+0000", 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 526.73981687100002, 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.426658829, 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.499 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173493 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675265+0000", 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675265+0000", 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675266+0000", 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0500000000000004e-07 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675268+0000", 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.774e-06 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675279+0000", 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0543999999999999e-05 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.675285+0000", 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8549999999999998e-06 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.681973+0000", 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066879540000000003 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682093+0000", 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120488 2024-09-21T10:21:11.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.682203+0000", 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109916 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703356+0000", 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.021152608999999999 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703367+0000", 2024-09-21T10:21:11.503 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0736e-05 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695780+0000", 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99241349599999995 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695792+0000", 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1928e-05 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917420+0000", 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2216284660000001 2024-09-21T10:21:11.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.917536+0000", 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011577400000000001 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.101842+0000", 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18430565700000001 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.506 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.101924+0000", 2024-09-21T10:21:11.506 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2126999999999995e-05 2024-09-21T10:21:11.506 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.506 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.506 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.506 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.506 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.506 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173503 9.1 9:8faf9e71:::benchmark_data_smithi017_237924_object80499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:21:11.507 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.688561+0000", 2024-09-21T10:21:11.507 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 526.72652102300003, 2024-09-21T10:21:11.507 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4121262919999999, 2024-09-21T10:21:11.507 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.507 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.507 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.507 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173503 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688561+0000", 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688561+0000", 2024-09-21T10:21:11.508 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688563+0000", 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5820000000000001e-06 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688567+0000", 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4089999999999996e-06 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688580+0000", 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3081999999999999e-05 2024-09-21T10:21:11.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.688590+0000", 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7960000000000007e-06 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703388+0000", 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014798443999999999 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.510 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703395+0000", 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4420000000000003e-06 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695908+0000", 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99251281199999997 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695917+0000", 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9579999999999996e-06 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.511 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.924501+0000", 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2285848580000001 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.924586+0000", 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4695000000000001e-05 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.924673+0000", 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6879999999999995e-05 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929451+0000", 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047781400000000002 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929490+0000", 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8472e-05 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.932192+0000", 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027021039999999999 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100434+0000", 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.168242063 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100534+0000", 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9809000000000006e-05 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.514 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100687+0000", 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015374599999999999 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173507 9.1 9:8f5fdfea:::benchmark_data_smithi017_237924_object80503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1266)", 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.702211+0000", 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 526.712871382, 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3987571480000001, 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.515 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173507 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702211+0000", 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702211+0000", 2024-09-21T10:21:11.516 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702212+0000", 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5659999999999999e-06 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702217+0000", 2024-09-21T10:21:11.517 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9010000000000002e-06 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702230+0000", 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2894000000000001e-05 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.702239+0000", 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9600000000000006e-06 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.518 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703414+0000", 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001175075 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.703420+0000", 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3559999999999998e-06 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695939+0000", 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99251901499999995 2024-09-21T10:21:11.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.695947+0000", 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2259999999999999e-06 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929558+0000", 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2336112850000001 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.520 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929714+0000", 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015599900000000001 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6,7", 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.929843+0000", 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012873499999999999 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.935021+0000", 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051778010000000001 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.935122+0000", 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000101073 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.097445+0000", 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16232375099999999 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100785+0000", 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033395149999999999 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100844+0000", 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9632000000000001e-05 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:27.100968+0000", 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012336399999999999 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45851.0:173514 9.7 9:ecfd9956:::benchmark_data_smithi017_237924_object80510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:12:24.723765+0000", 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 526.69131757100001, 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1725742700000001, 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45851", 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 173514 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.524 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723765+0000", 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723765+0000", 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723766+0000", 2024-09-21T10:21:11.525 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6390000000000001e-06 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723773+0000", 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3810000000000004e-06 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723790+0000", 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7751999999999999e-05 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.526 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723800+0000", 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2159999999999995e-06 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723863+0000", 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3297000000000001e-05 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:24.723873+0000", 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8500000000000006e-06 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.687969+0000", 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.96409637599999998 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:25.687981+0000", 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1549000000000001e-05 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.884912+0000", 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.196931465 2024-09-21T10:21:11.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.885045+0000", 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013280099999999999 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5,7", 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.885179+0000", 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013392699999999999 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.890945+0000", 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057662329999999999 2024-09-21T10:21:11.529 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.891473+0000", 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052811299999999998 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.892458+0000", 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0009847320000000001 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.895831+0000", 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033727309999999999 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.895893+0000", 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2228000000000001e-05 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:12:26.896339+0000", 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00044598000000000001 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.531 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:146862 10.15 10:abd97525:::benchmark_data_smithi017_268344_object20931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1317)", 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:15:30.837523+0000", 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 340.577558909, 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9509500950000001, 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 146862 2024-09-21T10:21:11.532 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.837523+0000", 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.837523+0000", 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.837524+0000", 2024-09-21T10:21:11.533 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8500000000000001e-07 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.837527+0000", 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8279999999999999e-06 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.837533+0000", 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6930000000000001e-06 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.534 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.837539+0000", 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2159999999999997e-06 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.837612+0000", 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3779e-05 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.837625+0000", 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3213999999999999e-05 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:31.625715+0000", 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.78808974099999995 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:31.625732+0000", 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.681e-05 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.767799+0000", 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1420668540000001 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.767933+0000", 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013378700000000001 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.768303+0000", 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000370582 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.773737+0000", 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005433645 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.776633+0000", 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028961 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.786261+0000", 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0096278560000000006 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.786339+0000", 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8073999999999998e-05 2024-09-21T10:21:11.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.788473+0000", 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021343310000000002 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:146863 10.15 10:a9dc7bb6:::benchmark_data_smithi017_268344_object20932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1317)", 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:15:30.839827+0000", 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 340.57525549399998, 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9558895920000001, 2024-09-21T10:21:11.539 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 146863 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.839827+0000", 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.540 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.839827+0000", 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.839828+0000", 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.39e-06 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.839834+0000", 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5999999999999997e-06 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.541 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.839848+0000", 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4096e-05 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.839857+0000", 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5019999999999998e-06 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.839929+0000", 2024-09-21T10:21:11.542 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1601999999999996e-05 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.839941+0000", 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2289999999999999e-05 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:31.625760+0000", 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.78581910899999996 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.543 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:31.625768+0000", 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8790000000000003e-06 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.771254+0000", 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1454856840000001 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.771363+0000", 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109503 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.544 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.771670+0000", 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030649200000000001 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.776779+0000", 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051096789999999998 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.776796+0000", 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6557000000000001e-05 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.545 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.793641+0000", 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016844726000000001 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.793706+0000", 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4963999999999997e-05 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.546 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.795716+0000", 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020105190000000001 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46058.0:146876 10.15 10:af375f79:::benchmark_data_smithi017_268344_object20945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1317)", 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:15:30.855462+0000", 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 340.55962043800002, 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9509078419999999, 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46058", 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 146876 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.855462+0000", 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.855462+0000", 2024-09-21T10:21:11.548 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.855463+0000", 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3540000000000001e-06 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.855468+0000", 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3700000000000003e-06 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.549 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.855479+0000", 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0776e-05 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.855486+0000", 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.793e-06 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.855545+0000", 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8709000000000003e-05 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.550 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:30.855561+0000", 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6365999999999999e-05 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:31.625791+0000", 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.77023013900000004 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:31.625798+0000", 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5649999999999998e-06 2024-09-21T10:21:11.551 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.773787+0000", 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1479894509999999 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.773952+0000", 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016458 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.774326+0000", 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037445099999999998 2024-09-21T10:21:11.552 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.780891+0000", 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065648410000000001 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.780957+0000", 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5994000000000005e-05 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.553 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.804200+0000", 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023242842 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.804294+0000", 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4598000000000005e-05 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:15:32.806369+0000", 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020750130000000001 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.554 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186423 11.10 11:0ae19b5d:::benchmark_data_smithi017_304926_object56216:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741571+0000", 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673510870999998, 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.08998332, 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.555 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186423 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741571+0000", 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741571+0000", 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.556 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741572+0000", 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1699999999999999e-07 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741574+0000", 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3029999999999998e-06 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741581+0000", 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3150000000000003e-06 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.557 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741587+0000", 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5500000000000002e-06 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762176+0000", 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020589471000000002 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762184+0000", 2024-09-21T10:21:11.558 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2609999999999996e-06 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.828713+0000", 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.066529448 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.828812+0000", 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9426000000000007e-05 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.829016+0000", 2024-09-21T10:21:11.559 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000203978 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.830388+0000", 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013720990000000001 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.831160+0000", 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00077147299999999995 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.831247+0000", 2024-09-21T10:21:11.560 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7342999999999994e-05 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.831309+0000", 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1545999999999997e-05 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.831379+0000", 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0028000000000001e-05 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.561 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.831440+0000", 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1431000000000002e-05 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.831555+0000", 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114231 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186459 11.10 11:0ab9d195:::benchmark_data_smithi017_304926_object56252:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741593+0000", 2024-09-21T10:21:11.562 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673489003, 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0951399070000001, 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186459 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.563 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741593+0000", 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741593+0000", 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741593+0000", 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1300000000000001e-07 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741595+0000", 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9240000000000001e-06 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741602+0000", 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1820000000000003e-06 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741614+0000", 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2707e-05 2024-09-21T10:21:11.565 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762208+0000", 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020593390999999999 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762211+0000", 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7780000000000001e-06 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.831614+0000", 2024-09-21T10:21:11.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0694022249999999 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.831735+0000", 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121382 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.831928+0000", 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000193007 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.567 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836282+0000", 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043541789999999997 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836326+0000", 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3445999999999997e-05 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836427+0000", 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010142899999999999 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.568 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836477+0000", 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9691000000000002e-05 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836521+0000", 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4524000000000003e-05 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836599+0000", 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7953000000000004e-05 2024-09-21T10:21:11.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836733+0000", 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133676 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186582 11.10 11:0a35e089:::benchmark_data_smithi017_304926_object56375:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741622+0000", 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673460370000001, 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.095556693, 2024-09-21T10:21:11.570 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186582 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741622+0000", 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741622+0000", 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741622+0000", 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0400000000000002e-07 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741625+0000", 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4949999999999998e-06 2024-09-21T10:21:11.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741631+0000", 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6059999999999999e-06 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741637+0000", 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2780000000000001e-06 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762242+0000", 2024-09-21T10:21:11.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020605836999999998 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762249+0000", 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0439999999999998e-06 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.833299+0000", 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.071049841 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.833430+0000", 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013074399999999999 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.833672+0000", 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000241962 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836343+0000", 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026711780000000002 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836811+0000", 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000467777 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836870+0000", 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9089999999999998e-05 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836928+0000", 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.821e-05 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836982+0000", 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4129000000000003e-05 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837043+0000", 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0375999999999999e-05 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837178+0000", 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013572200000000001 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.577 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186585 11.10 11:0fc14e99:::benchmark_data_smithi017_304926_object56378:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741644+0000", 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673437690999997, 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0960476809999999, 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186585 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.578 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741644+0000", 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741644+0000", 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741645+0000", 2024-09-21T10:21:11.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9499999999999998e-07 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741647+0000", 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.075e-06 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741654+0000", 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0759999999999997e-06 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741659+0000", 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4240000000000001e-06 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762272+0000", 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020612943000000002 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762276+0000", 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6389999999999998e-06 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.834925+0000", 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0726494019999999 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.835040+0000", 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011451399999999999 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.835246+0000", 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020659300000000001 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.836888+0000", 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001641104 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837245+0000", 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035701500000000002 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837334+0000", 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9822999999999997e-05 2024-09-21T10:21:11.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837397+0000", 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2867999999999995e-05 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837453+0000", 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5547000000000002e-05 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837514+0000", 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0841999999999998e-05 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837692+0000", 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000178421 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186596 11.10 11:09c734b0:::benchmark_data_smithi017_304926_object56389:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741666+0000", 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673415712999997, 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.10118834, 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.585 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186596 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741666+0000", 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741666+0000", 2024-09-21T10:21:11.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741667+0000", 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7899999999999999e-07 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741669+0000", 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1689999999999999e-06 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741675+0000", 2024-09-21T10:21:11.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0800000000000002e-06 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741680+0000", 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8189999999999999e-06 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762304+0000", 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020624267000000002 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762311+0000", 2024-09-21T10:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.063e-06 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837734+0000", 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0754230929999999 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.837878+0000", 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014411900000000001 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.838133+0000", 2024-09-21T10:21:11.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025495500000000001 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842534+0000", 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.004400887 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842547+0000", 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3115000000000001e-05 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842590+0000", 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2339999999999998e-05 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842634+0000", 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4694999999999998e-05 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842672+0000", 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7703999999999997e-05 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842727+0000", 2024-09-21T10:21:11.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4580999999999998e-05 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842855+0000", 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012807399999999999 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186613 11.10 11:0c0357ce:::benchmark_data_smithi017_304926_object56406:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741686+0000", 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673396373999999, 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1067936220000001, 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.592 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186613 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741686+0000", 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741686+0000", 2024-09-21T10:21:11.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741686+0000", 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5600000000000001e-07 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741688+0000", 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9269999999999999e-06 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741694+0000", 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7559999999999996e-06 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741698+0000", 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6380000000000003e-06 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762334+0000", 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020635206999999999 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762337+0000", 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.489e-06 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.839427+0000", 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.077089567 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.839558+0000", 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131383 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.839795+0000", 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000236908 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842555+0000", 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002760499 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842916+0000", 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036021700000000002 2024-09-21T10:21:11.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846335+0000", 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034191339999999999 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846377+0000", 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1887000000000001e-05 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848279+0000", 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019023600000000001 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848342+0000", 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3402e-05 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848479+0000", 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000136892 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.599 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186623 11.10 11:0c8358e9:::benchmark_data_smithi017_304926_object56416:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741704+0000", 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673377975999998, 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1069793999999999, 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186623 2024-09-21T10:21:11.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741704+0000", 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741704+0000", 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741704+0000", 2024-09-21T10:21:11.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.34e-07 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741706+0000", 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.778e-06 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741712+0000", 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5199999999999997e-06 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741718+0000", 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2110000000000003e-06 2024-09-21T10:21:11.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762366+0000", 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020648006999999999 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762374+0000", 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6680000000000001e-06 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.840975+0000", 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0786009240000001 2024-09-21T10:21:11.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.841108+0000", 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013379200000000001 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.841338+0000", 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022964 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846416+0000", 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050782229999999998 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846454+0000", 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8074000000000002e-05 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846492+0000", 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8207999999999997e-05 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848224+0000", 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017314629999999999 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848528+0000", 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030429099999999999 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848583+0000", 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4676999999999998e-05 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848684+0000", 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010059 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.606 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:11.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:11.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:11 smithi017 ceph-mon[102913]: pgmap v2306: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 127 active+clean; 3.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1984/153465 objects misplaced (1.293%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:21:11.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:11.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:11.607 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:11.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:11 smithi017 ceph-mon[105151]: pgmap v2306: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 127 active+clean; 3.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1984/153465 objects misplaced (1.293%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:21:11.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:11.608 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186663 11.10 11:0845ff8e:::benchmark_data_smithi017_304926_object56456:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.608 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741723+0000", 2024-09-21T10:21:11.608 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673358855000004, 2024-09-21T10:21:11.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.116520365, 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186663 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741723+0000", 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741723+0000", 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741724+0000", 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0899999999999997e-07 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741725+0000", 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.621e-06 2024-09-21T10:21:11.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741730+0000", 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3120000000000001e-06 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741735+0000", 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7500000000000003e-06 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762397+0000", 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020662125999999999 2024-09-21T10:21:11.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762401+0000", 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5810000000000001e-06 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.842977+0000", 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0805758110000001 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.843123+0000", 2024-09-21T10:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000145785 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.843396+0000", 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000273835 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846536+0000", 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003140065 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846573+0000", 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6702999999999997e-05 2024-09-21T10:21:11.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848211+0000", 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016377220000000001 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848232+0000", 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1245000000000001e-05 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.857984+0000", 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0097522440000000002 2024-09-21T10:21:11.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858105+0000", 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012038 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858244+0000", 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013887599999999999 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.615 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186665 11.10 11:0ece7d16:::benchmark_data_smithi017_304926_object56458:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741742+0000", 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673339841999997, 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1169009590000001, 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186665 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741742+0000", 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741742+0000", 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741743+0000", 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.15e-07 2024-09-21T10:21:11.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741749+0000", 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9410000000000004e-06 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741754+0000", 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.84e-06 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741759+0000", 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.831e-06 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762433+0000", 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020673546000000001 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762440+0000", 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3690000000000001e-06 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.844756+0000", 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.082315758 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.844881+0000", 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012493500000000001 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.845112+0000", 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023088199999999999 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846623+0000", 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015118130000000001 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848239+0000", 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001615326 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848738+0000", 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00049877600000000004 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858293+0000", 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095552399999999996 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858334+0000", 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0763000000000003e-05 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858506+0000", 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000172478 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858643+0000", 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013714599999999999 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186681 11.10 11:0d194d6d:::benchmark_data_smithi017_304926_object56474:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741765+0000", 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673317216999997, 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1171651549999999, 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.623 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186681 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741765+0000", 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741765+0000", 2024-09-21T10:21:11.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741765+0000", 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6100000000000002e-07 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741767+0000", 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.734e-06 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741772+0000", 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4480000000000002e-06 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741777+0000", 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5090000000000004e-06 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762464+0000", 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020686942 2024-09-21T10:21:11.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762467+0000", 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5619999999999999e-06 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846653+0000", 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.084185459 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846775+0000", 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012176300000000001 2024-09-21T10:21:11.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.846998+0000", 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022334699999999999 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848753+0000", 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017551229999999999 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848788+0000", 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4984000000000003e-05 2024-09-21T10:21:11.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.848832+0000", 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4186e-05 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858710+0000", 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0098774299999999995 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858762+0000", 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2539e-05 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858818+0000", 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5380999999999997e-05 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.858930+0000", 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011238700000000001 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186724 11.10 11:08b49435:::benchmark_data_smithi017_304926_object56517:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.630 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741782+0000", 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673299964999998, 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.121099179, 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186724 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741782+0000", 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741782+0000", 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741782+0000", 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8700000000000002e-07 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741784+0000", 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.725e-06 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741790+0000", 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3700000000000003e-06 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741794+0000", 2024-09-21T10:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4510000000000002e-06 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762496+0000", 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020702254 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762503+0000", 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7710000000000001e-06 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.856246+0000", 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0937432650000001 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.856357+0000", 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000110438 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.856659+0000", 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000302298 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.859004+0000", 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002345195 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.859051+0000", 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6585999999999997e-05 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.859105+0000", 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3881999999999997e-05 2024-09-21T10:21:11.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.859163+0000", 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8031000000000001e-05 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.862679+0000", 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035159340000000001 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.862741+0000", 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1953999999999997e-05 2024-09-21T10:21:11.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.862881+0000", 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014073800000000001 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186731 11.10 11:0cedf5c4:::benchmark_data_smithi017_304926_object56524:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741799+0000", 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673282853000003, 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1241635400000001, 2024-09-21T10:21:11.638 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186731 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741799+0000", 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741799+0000", 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741800+0000", 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9499999999999998e-07 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741801+0000", 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7120000000000001e-06 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741809+0000", 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8669999999999994e-06 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741814+0000", 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.955e-06 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762526+0000", 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020711731000000001 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762529+0000", 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.569e-06 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.859295+0000", 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.096765599 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.859409+0000", 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114509 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.859705+0000", 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029545299999999999 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.864514+0000", 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0048090019999999997 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.864571+0000", 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7459e-05 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.864628+0000", 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6471999999999997e-05 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.865724+0000", 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001096098 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.865763+0000", 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8664000000000001e-05 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.865833+0000", 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0193999999999999e-05 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.865963+0000", 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129961 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.36507.0:186786 11.10 11:0c87887a:::benchmark_data_smithi017_304926_object56579:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1380)", 2024-09-21T10:21:11.645 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2024-09-21T10:20:37.741820+0000", 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 33.673262043000001, 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.124569285, 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "continuous": false, 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.36507", 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 186786 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2024-09-21T10:21:11.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741820+0000", 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741820+0000", 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741820+0000", 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9799999999999999e-07 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2024-09-21T10:21:11.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741822+0000", 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.697e-06 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741828+0000", 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5720000000000002e-06 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.741832+0000", 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4880000000000001e-06 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762551+0000", 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020718625000000001 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:37.762557+0000", 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7139999999999996e-06 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.861056+0000", 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0984990290000001 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.861172+0000", 2024-09-21T10:21:11.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011547700000000001 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2,6,7", 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.861424+0000", 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025162999999999997 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.864673+0000", 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003249114 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.865707+0000", 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001034182 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.865730+0000", 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3365000000000001e-05 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.866115+0000", 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038455500000000002 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.866160+0000", 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4793000000000003e-05 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.866262+0000", 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010219 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2024-09-21T10:20:38.866389+0000", 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000127556 2024-09-21T10:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.653 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.653 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.653 INFO:teuthology.orchestra.run.smithi017.stdout: } 2024-09-21T10:21:11.653 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2024-09-21T10:21:11.653 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:21:11.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:11.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:11.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:11 smithi134 ceph-mon[90085]: pgmap v2306: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 127 active+clean; 3.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1984/153465 objects misplaced (1.293%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:21:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:11.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:11.827 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:11.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:11.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:11.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:11 smithi017 podman[321938]: 2024-09-21 10:21:11.691932835 +0000 UTC m=+5.585551223 container init d5f81ff1b44049ee3d44701860af266987c3b1046776f34aa11549b17cfe1fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T10:21:11.951 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:11 smithi017 podman[321938]: 2024-09-21 10:21:11.702674233 +0000 UTC m=+5.596292608 container start d5f81ff1b44049ee3d44701860af266987c3b1046776f34aa11549b17cfe1fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T10:21:11.964 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:11.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:11.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:21:12.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.102 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:12.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:12.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.245 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:12.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:12.245 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:11 smithi017 bash[321938]: d5f81ff1b44049ee3d44701860af266987c3b1046776f34aa11549b17cfe1fa7 2024-09-21T10:21:12.246 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:11 smithi017 systemd[1]: Started Ceph osd.1 for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:21:12.246 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:12 smithi017 ceph-osd[322513]: -- 172.21.15.17:0/1598572717 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b2771c7a40 con 0x55b277f96000 2024-09-21T10:21:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:21:12.302 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:21:12.345 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:12.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.369 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:12.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:12.470 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:12.506 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:12.608 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:12.642 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:12.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:12.744 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:12.783 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:12.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:12.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:12.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:12.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:12 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-21T10:21:12.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:12 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[322267]: 2024-09-21T10:21:12.883+0000 7f28c0a78640 -1 Falling back to public interface 2024-09-21T10:21:12.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:12 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-21T10:21:12.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:12 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-21T10:21:13.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:13.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:13.141 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:13.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.165 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:13.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:13.266 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:13.290 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:13.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:13.391 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:13.415 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:13.516 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:13.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.538 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:13.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:13.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:13.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:13.757 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:21:13.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:13.789 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:13.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:13.820 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:13.890 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:13.911 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:13.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:13.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:13 smithi017 ceph-mon[105151]: pgmap v2307: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 127 active+clean; 2.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 37 op/s; 1254/144873 objects misplaced (0.866%); 22 MiB/s, 349 objects/s recovering 2024-09-21T10:21:13.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-21T10:21:13.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:13 smithi017 ceph-mon[105151]: osdmap e1405: 8 total, 7 up, 7 in 2024-09-21T10:21:13.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:13 smithi017 ceph-mon[102913]: pgmap v2307: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 127 active+clean; 2.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 37 op/s; 1254/144873 objects misplaced (0.866%); 22 MiB/s, 349 objects/s recovering 2024-09-21T10:21:13.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-21T10:21:13.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:13 smithi017 ceph-mon[102913]: osdmap e1405: 8 total, 7 up, 7 in 2024-09-21T10:21:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:13 smithi134 ceph-mon[90085]: pgmap v2307: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 127 active+clean; 2.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 37 op/s; 1254/144873 objects misplaced (0.866%); 22 MiB/s, 349 objects/s recovering 2024-09-21T10:21:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-21T10:21:13.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:13 smithi134 ceph-mon[90085]: osdmap e1405: 8 total, 7 up, 7 in 2024-09-21T10:21:14.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:14.017 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:14.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:14.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:14.099 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:14.138 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:14.160 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:14.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:14.261 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:14.282 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:14.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:21:14.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:14.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:14.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:14.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:14.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:14.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:14.634 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:14.657 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:14.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:14.783 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:14.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:14.884 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:14.916 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[102913]: osdmap e1406: 8 total, 7 up, 7 in 2024-09-21T10:21:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[102913]: 1.b scrub starts 2024-09-21T10:21:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[102913]: 1.b scrub ok 2024-09-21T10:21:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:21:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:21:14.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[102913]: osdmap e1407: 8 total, 7 up, 7 in 2024-09-21T10:21:14.952 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:21:14] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-21T10:21:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[105151]: osdmap e1406: 8 total, 7 up, 7 in 2024-09-21T10:21:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[105151]: 1.b scrub starts 2024-09-21T10:21:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[105151]: 1.b scrub ok 2024-09-21T10:21:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:21:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:21:14.952 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:14 smithi017 ceph-mon[105151]: osdmap e1407: 8 total, 7 up, 7 in 2024-09-21T10:21:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:14 smithi134 ceph-mon[90085]: osdmap e1406: 8 total, 7 up, 7 in 2024-09-21T10:21:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:14 smithi134 ceph-mon[90085]: 1.b scrub starts 2024-09-21T10:21:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:14 smithi134 ceph-mon[90085]: 1.b scrub ok 2024-09-21T10:21:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:14 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:21:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:14 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:21:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:14 smithi134 ceph-mon[90085]: osdmap e1407: 8 total, 7 up, 7 in 2024-09-21T10:21:15.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:15.052 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:15.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:15.153 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:15.181 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:15.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.310 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:15.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:15.411 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:15.435 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:15.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:15.561 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:15.663 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:15.684 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:15.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:15.786 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:15.808 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:15.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:15.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:15.937 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:15.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:15 smithi017 ceph-mon[105151]: pgmap v2310: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 127 active+clean; 2.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 1254/140946 objects misplaced (0.890%); 22 MiB/s, 350 objects/s recovering 2024-09-21T10:21:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:15 smithi017 ceph-mon[105151]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 1404'12348 2024-09-21T10:21:15.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:15 smithi017 ceph-mon[105151]: osdmap e1408: 8 total, 7 up, 7 in 2024-09-21T10:21:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:15 smithi017 ceph-mon[102913]: pgmap v2310: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 127 active+clean; 2.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 1254/140946 objects misplaced (0.890%); 22 MiB/s, 350 objects/s recovering 2024-09-21T10:21:15.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:15 smithi017 ceph-mon[102913]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 1404'12348 2024-09-21T10:21:15.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:15 smithi017 ceph-mon[102913]: osdmap e1408: 8 total, 7 up, 7 in 2024-09-21T10:21:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:15 smithi134 ceph-mon[90085]: pgmap v2310: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 127 active+clean; 2.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 1254/140946 objects misplaced (0.890%); 22 MiB/s, 350 objects/s recovering 2024-09-21T10:21:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:15 smithi134 ceph-mon[90085]: 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 1404'12348 2024-09-21T10:21:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:15 smithi134 ceph-mon[90085]: osdmap e1408: 8 total, 7 up, 7 in 2024-09-21T10:21:16.009 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:21:16.009 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:21:16.009 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 0 2024-09-21T10:21:16.009 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:21:16.039 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:16.064 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:16.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:16.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:16.197 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:16.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:16.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:16.323 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:16.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:16.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:16.442 INFO:teuthology.orchestra.run.smithi017.stderr:no valid command found; 10 closest matches: 2024-09-21T10:21:16.442 INFO:teuthology.orchestra.run.smithi017.stderr:0 2024-09-21T10:21:16.442 INFO:teuthology.orchestra.run.smithi017.stderr:1 2024-09-21T10:21:16.442 INFO:teuthology.orchestra.run.smithi017.stderr:2 2024-09-21T10:21:16.442 INFO:teuthology.orchestra.run.smithi017.stderr:abort 2024-09-21T10:21:16.442 INFO:teuthology.orchestra.run.smithi017.stderr:assert 2024-09-21T10:21:16.442 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:21:16.442 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs files list 2024-09-21T10:21:16.443 INFO:teuthology.orchestra.run.smithi017.stderr:bluefs stats 2024-09-21T10:21:16.443 INFO:teuthology.orchestra.run.smithi017.stderr:bluestore bluefs device info [] 2024-09-21T10:21:16.443 INFO:teuthology.orchestra.run.smithi017.stderr:config diff 2024-09-21T10:21:16.443 INFO:teuthology.orchestra.run.smithi017.stderr:admin_socket: invalid command 2024-09-21T10:21:16.453 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:16.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:16.554 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:16.577 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:16.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:16.652 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:16.678 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:16.700 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:16.738 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:21:16.803 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:16.846 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:16.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:16.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:16.990 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:16.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:17.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:17.135 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:17.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:17.167 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:17.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:17.274 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:17.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:17.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:17.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.416 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:17.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:21:17.434 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2024-09-21T10:21:17.517 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:17.561 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:17.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:17.663 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:17.695 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:17.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:17.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:17.837 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:17.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:17.938 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:17 smithi017 ceph-mon[102913]: pgmap v2313: 129 pgs: 1 active+clean+scrubbing, 128 active+clean; 2.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:21:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:17 smithi017 ceph-mon[102913]: osdmap e1409: 8 total, 7 up, 7 in 2024-09-21T10:21:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:17 smithi017 ceph-mon[102913]: 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 1405'12210 2024-09-21T10:21:17.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:17 smithi017 ceph-mon[102913]: 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 1405'12210 2024-09-21T10:21:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:17 smithi017 ceph-mon[105151]: pgmap v2313: 129 pgs: 1 active+clean+scrubbing, 128 active+clean; 2.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:21:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:17 smithi017 ceph-mon[105151]: osdmap e1409: 8 total, 7 up, 7 in 2024-09-21T10:21:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:17 smithi017 ceph-mon[105151]: 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 1405'12210 2024-09-21T10:21:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:17 smithi017 ceph-mon[105151]: 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 1405'12210 2024-09-21T10:21:17.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.968 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:17.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:17 smithi134 ceph-mon[90085]: pgmap v2313: 129 pgs: 1 active+clean+scrubbing, 128 active+clean; 2.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:21:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:17 smithi134 ceph-mon[90085]: osdmap e1409: 8 total, 7 up, 7 in 2024-09-21T10:21:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:17 smithi134 ceph-mon[90085]: 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 1405'12210 2024-09-21T10:21:17.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:17 smithi134 ceph-mon[90085]: 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 1405'12210 2024-09-21T10:21:18.069 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:18.092 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:18.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:18.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:18.220 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:18.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:18.322 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.343 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:18.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:18.444 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:18.465 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:18.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:18.593 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:18.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:18.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:18.723 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:18.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:18.824 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:18.846 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:18.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:18.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:18.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.983 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:18.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:18 smithi134 ceph-mon[90085]: osdmap e1410: 8 total, 7 up, 7 in 2024-09-21T10:21:19.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:19.122 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:19.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:19.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:18 smithi017 ceph-mon[102913]: osdmap e1410: 8 total, 7 up, 7 in 2024-09-21T10:21:19.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:18 smithi017 ceph-mon[105151]: osdmap e1410: 8 total, 7 up, 7 in 2024-09-21T10:21:19.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:19.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:19.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:19.349 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:19.382 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:19.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:19.483 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:19.519 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:19.621 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:19.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:19.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:19.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:19.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:19.892 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:19.926 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:19.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:19 smithi134 ceph-mon[90085]: pgmap v2316: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 127 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 1754/132330 objects misplaced (1.325%); 38 MiB/s, 613 objects/s recovering 2024-09-21T10:21:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:19 smithi134 ceph-mon[90085]: osdmap e1411: 8 total, 7 up, 7 in 2024-09-21T10:21:20.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:20.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:20.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:20.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:20.191 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:20.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:19 smithi017 ceph-mon[102913]: pgmap v2316: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 127 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 1754/132330 objects misplaced (1.325%); 38 MiB/s, 613 objects/s recovering 2024-09-21T10:21:20.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:19 smithi017 ceph-mon[102913]: osdmap e1411: 8 total, 7 up, 7 in 2024-09-21T10:21:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:19 smithi017 ceph-mon[105151]: pgmap v2316: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 127 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 1754/132330 objects misplaced (1.325%); 38 MiB/s, 613 objects/s recovering 2024-09-21T10:21:20.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:19 smithi017 ceph-mon[105151]: osdmap e1411: 8 total, 7 up, 7 in 2024-09-21T10:21:20.292 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:20.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.326 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:20.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:20.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:20.464 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:21:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:21:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:21:20.565 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:20.601 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:20.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:20.702 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:20.734 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:20.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:20.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:20.873 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:20.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:20 smithi017 ceph-mon[102913]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T10:21:20.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:20 smithi017 ceph-mon[105151]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T10:21:20.951 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:20 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[322267]: 2024-09-21T10:21:20.529+0000 7f28c0a78640 -1 osd.1 1368 log_to_monitors true 2024-09-21T10:21:20.975 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:20 smithi134 ceph-mon[90085]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T10:21:20.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.998 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:20.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:21.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:21.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:21.238 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:21.271 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:21.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:21.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:21.407 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:21.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:21.450 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[322267]: 2024-09-21T10:21:21.099+0000 7f28b481a640 -1 osd.1 1412 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:21:21.509 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:21.549 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:21.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:21.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:21.687 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:21.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:21.788 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:21.820 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:21.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:21.922 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:21.958 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:21 smithi134 ceph-mon[90085]: 1.6 scrub starts 2024-09-21T10:21:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:21 smithi134 ceph-mon[90085]: 1.6 scrub ok 2024-09-21T10:21:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:21 smithi134 ceph-mon[90085]: pgmap v2318: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 127 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 1754/128379 objects misplaced (1.366%); 31 MiB/s, 498 objects/s recovering 2024-09-21T10:21:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:21 smithi134 ceph-mon[90085]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:21:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:21 smithi134 ceph-mon[90085]: osdmap e1412: 8 total, 7 up, 7 in 2024-09-21T10:21:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:21 smithi134 ceph-mon[90085]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T10:21:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:21 smithi134 ceph-mon[90085]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' 2024-09-21T10:21:22.060 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:22.100 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:22.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[102913]: 1.6 scrub starts 2024-09-21T10:21:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[102913]: 1.6 scrub ok 2024-09-21T10:21:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[102913]: pgmap v2318: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 127 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 1754/128379 objects misplaced (1.366%); 31 MiB/s, 498 objects/s recovering 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[102913]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[102913]: osdmap e1412: 8 total, 7 up, 7 in 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[102913]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[102913]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[105151]: 1.6 scrub starts 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[105151]: 1.6 scrub ok 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[105151]: pgmap v2318: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 127 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 1754/128379 objects misplaced (1.366%); 31 MiB/s, 498 objects/s recovering 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[105151]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[105151]: osdmap e1412: 8 total, 7 up, 7 in 2024-09-21T10:21:22.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[105151]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]} : dispatch 2024-09-21T10:21:22.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:22.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:21 smithi017 ceph-mon[105151]: from='osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258]' entity='osd.1' 2024-09-21T10:21:22.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:22.238 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:22.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:22.340 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:22.376 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:22.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:22.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:21:22.478 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:22.515 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:22.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:22.617 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:22.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.659 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:22.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:22.760 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:22.798 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:22.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:22.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:22.933 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:22.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:23.034 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:23.073 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:23.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:23.124 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:21:23.125 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:21:23.125 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:21:23.125 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:21:23.125 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:21:23.175 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[102913]: 11.15 scrub starts 2024-09-21T10:21:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[102913]: osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258] boot 2024-09-21T10:21:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[102913]: osdmap e1413: 8 total, 8 up, 7 in 2024-09-21T10:21:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T10:21:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[102913]: 11.15 scrub ok 2024-09-21T10:21:23.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[102913]: pgmap v2321: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 2.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 1754/120741 objects misplaced (1.453%); 6.0 MiB/s, 95 objects/s recovering 2024-09-21T10:21:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[105151]: 11.15 scrub starts 2024-09-21T10:21:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[105151]: osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258] boot 2024-09-21T10:21:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[105151]: osdmap e1413: 8 total, 8 up, 7 in 2024-09-21T10:21:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T10:21:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[105151]: 11.15 scrub ok 2024-09-21T10:21:23.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:23 smithi017 ceph-mon[105151]: pgmap v2321: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 2.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 1754/120741 objects misplaced (1.453%); 6.0 MiB/s, 95 objects/s recovering 2024-09-21T10:21:23.210 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:23.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:23 smithi134 ceph-mon[90085]: 11.15 scrub starts 2024-09-21T10:21:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:23 smithi134 ceph-mon[90085]: osd.1 [v2:172.21.15.17:6810/2519043258,v1:172.21.15.17:6811/2519043258] boot 2024-09-21T10:21:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:23 smithi134 ceph-mon[90085]: osdmap e1413: 8 total, 8 up, 7 in 2024-09-21T10:21:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:23 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T10:21:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:23 smithi134 ceph-mon[90085]: 11.15 scrub ok 2024-09-21T10:21:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:23 smithi134 ceph-mon[90085]: pgmap v2321: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 2.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 1754/120741 objects misplaced (1.453%); 6.0 MiB/s, 95 objects/s recovering 2024-09-21T10:21:23.312 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:23.354 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:23.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:23.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:23.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:23.596 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:23.628 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:23.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:23.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:23.763 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:23.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:23.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:23.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:23.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:23.954 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:21:23.954 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:23.989 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:21:23.990 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:24.008 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:24.027 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:21:24.027 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:24.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.047 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:24.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:24.058 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:21:24.058 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:24.090 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:21:24.090 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:24.130 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:21:24.130 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:24.148 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:24.165 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2517367063' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:24.165 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:21:24.165 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:24.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2517367063' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:24.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:21:24.195 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:24.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:24.201 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:24.207 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:21:24.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:24.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:24.345 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:24.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:24.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:24.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2517367063' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:24.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.495 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:24.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:24.570 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:24.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:24.639 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:24.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:24.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:24.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:24.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:24.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:24.923 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:24.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:24.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:21:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:21:24] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-21T10:21:25.025 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:25.063 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:25.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:25.119 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:21:25.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:25.200 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:25.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:25.268 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:25 smithi017 ceph-mon[102913]: pgmap v2322: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 2.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 332/116007 objects misplaced (0.286%) 2024-09-21T10:21:25.268 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:25 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1663677729' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:25.268 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:25 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:25.268 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:25 smithi017 ceph-mon[105151]: pgmap v2322: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 2.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 332/116007 objects misplaced (0.286%) 2024-09-21T10:21:25.268 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:25 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1663677729' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:25.268 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:25 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:25.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:25.337 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:25.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:25.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:25.471 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:25.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:25 smithi134 ceph-mon[90085]: pgmap v2322: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 2.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 332/116007 objects misplaced (0.286%) 2024-09-21T10:21:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:25 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1663677729' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:25 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:25.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:25.596 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:25.698 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:25.719 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:25.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:25.821 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:25.841 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:25.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:25.943 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:25.964 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:25.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:26.065 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:26.086 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:26.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:26.221 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:26.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:26 smithi134 ceph-mon[90085]: 1.c scrub starts 2024-09-21T10:21:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:26 smithi134 ceph-mon[90085]: 1.c scrub ok 2024-09-21T10:21:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:26 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:26 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:21:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:26 smithi134 ceph-mon[90085]: osdmap e1414: 8 total, 8 up, 7 in 2024-09-21T10:21:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:26.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:26 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:26.323 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:26.361 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[102913]: 1.c scrub starts 2024-09-21T10:21:26.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[102913]: 1.c scrub ok 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[102913]: osdmap e1414: 8 total, 8 up, 7 in 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[105151]: 1.c scrub starts 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[105151]: 1.c scrub ok 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[105151]: osdmap e1414: 8 total, 8 up, 7 in 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:26.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:26 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:26.463 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:26.498 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:26.576 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:21:26.576 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:21:26.599 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:26.636 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:26.737 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:26.774 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:26.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:26.876 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:26.898 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:26.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:27.000 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:27.032 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:27.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:27.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:27.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:27.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:27.272 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:27.313 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:27.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:27.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:27 smithi017 ceph-mon[102913]: osdmap e1415: 8 total, 8 up, 7 in 2024-09-21T10:21:27.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:27 smithi017 ceph-mon[102913]: pgmap v2325: 129 pgs: 1 peering, 128 active+clean; 2.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 58 op/s 2024-09-21T10:21:27.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:27 smithi017 ceph-mon[105151]: osdmap e1415: 8 total, 8 up, 7 in 2024-09-21T10:21:27.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:27 smithi017 ceph-mon[105151]: pgmap v2325: 129 pgs: 1 peering, 128 active+clean; 2.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 58 op/s 2024-09-21T10:21:27.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:27.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:27.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:27 smithi134 ceph-mon[90085]: osdmap e1415: 8 total, 8 up, 7 in 2024-09-21T10:21:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:27 smithi134 ceph-mon[90085]: pgmap v2325: 129 pgs: 1 peering, 128 active+clean; 2.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 58 op/s 2024-09-21T10:21:27.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:27.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.594 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:27.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:27.667 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:21:27.667 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:21:27.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:27.736 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:27.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:27.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:27.870 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:27.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:27.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:21:27.972 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:28.020 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:28.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:28.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:21:28.122 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:28.171 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:28.272 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:28.303 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:28.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.322 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:28.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:28.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:28.470 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:28.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:28.494 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:28.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:28.620 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:28.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:28.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:28.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.757 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:28.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:28.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:28.897 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:28.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:29.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:29.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:29.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:21:29.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:29.198 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:29.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:29.336 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:29.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:29.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.482 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:29.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:29.583 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:29.611 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:29.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:29.713 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:29.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.746 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:29.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:29.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:29.884 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:29.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:29 smithi017 ceph-mon[102913]: pgmap v2326: 129 pgs: 1 peering, 128 active+clean; 2.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 202 op/s 2024-09-21T10:21:29.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:29 smithi017 ceph-mon[105151]: pgmap v2326: 129 pgs: 1 peering, 128 active+clean; 2.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 202 op/s 2024-09-21T10:21:29.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:29 smithi134 ceph-mon[90085]: pgmap v2326: 129 pgs: 1 peering, 128 active+clean; 2.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 202 op/s 2024-09-21T10:21:30.041 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:30.087 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:30.142 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:30.179 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:30.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:30.281 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:30.320 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:30.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:30.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:30.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:30.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:21:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:21:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:21:30.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:30.592 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:30.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:30.694 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:30.734 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:30.836 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:30.865 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:30.962 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:21:30.962 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:30.966 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:30.970 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:21:30.970 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:31.000 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:21:31.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:31.004 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:31.025 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:21:31.025 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:31.079 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:21:31.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:31.106 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:31.109 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:21:31.109 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:31.141 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:31.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:31.143 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:21:31.143 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:31.178 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:21:31.178 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:31.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:31.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:31.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:31.381 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:31.401 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:31.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:31.503 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:31.537 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:31.639 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:31.671 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:31.773 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:31.810 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:31.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:31.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:31.945 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:31 smithi017 ceph-mon[102913]: pgmap v2327: 129 pgs: 1 peering, 128 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 302 op/s 2024-09-21T10:21:31.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:31 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/25448361' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:31 smithi017 ceph-mon[105151]: pgmap v2327: 129 pgs: 1 peering, 128 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 302 op/s 2024-09-21T10:21:31.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:31 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/25448361' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:31.973 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:31 smithi134 ceph-mon[90085]: pgmap v2327: 129 pgs: 1 peering, 128 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 302 op/s 2024-09-21T10:21:31.973 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:31 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/25448361' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:32.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:32.088 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:32.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:32.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:32.330 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:32.365 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:32.467 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:32.515 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:32.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:32.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:32.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:32.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:32.679 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:21:32.679 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:21:32.679 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.679 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:235779 11.18 e1415/1410 11:1a3bf70a:::benchmark_data_smithi017_304926_object105572:head v 1415'13005, mlcod=1415'13003)", 2024-09-21T10:21:32.679 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:32.667960+0000", 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0084191060000000009, 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0084475609999999993, 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.667960+0000", 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:32.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.667960+0000", 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.667959+0000", 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.667966+0000", 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9609999999999999e-06 2024-09-21T10:21:32.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.667976+0000", 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5189999999999998e-06 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.667981+0000", 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0429999999999998e-06 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.670592+0000", 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0026114290000000002 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.670611+0000", 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8672999999999999e-05 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:235783 11.a e1415/1367 11:585bfe2e:::benchmark_data_smithi017_304926_object105576:head v 1415'14756, mlcod=1415'14754)", 2024-09-21T10:21:32.683 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:32.670809+0000", 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0055705199999999998, 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0056653470000000003, 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.670809+0000", 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.684 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.670809+0000", 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.670808+0000", 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.670818+0000", 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0369e-05 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.670829+0000", 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1531e-05 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.670835+0000", 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6720000000000001e-06 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.673632+0000", 2024-09-21T10:21:32.686 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002797234 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.673652+0000", 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9378e-05 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.676397+0000", 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.002745543 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.687 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_repop(client.36507.0:235776 11.17 e1415/1406 11:ef02fd38:::benchmark_data_smithi017_304926_object105569:head v 1415'12979, mlcod=1415'12977)", 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:32.671898+0000", 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0044813129999999998, 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0046397189999999996, 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "started", 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.671898+0000", 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.688 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.671898+0000", 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.671897+0000", 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.671905+0000", 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.6909999999999997e-06 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.689 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.671917+0000", 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1739999999999999e-05 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.671923+0000", 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0109999999999997e-06 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.671999+0000", 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5964000000000007e-05 2024-09-21T10:21:32.690 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.672018+0000", 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9899999999999999e-05 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:235785 11.3 11:c4a0bbdb:::benchmark_data_smithi017_304926_object105578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1415)", 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:32.676283+0000", 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 9.5781999999999999e-05, 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00031600699999999998, 2024-09-21T10:21:32.691 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 235785 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.676283+0000", 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.692 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.676283+0000", 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.676281+0000", 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.676289+0000", 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3429999999999998e-06 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.693 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.676307+0000", 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8680000000000001e-05 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.676314+0000", 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7630000000000003e-06 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:32.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.676365+0000", 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0488000000000002e-05 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:235777 11.3 11:c45b67bc:::benchmark_data_smithi017_304926_object105570:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1415)", 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:32.666603+0000", 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 0.0097762690000000006, 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010062699, 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:32.695 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 235777 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.666603+0000", 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.696 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.666603+0000", 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.666601+0000", 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999981 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.666611+0000", 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8470000000000008e-06 2024-09-21T10:21:32.697 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.666629+0000", 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8261999999999999e-05 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.666636+0000", 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.9609999999999999e-06 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.666687+0000", 2024-09-21T10:21:32.698 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.1134999999999997e-05 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.666801+0000", 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011395000000000001 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 5,7", 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.667241+0000", 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00043969700000000002 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.699 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.670966+0000", 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0037253960000000002 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:32.674321+0000", 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0033549280000000001 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:32.700 INFO:teuthology.orchestra.run.smithi134.stdout: ], 2024-09-21T10:21:32.701 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 5 2024-09-21T10:21:32.701 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:21:32.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:32.811 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:32.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:32.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:32.947 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:32.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:32.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:32 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:21:32.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:32 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:21:32.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:32 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:21:33.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:33.089 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:33.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:33.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:33.228 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:33.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:33.330 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:33.367 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:33.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:33.396 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 2, 3, 5, 6, 7, 4, 1] 2024-09-21T10:21:33.396 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:21:33.396 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:21:33.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 110 3.0 2024-09-21T10:21:33.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:33.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:33.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:33.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:33.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.691 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:33.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:33.792 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:33.801 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:21:33.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.833 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:33.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:33.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:33.965 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:33.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:33 smithi134 ceph-mon[90085]: pgmap v2328: 129 pgs: 129 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.41k op/s; 1.3 MiB/s, 21 objects/s recovering 2024-09-21T10:21:34.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:33 smithi017 ceph-mon[102913]: pgmap v2328: 129 pgs: 129 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.41k op/s; 1.3 MiB/s, 21 objects/s recovering 2024-09-21T10:21:34.057 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:33 smithi017 ceph-mon[105151]: pgmap v2328: 129 pgs: 129 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.41k op/s; 1.3 MiB/s, 21 objects/s recovering 2024-09-21T10:21:34.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.122 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:34.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:34.223 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:34.260 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:34.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:34.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:34.397 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:34.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:34.499 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:34.550 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:34.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:34.652 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:34.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:21:34.686 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:34.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:34.720 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:21:34.788 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:34.827 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:34.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:34.880 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:34.929 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:34.975 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:34.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:34.996 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2795306955' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:35.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:35.116 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:35.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:35.190 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2795306955' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:35.190 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:21:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:21:34] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.51.0" 2024-09-21T10:21:35.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2795306955' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:35.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:35.247 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:35.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:35.349 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:35.384 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:35.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:35.486 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:35.522 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:35.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:35.624 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:35.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:35.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:35.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:35.795 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:35.809 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:35.825 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:35.927 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:35 smithi017 ceph-mon[102913]: pgmap v2329: 129 pgs: 129 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.35k op/s; 1.1 MiB/s, 18 objects/s recovering 2024-09-21T10:21:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:35 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2795306955' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:35.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:35 smithi017 ceph-mon[102913]: osdmap e1416: 8 total, 8 up, 7 in 2024-09-21T10:21:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:35 smithi017 ceph-mon[105151]: pgmap v2329: 129 pgs: 129 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.35k op/s; 1.1 MiB/s, 18 objects/s recovering 2024-09-21T10:21:35.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:35 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2795306955' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:35.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:35 smithi017 ceph-mon[105151]: osdmap e1416: 8 total, 8 up, 7 in 2024-09-21T10:21:35.966 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:35.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:35 smithi134 ceph-mon[90085]: pgmap v2329: 129 pgs: 129 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.35k op/s; 1.1 MiB/s, 18 objects/s recovering 2024-09-21T10:21:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:35 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2795306955' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:35 smithi134 ceph-mon[90085]: osdmap e1416: 8 total, 8 up, 7 in 2024-09-21T10:21:36.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:36.111 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:36.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:36.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:36.244 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:36.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:36.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:36.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.385 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:36.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:36.486 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:36.547 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:36.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:36.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:36.689 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:36.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:36.814 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:36.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:36.915 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:36.952 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:36.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:37.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.087 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:37.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:37.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:37.223 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:37.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:37.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:37.359 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:37.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:37.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:37.493 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:37.558 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:21:37.558 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:21:37.558 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:21:37.558 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:21:37.559 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:21:37.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:37.625 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:37.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:21:37.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:37.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:37.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:37.892 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.940 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:37.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:37 smithi017 ceph-mon[102913]: pgmap v2331: 129 pgs: 129 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.30k op/s 2024-09-21T10:21:37.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:37 smithi017 ceph-mon[105151]: pgmap v2331: 129 pgs: 129 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.30k op/s 2024-09-21T10:21:37.977 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:37 smithi134 ceph-mon[90085]: pgmap v2331: 129 pgs: 129 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.30k op/s 2024-09-21T10:21:38.042 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.098 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:38.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:38.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:38.230 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:38.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:38.332 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:38.368 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:38.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:38.470 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:38.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.504 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:38.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:38.605 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:38.643 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:38.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:38.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.790 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:38.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:38.891 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:38.937 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:38.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:38.958 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:21:39.039 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:39.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.099 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:39.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:39.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:39.218 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:21:39.218 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:39.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.242 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:39.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:39.254 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:21:39.254 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:39.279 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:21:39.279 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:39.309 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:21:39.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:39.339 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:21:39.340 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:39.343 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:39.364 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:21:39.364 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:39.388 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:21:39.389 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:39.401 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:39.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:39.424 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:21:39.425 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:39.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:39.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:39.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:39.673 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:39.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:39.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:39.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.822 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:39.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:39.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:39.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:39 smithi134 ceph-mon[90085]: pgmap v2332: 129 pgs: 129 active+clean; 1.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s 2024-09-21T10:21:39.967 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:39 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2886644385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:39.967 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:39.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:40.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:40.099 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:40.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:39 smithi017 ceph-mon[102913]: pgmap v2332: 129 pgs: 129 active+clean; 1.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s 2024-09-21T10:21:40.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:39 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2886644385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:39 smithi017 ceph-mon[105151]: pgmap v2332: 129 pgs: 129 active+clean; 1.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.31k op/s 2024-09-21T10:21:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:39 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2886644385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:40.201 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:40.245 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:21:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:21:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:21:40.253 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:40.296 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:40.355 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:40.406 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:40.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:40.545 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:40.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:40.647 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.690 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:40.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:40.791 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:40.818 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:40.919 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:40.981 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:40.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:41.083 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:41.127 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:41.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:41.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:41.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:41.229 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:41.267 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:41.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.412 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:41.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:41.513 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:41.556 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:41.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:41.658 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:41.695 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:41.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:41.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:41.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:41.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:41.969 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:41.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:42.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:42.107 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:42.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:42.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:41 smithi017 ceph-mon[102913]: pgmap v2333: 129 pgs: 129 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.33k op/s 2024-09-21T10:21:42.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:42.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:41 smithi017 ceph-mon[105151]: pgmap v2333: 129 pgs: 129 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.33k op/s 2024-09-21T10:21:42.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:42.209 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:41 smithi134 ceph-mon[90085]: pgmap v2333: 129 pgs: 129 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.33k op/s 2024-09-21T10:21:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:42.246 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:42.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:42.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.393 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:42.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:42.494 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:42.502 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:42.531 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:42.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.672 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:42.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:42.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:42.833 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:42.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:42.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:42.975 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:42.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:43.077 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:43.111 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:43.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:43.213 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:43.246 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:43.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:43.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:43.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.380 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:43.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:43.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:43.534 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:43.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:43.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:43.672 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:43.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:43.808 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:43.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:43.910 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:43.949 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:43.949 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:43 smithi134 ceph-mon[90085]: pgmap v2334: 129 pgs: 129 active+clean; 877 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.33k op/s 2024-09-21T10:21:43.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:44.051 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.089 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:44.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:44.177 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:21:44.178 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:21:44.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:44.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:43 smithi017 ceph-mon[102913]: pgmap v2334: 129 pgs: 129 active+clean; 877 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.33k op/s 2024-09-21T10:21:44.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:43 smithi017 ceph-mon[105151]: pgmap v2334: 129 pgs: 129 active+clean; 877 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.33k op/s 2024-09-21T10:21:44.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.227 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:44.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:44.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.370 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:44.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:44.470 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:44.498 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:44.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:44.544 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:21:44.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:44.632 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:44.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:44.734 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:44.772 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:44.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:44.863 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:21:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:21:44] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.51.0" 2024-09-21T10:21:44.874 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:44.902 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:44.978 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:21:45.004 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:45.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.069 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:45.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:45.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:45.205 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:45.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:45.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:45.343 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:45.362 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:45.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.483 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:45.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:45 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:45 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:45.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:45 smithi134 ceph-mon[90085]: pgmap v2335: 129 pgs: 129 active+clean; 784 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s 2024-09-21T10:21:45.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:45.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.622 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:45.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:45 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:45 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:45 smithi017 ceph-mon[105151]: pgmap v2335: 129 pgs: 129 active+clean; 784 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s 2024-09-21T10:21:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:45 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:21:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:45 smithi017 ceph-mon[102913]: pgmap v2335: 129 pgs: 129 active+clean; 784 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.32k op/s 2024-09-21T10:21:45.701 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:21:45 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T10:21:45.281+0000 7f3131349640 -1 mgr.server reply reply (4) Interrupted system call no change 2024-09-21T10:21:45.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:45.750 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:45.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:45.852 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:45.906 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:45.982 INFO:teuthology.orchestra.run.smithi017.stdout:moved 37 / 387 (9.56072%) 2024-09-21T10:21:45.982 INFO:teuthology.orchestra.run.smithi017.stdout:avg 55.2857 2024-09-21T10:21:45.982 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 15.4616 -> 10.7 (expected baseline 6.88388) 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 20 -> 37 pgs (0.361757 -> 0.669251 * mean) 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.6 with 69 -> 73 pgs (1.24806 -> 1.32041 * mean) 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 3 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0162 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0178 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 0.4129 -> 0.5947 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 weight 1.0000 -> 0.8221 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 weight 0.9074 -> 0.7638 2024-09-21T10:21:45.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 weight 1.0000 -> 0.8786 2024-09-21T10:21:45.984 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2024-09-21T10:21:46.008 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:46.035 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:46.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:46.137 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:46.177 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:46.279 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:46.315 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:46.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:46.449 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:46.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:46.572 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:46.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:46.674 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:46.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:46 smithi017 ceph-mon[102913]: from='client.36657 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:21:46.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:46 smithi017 ceph-mon[105151]: from='client.36657 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:21:46.706 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:46 smithi134 ceph-mon[90085]: from='client.36657 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:21:46.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.712 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:46.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:46.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.857 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:46.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:46.958 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:47.005 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:47.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:47.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.144 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:47.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:47.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:47.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:47.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:47.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:47.413 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:47.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:47.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:47 smithi134 ceph-mon[90085]: pgmap v2336: 129 pgs: 129 active+clean; 572 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.40k op/s 2024-09-21T10:21:47.515 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.555 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:47.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:47.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:47.672 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:21:47.673 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:21:47.688 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:47.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:47.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:47 smithi017 ceph-mon[102913]: pgmap v2336: 129 pgs: 129 active+clean; 572 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.40k op/s 2024-09-21T10:21:47.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:47 smithi017 ceph-mon[105151]: pgmap v2336: 129 pgs: 129 active+clean; 572 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 1.40k op/s 2024-09-21T10:21:47.790 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:47.827 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:47.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:47.929 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:47.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.971 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:47.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:48.072 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:48.110 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:48.212 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:48.246 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:48.348 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:48.381 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:48.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:48.483 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:48.505 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:48.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:48.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:48.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.640 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:48.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:48.740 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:48.761 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:48.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:48.862 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:48.889 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:48.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:48.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:49.029 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:49.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:49.067 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-21T10:21:49.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 110 3.0 2024-09-21T10:21:49.095 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:21:49.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:49.191 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:49.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:49.292 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:49.327 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:49.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:49.458 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:49.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:49.534 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:49.556 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:49.560 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:49.591 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:49.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:49.693 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:49.731 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:49.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:49.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:49.872 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:49.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:49.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:49 smithi017 ceph-mon[102913]: pgmap v2337: 129 pgs: 129 active+clean; 443 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-21T10:21:49.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:49 smithi017 ceph-mon[105151]: pgmap v2337: 129 pgs: 129 active+clean; 443 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-21T10:21:49.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:49.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:49 smithi134 ceph-mon[90085]: pgmap v2337: 129 pgs: 129 active+clean; 443 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-21T10:21:50.014 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:50.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:50.115 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:50.154 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:50.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:50.286 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:50.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:50.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:50.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.430 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:50.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:21:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:21:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:21:50.532 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.567 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:50.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:50.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:50.711 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:50.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:50.779 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:50.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:50.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.854 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:50.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:50.954 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:50.990 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:50.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:51.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:51.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.131 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:51.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:51.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.265 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:51.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:51.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 130204 objects 2024-09-21T10:21:51.328 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :118.211 2024-09-21T10:21:51.366 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:51.387 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:51.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:51.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:51.512 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:51.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:51.613 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:51.635 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:51.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:51.688 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:21:51.737 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:51.786 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:51.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:51 smithi017 ceph-mon[102913]: pgmap v2338: 129 pgs: 129 active+clean; 272 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.34k op/s 2024-09-21T10:21:51.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:51 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4162191664' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:51.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:51 smithi017 ceph-mon[105151]: pgmap v2338: 129 pgs: 129 active+clean; 272 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.34k op/s 2024-09-21T10:21:51.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:51 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4162191664' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:51.883 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:51 smithi134 ceph-mon[90085]: pgmap v2338: 129 pgs: 129 active+clean; 272 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.34k op/s 2024-09-21T10:21:51.883 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:51 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4162191664' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:21:51.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:51.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:51.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:52.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:52.035 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:52.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:52.136 INFO:teuthology.orchestra.run.smithi134.stdout:ERROR: (22) Invalid argument 2024-09-21T10:21:52.137 INFO:teuthology.orchestra.run.smithi134.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:21:52.137 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.159 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:52.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:52.260 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:52.281 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:52.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:52.383 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:52.405 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:52.507 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:52.529 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:52.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:52.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:52.759 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:52.781 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:52.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:52.883 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:52.906 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:52.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:52 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:52 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4162191664' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:21:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:52 smithi017 ceph-mon[102913]: osdmap e1417: 8 total, 8 up, 7 in 2024-09-21T10:21:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:52 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:52 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4162191664' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:21:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:52 smithi017 ceph-mon[105151]: osdmap e1417: 8 total, 8 up, 7 in 2024-09-21T10:21:52.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:52 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:52.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:52 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4162191664' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:21:52.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:52 smithi134 ceph-mon[90085]: osdmap e1417: 8 total, 8 up, 7 in 2024-09-21T10:21:53.008 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:53.030 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:53.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:53.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:53.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:53.283 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:53.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:53.384 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:53.410 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:53.512 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:53.537 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:53.639 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:53.664 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:53.766 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:53.791 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:53.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:53.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:53.918 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:53.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:53.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:53 smithi017 ceph-mon[102913]: pgmap v2340: 129 pgs: 129 active+clean; 89 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-21T10:21:53.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:53 smithi017 ceph-mon[105151]: pgmap v2340: 129 pgs: 129 active+clean; 89 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-21T10:21:53.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:53 smithi134 ceph-mon[90085]: pgmap v2340: 129 pgs: 129 active+clean; 89 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-21T10:21:54.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:54.048 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:54.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:54.149 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:54.175 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:54.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:54.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:54.302 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:54.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:54.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:54.432 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:54.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:54.534 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:54.559 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:54.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:54.676 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:21:54.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.689 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:54.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:54.789 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:54.811 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:54.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:54.912 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:54.930 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:54.934 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:54 smithi017 ceph-mon[102913]: 11.12 scrub starts 2024-09-21T10:21:54.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:54 smithi017 ceph-mon[102913]: 11.12 scrub ok 2024-09-21T10:21:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:54 smithi017 ceph-mon[105151]: 11.12 scrub starts 2024-09-21T10:21:54.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:54 smithi017 ceph-mon[105151]: 11.12 scrub ok 2024-09-21T10:21:54.951 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:21:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:21:54] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.51.0" 2024-09-21T10:21:54.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:21:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:54 smithi134 ceph-mon[90085]: 11.12 scrub starts 2024-09-21T10:21:54.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:54 smithi134 ceph-mon[90085]: 11.12 scrub ok 2024-09-21T10:21:55.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:55.059 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:55.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:55.161 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:55.183 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:55.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:55.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:55.306 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:55.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:55.408 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:55.429 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:55.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:55.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:55.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:55.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:55.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:55.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:55.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:55.802 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:55.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:55.903 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:55.925 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:55 smithi134 ceph-mon[90085]: pgmap v2341: 129 pgs: 129 active+clean; 42 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-21T10:21:55.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:56.026 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:56.048 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:56.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:56.074 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:55 smithi017 ceph-mon[102913]: pgmap v2341: 129 pgs: 129 active+clean; 42 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-21T10:21:56.074 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:56.074 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:55 smithi017 ceph-mon[105151]: pgmap v2341: 129 pgs: 129 active+clean; 42 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 1.20k op/s 2024-09-21T10:21:56.074 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:56.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:56.173 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:56.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:56.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:56.297 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:56.398 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:56.421 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:56.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:56.523 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:56.545 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:56.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:56.546 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:21:56.546 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:56.552 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:56.556 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:21:56.557 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:56.574 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:21:56.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:56.585 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2024-09-21T10:21:56.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2024-09-21T10:21:56.591 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:21:56.591 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:56.607 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:21:56.608 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:56.623 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:21:56.624 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:56.639 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:21:56.639 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:56.647 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:56.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:21:56.656 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:56.668 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:56.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:173491 9.3 9.22c85303 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:24.703930+0000", 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 571.965881545, 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.98604968599999998, 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.673 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 173491 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703930+0000", 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.674 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703930+0000", 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703929+0000", 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703935+0000", 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7830000000000004e-06 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.675 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703947+0000", 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2495e-05 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.703955+0000", 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0749999999999998e-06 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.704021+0000", 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5544999999999997e-05 2024-09-21T10:21:56.676 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.704032+0000", 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1714e-05 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689967+0000", 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.98593412800000002 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.677 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689980+0000", 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3451e-05 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:197588 10.13 10:cf2543b8:::benchmark_data_smithi017_268344_object71657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:16:26.380628+0000", 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 330.28918432199998, 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2115995979999998, 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.678 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 197588 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.380628+0000", 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.380628+0000", 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.679 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.380627+0000", 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.380631+0000", 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8539999999999999e-06 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.680 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.380638+0000", 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5100000000000001e-06 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.380642+0000", 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5549999999999998e-06 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.380683+0000", 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1264999999999999e-05 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.681 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.380689+0000", 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4019999999999999e-06 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:27.305781+0000", 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.92509209400000003 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:27.305792+0000", 2024-09-21T10:21:56.682 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0311e-05 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.579818+0000", 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2740259389999999 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.579966+0000", 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000148577 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.683 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.580477+0000", 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00051035800000000004 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.587340+0000", 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.006863442 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.588172+0000", 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00083246600000000002 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.684 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.591702+0000", 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003529295 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.591758+0000", 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6326000000000002e-05 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.592227+0000", 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00046906200000000002 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.685 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187385 11.12 11.2b39a592 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:40.866254+0000", 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.803558187999997, 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.88790036299999997, 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187385 2024-09-21T10:21:56.686 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.687 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.687 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.687 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.687 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.866254+0000", 2024-09-21T10:21:56.687 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.687 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.687 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.866254+0000", 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.866253+0000", 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.688 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.866258+0000", 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8980000000000004e-06 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.866268+0000", 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0497e-05 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.866273+0000", 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3120000000000004e-06 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.689 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.866314+0000", 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1461999999999997e-05 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.866321+0000", 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7240000000000003e-06 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754144+0000", 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.88782289700000006 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.690 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754154+0000", 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0332999999999999e-05 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187421 11.12 11.981eab2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:40.888563+0000", 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.781248403000006, 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.86561988599999995, 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.691 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187421 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.888563+0000", 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.888563+0000", 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.692 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.888562+0000", 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.888569+0000", 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5370000000000003e-06 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.888581+0000", 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2618e-05 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.693 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.888587+0000", 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3689999999999998e-06 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.888636+0000", 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8922999999999999e-05 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.888642+0000", 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3520000000000003e-06 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.694 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754177+0000", 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.86553492600000004 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754183+0000", 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4919999999999998e-06 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187434 11.12 11.516ea712 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:40.893664+0000", 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.776147590999997, 2024-09-21T10:21:56.695 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.86054533499999997, 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187434 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.893664+0000", 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.696 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.893664+0000", 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.893663+0000", 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.893672+0000", 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5339999999999999e-06 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.697 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.893684+0000", 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2123000000000001e-05 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.893690+0000", 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8510000000000004e-06 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.894674+0000", 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0009848420000000001 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.698 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.894681+0000", 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8079999999999999e-06 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754203+0000", 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.85952212100000003 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754210+0000", 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.28e-06 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.699 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187442 11.12 11.c53012d2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:40.899245+0000", 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.770566565999999, 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.85499022599999996, 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187442 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.700 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.899245+0000", 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.899245+0000", 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.899245+0000", 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.701 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.899247+0000", 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.2350000000000002e-06 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.899254+0000", 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.596e-06 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.899258+0000", 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.9029999999999997e-06 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.702 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.899366+0000", 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00010833899999999999 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.899372+0000", 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2299999999999996e-06 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754229+0000", 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.85485711099999995 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.703 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754236+0000", 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2029999999999997e-06 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187446 11.12 11.8250be92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:40.902630+0000", 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.767182228999999, 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.85163300799999997, 2024-09-21T10:21:56.704 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187446 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.902630+0000", 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.705 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.902630+0000", 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.902629+0000", 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.902631+0000", 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.643e-06 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.706 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.902637+0000", 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3570000000000003e-06 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.902641+0000", 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7519999999999998e-06 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.902677+0000", 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6062999999999997e-05 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.707 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.902683+0000", 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4940000000000003e-06 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754256+0000", 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.85157368 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754263+0000", 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3180000000000003e-06 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.708 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187463 11.12 11.ff1d93d2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:40.913397+0000", 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.756415356999995, 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.84089497599999996, 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187463 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.709 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.913397+0000", 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.913397+0000", 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.913396+0000", 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.710 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.913401+0000", 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2399999999999998e-06 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.913410+0000", 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.4420000000000004e-06 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.913416+0000", 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5180000000000004e-06 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.913471+0000", 2024-09-21T10:21:56.711 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5618999999999997e-05 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.913478+0000", 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.421e-06 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754284+0000", 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.84080641899999997 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.712 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754292+0000", 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2860000000000002e-06 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187478 11.12 11.e1466fd2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:40.922635+0000", 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.747176830000001, 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.83168278100000004, 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.713 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187478 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.922635+0000", 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.922635+0000", 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.714 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.922634+0000", 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.922639+0000", 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.6879999999999998e-06 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.922649+0000", 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.022e-05 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.715 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.922655+0000", 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7849999999999997e-06 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.922693+0000", 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7539000000000001e-05 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.922697+0000", 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3250000000000001e-06 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.716 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754311+0000", 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.83161448500000001 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754318+0000", 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4509999999999997e-06 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.717 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187480 11.12 11.34be4552 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:40.924607+0000", 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.745204635999997, 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.82973738600000002, 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187480 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.718 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.924607+0000", 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.924607+0000", 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.924606+0000", 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.719 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.924611+0000", 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3989999999999997e-06 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.924621+0000", 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0611999999999999e-05 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.924626+0000", 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.408e-06 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.720 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.924662+0000", 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5655999999999999e-05 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.924666+0000", 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0040000000000001e-06 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754338+0000", 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.82967264500000004 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.721 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754345+0000", 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4620000000000001e-06 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187522 11.12 11.1cd5572 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:40.960204+0000", 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.709607500000004, 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.79416623600000003, 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.722 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187522 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.960204+0000", 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.960204+0000", 2024-09-21T10:21:56.723 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.960203+0000", 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.960208+0000", 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.33e-06 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.960215+0000", 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.3100000000000003e-06 2024-09-21T10:21:56.724 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.960218+0000", 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.2049999999999998e-06 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.960255+0000", 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6953999999999999e-05 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:40.960261+0000", 2024-09-21T10:21:56.725 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4450000000000004e-06 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754364+0000", 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.79410357600000003 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754371+0000", 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3849999999999999e-06 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.726 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187583 11.12 11.1f2c6c12 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:41.006344+0000", 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.663467603000001, 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.74805460700000004, 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187583 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.727 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.006344+0000", 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.006344+0000", 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.006343+0000", 2024-09-21T10:21:56.728 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.006349+0000", 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0310000000000004e-06 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.006363+0000", 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4153e-05 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.006369+0000", 2024-09-21T10:21:56.729 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9699999999999996e-06 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.006414+0000", 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4842000000000002e-05 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.006421+0000", 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8129999999999998e-06 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.730 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754393+0000", 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.74797195500000002 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754399+0000", 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2149999999999998e-06 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187619 11.12 11.53b269b2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:41.036440+0000", 2024-09-21T10:21:56.731 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 75.633371482000001, 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.71798508800000005, 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187619 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.036440+0000", 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.732 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.036440+0000", 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.036440+0000", 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.036444+0000", 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3490000000000002e-06 2024-09-21T10:21:56.733 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.036454+0000", 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.7440000000000002e-06 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.036459+0000", 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5230000000000003e-06 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.036508+0000", 2024-09-21T10:21:56.734 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.8173000000000001e-05 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.036514+0000", 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8920000000000003e-06 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754419+0000", 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.717904616 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:41.754426+0000", 2024-09-21T10:21:56.735 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4869999999999999e-06 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:215498 11.19 11.6eae6739 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1407)", 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:14.784563+0000", 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 41.885248418000003, 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.90930455200000004, 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215498 2024-09-21T10:21:56.736 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.784563+0000", 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.784563+0000", 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.784562+0000", 2024-09-21T10:21:56.737 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.784568+0000", 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0789999999999997e-06 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.784581+0000", 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.236e-05 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.738 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.784586+0000", 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.412e-06 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.784633+0000", 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7070000000000002e-05 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.784642+0000", 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.5869999999999993e-06 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.739 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.693856+0000", 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.90921414599999995 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.693868+0000", 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2179e-05 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:215601 11.19 11.c877eef9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1407)", 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:14.861563+0000", 2024-09-21T10:21:56.740 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 41.808249007999997, 2024-09-21T10:21:56.741 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.83234685200000003, 2024-09-21T10:21:56.741 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.741 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.741 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.741 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.741 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.741 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215601 2024-09-21T10:21:56.741 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.741 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.861563+0000", 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.861563+0000", 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.861562+0000", 2024-09-21T10:21:56.742 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.861565+0000", 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9730000000000002e-06 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.861571+0000", 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9560000000000002e-06 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.743 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.861575+0000", 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5499999999999999e-06 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.861621+0000", 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5936000000000002e-05 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.861627+0000", 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.224e-06 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.693898+0000", 2024-09-21T10:21:56.744 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.83227141800000004 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.693910+0000", 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1348e-05 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:215637 11.19 11.2672dcf9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1407)", 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:14.884401+0000", 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 41.785410857999999, 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.80955152900000005, 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.745 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215637 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.884401+0000", 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.746 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.884401+0000", 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.884400+0000", 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.884407+0000", 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.4410000000000004e-06 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.747 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.884421+0000", 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3385e-05 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.884426+0000", 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.203e-06 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.884473+0000", 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7416000000000001e-05 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.748 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.884481+0000", 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.7840000000000004e-06 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.693942+0000", 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.80946099100000002 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.693953+0000", 2024-09-21T10:21:56.749 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0538e-05 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:215678 11.19 11.e6b8fb99 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1407)", 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:14.913680+0000", 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 41.756132102999999, 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.78031345699999999, 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.750 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215678 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.913680+0000", 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.913680+0000", 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.751 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.913678+0000", 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.913683+0000", 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.109e-06 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.913691+0000", 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8480000000000007e-06 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.752 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.913696+0000", 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1690000000000002e-06 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.914152+0000", 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00045675799999999998 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.914160+0000", 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2439999999999996e-06 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.753 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.693987+0000", 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.77982743799999998 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.693993+0000", 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1890000000000004e-06 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:215733 11.19 11.ff701779 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1407)", 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:14.952074+0000", 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 41.717737944, 2024-09-21T10:21:56.754 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.74195440000000001, 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215733 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.952074+0000", 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.755 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.952074+0000", 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.952073+0000", 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.952078+0000", 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.507e-06 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.756 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.952089+0000", 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0635000000000001e-05 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.952094+0000", 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3970000000000002e-06 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.952137+0000", 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2658999999999999e-05 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.757 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.952143+0000", 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2920000000000001e-06 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.694012+0000", 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.74186890800000005 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.694028+0000", 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.6254000000000001e-05 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.758 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:215743 11.19 11.9299d479 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1407)", 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:14.958108+0000", 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 41.711703886000002, 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.73594792799999997, 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215743 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.759 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.958108+0000", 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.958108+0000", 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.958107+0000", 2024-09-21T10:21:56.760 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.958116+0000", 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8349999999999993e-06 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.958127+0000", 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1637999999999999e-05 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.761 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.958132+0000", 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3730000000000003e-06 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.958186+0000", 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4518999999999998e-05 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.958195+0000", 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.4349999999999997e-06 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.762 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.694049+0000", 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.73585459200000003 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.694056+0000", 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7970000000000003e-06 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:215771 11.19 11.2aaaec99 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1407)", 2024-09-21T10:21:56.763 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:21:14.978490+0000", 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 41.691322339999999, 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.71558946999999995, 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 215771 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.978490+0000", 2024-09-21T10:21:56.764 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.978490+0000", 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.978488+0000", 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:21:56.765 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.978494+0000", 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4859999999999997e-06 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.978506+0000", 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.188e-05 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.978512+0000", 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5830000000000003e-06 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.766 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.978566+0000", 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4194000000000003e-05 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:14.978573+0000", 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7689999999999999e-06 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.694073+0000", 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.71550020299999995 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:21:56.767 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:21:56.768 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:21:15.694079+0000", 2024-09-21T10:21:56.768 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6250000000000004e-06 2024-09-21T10:21:56.768 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.768 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.768 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.768 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:21:56.768 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:21:56.768 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:21:56.769 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:56.790 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:56.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:56.914 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:56.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:56 smithi017 ceph-mon[105151]: 11.13 scrub starts 2024-09-21T10:21:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:56 smithi017 ceph-mon[105151]: 11.13 scrub ok 2024-09-21T10:21:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:56 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2372202421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:56 smithi017 ceph-mon[102913]: 11.13 scrub starts 2024-09-21T10:21:56.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:56 smithi017 ceph-mon[102913]: 11.13 scrub ok 2024-09-21T10:21:56.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:56 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2372202421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:56.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:56 smithi134 ceph-mon[90085]: 11.13 scrub starts 2024-09-21T10:21:56.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:56 smithi134 ceph-mon[90085]: 11.13 scrub ok 2024-09-21T10:21:56.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:56 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2372202421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:57.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:57.038 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:57.140 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:57.161 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:57.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:57.263 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:57.284 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:57.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:57.386 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:57.406 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:57.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:57.508 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:57.529 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:57.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:57.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:57.641 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:21:57.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.652 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:57.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:moved 31 / 387 (8.01034%) 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:avg 55.2857 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 15.4616 -> 11.3479 (expected baseline 6.88388) 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 20 -> 34 pgs (0.361757 -> 0.614987 * mean) 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.6 with 69 -> 72 pgs (1.24806 -> 1.30233 * mean) 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 3 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0120 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0132 2024-09-21T10:21:57.748 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 weight 0.9074 -> 0.7640 2024-09-21T10:21:57.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 0.4129 -> 0.5051 2024-09-21T10:21:57.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 weight 1.0000 -> 0.8707 2024-09-21T10:21:57.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 weight 1.0000 -> 0.9065 2024-09-21T10:21:57.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:57.770 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:57 smithi017 ceph-mon[102913]: pgmap v2342: 129 pgs: 129 active+clean; 20 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 75 op/s 2024-09-21T10:21:57.771 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:57 smithi017 ceph-mon[102913]: from='client.46511 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:21:57.771 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:57 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'57061','3':'50068','5':'33104','7':'59411'}"} : dispatch 2024-09-21T10:21:57.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:57 smithi017 ceph-mon[105151]: pgmap v2342: 129 pgs: 129 active+clean; 20 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 75 op/s 2024-09-21T10:21:57.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:57 smithi017 ceph-mon[105151]: from='client.46511 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:21:57.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:57 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'57061','3':'50068','5':'33104','7':'59411'}"} : dispatch 2024-09-21T10:21:57.782 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:57.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:57.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:57.905 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:57.907 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:57 smithi134 ceph-mon[90085]: pgmap v2342: 129 pgs: 129 active+clean; 20 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 75 op/s 2024-09-21T10:21:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:57 smithi134 ceph-mon[90085]: from='client.46511 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:21:57.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:57 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'2':'57061','3':'50068','5':'33104','7':'59411'}"} : dispatch 2024-09-21T10:21:58.009 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:58.031 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:58.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:58.132 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:58.156 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:58.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:58.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:58.281 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:58.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:58.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:58.409 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:58.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:58.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:58.533 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:58.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:58.635 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.658 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:58.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:58.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:58.781 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:58.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:58.883 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:58.905 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:58.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:59.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:59.031 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:59.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:58 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'57061','3':'50068','5':'33104','7':'59411'}"}]': finished 2024-09-21T10:21:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:58 smithi134 ceph-mon[90085]: osdmap e1418: 8 total, 8 up, 7 in 2024-09-21T10:21:59.054 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:58 smithi134 ceph-mon[90085]: osdmap e1419: 8 total, 8 up, 7 in 2024-09-21T10:21:59.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:58 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'57061','3':'50068','5':'33104','7':'59411'}"}]': finished 2024-09-21T10:21:59.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:58 smithi017 ceph-mon[102913]: osdmap e1418: 8 total, 8 up, 7 in 2024-09-21T10:21:59.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:58 smithi017 ceph-mon[102913]: osdmap e1419: 8 total, 8 up, 7 in 2024-09-21T10:21:59.116 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:58 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'57061','3':'50068','5':'33104','7':'59411'}"}]': finished 2024-09-21T10:21:59.116 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:58 smithi017 ceph-mon[105151]: osdmap e1418: 8 total, 8 up, 7 in 2024-09-21T10:21:59.116 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:58 smithi017 ceph-mon[105151]: osdmap e1419: 8 total, 8 up, 7 in 2024-09-21T10:21:59.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:59.155 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:59.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:59.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:59.279 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:59.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:59.403 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:59.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:59.456 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:21:59.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 110 0.05 2024-09-21T10:21:59.505 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:59.530 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:59.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:59.631 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:59.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.654 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:59.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:59.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:59.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:21:59.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:59.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:59.920 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:21:59.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:00.022 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:00.043 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:00.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: 11.14 scrub starts 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: 11.14 scrub ok 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: 11.19 scrub starts 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: 11.19 scrub ok 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: pgmap v2344: 129 pgs: 129 active+clean; 20 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: 1.17 scrub starts 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: 1.17 scrub ok 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:00.094 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:21:59 smithi134 ceph-mon[90085]: osdmap e1420: 8 total, 8 up, 7 in 2024-09-21T10:22:00.145 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:00.166 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:00.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: 11.14 scrub starts 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: 11.14 scrub ok 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: 11.19 scrub starts 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: 11.19 scrub ok 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: pgmap v2344: 129 pgs: 129 active+clean; 20 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: 1.17 scrub starts 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: 1.17 scrub ok 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[105151]: osdmap e1420: 8 total, 8 up, 7 in 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: 11.14 scrub starts 2024-09-21T10:22:00.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: 11.14 scrub ok 2024-09-21T10:22:00.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: 11.19 scrub starts 2024-09-21T10:22:00.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: 11.19 scrub ok 2024-09-21T10:22:00.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: pgmap v2344: 129 pgs: 129 active+clean; 20 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:00.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: 1.17 scrub starts 2024-09-21T10:22:00.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: 1.17 scrub ok 2024-09-21T10:22:00.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:00.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:00.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:21:59 smithi017 ceph-mon[102913]: osdmap e1420: 8 total, 8 up, 7 in 2024-09-21T10:22:00.268 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:00.289 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:00.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:00.364 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:22:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:22:00.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:00.414 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:00.472 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:22:00.472 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:22:00.472 INFO:teuthology.orchestra.run.smithi134.stdout: "num_ops": 0 2024-09-21T10:22:00.472 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:22:00.487 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:00.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:00.539 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:00.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:00.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:00.663 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:00.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:00.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:00.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:00.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:00.857 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:22:00.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.8 starting backfill to osd.4 from (0'0,0'0] MAX to 1416'13719 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.17 starting backfill to osd.3 from (0'0,0'0] MAX to 1416'13734 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1416'13825 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.d starting backfill to osd.5 from (0'0,0'0] MAX to 1416'16078 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1416'13733 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.12 starting backfill to osd.3 from (1372'6300,1394'11798] MIN to 1416'13871 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.5 starting backfill to osd.0 from (0'0,0'0] MAX to 1416'13648 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.16 starting backfill to osd.4 from (1370'4700,1401'12126] MIN to 1416'13838 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.8 scrub starts 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.8 scrub ok 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.12 scrub starts 2024-09-21T10:22:00.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: 11.12 scrub ok 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2460851139' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.8 starting backfill to osd.4 from (0'0,0'0] MAX to 1416'13719 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.17 starting backfill to osd.3 from (0'0,0'0] MAX to 1416'13734 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1416'13825 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.d starting backfill to osd.5 from (0'0,0'0] MAX to 1416'16078 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1416'13733 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.12 starting backfill to osd.3 from (1372'6300,1394'11798] MIN to 1416'13871 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.5 starting backfill to osd.0 from (0'0,0'0] MAX to 1416'13648 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.16 starting backfill to osd.4 from (1370'4700,1401'12126] MIN to 1416'13838 2024-09-21T10:22:00.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:22:00.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:22:00.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:00.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.8 scrub starts 2024-09-21T10:22:00.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.8 scrub ok 2024-09-21T10:22:00.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.12 scrub starts 2024-09-21T10:22:00.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: 11.12 scrub ok 2024-09-21T10:22:00.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2460851139' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:00.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:00 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:00.889 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:00.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:01.014 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:01.037 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:01.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:01.139 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:01.160 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:01.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:01.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.8 starting backfill to osd.4 from (0'0,0'0] MAX to 1416'13719 2024-09-21T10:22:01.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.17 starting backfill to osd.3 from (0'0,0'0] MAX to 1416'13734 2024-09-21T10:22:01.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1416'13825 2024-09-21T10:22:01.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.d starting backfill to osd.5 from (0'0,0'0] MAX to 1416'16078 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1416'13733 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.12 starting backfill to osd.3 from (1372'6300,1394'11798] MIN to 1416'13871 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.5 starting backfill to osd.0 from (0'0,0'0] MAX to 1416'13648 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.16 starting backfill to osd.4 from (1370'4700,1401'12126] MIN to 1416'13838 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.8 scrub starts 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.8 scrub ok 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.12 scrub starts 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: 11.12 scrub ok 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2460851139' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:01.245 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:00 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:01.262 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:01.282 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:01.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:01.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:22:01.384 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:01.424 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:01.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:01.526 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:01.548 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:01.605 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:22:01.651 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:01.672 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:01.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:01.774 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:01.794 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:01.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:01.896 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:01.924 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:01.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:01.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[105151]: 11.16 scrub starts 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[105151]: 11.16 scrub ok 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[105151]: pgmap v2347: 129 pgs: 2 remapped+peering, 127 active+clean; 13 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 5.7 KiB/s rd, 2 op/s 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[105151]: osdmap e1421: 8 total, 8 up, 7 in 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[102913]: 11.16 scrub starts 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[102913]: 11.16 scrub ok 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[102913]: pgmap v2347: 129 pgs: 2 remapped+peering, 127 active+clean; 13 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 5.7 KiB/s rd, 2 op/s 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:22:01.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:01.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:01 smithi017 ceph-mon[102913]: osdmap e1421: 8 total, 8 up, 7 in 2024-09-21T10:22:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:01 smithi134 ceph-mon[90085]: 11.16 scrub starts 2024-09-21T10:22:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:01 smithi134 ceph-mon[90085]: 11.16 scrub ok 2024-09-21T10:22:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:01 smithi134 ceph-mon[90085]: pgmap v2347: 129 pgs: 2 remapped+peering, 127 active+clean; 13 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 5.7 KiB/s rd, 2 op/s 2024-09-21T10:22:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:01 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:01 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:22:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:01 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:01.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:01 smithi134 ceph-mon[90085]: osdmap e1421: 8 total, 8 up, 7 in 2024-09-21T10:22:02.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:02.051 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:02.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:02.153 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:02.175 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:02.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:02.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:22:02.276 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:02.285 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:02.297 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:02.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:02.420 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:02.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:02.544 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:02.646 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:02.669 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:02.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:02.770 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:02.792 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:02.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:02.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:02.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:02.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:02 smithi017 ceph-mon[102913]: osdmap e1422: 8 total, 8 up, 7 in 2024-09-21T10:22:02.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:02 smithi017 ceph-mon[105151]: osdmap e1422: 8 total, 8 up, 7 in 2024-09-21T10:22:03.018 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:03.039 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:03.141 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:03.144 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:02 smithi134 ceph-mon[90085]: osdmap e1422: 8 total, 8 up, 7 in 2024-09-21T10:22:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.163 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:03.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:03.264 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:03.287 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:03.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:03.388 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:03.389 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:22:03.389 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [], 2024-09-21T10:22:03.390 INFO:teuthology.orchestra.run.smithi134.stdout: "complaint_time": 30, 2024-09-21T10:22:03.390 INFO:teuthology.orchestra.run.smithi134.stdout: "num_blocked_ops": 0 2024-09-21T10:22:03.390 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:22:03.410 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:03.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:03.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:03.535 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:03.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:03.636 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:03.658 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:03.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:03.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.784 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:03.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:03.885 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:03.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.907 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:03.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:03.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:03 smithi017 ceph-mon[102913]: pgmap v2350: 129 pgs: 3 peering, 2 remapped+peering, 124 active+clean; 11 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 8.5 KiB/s rd, 3 op/s 2024-09-21T10:22:03.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:03 smithi017 ceph-mon[105151]: pgmap v2350: 129 pgs: 3 peering, 2 remapped+peering, 124 active+clean; 11 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 8.5 KiB/s rd, 3 op/s 2024-09-21T10:22:04.008 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:04.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.030 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:04.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:04.052 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:03 smithi134 ceph-mon[90085]: pgmap v2350: 129 pgs: 3 peering, 2 remapped+peering, 124 active+clean; 11 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 8.5 KiB/s rd, 3 op/s 2024-09-21T10:22:04.058 INFO:teuthology.orchestra.run.smithi017.stdout:moved 10 / 387 (2.58398%) 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:avg 55.2857 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 11.3479 -> 10.5656 (expected baseline 6.88388) 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 34 -> 38 pgs (0.614987 -> 0.687339 * mean) 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.6 with 72 -> 73 pgs (1.30233 -> 1.32041 * mean) 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0123 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0136 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 weight 0.9065 -> 0.8566 2024-09-21T10:22:04.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 0.5051 -> 0.5551 2024-09-21T10:22:04.060 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 weight 0.7640 -> 0.7140 2024-09-21T10:22:04.060 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 weight 0.8707 -> 0.8207 2024-09-21T10:22:04.060 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2024-09-21T10:22:04.076 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:22:04.132 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.182 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:04.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:04.283 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:04.305 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:04.321 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:22:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T10:22:04.056+0000 7f3131349640 -1 mgr.server reply reply (4) Interrupted system call no change 2024-09-21T10:22:04.362 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:22:04.407 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:04.428 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:04.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:04.553 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:04.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:04.655 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:04.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:04.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:04.799 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:04.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:04.814 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:04.865 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:22:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:22:04] "GET /metrics HTTP/1.1" 200 36723 "" "Prometheus/2.51.0" 2024-09-21T10:22:04.900 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:04.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:04.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:05.024 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:05.047 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:05.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:05.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:05.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:05.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:05.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:04 smithi017 ceph-mon[102913]: 11.17 scrub starts 2024-09-21T10:22:05.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:04 smithi017 ceph-mon[102913]: 11.17 scrub ok 2024-09-21T10:22:05.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:04 smithi017 ceph-mon[105151]: 11.17 scrub starts 2024-09-21T10:22:05.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:04 smithi017 ceph-mon[105151]: 11.17 scrub ok 2024-09-21T10:22:05.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:04 smithi134 ceph-mon[90085]: 11.17 scrub starts 2024-09-21T10:22:05.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:04 smithi134 ceph-mon[90085]: 11.17 scrub ok 2024-09-21T10:22:05.273 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:05.294 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:05.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:05.396 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:05.413 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:22:05.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 1000 1.0 2024-09-21T10:22:05.417 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:05.519 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.541 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:05.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:05.642 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:05.664 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:05.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:05.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:22:05.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:05.788 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:05.869 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_6' removed 2024-09-21T10:22:05.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:05.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:05 smithi017 ceph-mon[102913]: from='client.46523 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:22:05.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:05 smithi017 ceph-mon[102913]: pgmap v2351: 129 pgs: 3 peering, 126 active+clean; 11 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 5.8 KiB/s rd, 2 op/s 2024-09-21T10:22:05.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:05 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1132038482' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:22:05.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:05 smithi017 ceph-mon[105151]: from='client.46523 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:22:05.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:05 smithi017 ceph-mon[105151]: pgmap v2351: 129 pgs: 3 peering, 126 active+clean; 11 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 5.8 KiB/s rd, 2 op/s 2024-09-21T10:22:05.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:05 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1132038482' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:22:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.956 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:05.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:06.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:06.078 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:06.180 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:06.201 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:05 smithi134 ceph-mon[90085]: from='client.46523 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:22:06.201 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:05 smithi134 ceph-mon[90085]: pgmap v2351: 129 pgs: 3 peering, 126 active+clean; 11 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 5.8 KiB/s rd, 2 op/s 2024-09-21T10:22:06.201 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:05 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1132038482' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:22:06.203 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:06.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:06.219 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "size": 20, 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 600, 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "ops": [ 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.45851.0:173513 9.e 9:71c17054:::benchmark_data_smithi017_237924_object80509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:12:24.724062+0000", 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 581.47407751100002, 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.4376691149999998, 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.45851", 2024-09-21T10:22:06.220 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/3001216500", 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 173513 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724062+0000", 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724062+0000", 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.221 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724060+0000", 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999976 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724071+0000", 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1567e-05 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724093+0000", 2024-09-21T10:22:06.222 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.1767999999999999e-05 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724101+0000", 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.4549999999999995e-06 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724163+0000", 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1303000000000006e-05 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.223 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:24.724176+0000", 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2779000000000001e-05 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689519+0000", 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96534338099999994 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:25.689531+0000", 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1839e-05 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.224 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.123518+0000", 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.4339874020000001 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.123669+0000", 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015062300000000001 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,1,7", 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.123813+0000", 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000144487 2024-09-21T10:22:06.225 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.134005+0000", 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010191817000000001 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.134077+0000", 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1769999999999999e-05 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.147373+0000", 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013295968999999999 2024-09-21T10:22:06.226 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161450+0000", 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014077265 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161485+0000", 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.5074000000000003e-05 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:12:27.161731+0000", 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000246016 2024-09-21T10:22:06.227 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.228 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.228 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.228 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.229 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.229 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62006 10.10 10:0a3d3700:::benchmark_data_smithi017_268344_object62005:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.229 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.010013+0000", 2024-09-21T10:22:06.229 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.18812688399998, 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.063320321, 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62006 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010013+0000", 2024-09-21T10:22:06.230 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010013+0000", 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010011+0000", 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010189+0000", 2024-09-21T10:22:06.231 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017731600000000001 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010201+0000", 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2072000000000001e-05 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010207+0000", 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7780000000000004e-06 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.232 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010266+0000", 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8933e-05 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.010274+0000", 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.0260000000000007e-06 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997736+0000", 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.98746292599999996 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.233 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997747+0000", 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0535e-05 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.057043+0000", 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0592958029999999 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.057156+0000", 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00011309 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.234 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.057287+0000", 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000131483 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060730+0000", 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0034428140000000002 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072326+0000", 2024-09-21T10:22:06.235 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011595764999999999 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072522+0000", 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00019573 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072558+0000", 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6087999999999997e-05 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.236 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073333+0000", 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00077519299999999998 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62045 10.10 10:09934317:::benchmark_data_smithi017_268344_object62044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.028735+0000", 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.16940487300002, 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.045632221, 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.237 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62045 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028735+0000", 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.238 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028735+0000", 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028734+0000", 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028791+0000", 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7380999999999998e-05 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.239 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028802+0000", 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0868e-05 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028807+0000", 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9620000000000001e-06 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028858+0000", 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0646999999999998e-05 2024-09-21T10:22:06.240 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028866+0000", 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1000000000000004e-06 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997771+0000", 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96890569500000001 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997778+0000", 2024-09-21T10:22:06.241 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0460000000000002e-06 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.057972+0000", 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.060194053 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.058100+0000", 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000128169 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.242 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.058264+0000", 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000164649 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.061464+0000", 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0031997639999999999 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072356+0000", 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.010891616 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.243 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073623+0000", 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001267114 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073680+0000", 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6700999999999998e-05 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.074367+0000", 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00068730600000000005 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.244 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62049 10.10 10:0ccbb6b8:::benchmark_data_smithi017_268344_object62048:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.028883+0000", 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.16925609100002, 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.046507053, 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62049 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.245 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028883+0000", 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028883+0000", 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028883+0000", 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:22:06.246 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028947+0000", 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4085999999999996e-05 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028955+0000", 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.9500000000000001e-06 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.028959+0000", 2024-09-21T10:22:06.247 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.3000000000000003e-06 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.029019+0000", 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9722000000000002e-05 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.029025+0000", 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7610000000000004e-06 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.248 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997799+0000", 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96877391599999996 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997805+0000", 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2349999999999996e-06 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.058878+0000", 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.061073696 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.249 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.059013+0000", 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013482699999999999 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.059166+0000", 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015248900000000001 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.062458+0000", 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032921339999999999 2024-09-21T10:22:06.250 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072397+0000", 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0099391219999999999 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.074478+0000", 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0020814050000000001 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.074526+0000", 2024-09-21T10:22:06.251 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7168e-05 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.075391+0000", 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00086493200000000001 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62065 10.10 10:0ecadf09:::benchmark_data_smithi017_268344_object62064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.036833+0000", 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.16130620799998, 2024-09-21T10:22:06.252 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.039298166, 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62065 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036833+0000", 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.253 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036833+0000", 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036832+0000", 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036959+0000", 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012738300000000001 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.254 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036970+0000", 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.1007e-05 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.036976+0000", 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4369999999999998e-06 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.037047+0000", 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.1067000000000002e-05 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.255 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.037056+0000", 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6930000000000002e-06 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997826+0000", 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.96076999299999999 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997832+0000", 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9649999999999996e-06 2024-09-21T10:22:06.256 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.059887+0000", 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.062054651 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060031+0000", 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000144122 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060202+0000", 2024-09-21T10:22:06.257 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000170421 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064305+0000", 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0041030090000000003 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072427+0000", 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0081220770000000001 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.258 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.075436+0000", 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0030096419999999999 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.075474+0000", 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7456999999999997e-05 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.076132+0000", 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000657735 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.259 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62079 10.10 10:08d2baad:::benchmark_data_smithi017_268344_object62078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.042330+0000", 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.15581000499998, 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0347306949999999, 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62079 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.260 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042330+0000", 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042330+0000", 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042328+0000", 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:22:06.261 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042375+0000", 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7429999999999998e-05 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042388+0000", 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2347000000000001e-05 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042393+0000", 2024-09-21T10:22:06.262 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.921e-06 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042453+0000", 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0038999999999999e-05 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.042462+0000", 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6180000000000001e-06 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997855+0000", 2024-09-21T10:22:06.263 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.95539260199999998 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997861+0000", 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1670000000000004e-06 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060760+0000", 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0628989769999999 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.264 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060892+0000", 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013164200000000001 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.060987+0000", 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5452999999999996e-05 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064317+0000", 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0033294599999999998 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.265 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072451+0000", 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0081343519999999992 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.076247+0000", 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0037959030000000002 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.076302+0000", 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5504000000000003e-05 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077060+0000", 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000757794 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62118 10.10 10:0a3d075f:::benchmark_data_smithi017_268344_object62117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.066075+0000", 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.13206406400002, 2024-09-21T10:22:06.267 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0117501670000002, 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62118 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066075+0000", 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066075+0000", 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066074+0000", 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066104+0000", 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9998999999999999e-05 2024-09-21T10:22:06.269 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066114+0000", 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.3149999999999998e-06 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066118+0000", 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5990000000000004e-06 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.270 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066173+0000", 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4737999999999997e-05 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.066181+0000", 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8490000000000006e-06 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997893+0000", 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.93171239400000005 2024-09-21T10:22:06.271 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997900+0000", 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2829999999999998e-06 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.061493+0000", 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.063593528 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.061625+0000", 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132127 2024-09-21T10:22:06.272 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.061796+0000", 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017107900000000001 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.065992+0000", 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0041953980000000004 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.273 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072475+0000", 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.006483707 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077124+0000", 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0046483269999999998 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077183+0000", 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.897e-05 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.274 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077826+0000", 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00064299799999999999 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62139 10.10 10:0ba8e010:::benchmark_data_smithi017_268344_object62138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.077457+0000", 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.12068300499999, 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.0014778230000001, 2024-09-21T10:22:06.275 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62139 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077457+0000", 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.276 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077457+0000", 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077456+0000", 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077533+0000", 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.7880000000000007e-05 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.277 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077547+0000", 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.401e-05 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077554+0000", 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2280000000000003e-06 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077619+0000", 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.4863000000000006e-05 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.278 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.077628+0000", 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.6630000000000005e-06 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997923+0000", 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.92029435400000004 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997929+0000", 2024-09-21T10:22:06.279 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1510000000000001e-06 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.062497+0000", 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.064568411 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.062619+0000", 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012207599999999999 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.280 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.062783+0000", 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000164072 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066011+0000", 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0032278200000000002 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072503+0000", 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0064917580000000003 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.281 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.078076+0000", 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0055731849999999996 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.078142+0000", 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.5740000000000004e-05 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.078934+0000", 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00079261100000000003 2024-09-21T10:22:06.282 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62252 10.10 10:0bdce5c9:::benchmark_data_smithi017_268344_object62251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.141789+0000", 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.05635068100003, 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9393023760000001, 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.283 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62252 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141789+0000", 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141789+0000", 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.284 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141788+0000", 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141836+0000", 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.7958999999999999e-05 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141851+0000", 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5679e-05 2024-09-21T10:22:06.285 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141857+0000", 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6130000000000003e-06 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141918+0000", 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0735e-05 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.141928+0000", 2024-09-21T10:22:06.286 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0206e-05 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997949+0000", 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.85602108099999996 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997955+0000", 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0800000000000002e-06 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.063480+0000", 2024-09-21T10:22:06.287 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0655249870000001 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.063614+0000", 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000133984 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.063768+0000", 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015351300000000001 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067623+0000", 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0038550450000000001 2024-09-21T10:22:06.288 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073393+0000", 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0057700340000000003 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.080053+0000", 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0066600729999999999 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.289 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.080109+0000", 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6363000000000002e-05 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.081091+0000", 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00098213399999999996 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62285 10.10 10:0b189abc:::benchmark_data_smithi017_268344_object62284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.159865+0000", 2024-09-21T10:22:06.290 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.03827478199997, 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9218552840000001, 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62285 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159865+0000", 2024-09-21T10:22:06.291 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159865+0000", 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159863+0000", 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.292 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159921+0000", 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.7759e-05 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159935+0000", 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.401e-05 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159941+0000", 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1940000000000003e-06 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.293 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.159998+0000", 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.715e-05 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.160005+0000", 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2089999999999999e-06 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997977+0000", 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.83797192300000001 2024-09-21T10:22:06.294 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.997984+0000", 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1310000000000003e-06 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064350+0000", 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.066366404 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064432+0000", 2024-09-21T10:22:06.295 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.1909000000000005e-05 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.064526+0000", 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.4266999999999997e-05 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067633+0000", 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0031064310000000002 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.296 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073451+0000", 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0058181439999999999 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.081135+0000", 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0076844399999999998 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.297 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.081172+0000", 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.6900000000000002e-05 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.081720+0000", 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00054806099999999997 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.298 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62298 10.10 10:0dc9853e:::benchmark_data_smithi017_268344_object62297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.167360+0000", 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.030779709, 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9187170839999999, 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62298 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.299 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167360+0000", 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167360+0000", 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167355+0000", 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999948 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.300 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167433+0000", 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8127000000000006e-05 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167446+0000", 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.2996000000000001e-05 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167452+0000", 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9039999999999997e-06 2024-09-21T10:22:06.301 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167496+0000", 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4366999999999997e-05 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.167503+0000", 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.8900000000000001e-06 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.302 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998005+0000", 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.83050180600000001 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998010+0000", 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4929999999999998e-06 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.065213+0000", 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0672026370000001 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.304 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.065334+0000", 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012127499999999999 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.065466+0000", 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013228900000000001 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069466+0000", 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0039995300000000003 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.305 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073499+0000", 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0040332170000000004 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.084549+0000", 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.011049892 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.084604+0000", 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4500000000000003e-05 2024-09-21T10:22:06.306 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.086077+0000", 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0014733599999999999 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62326 10.10 10:09ee21e6:::benchmark_data_smithi017_268344_object62325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.184514+0000", 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.013625902, 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.9027839289999999, 2024-09-21T10:22:06.307 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62326 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184514+0000", 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.308 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184514+0000", 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184513+0000", 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184562+0000", 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9391000000000002e-05 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184572+0000", 2024-09-21T10:22:06.309 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0069e-05 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184576+0000", 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4510000000000002e-06 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184618+0000", 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.1137000000000003e-05 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.310 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.184623+0000", 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6400000000000002e-06 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998032+0000", 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.813408833 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998038+0000", 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.0700000000000003e-06 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.311 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066048+0000", 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0680093959999999 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066180+0000", 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132409 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066359+0000", 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00017886399999999999 2024-09-21T10:22:06.312 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069474+0000", 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0031148870000000002 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073545+0000", 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0040713709999999998 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.086141+0000", 2024-09-21T10:22:06.313 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.012596424 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.086217+0000", 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.5721000000000002e-05 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.087298+0000", 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0010804 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.314 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62337 10.10 10:0ed49e86:::benchmark_data_smithi017_268344_object62336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.191812+0000", 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 475.006327951, 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.89657432, 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62337 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.315 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191812+0000", 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191812+0000", 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191811+0000", 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191863+0000", 2024-09-21T10:22:06.316 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2701999999999998e-05 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191879+0000", 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5159e-05 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191883+0000", 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4700000000000004e-06 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.317 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191926+0000", 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.2605999999999998e-05 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.191932+0000", 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.2020000000000001e-06 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998059+0000", 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.80612681600000002 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.318 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998064+0000", 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5809999999999996e-06 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.066911+0000", 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.068846288 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067004+0000", 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.3441000000000002e-05 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.319 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067107+0000", 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000103467 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.071083+0000", 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0039751409999999997 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.073578+0000", 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0024952199999999998 2024-09-21T10:22:06.320 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.087347+0000", 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013768786 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.087406+0000", 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.9669000000000001e-05 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.321 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.088386+0000", 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00097964100000000006 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62347 10.10 10:081c52cb:::benchmark_data_smithi017_268344_object62346:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.198848+0000", 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 474.99929125300002, 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8906097960000001, 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.322 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62347 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198848+0000", 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198848+0000", 2024-09-21T10:22:06.323 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198848+0000", 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198886+0000", 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.8263999999999998e-05 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.324 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198892+0000", 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5369999999999997e-06 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198895+0000", 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.9189999999999999e-06 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198940+0000", 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5080999999999997e-05 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.325 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.198945+0000", 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.5709999999999997e-06 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998087+0000", 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.79914121199999999 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998092+0000", 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.8810000000000001e-06 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.326 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067678+0000", 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0695851270000001 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067813+0000", 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013508699999999999 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.067970+0000", 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00015738500000000001 2024-09-21T10:22:06.327 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072284+0000", 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0043139349999999996 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.074428+0000", 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021445000000000001 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.088430+0000", 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014001764 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.328 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.088518+0000", 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.7372000000000003e-05 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.089458+0000", 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00094053099999999998 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62353 10.10 10:0b0475b7:::benchmark_data_smithi017_268344_object62352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.329 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.203016+0000", 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 474.99512344700003, 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.8887158500000001, 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62353 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.330 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203016+0000", 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203016+0000", 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203015+0000", 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.331 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203066+0000", 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.0473000000000003e-05 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203073+0000", 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.8900000000000007e-06 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203077+0000", 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.365e-06 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.332 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203131+0000", 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4598999999999997e-05 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.203141+0000", 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.5100000000000004e-06 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998137+0000", 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.79499567100000001 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.333 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998146+0000", 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.9789999999999999e-06 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.068505+0000", 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.070359587 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.068634+0000", 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000128956 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.334 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.068765+0000", 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00013051800000000001 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072361+0000", 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003596821 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.076196+0000", 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.003834414 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.090509+0000", 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014313090000000001 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.090560+0000", 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.151e-05 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.336 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.091732+0000", 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.001171524 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62385 10.10 10:0e102290:::benchmark_data_smithi017_268344_object62384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.241726+0000", 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 474.95641370599998, 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.850994201, 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.337 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62385 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241726+0000", 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241726+0000", 2024-09-21T10:22:06.338 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241725+0000", 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241778+0000", 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3606000000000001e-05 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.339 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241787+0000", 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1309999999999995e-06 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241792+0000", 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.126e-06 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241832+0000", 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.0929000000000001e-05 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.340 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.241838+0000", 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.519e-06 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998181+0000", 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.75634329300000003 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998186+0000", 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.498e-06 2024-09-21T10:22:06.341 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069517+0000", 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0713316340000001 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069650+0000", 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132099 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.342 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.069825+0000", 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000175415 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072367+0000", 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0025424760000000001 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077894+0000", 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0055264299999999997 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.343 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.091783+0000", 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.013889225 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.091836+0000", 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3300000000000001e-05 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.092720+0000", 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00088369999999999996 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.344 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:62487 10.10 10:0b19b1b2:::benchmark_data_smithi017_268344_object62486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1294)", 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:11.462833+0000", 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 474.73530690199999, 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.630773469, 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 62487 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.345 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462833+0000", 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462833+0000", 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462828+0000", 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999952 2024-09-21T10:22:06.346 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462882+0000", 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.4044e-05 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462892+0000", 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.8230000000000006e-06 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462897+0000", 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.9450000000000001e-06 2024-09-21T10:22:06.347 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462942+0000", 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5859999999999998e-05 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.462949+0000", 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.7689999999999999e-06 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998215+0000", 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.53526555300000001 2024-09-21T10:22:06.348 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:11.998222+0000", 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 7.2099999999999996e-06 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.070360+0000", 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.072138456 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.070487+0000", 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012635700000000001 2024-09-21T10:22:06.349 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 6,7", 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.070612+0000", 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00012507600000000001 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.072371+0000", 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00175961 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.077947+0000", 2024-09-21T10:22:06.350 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0055752609999999998 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.092779+0000", 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.014832675999999999 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.092836+0000", 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6518000000000003e-05 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:13.093606+0000", 2024-09-21T10:22:06.351 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000770216 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:107632 10.d 10:bae075aa:::benchmark_data_smithi017_268344_object107631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1309)", 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:14:53.648790+0000", 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 432.54934949300002, 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0769366060000001, 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.352 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 107632 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.648790+0000", 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.648790+0000", 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.648788+0000", 2024-09-21T10:22:06.353 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999986 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.650952+0000", 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0021638780000000002 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.650970+0000", 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.7306e-05 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.354 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.650976+0000", 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.1099999999999999e-06 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.651039+0000", 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.3382999999999999e-05 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:53.651048+0000", 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 8.8189999999999998e-06 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.355 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.666773+0000", 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0157253959999999 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for missing object", 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.666919+0000", 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000145599 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.696188+0000", 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029269046 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.356 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for degraded object", 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.696233+0000", 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.4929999999999998e-05 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.725635+0000", 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.029402333999999999 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:14:54.725727+0000", 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 9.1385999999999995e-05 2024-09-21T10:22:06.357 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.46058.0:197548 10.2 10:40a3616b:::benchmark_data_smithi017_268344_object71617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:16:26.328429+0000", 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 339.869710183, 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 2.330358935, 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.46058", 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1021486797", 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 197548 2024-09-21T10:22:06.358 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328429+0000", 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328429+0000", 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.359 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328429+0000", 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.9999995 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328432+0000", 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.061e-06 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328445+0000", 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.3256e-05 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.360 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328450+0000", 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4.5279999999999997e-06 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328484+0000", 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.4725999999999999e-05 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:26.328490+0000", 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.6239999999999999e-06 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.361 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:27.306205+0000", 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.97771489 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:27.306215+0000", 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0118e-05 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.639779+0000", 2024-09-21T10:22:06.362 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.333564252 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "started", 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.639912+0000", 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.000132662 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.640308+0000", 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00039639500000000002 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "op_commit", 2024-09-21T10:22:06.363 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.652812+0000", 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.012503284 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.657104+0000", 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.0042921979999999997 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.657983+0000", 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00087909900000000005 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "commit_sent", 2024-09-21T10:22:06.364 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.658020+0000", 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 3.7431999999999997e-05 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:16:28.658788+0000", 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.00076788599999999996 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "description": "osd_op(client.36507.0:187867 11.13 11.ec033fb3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1385)", 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "initiated_at": "2024-09-21T10:20:42.855288+0000", 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "age": 83.342851816999996, 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.86141847100000002, 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "type_data": { 2024-09-21T10:22:06.365 INFO:teuthology.orchestra.run.smithi134.stdout: "flag_point": "reached pg", 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "client_info": { 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "client": "client.36507", 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "client_addr": "172.21.15.17:0/1210895786", 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "tid": 187867 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "events": [ 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "initiated", 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:42.855288+0000", 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "throttled", 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:42.855288+0000", 2024-09-21T10:22:06.366 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "header_read", 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:42.855287+0000", 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 4294967295.999999 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "all_read", 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:42.855293+0000", 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.302e-06 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "dispatched", 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:42.855308+0000", 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.5281000000000002e-05 2024-09-21T10:22:06.367 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "queued_for_pg", 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:42.855315+0000", 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 6.37e-06 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:42.855368+0000", 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.3207000000000001e-05 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "waiting for peered", 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:42.855373+0000", 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 5.2009999999999998e-06 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.368 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "reached_pg", 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:43.716696+0000", 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 0.86132297099999999 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: { 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: "event": "done", 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: "time": "2024-09-21T10:20:43.716706+0000", 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: "duration": 1.0179e-05 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-09-21T10:22:06.369 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-09-21T10:22:06.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:06.393 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:06.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:06.495 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:06.516 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:06.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:06.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:06.641 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:06.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:06.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:06.766 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:06.868 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.890 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:06.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:06.991 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:07.011 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:07.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:07.136 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:06 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1132038482' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:22:07.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:06 smithi017 ceph-mon[102913]: osdmap e1423: 8 total, 8 up, 7 in 2024-09-21T10:22:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:06 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1132038482' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:22:07.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:06 smithi017 ceph-mon[105151]: osdmap e1423: 8 total, 8 up, 7 in 2024-09-21T10:22:07.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:06 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1132038482' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:22:07.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:06 smithi134 ceph-mon[90085]: osdmap e1423: 8 total, 8 up, 7 in 2024-09-21T10:22:07.251 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T10:22:07.251 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-21T10:22:07.251 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-21T10:22:07.256 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-21T10:22:07.257 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-21T10:22:07.257 DEBUG:teuthology.orchestra.run.smithi017:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-21T10:22:07.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.261 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:07.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:07.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.284 INFO:teuthology.orchestra.run.smithi017.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-21T10:22:07.284 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T10:22:07.358 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-21T10:22:07.358 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-21T10:22:07.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:07.414 INFO:tasks.workunit:timeout=3h 2024-09-21T10:22:07.414 INFO:tasks.workunit:cleanup=True 2024-09-21T10:22:07.414 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-21T10:22:07.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.421 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:07.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:07.446 INFO:tasks.workunit.client.0.smithi017.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-21T10:22:07.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:22:07.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:07.552 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:07.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:07.653 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.677 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:07.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:07.778 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:07.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.800 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:07.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:07.897 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:07.901 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:07.922 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:07.922 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:08.024 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:08.045 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:08.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:08.147 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:08.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:07 smithi017 ceph-mon[102913]: pgmap v2353: 103 pgs: 103 active+clean; 626 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:08.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:07 smithi017 ceph-mon[105151]: pgmap v2353: 103 pgs: 103 active+clean; 626 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.172 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:08.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:08.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:07 smithi134 ceph-mon[90085]: pgmap v2353: 103 pgs: 103 active+clean; 626 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:08.274 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:08.296 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:08.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:08.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:08.420 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:08.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:08.544 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:08.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:08.645 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:08.667 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:08.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:08.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:08.791 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:08.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:08.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:08.915 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:08.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:09.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:09.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:09.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:09.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:09.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.164 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:09.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:09.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:09.288 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:09.390 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:09.413 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:09.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:09.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:09.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:22:09.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:22:09.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:22:09.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:22:09.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:22:09.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:22:09.536 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:22:09.536 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:22:09.536 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:22:09.536 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:22:09.536 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:22:09.536 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:22:09.537 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:09.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:09.543 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:09.553 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:22:09.553 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:22:09.570 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:22:09.570 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:22:09.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:09.662 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:09.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:09.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:09.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:09.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:09.887 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:09.912 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:10.014 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:10.038 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:10.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:10.162 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:10.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:09 smithi017 ceph-mon[102913]: 1.2 deep-scrub starts 2024-09-21T10:22:10.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:09 smithi017 ceph-mon[102913]: 1.2 deep-scrub ok 2024-09-21T10:22:10.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:09 smithi017 ceph-mon[102913]: pgmap v2354: 103 pgs: 103 active+clean; 626 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:10.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:09 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1124270896' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:09 smithi017 ceph-mon[105151]: 1.2 deep-scrub starts 2024-09-21T10:22:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:09 smithi017 ceph-mon[105151]: 1.2 deep-scrub ok 2024-09-21T10:22:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:09 smithi017 ceph-mon[105151]: pgmap v2354: 103 pgs: 103 active+clean; 626 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:10.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:09 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1124270896' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:10.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:09 smithi134 ceph-mon[90085]: 1.2 deep-scrub starts 2024-09-21T10:22:10.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:09 smithi134 ceph-mon[90085]: 1.2 deep-scrub ok 2024-09-21T10:22:10.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:09 smithi134 ceph-mon[90085]: pgmap v2354: 103 pgs: 103 active+clean; 626 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:10.215 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:09 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1124270896' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:10.265 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:10.290 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:10.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:10.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:22:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:22:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:22:10.516 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:10.542 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:10.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:10.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:10.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:10.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:10.768 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:10.793 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:10.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:10.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:10.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:11.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:11.045 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:11.146 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:11.171 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:11.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:10 smithi017 ceph-mon[102913]: 1.13 scrub starts 2024-09-21T10:22:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:10 smithi017 ceph-mon[102913]: 1.13 scrub ok 2024-09-21T10:22:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:11.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:10 smithi017 ceph-mon[105151]: 1.13 scrub starts 2024-09-21T10:22:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:10 smithi017 ceph-mon[105151]: 1.13 scrub ok 2024-09-21T10:22:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:11.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:10 smithi134 ceph-mon[90085]: 1.13 scrub starts 2024-09-21T10:22:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:10 smithi134 ceph-mon[90085]: 1.13 scrub ok 2024-09-21T10:22:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:11.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:11.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:11.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:11.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:11.420 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:11.522 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:11.546 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:11.647 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:11.670 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:11.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:11.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:11.794 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:11.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:11.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:11.918 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:11.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:12.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:12.042 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:12.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:12.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:11 smithi017 ceph-mon[102913]: pgmap v2355: 103 pgs: 103 active+clean; 599 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:12.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:11 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:12.110 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:11 smithi017 ceph-mon[105151]: pgmap v2355: 103 pgs: 103 active+clean; 599 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:12.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:11 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:12.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:12.166 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:12.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:11 smithi134 ceph-mon[90085]: pgmap v2355: 103 pgs: 103 active+clean; 599 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:12.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:11 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:12.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:12.290 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:12.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:12.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.415 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:12.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:12.516 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:12.542 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:12.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:12.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:12.666 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:12.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:12.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:12.792 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:12.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:12.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:12.916 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:12.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:13.017 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:13.044 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:13.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:13.170 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:13.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:13.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:13.295 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:13.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:13.397 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:13.423 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:13.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:13.525 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:13.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:22:13.551 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:13.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:13.675 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:13.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:13.777 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:13.802 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:13.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:13.904 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:13.929 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:13.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:13.952 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2024-09-21T10:22:14.030 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:14.054 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:14.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:14.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:14.179 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:14.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:13 smithi017 ceph-mon[102913]: pgmap v2356: 103 pgs: 103 active+clean; 599 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:14.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:13 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3440620396' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:13 smithi017 ceph-mon[105151]: pgmap v2356: 103 pgs: 103 active+clean; 599 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:14.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:13 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3440620396' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:14.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:13 smithi134 ceph-mon[90085]: pgmap v2356: 103 pgs: 103 active+clean; 599 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:14.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:13 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3440620396' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:14.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:14.303 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:14.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:14.404 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:14.431 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:14.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:14.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:14.554 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:14.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:14.597 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-21T10:22:14.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:moved 3 / 309 (0.970874%) 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:avg 44.1429 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 9.15624 -> 8.39096 (expected baseline 6.15116) 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 29 -> 32 pgs (0.656958 -> 0.724919 * mean) 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.6 with 57 -> 57 pgs (1.29126 -> 1.29126 * mean) 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0121 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.1215 2024-09-21T10:22:14.663 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 0.5051 -> 0.6323 2024-09-21T10:22:14.664 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2024-09-21T10:22:14.679 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:14.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:14.684 INFO:tasks.workunit.client.0.smithi017.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-21T10:22:14.686 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T10:22:14.687 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-21T10:22:14.781 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:14.806 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:14.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:14.907 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:14.932 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:14.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:22:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T10:22:14.660+0000 7f3131349640 -1 mgr.server reply reply (1) Operation not permitted no change 2024-09-21T10:22:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:22:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:22:14] "GET /metrics HTTP/1.1" 200 34353 "" "Prometheus/2.51.0" 2024-09-21T10:22:15.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:15.057 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:15.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:15.159 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:15.183 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:14 smithi134 ceph-mon[90085]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3440620396' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:22:15.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:14 smithi134 ceph-mon[90085]: osdmap e1424: 8 total, 8 up, 7 in 2024-09-21T10:22:15.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:15.309 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:15.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:15.435 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:14 smithi017 ceph-mon[105151]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3440620396' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:22:15.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:14 smithi017 ceph-mon[105151]: osdmap e1424: 8 total, 8 up, 7 in 2024-09-21T10:22:15.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:14 smithi017 ceph-mon[102913]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:15.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3440620396' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:22:15.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:14 smithi017 ceph-mon[102913]: osdmap e1424: 8 total, 8 up, 7 in 2024-09-21T10:22:15.537 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:15.564 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:15.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:15.665 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:15.690 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:15.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:15.791 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:15.816 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:15.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:15.917 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:15.945 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:15.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:16.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:16.067 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:22:16.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 1000 1.0 2024-09-21T10:22:16.072 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:16.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:16.199 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:16.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:16.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:15 smithi134 ceph-mon[90085]: pgmap v2358: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:16.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:15 smithi134 ceph-mon[90085]: from='client.36711 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:22:16.301 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:16.327 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:16.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:16.428 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:15 smithi017 ceph-mon[102913]: pgmap v2358: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:15 smithi017 ceph-mon[102913]: from='client.36711 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:22:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:15 smithi017 ceph-mon[105151]: pgmap v2358: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:16.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:15 smithi017 ceph-mon[105151]: from='client.36711 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:22:16.453 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:16.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:16.555 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:16.580 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:16.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:16.682 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:16.707 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:16.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:16.834 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:16.909 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2024-09-21T10:22:16.909 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:22:16.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:16.958 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:17.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:17.082 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:17.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:17.184 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:17.209 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:17.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:17.248 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:17.266 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:17.310 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:17.335 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:17.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:17.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:17.459 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:17.511 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-21T10:22:17.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:17.582 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:17.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:17.684 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:17.707 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:17.770 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-21T10:22:17.809 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:17.823 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T10:22:17.824 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T10:22:17.824 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-21T10:22:17.831 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:17.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:17.865 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T10:22:17.898 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2024-09-21T10:22:17.898 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-21T10:22:17.933 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:17.943 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-09-21T10:22:17.944 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-21T10:22:17.944 DEBUG:teuthology.orchestra.run.smithi017:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-21T10:22:17.956 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:18.058 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:18.080 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:18.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:18.182 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:18.192 INFO:tasks.workunit.client.0.smithi017.stdout:Running main() from gmock_main.cc 2024-09-21T10:22:18.192 INFO:tasks.workunit.client.0.smithi017.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-21T10:22:18.192 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] Global test environment set-up. 2024-09-21T10:22:18.192 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] 70 tests from TestClsRbd 2024-09-21T10:22:18.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:17 smithi017 ceph-mon[105151]: pgmap v2359: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:18.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:17 smithi017 ceph-mon[102913]: pgmap v2359: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:18.204 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:18.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:18.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:22:18.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:17 smithi134 ceph-mon[90085]: pgmap v2359: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:18.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:22:18.306 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:18.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.330 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:18.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:18.431 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:18.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:18.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:18.556 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:18.580 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:18.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:18.682 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:18.705 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:18.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:18.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:18.829 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:18.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:18.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:18.953 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:18.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:19.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:19.076 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:19.178 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:19.203 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:19.305 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:19.331 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:19.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:19.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:19.455 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:19.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:19.580 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:19.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:19.682 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:19.707 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:19.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:19.810 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:19.837 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:19.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:19.964 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:19.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:19.974 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-21T10:22:20.007 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_all_features (34 ms) 2024-09-21T10:22:20.007 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.copyup 2024-09-21T10:22:20.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:20.092 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:20.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:20.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:20.211 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.copyup (204 ms) 2024-09-21T10:22:20.211 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-21T10:22:20.217 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:20.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:20.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:19 smithi134 ceph-mon[90085]: pgmap v2360: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:20.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:19 smithi134 ceph-mon[90085]: osdmap e1425: 8 total, 8 up, 7 in 2024-09-21T10:22:20.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:19 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4016904340' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-331449-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:22:20.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:22:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:22:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:22:20.251 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:20.251 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:20.253 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.sparse_copyup (42 ms) 2024-09-21T10:22:20.253 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-21T10:22:20.261 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2024-09-21T10:22:20.261 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-21T10:22:20.284 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.add_remove_child (23 ms) 2024-09-21T10:22:20.285 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-21T10:22:20.312 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.directory_methods (28 ms) 2024-09-21T10:22:20.312 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.create 2024-09-21T10:22:20.319 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:20.325 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.create (13 ms) 2024-09-21T10:22:20.325 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_features 2024-09-21T10:22:20.327 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-21T10:22:20.327 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-21T10:22:20.329 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-09-21T10:22:20.329 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-21T10:22:20.330 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2024-09-21T10:22:20.330 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-21T10:22:20.332 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-09-21T10:22:20.332 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-21T10:22:20.333 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-21T10:22:20.333 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-21T10:22:20.336 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-09-21T10:22:20.337 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_size 2024-09-21T10:22:20.340 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-21T10:22:20.340 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.set_size 2024-09-21T10:22:20.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:20.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:20.344 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-09-21T10:22:20.345 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-21T10:22:20.362 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2024-09-21T10:22:20.362 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-21T10:22:20.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:19 smithi017 ceph-mon[102913]: pgmap v2360: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:20.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:19 smithi017 ceph-mon[102913]: osdmap e1425: 8 total, 8 up, 7 in 2024-09-21T10:22:20.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:19 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4016904340' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-331449-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:22:20.364 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:19 smithi017 ceph-mon[105151]: pgmap v2360: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:20.364 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:19 smithi017 ceph-mon[105151]: osdmap e1425: 8 total, 8 up, 7 in 2024-09-21T10:22:20.364 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:19 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4016904340' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-331449-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:22:20.369 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-09-21T10:22:20.369 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-21T10:22:20.421 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.parents_v1 (52 ms) 2024-09-21T10:22:20.421 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-21T10:22:20.443 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:20.450 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2024-09-21T10:22:20.450 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-21T10:22:20.465 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:20.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:20.475 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.snapshots (24 ms) 2024-09-21T10:22:20.475 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-21T10:22:20.478 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.snapid_race (4 ms) 2024-09-21T10:22:20.479 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-21T10:22:20.487 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2024-09-21T10:22:20.488 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-21T10:22:20.492 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_save (5 ms) 2024-09-21T10:22:20.492 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-21T10:22:20.497 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_resize (4 ms) 2024-09-21T10:22:20.497 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-21T10:22:20.504 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_update (7 ms) 2024-09-21T10:22:20.504 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-21T10:22:20.504 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-21T10:22:20.504 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-21T10:22:20.508 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_snap_add (5 ms) 2024-09-21T10:22:20.508 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-21T10:22:20.515 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_snap_remove (6 ms) 2024-09-21T10:22:20.515 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.flags 2024-09-21T10:22:20.519 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-09-21T10:22:20.519 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.metadata 2024-09-21T10:22:20.551 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.metadata (32 ms) 2024-09-21T10:22:20.551 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.set_features 2024-09-21T10:22:20.558 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-09-21T10:22:20.558 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror 2024-09-21T10:22:20.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:20.589 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:20.592 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror (35 ms) 2024-09-21T10:22:20.593 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-21T10:22:20.608 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2024-09-21T10:22:20.609 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-21T10:22:20.692 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:20.716 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:20.819 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:20.842 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:20.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:20.944 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:20.966 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:21.067 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:21.068 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:20 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/4016904340' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-331449-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:22:21.068 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:20 smithi017 ceph-mon[102913]: osdmap e1426: 8 total, 8 up, 7 in 2024-09-21T10:22:21.068 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:20 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/4016904340' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-331449-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:22:21.068 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:20 smithi017 ceph-mon[105151]: osdmap e1426: 8 total, 8 up, 7 in 2024-09-21T10:22:21.092 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:21.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:21.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:21.216 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:21.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:21.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:20 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/4016904340' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-331449-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:22:21.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:20 smithi134 ceph-mon[90085]: osdmap e1426: 8 total, 8 up, 7 in 2024-09-21T10:22:21.317 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:21.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:21.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:21.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:21.465 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:21.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:21.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:21.588 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:21.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:21.691 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:21.715 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:21.817 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:21.840 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:21.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:21.907 INFO:teuthology.orchestra.run.smithi017.stdout:moved 4 / 405 (0.987654%) 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:avg 57.8571 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 12.3916 -> 11.5935 (expected baseline 7.04215) 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.5 with 38 -> 42 pgs (0.65679 -> 0.725926 * mean) 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.6 with 77 -> 77 pgs (1.33086 -> 1.33086 * mean) 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0121 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.1215 2024-09-21T10:22:21.908 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 0.5051 -> 0.6325 2024-09-21T10:22:21.909 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2024-09-21T10:22:21.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:21.964 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:21.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:22.066 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:22.089 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:22.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:22.191 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:21 smithi017 ceph-mon[102913]: pgmap v2363: 135 pgs: 26 unknown, 109 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 18 KiB/s rd, 4.2 KiB/s wr, 25 op/s 2024-09-21T10:22:22.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:21 smithi017 ceph-mon[102913]: osdmap e1427: 8 total, 8 up, 7 in 2024-09-21T10:22:22.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:21 smithi017 ceph-mon[105151]: pgmap v2363: 135 pgs: 26 unknown, 109 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 18 KiB/s rd, 4.2 KiB/s wr, 25 op/s 2024-09-21T10:22:22.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:21 smithi017 ceph-mon[105151]: osdmap e1427: 8 total, 8 up, 7 in 2024-09-21T10:22:22.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:22:21 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T10:22:21.904+0000 7f3131349640 -1 mgr.server reply reply (1) Operation not permitted no change 2024-09-21T10:22:22.215 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:22.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:22.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:21 smithi134 ceph-mon[90085]: pgmap v2363: 135 pgs: 26 unknown, 109 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 18 KiB/s rd, 4.2 KiB/s wr, 25 op/s 2024-09-21T10:22:22.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:21 smithi134 ceph-mon[90085]: osdmap e1427: 8 total, 8 up, 7 in 2024-09-21T10:22:22.317 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:22.341 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:22.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:22.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:22.467 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:22.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:22.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:22.661 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:22.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:22.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:22.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.786 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:22.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:22.887 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:22.909 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:22.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:23.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:23.032 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:23.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:23.063 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_image_status (2455 ms) 2024-09-21T10:22:23.063 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-21T10:22:23.135 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:23.152 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_image_map (89 ms) 2024-09-21T10:22:23.152 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-21T10:22:23.156 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:23.159 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_instances (7 ms) 2024-09-21T10:22:23.159 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-21T10:22:23.179 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_snapshot (20 ms) 2024-09-21T10:22:23.179 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-21T10:22:23.181 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-09-21T10:22:23.182 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-21T10:22:23.183 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-21T10:22:23.184 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-21T10:22:23.186 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-21T10:22:23.186 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-21T10:22:23.192 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-09-21T10:22:23.192 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-21T10:22:23.195 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-21T10:22:23.195 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-21T10:22:23.199 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-09-21T10:22:23.199 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-21T10:22:23.202 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2024-09-21T10:22:23.202 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-21T10:22:23.205 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-21T10:22:23.205 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-21T10:22:23.208 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-21T10:22:23.208 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-21T10:22:23.213 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2024-09-21T10:22:23.213 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-21T10:22:23.216 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2024-09-21T10:22:23.216 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-21T10:22:23.219 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-21T10:22:23.219 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-21T10:22:23.222 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-21T10:22:23.222 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-21T10:22:23.224 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-21T10:22:23.224 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-21T10:22:23.226 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-21T10:22:23.226 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-21T10:22:23.229 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (4 ms) 2024-09-21T10:22:23.229 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-21T10:22:23.231 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2024-09-21T10:22:23.231 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-21T10:22:23.234 INFO:tasks.workunit.client.0.smithi017.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1649-gfbd1a6ab/rpm/el9/BUILD/ceph-17.2.7-1649-gfbd1a6ab/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-21T10:22:23.234 INFO:tasks.workunit.client.0.smithi017.stdout:Expected equality of these values: 2024-09-21T10:22:23.234 INFO:tasks.workunit.client.0.smithi017.stdout: 1U 2024-09-21T10:22:23.234 INFO:tasks.workunit.client.0.smithi017.stdout: Which is: 1 2024-09-21T10:22:23.234 INFO:tasks.workunit.client.0.smithi017.stdout: keys.size() 2024-09-21T10:22:23.234 INFO:tasks.workunit.client.0.smithi017.stdout: Which is: 3 2024-09-21T10:22:23.235 INFO:tasks.workunit.client.0.smithi017.stdout:[ FAILED ] TestClsRbd.group_snap_set (3 ms) 2024-09-21T10:22:23.235 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-21T10:22:23.237 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-21T10:22:23.237 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-21T10:22:23.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:22 smithi134 ceph-mon[90085]: from='client.46559 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:22:23.253 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:22 smithi017 ceph-mon[102913]: from='client.46559 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:22:23.253 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:22 smithi017 ceph-mon[105151]: from='client.46559 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:22:23.255 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (18 ms) 2024-09-21T10:22:23.255 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-21T10:22:23.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:23.259 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2024-09-21T10:22:23.259 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:22:23.276 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2024-09-21T10:22:23.276 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:22:23.285 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:23.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:23.297 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2024-09-21T10:22:23.298 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:22:23.316 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2024-09-21T10:22:23.316 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:22:23.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2024-09-21T10:22:23.335 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:22:23.353 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2024-09-21T10:22:23.353 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:22:23.371 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2024-09-21T10:22:23.371 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:22:23.387 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:23.390 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2024-09-21T10:22:23.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:22:23.408 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:23.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:23.479 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (224 ms) 2024-09-21T10:22:23.479 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-21T10:22:23.481 INFO:tasks.workunit.client.0.smithi017.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1649-gfbd1a6ab/rpm/el9/BUILD/ceph-17.2.7-1649-gfbd1a6ab/src/test/cls_rbd/test_cls_rbd.cc:2792: Failure 2024-09-21T10:22:23.481 INFO:tasks.workunit.client.0.smithi017.stdout:Expected equality of these values: 2024-09-21T10:22:23.481 INFO:tasks.workunit.client.0.smithi017.stdout: 1U 2024-09-21T10:22:23.481 INFO:tasks.workunit.client.0.smithi017.stdout: Which is: 1 2024-09-21T10:22:23.481 INFO:tasks.workunit.client.0.smithi017.stdout: keys.size() 2024-09-21T10:22:23.482 INFO:tasks.workunit.client.0.smithi017.stdout: Which is: 3 2024-09-21T10:22:23.482 INFO:tasks.workunit.client.0.smithi017.stdout:[ FAILED ] TestClsRbd.group_snap_remove (2 ms) 2024-09-21T10:22:23.482 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-21T10:22:23.483 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-21T10:22:23.483 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-21T10:22:23.490 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-09-21T10:22:23.490 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.op_features 2024-09-21T10:22:23.496 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.op_features (6 ms) 2024-09-21T10:22:23.496 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-21T10:22:23.510 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:23.516 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2024-09-21T10:22:23.516 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-21T10:22:23.524 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-21T10:22:23.524 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-21T10:22:23.531 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:23.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:23.536 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2024-09-21T10:22:23.536 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-21T10:22:23.542 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2024-09-21T10:22:23.543 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.migration 2024-09-21T10:22:23.552 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-09-21T10:22:23.552 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-21T10:22:23.563 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.migration_v1 (11 ms) 2024-09-21T10:22:23.563 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-21T10:22:23.633 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:23.654 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:23.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:23.756 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:23.777 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:23.879 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:23.899 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:23.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:24.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:24.024 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:24.126 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:24.147 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:24.174 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:23 smithi017 ceph-mon[102913]: pgmap v2365: 135 pgs: 9 unknown, 126 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1.6 MiB/s rd, 816 KiB/s wr, 408 op/s 2024-09-21T10:22:24.174 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:23 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3167202075' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:24.174 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:23 smithi017 ceph-mon[105151]: pgmap v2365: 135 pgs: 9 unknown, 126 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1.6 MiB/s rd, 816 KiB/s wr, 408 op/s 2024-09-21T10:22:24.175 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:23 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3167202075' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:24.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:23 smithi134 ceph-mon[90085]: pgmap v2365: 135 pgs: 9 unknown, 126 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1.6 MiB/s rd, 816 KiB/s wr, 408 op/s 2024-09-21T10:22:24.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:23 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3167202075' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:24.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:24.272 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:24.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:24.374 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:24.395 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:24.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:24.497 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:24.518 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:24.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:24.620 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:24.641 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:24.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:24.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:24.764 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:24.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:24.866 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:24.888 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:24.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:24.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:22:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:22:24] "GET /metrics HTTP/1.1" 200 34353 "" "Prometheus/2.51.0" 2024-09-21T10:22:24.990 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:25.001 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1439 ms) 2024-09-21T10:22:25.002 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-21T10:22:25.017 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:25.035 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.sparsify (34 ms) 2024-09-21T10:22:25.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:25.143 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:25.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:25.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:25.269 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:25.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:25.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:25.394 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:25.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:25.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:24 smithi017 ceph-mon[102913]: osdmap e1428: 8 total, 8 up, 7 in 2024-09-21T10:22:25.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:24 smithi017 ceph-mon[105151]: osdmap e1428: 8 total, 8 up, 7 in 2024-09-21T10:22:25.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:24 smithi134 ceph-mon[90085]: osdmap e1428: 8 total, 8 up, 7 in 2024-09-21T10:22:25.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:25.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.521 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:25.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:25.622 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:25.645 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:25.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:25.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:25.769 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:25.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:25.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:25.893 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:25.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:25.995 DEBUG:teuthology.orchestra.run.smithi134:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:26.007 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] 70 tests from TestClsRbd (5062 ms total) 2024-09-21T10:22:26.007 INFO:tasks.workunit.client.0.smithi017.stdout: 2024-09-21T10:22:26.007 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] Global test environment tear-down 2024-09-21T10:22:26.007 INFO:tasks.workunit.client.0.smithi017.stdout:[==========] 70 tests from 1 test suite ran. (7815 ms total) 2024-09-21T10:22:26.007 INFO:tasks.workunit.client.0.smithi017.stdout:[ PASSED ] 68 tests. 2024-09-21T10:22:26.008 INFO:tasks.workunit.client.0.smithi017.stdout:[ FAILED ] 2 tests, listed below: 2024-09-21T10:22:26.008 INFO:tasks.workunit.client.0.smithi017.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-21T10:22:26.008 INFO:tasks.workunit.client.0.smithi017.stdout:[ FAILED ] TestClsRbd.group_snap_remove 2024-09-21T10:22:26.008 INFO:tasks.workunit.client.0.smithi017.stdout: 2024-09-21T10:22:26.008 INFO:tasks.workunit.client.0.smithi017.stdout: 2 FAILED TESTS 2024-09-21T10:22:26.025 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: docker: command not found 2024-09-21T10:22:26.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi134 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.027 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-09-21T10:22:26.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-21T10:22:26.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:26.149 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: docker: command not found 2024-09-21T10:22:26.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 1: 'sudo docker kill -s 1 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:26.247 INFO:tasks.thrashosds:joining thrashosds 2024-09-21T10:22:26.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:25 smithi017 ceph-mon[102913]: pgmap v2367: 135 pgs: 135 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1.7 MiB/s rd, 877 KiB/s wr, 473 op/s 2024-09-21T10:22:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:25 smithi017 ceph-mon[102913]: osdmap e1429: 8 total, 8 up, 7 in 2024-09-21T10:22:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:26.287 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:25 smithi017 ceph-mon[105151]: pgmap v2367: 135 pgs: 135 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1.7 MiB/s rd, 877 KiB/s wr, 473 op/s 2024-09-21T10:22:26.287 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:25 smithi017 ceph-mon[105151]: osdmap e1429: 8 total, 8 up, 7 in 2024-09-21T10:22:26.287 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:26.287 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:25 smithi134 ceph-mon[90085]: pgmap v2367: 135 pgs: 135 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 1.7 MiB/s rd, 877 KiB/s wr, 473 op/s 2024-09-21T10:22:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:25 smithi134 ceph-mon[90085]: osdmap e1429: 8 total, 8 up, 7 in 2024-09-21T10:22:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:26.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:26.846 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:22:26.846 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2024-09-21T10:22:26.847 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2024-09-21T10:22:26.847 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2024-09-21T10:22:26.847 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:22:27.007 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2024-09-21T10:22:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:27 smithi017 ceph-mon[102913]: osdmap e1430: 8 total, 8 up, 7 in 2024-09-21T10:22:27.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:27 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/763615368' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:27.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:27 smithi017 ceph-mon[105151]: osdmap e1430: 8 total, 8 up, 7 in 2024-09-21T10:22:27.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:27 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/763615368' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:27 smithi134 ceph-mon[90085]: osdmap e1430: 8 total, 8 up, 7 in 2024-09-21T10:22:27.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:27 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/763615368' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:27.755 INFO:tasks.thrashosds.thrasher:inning osd 2024-09-21T10:22:27.756 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-09-21T10:22:27.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd in 1 2024-09-21T10:22:28.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:22:28.204 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:28.327 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:28 smithi017 ceph-mon[102913]: pgmap v2370: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:28.327 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:28 smithi017 ceph-mon[102913]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:28.327 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:28 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/763615368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:28.327 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:28 smithi017 ceph-mon[102913]: osdmap e1431: 8 total, 8 up, 7 in 2024-09-21T10:22:28.327 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:28 smithi017 ceph-mon[105151]: pgmap v2370: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:28.327 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:28 smithi017 ceph-mon[105151]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:28.327 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:28 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/763615368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:28.328 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:28 smithi017 ceph-mon[105151]: osdmap e1431: 8 total, 8 up, 7 in 2024-09-21T10:22:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:28 smithi134 ceph-mon[90085]: pgmap v2370: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:28 smithi134 ceph-mon[90085]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:28.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:28 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/763615368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:28.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:28 smithi134 ceph-mon[90085]: osdmap e1431: 8 total, 8 up, 7 in 2024-09-21T10:22:29.447 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:29.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:22:30.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:30 smithi017 ceph-mon[105151]: pgmap v2372: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:30.219 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:30 smithi017 ceph-mon[102913]: pgmap v2372: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:30.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:30 smithi134 ceph-mon[90085]: pgmap v2372: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:22:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:22:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:22:31.476 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:31.480 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2024-09-21T10:22:31.480 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2024-09-21T10:22:31.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2024-09-21T10:22:31.480 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [] 2024-09-21T10:22:31.480 INFO:teuthology.orchestra.run.smithi017.stdout:} 2024-09-21T10:22:31.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:31 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:22:31.418+0000 7f67e36e5640 -1 mon.a@0(leader).osd e1431 definitely_dead 0 2024-09-21T10:22:32.013 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.1. 2024-09-21T10:22:32.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:31 smithi017 ceph-mon[102913]: pgmap v2373: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:32.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:31 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:32.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:31 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2756961836' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:22:32.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:31 smithi017 ceph-mon[105151]: pgmap v2373: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:32.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:31 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:32.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:31 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2756961836' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:22:32.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:31 smithi134 ceph-mon[90085]: pgmap v2373: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:32.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:31 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:32.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:31 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2756961836' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-21T10:22:33.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:33 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2756961836' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:22:33.020 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:33 smithi017 ceph-mon[102913]: osdmap e1432: 8 total, 8 up, 8 in 2024-09-21T10:22:33.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:33 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2756961836' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:22:33.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:33 smithi017 ceph-mon[105151]: osdmap e1432: 8 total, 8 up, 8 in 2024-09-21T10:22:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:33 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2756961836' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-21T10:22:33.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:33 smithi134 ceph-mon[90085]: osdmap e1432: 8 total, 8 up, 8 in 2024-09-21T10:22:33.563 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-09-21T10:22:33.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-09-21T10:22:34.338 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:34 smithi017 ceph-mon[102913]: pgmap v2375: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:34.338 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:34 smithi017 ceph-mon[102913]: osdmap e1433: 8 total, 8 up, 8 in 2024-09-21T10:22:34.338 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:34 smithi017 ceph-mon[105151]: pgmap v2375: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:34.339 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:34 smithi017 ceph-mon[105151]: osdmap e1433: 8 total, 8 up, 8 in 2024-09-21T10:22:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:34 smithi134 ceph-mon[90085]: pgmap v2375: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2024-09-21T10:22:34.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:34 smithi134 ceph-mon[90085]: osdmap e1433: 8 total, 8 up, 8 in 2024-09-21T10:22:34.625 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:35.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:22:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:22:34] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-21T10:22:35.028 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2024-09-21T10:22:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/704793803' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:35.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:35 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:35.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/704793803' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:35.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:35 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/704793803' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:35.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:35 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:36.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:22:36.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:36 smithi134 ceph-mon[90085]: pgmap v2377: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:22:36.395 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:36 smithi134 ceph-mon[90085]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:36.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:36 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:36.396 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:36 smithi134 ceph-mon[90085]: osdmap e1434: 8 total, 8 up, 8 in 2024-09-21T10:22:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:36 smithi017 ceph-mon[102913]: pgmap v2377: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:22:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:36 smithi017 ceph-mon[102913]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:36 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:36.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:36 smithi017 ceph-mon[102913]: osdmap e1434: 8 total, 8 up, 8 in 2024-09-21T10:22:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:36 smithi017 ceph-mon[105151]: pgmap v2377: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:22:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:36 smithi017 ceph-mon[105151]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:36 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:36.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:36 smithi017 ceph-mon[105151]: osdmap e1434: 8 total, 8 up, 8 in 2024-09-21T10:22:36.865 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:37.346 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a: {} 2024-09-21T10:22:37.347 INFO:teuthology.orchestra.run.smithi017.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:22:37.355 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b: {} 2024-09-21T10:22:37.355 INFO:teuthology.orchestra.run.smithi017.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:22:37.364 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c: {} 2024-09-21T10:22:37.364 INFO:teuthology.orchestra.run.smithi017.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:22:38.128 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:38 smithi017 ceph-mon[102913]: pgmap v2379: 103 pgs: 3 peering, 100 active+clean; 639 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 27 KiB/s wr, 7 op/s; 9.5 KiB/s, 0 objects/s recovering 2024-09-21T10:22:38.128 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:38 smithi017 ceph-mon[102913]: 1.16 scrub starts 2024-09-21T10:22:38.128 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:38 smithi017 ceph-mon[102913]: 1.16 scrub ok 2024-09-21T10:22:38.128 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:38 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2100772848' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:22:38.128 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:38 smithi017 ceph-mon[105151]: pgmap v2379: 103 pgs: 3 peering, 100 active+clean; 639 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 27 KiB/s wr, 7 op/s; 9.5 KiB/s, 0 objects/s recovering 2024-09-21T10:22:38.128 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:38 smithi017 ceph-mon[105151]: 1.16 scrub starts 2024-09-21T10:22:38.128 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:38 smithi017 ceph-mon[105151]: 1.16 scrub ok 2024-09-21T10:22:38.128 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:38 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2100772848' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:22:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:38 smithi134 ceph-mon[90085]: pgmap v2379: 103 pgs: 3 peering, 100 active+clean; 639 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 27 KiB/s wr, 7 op/s; 9.5 KiB/s, 0 objects/s recovering 2024-09-21T10:22:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:38 smithi134 ceph-mon[90085]: 1.16 scrub starts 2024-09-21T10:22:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:38 smithi134 ceph-mon[90085]: 1.16 scrub ok 2024-09-21T10:22:38.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:38 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2100772848' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:22:38.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 0 1 2024-09-21T10:22:39.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:39 smithi017 ceph-mon[102913]: 1.3 scrub starts 2024-09-21T10:22:39.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:39 smithi017 ceph-mon[102913]: 1.3 scrub ok 2024-09-21T10:22:39.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:39 smithi017 ceph-mon[102913]: 1.18 scrub starts 2024-09-21T10:22:39.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:39 smithi017 ceph-mon[102913]: 1.18 scrub ok 2024-09-21T10:22:39.379 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:39 smithi017 ceph-mon[105151]: 1.3 scrub starts 2024-09-21T10:22:39.379 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:39 smithi017 ceph-mon[105151]: 1.3 scrub ok 2024-09-21T10:22:39.379 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:39 smithi017 ceph-mon[105151]: 1.18 scrub starts 2024-09-21T10:22:39.379 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:39 smithi017 ceph-mon[105151]: 1.18 scrub ok 2024-09-21T10:22:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:39 smithi134 ceph-mon[90085]: 1.3 scrub starts 2024-09-21T10:22:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:39 smithi134 ceph-mon[90085]: 1.3 scrub ok 2024-09-21T10:22:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:39 smithi134 ceph-mon[90085]: 1.18 scrub starts 2024-09-21T10:22:39.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:39 smithi134 ceph-mon[90085]: 1.18 scrub ok 2024-09-21T10:22:39.631 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:40.043 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2024-09-21T10:22:40.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:40 smithi017 ceph-mon[102913]: pgmap v2380: 103 pgs: 103 active+clean; 639 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 11 KiB/s rd, 24 KiB/s wr, 6 op/s; 47 KiB/s, 0 objects/s recovering 2024-09-21T10:22:40.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:40 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1136626837' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:40.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:40 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:40.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:40 smithi017 ceph-mon[105151]: pgmap v2380: 103 pgs: 103 active+clean; 639 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 11 KiB/s rd, 24 KiB/s wr, 6 op/s; 47 KiB/s, 0 objects/s recovering 2024-09-21T10:22:40.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:40 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1136626837' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:40.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:40 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:40 smithi134 ceph-mon[90085]: pgmap v2380: 103 pgs: 103 active+clean; 639 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 11 KiB/s rd, 24 KiB/s wr, 6 op/s; 47 KiB/s, 0 objects/s recovering 2024-09-21T10:22:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:40 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1136626837' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:40.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:40 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:22:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:22:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: 1.13 scrub starts 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: 1.13 scrub ok 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: Cluster is now healthy 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: osdmap e1435: 8 total, 8 up, 8 in 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-09-21T10:22:41.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]} : dispatch 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: 1.13 scrub starts 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: 1.13 scrub ok 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: Cluster is now healthy 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: osdmap e1435: 8 total, 8 up, 8 in 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-21T10:22:41.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:41 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]} : dispatch 2024-09-21T10:22:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: 1.13 scrub starts 2024-09-21T10:22:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: 1.13 scrub ok 2024-09-21T10:22:41.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:41.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: Cluster is now healthy 2024-09-21T10:22:41.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:41.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: osdmap e1435: 8 total, 8 up, 8 in 2024-09-21T10:22:41.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:41.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:41.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-09-21T10:22:41.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-21T10:22:41.495 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:41 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]} : dispatch 2024-09-21T10:22:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[102913]: pgmap v2382: 103 pgs: 103 active+clean; 639 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 9.6 KiB/s rd, 21 KiB/s wr, 5 op/s; 41 KiB/s, 0 objects/s recovering 2024-09-21T10:22:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T10:22:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-21T10:22:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-21T10:22:42.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[102913]: osdmap e1436: 8 total, 8 up, 8 in 2024-09-21T10:22:42.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[105151]: pgmap v2382: 103 pgs: 103 active+clean; 639 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 9.6 KiB/s rd, 21 KiB/s wr, 5 op/s; 41 KiB/s, 0 objects/s recovering 2024-09-21T10:22:42.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T10:22:42.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-21T10:22:42.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-21T10:22:42.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:42 smithi017 ceph-mon[105151]: osdmap e1436: 8 total, 8 up, 8 in 2024-09-21T10:22:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:42 smithi134 ceph-mon[90085]: pgmap v2382: 103 pgs: 103 active+clean; 639 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 9.6 KiB/s rd, 21 KiB/s wr, 5 op/s; 41 KiB/s, 0 objects/s recovering 2024-09-21T10:22:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T10:22:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-21T10:22:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:42 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-21T10:22:42.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:42 smithi134 ceph-mon[90085]: osdmap e1436: 8 total, 8 up, 8 in 2024-09-21T10:22:43.320 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2024-09-21T10:22:43.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:43 smithi017 ceph-mon[102913]: osdmap e1437: 8 total, 8 up, 8 in 2024-09-21T10:22:43.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:43 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3552155004' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:43.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:43 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:43.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:43 smithi017 ceph-mon[102913]: pgmap v2385: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:22:43.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:43 smithi017 ceph-mon[105151]: osdmap e1437: 8 total, 8 up, 8 in 2024-09-21T10:22:43.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:43 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3552155004' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:43.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:43 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:43.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:43 smithi017 ceph-mon[105151]: pgmap v2385: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:22:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:43 smithi134 ceph-mon[90085]: osdmap e1437: 8 total, 8 up, 8 in 2024-09-21T10:22:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:43 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3552155004' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:43 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:43.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:43 smithi134 ceph-mon[90085]: pgmap v2385: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:22:44.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 0 1 2024-09-21T10:22:44.369 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:44 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:22:44.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:44 smithi017 ceph-mon[102913]: osdmap e1438: 8 total, 8 up, 8 in 2024-09-21T10:22:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:44 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:22:44.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:44 smithi017 ceph-mon[105151]: osdmap e1438: 8 total, 8 up, 8 in 2024-09-21T10:22:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:44 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:22:44.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:44 smithi134 ceph-mon[90085]: osdmap e1438: 8 total, 8 up, 8 in 2024-09-21T10:22:44.807 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:22:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:22:44] "GET /metrics HTTP/1.1" 200 34358 "" "Prometheus/2.51.0" 2024-09-21T10:22:45.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:45 smithi017 ceph-mon[102913]: osdmap e1439: 8 total, 8 up, 8 in 2024-09-21T10:22:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:45 smithi017 ceph-mon[102913]: pgmap v2388: 103 pgs: 1 peering, 102 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:22:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:45 smithi017 ceph-mon[102913]: 1.19 scrub starts 2024-09-21T10:22:45.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:45 smithi017 ceph-mon[102913]: 1.19 scrub ok 2024-09-21T10:22:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:45 smithi017 ceph-mon[105151]: osdmap e1439: 8 total, 8 up, 8 in 2024-09-21T10:22:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:45 smithi017 ceph-mon[105151]: pgmap v2388: 103 pgs: 1 peering, 102 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:22:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:45 smithi017 ceph-mon[105151]: 1.19 scrub starts 2024-09-21T10:22:45.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:45 smithi017 ceph-mon[105151]: 1.19 scrub ok 2024-09-21T10:22:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:45 smithi134 ceph-mon[90085]: osdmap e1439: 8 total, 8 up, 8 in 2024-09-21T10:22:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:45 smithi134 ceph-mon[90085]: pgmap v2388: 103 pgs: 1 peering, 102 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:22:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:45 smithi134 ceph-mon[90085]: 1.19 scrub starts 2024-09-21T10:22:45.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:45 smithi134 ceph-mon[90085]: 1.19 scrub ok 2024-09-21T10:22:46.972 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-21T10:22:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:46 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:47.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:46 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2307685204' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:47.360 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:46 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:47.360 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:46 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2307685204' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:47.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:46 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:47.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:46 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2307685204' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:22:47.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:22:48.048 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:48.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:47 smithi017 ceph-mon[105151]: pgmap v2389: 103 pgs: 1 peering, 102 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:22:48.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:47 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2307685204' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:22:48.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:47 smithi017 ceph-mon[105151]: osdmap e1440: 8 total, 8 up, 8 in 2024-09-21T10:22:48.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:47 smithi017 ceph-mon[102913]: pgmap v2389: 103 pgs: 1 peering, 102 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:22:48.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:47 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2307685204' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:22:48.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:47 smithi017 ceph-mon[102913]: osdmap e1440: 8 total, 8 up, 8 in 2024-09-21T10:22:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:47 smithi134 ceph-mon[90085]: pgmap v2389: 103 pgs: 1 peering, 102 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:22:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:47 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2307685204' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:22:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:47 smithi134 ceph-mon[90085]: osdmap e1440: 8 total, 8 up, 8 in 2024-09-21T10:22:49.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:48 smithi017 ceph-mon[102913]: osdmap e1441: 8 total, 8 up, 8 in 2024-09-21T10:22:49.168 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:48 smithi017 ceph-mon[105151]: osdmap e1441: 8 total, 8 up, 8 in 2024-09-21T10:22:49.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:48 smithi134 ceph-mon[90085]: osdmap e1441: 8 total, 8 up, 8 in 2024-09-21T10:22:49.987 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:22:50.375 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:49 smithi017 ceph-mon[105151]: pgmap v2392: 103 pgs: 103 active+clean; 771 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 13 KiB/s rd, 28 KiB/s wr, 7 op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-21T10:22:50.375 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:49 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3260252893' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:22:50.375 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:49 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:22:50.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:49 smithi017 ceph-mon[102913]: pgmap v2392: 103 pgs: 103 active+clean; 771 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 13 KiB/s rd, 28 KiB/s wr, 7 op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-21T10:22:50.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:49 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3260252893' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:22:50.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:49 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:22:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:49 smithi134 ceph-mon[90085]: pgmap v2392: 103 pgs: 103 active+clean; 771 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 13 KiB/s rd, 28 KiB/s wr, 7 op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-21T10:22:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:49 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3260252893' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:22:50.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:49 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:22:50.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:22:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:22:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:22:50.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 2 1 2024-09-21T10:22:51.040 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:51.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:51 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:22:51.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:51 smithi017 ceph-mon[102913]: osdmap e1442: 8 total, 8 up, 8 in 2024-09-21T10:22:51.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:51 smithi017 ceph-mon[102913]: osdmap e1443: 8 total, 8 up, 8 in 2024-09-21T10:22:51.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:51 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:22:51.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:51 smithi017 ceph-mon[105151]: osdmap e1442: 8 total, 8 up, 8 in 2024-09-21T10:22:51.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:51 smithi017 ceph-mon[105151]: osdmap e1443: 8 total, 8 up, 8 in 2024-09-21T10:22:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:51 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:22:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:51 smithi134 ceph-mon[90085]: osdmap e1442: 8 total, 8 up, 8 in 2024-09-21T10:22:51.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:51 smithi134 ceph-mon[90085]: osdmap e1443: 8 total, 8 up, 8 in 2024-09-21T10:22:52.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:52 smithi017 ceph-mon[102913]: pgmap v2394: 103 pgs: 103 active+clean; 771 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 6 op/s; 8.7 KiB/s, 0 objects/s recovering 2024-09-21T10:22:52.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:52 smithi017 ceph-mon[105151]: pgmap v2394: 103 pgs: 103 active+clean; 771 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 6 op/s; 8.7 KiB/s, 0 objects/s recovering 2024-09-21T10:22:52.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:52 smithi134 ceph-mon[90085]: pgmap v2394: 103 pgs: 103 active+clean; 771 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 11 KiB/s rd, 25 KiB/s wr, 6 op/s; 8.7 KiB/s, 0 objects/s recovering 2024-09-21T10:22:53.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:53 smithi017 ceph-mon[102913]: 1.6 scrub starts 2024-09-21T10:22:53.136 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:53 smithi017 ceph-mon[102913]: 1.6 scrub ok 2024-09-21T10:22:53.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:53 smithi017 ceph-mon[105151]: 1.6 scrub starts 2024-09-21T10:22:53.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:53 smithi017 ceph-mon[105151]: 1.6 scrub ok 2024-09-21T10:22:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:53 smithi134 ceph-mon[90085]: 1.6 scrub starts 2024-09-21T10:22:53.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:53 smithi134 ceph-mon[90085]: 1.6 scrub ok 2024-09-21T10:22:54.010 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-21T10:22:54.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:54 smithi017 ceph-mon[102913]: 1.17 scrub starts 2024-09-21T10:22:54.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:54 smithi017 ceph-mon[102913]: 1.17 scrub ok 2024-09-21T10:22:54.324 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:54 smithi017 ceph-mon[102913]: pgmap v2396: 103 pgs: 103 active+clean; 771 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 26 KiB/s wr, 6 op/s; 9.2 KiB/s, 0 objects/s recovering 2024-09-21T10:22:54.324 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:54 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3253405794' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:22:54.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:54 smithi017 ceph-mon[105151]: 1.17 scrub starts 2024-09-21T10:22:54.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:54 smithi017 ceph-mon[105151]: 1.17 scrub ok 2024-09-21T10:22:54.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:54 smithi017 ceph-mon[105151]: pgmap v2396: 103 pgs: 103 active+clean; 771 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 26 KiB/s wr, 6 op/s; 9.2 KiB/s, 0 objects/s recovering 2024-09-21T10:22:54.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:54 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3253405794' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:22:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:54 smithi134 ceph-mon[90085]: 1.17 scrub starts 2024-09-21T10:22:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:54 smithi134 ceph-mon[90085]: 1.17 scrub ok 2024-09-21T10:22:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:54 smithi134 ceph-mon[90085]: pgmap v2396: 103 pgs: 103 active+clean; 771 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 26 KiB/s wr, 6 op/s; 9.2 KiB/s, 0 objects/s recovering 2024-09-21T10:22:54.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:54 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3253405794' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:22:54.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:22:55.004 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:55.022 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:22:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:22:54] "GET /metrics HTTP/1.1" 200 34358 "" "Prometheus/2.51.0" 2024-09-21T10:22:55.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:55 smithi017 ceph-mon[102913]: 1.2 scrub starts 2024-09-21T10:22:55.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:55 smithi017 ceph-mon[102913]: 1.2 scrub ok 2024-09-21T10:22:55.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:55 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3253405794' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:22:55.280 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:55 smithi017 ceph-mon[102913]: osdmap e1444: 8 total, 8 up, 8 in 2024-09-21T10:22:55.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:55 smithi017 ceph-mon[105151]: 1.2 scrub starts 2024-09-21T10:22:55.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:55 smithi017 ceph-mon[105151]: 1.2 scrub ok 2024-09-21T10:22:55.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:55 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3253405794' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:22:55.280 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:55 smithi017 ceph-mon[105151]: osdmap e1444: 8 total, 8 up, 8 in 2024-09-21T10:22:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:55 smithi134 ceph-mon[90085]: 1.2 scrub starts 2024-09-21T10:22:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:55 smithi134 ceph-mon[90085]: 1.2 scrub ok 2024-09-21T10:22:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:55 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3253405794' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:22:55.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:55 smithi134 ceph-mon[90085]: osdmap e1444: 8 total, 8 up, 8 in 2024-09-21T10:22:56.255 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:56 smithi017 ceph-mon[102913]: pgmap v2398: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:22:56.255 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:56.255 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:56 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:56.256 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:56 smithi017 ceph-mon[105151]: pgmap v2398: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:22:56.256 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:56.256 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:56 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:56 smithi134 ceph-mon[90085]: pgmap v2398: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:22:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:56.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:56 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:22:57.034 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:22:57.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:57 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/32099220' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:22:57.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:57 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:22:57.346 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:57 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/32099220' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:22:57.346 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:57 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:22:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:57 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/32099220' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:22:57.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:57 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:22:57.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 3 1 2024-09-21T10:22:58.059 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:58.298 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:58 smithi017 ceph-mon[102913]: pgmap v2399: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:22:58.298 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:58 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:22:58.298 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:58 smithi017 ceph-mon[102913]: osdmap e1445: 8 total, 8 up, 8 in 2024-09-21T10:22:58.298 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:58 smithi017 ceph-mon[105151]: pgmap v2399: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:22:58.298 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:58 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:22:58.299 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:58 smithi017 ceph-mon[105151]: osdmap e1445: 8 total, 8 up, 8 in 2024-09-21T10:22:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:58 smithi134 ceph-mon[90085]: pgmap v2399: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:22:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:58 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:22:58.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:58 smithi134 ceph-mon[90085]: osdmap e1445: 8 total, 8 up, 8 in 2024-09-21T10:22:59.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:22:59 smithi017 ceph-mon[102913]: osdmap e1446: 8 total, 8 up, 8 in 2024-09-21T10:22:59.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:22:59 smithi017 ceph-mon[105151]: osdmap e1446: 8 total, 8 up, 8 in 2024-09-21T10:22:59.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:22:59 smithi134 ceph-mon[90085]: osdmap e1446: 8 total, 8 up, 8 in 2024-09-21T10:23:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:00 smithi017 ceph-mon[102913]: pgmap v2402: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:23:00.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:00 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:23:00.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:00 smithi017 ceph-mon[105151]: pgmap v2402: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:23:00.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:00 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:23:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:00 smithi134 ceph-mon[90085]: pgmap v2402: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:23:00.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:00 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:23:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:23:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:23:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:23:01.047 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-09-21T10:23:01.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:01 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/155060204' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:23:01.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:01 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:23:01.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:01 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:01.292 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:01 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:01.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:01 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/155060204' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:23:01.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:01 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:23:01.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:01 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:01.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:01 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:01.367 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:01 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/155060204' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:23:01.367 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:01 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:23:01.367 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:01 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:01.367 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:01 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:02.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:23:02.308 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:02.347 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:02 smithi134 ceph-mon[90085]: pgmap v2403: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:02.347 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:02 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:23:02.347 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:02 smithi134 ceph-mon[90085]: osdmap e1447: 8 total, 8 up, 8 in 2024-09-21T10:23:02.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:02 smithi017 ceph-mon[102913]: pgmap v2403: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:02.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:02 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:23:02.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:02 smithi017 ceph-mon[102913]: osdmap e1447: 8 total, 8 up, 8 in 2024-09-21T10:23:02.436 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:02 smithi017 ceph-mon[105151]: pgmap v2403: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:02.436 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:02 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:23:02.436 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:02 smithi017 ceph-mon[105151]: osdmap e1447: 8 total, 8 up, 8 in 2024-09-21T10:23:04.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:04 smithi134 ceph-mon[90085]: pgmap v2405: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:04 smithi017 ceph-mon[105151]: pgmap v2405: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:04.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:04 smithi017 ceph-mon[102913]: pgmap v2405: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:04.897 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:23:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:23:04] "GET /metrics HTTP/1.1" 200 34321 "" "Prometheus/2.51.0" 2024-09-21T10:23:05.321 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:23:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[102913]: pgmap v2406: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:05.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1031243686' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:05.452 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[105151]: pgmap v2406: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:05.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1031243686' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:05.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:05 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:05 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:05 smithi134 ceph-mon[90085]: pgmap v2406: 103 pgs: 103 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:05 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1031243686' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:05.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:05 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:06.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 5 1 2024-09-21T10:23:06.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:06 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:23:06.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:06 smithi017 ceph-mon[102913]: osdmap e1448: 8 total, 8 up, 8 in 2024-09-21T10:23:06.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:06 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:23:06.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:06 smithi017 ceph-mon[105151]: osdmap e1448: 8 total, 8 up, 8 in 2024-09-21T10:23:06.645 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:06 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:23:06.646 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:06 smithi134 ceph-mon[90085]: osdmap e1448: 8 total, 8 up, 8 in 2024-09-21T10:23:06.817 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[105151]: osdmap e1449: 8 total, 8 up, 8 in 2024-09-21T10:23:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[105151]: pgmap v2409: 103 pgs: 9 peering, 94 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:07.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:23:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:23:07.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:07.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[102913]: osdmap e1449: 8 total, 8 up, 8 in 2024-09-21T10:23:07.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[102913]: pgmap v2409: 103 pgs: 9 peering, 94 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:07.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:07.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:07.701 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:23:07.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:23:07.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:07 smithi134 ceph-mon[90085]: osdmap e1449: 8 total, 8 up, 8 in 2024-09-21T10:23:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:07 smithi134 ceph-mon[90085]: pgmap v2409: 103 pgs: 9 peering, 94 active+clean; 719 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:23:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:23:07.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:08 smithi017 ceph-mon[102913]: 1.8 scrub starts 2024-09-21T10:23:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:08 smithi017 ceph-mon[102913]: 1.8 scrub ok 2024-09-21T10:23:08.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:08 smithi017 ceph-mon[102913]: 1.1d scrub starts 2024-09-21T10:23:08.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:08 smithi017 ceph-mon[102913]: 1.1d scrub ok 2024-09-21T10:23:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:08 smithi017 ceph-mon[105151]: 1.8 scrub starts 2024-09-21T10:23:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:08 smithi017 ceph-mon[105151]: 1.8 scrub ok 2024-09-21T10:23:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:08 smithi017 ceph-mon[105151]: 1.1d scrub starts 2024-09-21T10:23:08.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:08 smithi017 ceph-mon[105151]: 1.1d scrub ok 2024-09-21T10:23:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:08 smithi134 ceph-mon[90085]: 1.8 scrub starts 2024-09-21T10:23:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:08 smithi134 ceph-mon[90085]: 1.8 scrub ok 2024-09-21T10:23:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:08 smithi134 ceph-mon[90085]: 1.1d scrub starts 2024-09-21T10:23:08.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:08 smithi134 ceph-mon[90085]: 1.1d scrub ok 2024-09-21T10:23:09.343 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-09-21T10:23:09.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:09 smithi017 ceph-mon[105151]: pgmap v2410: 103 pgs: 9 peering, 94 active+clean; 591 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 17 KiB/s, 0 objects/s recovering 2024-09-21T10:23:09.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:09 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1941099831' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:09.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:09 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:09.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:09 smithi017 ceph-mon[102913]: pgmap v2410: 103 pgs: 9 peering, 94 active+clean; 591 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 17 KiB/s, 0 objects/s recovering 2024-09-21T10:23:09.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:09 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1941099831' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:09.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:09 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:09 smithi134 ceph-mon[90085]: pgmap v2410: 103 pgs: 9 peering, 94 active+clean; 591 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 17 KiB/s, 0 objects/s recovering 2024-09-21T10:23:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:09 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1941099831' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:09.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:09 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:23:10.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:23:10.397 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:10 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:23:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:10 smithi134 ceph-mon[90085]: osdmap e1450: 8 total, 8 up, 8 in 2024-09-21T10:23:10.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:10.495 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:23:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:23:10.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:23:10.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:10 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:23:10.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:10 smithi017 ceph-mon[102913]: osdmap e1450: 8 total, 8 up, 8 in 2024-09-21T10:23:10.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:10.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:10 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:23:10.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:10 smithi017 ceph-mon[105151]: osdmap e1450: 8 total, 8 up, 8 in 2024-09-21T10:23:10.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:11.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:11 smithi017 ceph-mon[102913]: osdmap e1451: 8 total, 8 up, 8 in 2024-09-21T10:23:11.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:11 smithi017 ceph-mon[102913]: pgmap v2413: 103 pgs: 7 peering, 96 active+clean; 591 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 24 KiB/s, 0 objects/s recovering 2024-09-21T10:23:11.603 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:11 smithi017 ceph-mon[105151]: osdmap e1451: 8 total, 8 up, 8 in 2024-09-21T10:23:11.603 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:11 smithi017 ceph-mon[105151]: pgmap v2413: 103 pgs: 7 peering, 96 active+clean; 591 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 24 KiB/s, 0 objects/s recovering 2024-09-21T10:23:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:11 smithi134 ceph-mon[90085]: osdmap e1451: 8 total, 8 up, 8 in 2024-09-21T10:23:11.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:11 smithi134 ceph-mon[90085]: pgmap v2413: 103 pgs: 7 peering, 96 active+clean; 591 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 24 KiB/s, 0 objects/s recovering 2024-09-21T10:23:12.428 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:23:12.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:12 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/236152217' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:23:12.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:12 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/236152217' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:23:12.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:12 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/236152217' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:23:13.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 6 1 2024-09-21T10:23:13.386 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:13.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[102913]: 1.8 scrub starts 2024-09-21T10:23:13.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[102913]: 1.8 scrub ok 2024-09-21T10:23:13.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/236152217' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:23:13.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[102913]: osdmap e1452: 8 total, 8 up, 8 in 2024-09-21T10:23:13.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[102913]: pgmap v2415: 103 pgs: 103 active+clean; 745 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 5.7 KiB/s rd, 12 KiB/s wr, 3 op/s; 4.4 KiB/s, 0 objects/s recovering 2024-09-21T10:23:13.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[105151]: 1.8 scrub starts 2024-09-21T10:23:13.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[105151]: 1.8 scrub ok 2024-09-21T10:23:13.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/236152217' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:23:13.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[105151]: osdmap e1452: 8 total, 8 up, 8 in 2024-09-21T10:23:13.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:13 smithi017 ceph-mon[105151]: pgmap v2415: 103 pgs: 103 active+clean; 745 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 5.7 KiB/s rd, 12 KiB/s wr, 3 op/s; 4.4 KiB/s, 0 objects/s recovering 2024-09-21T10:23:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:13 smithi134 ceph-mon[90085]: 1.8 scrub starts 2024-09-21T10:23:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:13 smithi134 ceph-mon[90085]: 1.8 scrub ok 2024-09-21T10:23:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:13 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/236152217' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:23:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:13 smithi134 ceph-mon[90085]: osdmap e1452: 8 total, 8 up, 8 in 2024-09-21T10:23:13.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:13 smithi134 ceph-mon[90085]: pgmap v2415: 103 pgs: 103 active+clean; 745 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 5.7 KiB/s rd, 12 KiB/s wr, 3 op/s; 4.4 KiB/s, 0 objects/s recovering 2024-09-21T10:23:14.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:14 smithi017 ceph-mon[105151]: 1.b scrub starts 2024-09-21T10:23:14.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:14 smithi017 ceph-mon[105151]: 1.b scrub ok 2024-09-21T10:23:14.901 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:14 smithi017 ceph-mon[102913]: 1.b scrub starts 2024-09-21T10:23:14.901 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:14 smithi017 ceph-mon[102913]: 1.b scrub ok 2024-09-21T10:23:14.901 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:23:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:23:14] "GET /metrics HTTP/1.1" 200 34303 "" "Prometheus/2.51.0" 2024-09-21T10:23:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:14 smithi134 ceph-mon[90085]: 1.b scrub starts 2024-09-21T10:23:14.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:14 smithi134 ceph-mon[90085]: 1.b scrub ok 2024-09-21T10:23:15.608 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-21T10:23:15.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:15 smithi017 ceph-mon[102913]: pgmap v2416: 103 pgs: 103 active+clean; 745 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 5.7 KiB/s rd, 12 KiB/s wr, 3 op/s 2024-09-21T10:23:15.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:15 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2873727026' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:23:15.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:15 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:23:15.937 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:15 smithi017 ceph-mon[105151]: pgmap v2416: 103 pgs: 103 active+clean; 745 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 5.7 KiB/s rd, 12 KiB/s wr, 3 op/s 2024-09-21T10:23:15.937 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:15 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2873727026' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:23:15.937 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:15 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:23:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:15 smithi134 ceph-mon[90085]: pgmap v2416: 103 pgs: 103 active+clean; 745 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 5.7 KiB/s rd, 12 KiB/s wr, 3 op/s 2024-09-21T10:23:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:15 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2873727026' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:23:15.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:15 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:23:16.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 7 1 2024-09-21T10:23:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:16 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:23:16.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:16 smithi017 ceph-mon[102913]: osdmap e1453: 8 total, 8 up, 8 in 2024-09-21T10:23:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:16 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:23:16.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:16 smithi017 ceph-mon[105151]: osdmap e1453: 8 total, 8 up, 8 in 2024-09-21T10:23:16.745 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:16 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:23:16.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:16 smithi134 ceph-mon[90085]: osdmap e1453: 8 total, 8 up, 8 in 2024-09-21T10:23:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:17 smithi134 ceph-mon[90085]: osdmap e1454: 8 total, 8 up, 8 in 2024-09-21T10:23:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:17 smithi134 ceph-mon[90085]: pgmap v2419: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 3.8 KiB/s rd, 8.2 KiB/s wr, 2 op/s 2024-09-21T10:23:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:17 smithi134 ceph-mon[90085]: 1.7 deep-scrub starts 2024-09-21T10:23:17.744 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:17 smithi134 ceph-mon[90085]: 1.7 deep-scrub ok 2024-09-21T10:23:17.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:17 smithi017 ceph-mon[105151]: osdmap e1454: 8 total, 8 up, 8 in 2024-09-21T10:23:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:17 smithi017 ceph-mon[105151]: pgmap v2419: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 3.8 KiB/s rd, 8.2 KiB/s wr, 2 op/s 2024-09-21T10:23:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:17 smithi017 ceph-mon[105151]: 1.7 deep-scrub starts 2024-09-21T10:23:17.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:17 smithi017 ceph-mon[105151]: 1.7 deep-scrub ok 2024-09-21T10:23:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:17 smithi017 ceph-mon[102913]: osdmap e1454: 8 total, 8 up, 8 in 2024-09-21T10:23:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:17 smithi017 ceph-mon[102913]: pgmap v2419: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 3.8 KiB/s rd, 8.2 KiB/s wr, 2 op/s 2024-09-21T10:23:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:17 smithi017 ceph-mon[102913]: 1.7 deep-scrub starts 2024-09-21T10:23:17.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:17 smithi017 ceph-mon[102913]: 1.7 deep-scrub ok 2024-09-21T10:23:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:18 smithi017 ceph-mon[102913]: 1.6 scrub starts 2024-09-21T10:23:18.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:18 smithi017 ceph-mon[102913]: 1.6 scrub ok 2024-09-21T10:23:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:18 smithi017 ceph-mon[105151]: 1.6 scrub starts 2024-09-21T10:23:18.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:18 smithi017 ceph-mon[105151]: 1.6 scrub ok 2024-09-21T10:23:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:18 smithi134 ceph-mon[90085]: 1.6 scrub starts 2024-09-21T10:23:18.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:18 smithi134 ceph-mon[90085]: 1.6 scrub ok 2024-09-21T10:23:19.631 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2024-09-21T10:23:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:19 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1413993618' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:23:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:19 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:23:19.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:19 smithi017 ceph-mon[102913]: pgmap v2420: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:19 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1413993618' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:23:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:19 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:23:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:19 smithi017 ceph-mon[105151]: pgmap v2420: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:19 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1413993618' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:23:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:19 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:23:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:19 smithi134 ceph-mon[90085]: pgmap v2420: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:23:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:23:20.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:23:20.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 7 1 2024-09-21T10:23:20.816 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:20.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:20 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:23:20.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:20 smithi017 ceph-mon[102913]: osdmap e1455: 8 total, 8 up, 8 in 2024-09-21T10:23:20.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:20 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:23:20.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:20 smithi017 ceph-mon[105151]: osdmap e1455: 8 total, 8 up, 8 in 2024-09-21T10:23:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:20 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:23:20.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:20 smithi134 ceph-mon[90085]: osdmap e1455: 8 total, 8 up, 8 in 2024-09-21T10:23:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:21 smithi017 ceph-mon[102913]: osdmap e1456: 8 total, 8 up, 8 in 2024-09-21T10:23:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:21 smithi017 ceph-mon[102913]: pgmap v2423: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:21 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:21 smithi017 ceph-mon[105151]: osdmap e1456: 8 total, 8 up, 8 in 2024-09-21T10:23:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:21 smithi017 ceph-mon[105151]: pgmap v2423: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:21 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:21 smithi134 ceph-mon[90085]: osdmap e1456: 8 total, 8 up, 8 in 2024-09-21T10:23:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:21 smithi134 ceph-mon[90085]: pgmap v2423: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:21 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:23:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:22 smithi017 ceph-mon[102913]: 1.a scrub starts 2024-09-21T10:23:22.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:22 smithi017 ceph-mon[102913]: 1.a scrub ok 2024-09-21T10:23:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:22 smithi017 ceph-mon[105151]: 1.a scrub starts 2024-09-21T10:23:22.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:22 smithi017 ceph-mon[105151]: 1.a scrub ok 2024-09-21T10:23:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:22 smithi134 ceph-mon[90085]: 1.a scrub starts 2024-09-21T10:23:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:22 smithi134 ceph-mon[90085]: 1.a scrub ok 2024-09-21T10:23:23.645 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-09-21T10:23:23.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:23 smithi017 ceph-mon[105151]: pgmap v2424: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:23.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:23 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/867739119' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:23:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:23 smithi017 ceph-mon[102913]: pgmap v2424: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:23.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:23 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/867739119' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:23:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:23 smithi134 ceph-mon[90085]: pgmap v2424: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:23 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/867739119' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:23:24.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:23:24.784 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:24.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/867739119' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:23:24.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:24 smithi017 ceph-mon[102913]: osdmap e1457: 8 total, 8 up, 8 in 2024-09-21T10:23:24.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/867739119' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:23:24.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:24 smithi017 ceph-mon[105151]: osdmap e1457: 8 total, 8 up, 8 in 2024-09-21T10:23:24.906 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:23:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:23:24] "GET /metrics HTTP/1.1" 200 34303 "" "Prometheus/2.51.0" 2024-09-21T10:23:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/867739119' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:23:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:24 smithi134 ceph-mon[90085]: osdmap e1457: 8 total, 8 up, 8 in 2024-09-21T10:23:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:25 smithi017 ceph-mon[102913]: pgmap v2426: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:25.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:25 smithi017 ceph-mon[105151]: pgmap v2426: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:25 smithi134 ceph-mon[90085]: pgmap v2426: 103 pgs: 103 active+clean; 608 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:27.718 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:23:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:27 smithi134 ceph-mon[90085]: pgmap v2427: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:27 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/905064181' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:27 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:28.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:27 smithi017 ceph-mon[102913]: pgmap v2427: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:28.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:27 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/905064181' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:28.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:27 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:28.087 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:27 smithi017 ceph-mon[105151]: pgmap v2427: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:28.087 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:27 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/905064181' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:28.087 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:27 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:28.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 4 1 2024-09-21T10:23:28.657 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:28 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:23:28.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:28 smithi134 ceph-mon[90085]: osdmap e1458: 8 total, 8 up, 8 in 2024-09-21T10:23:29.081 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:28 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:23:29.081 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:28 smithi017 ceph-mon[102913]: osdmap e1458: 8 total, 8 up, 8 in 2024-09-21T10:23:29.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:28 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:23:29.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:28 smithi017 ceph-mon[105151]: osdmap e1458: 8 total, 8 up, 8 in 2024-09-21T10:23:29.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:29 smithi134 ceph-mon[90085]: pgmap v2429: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:30.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:29 smithi017 ceph-mon[102913]: pgmap v2429: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:30.046 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:29 smithi017 ceph-mon[105151]: pgmap v2429: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:23:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:23:30.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:23:30.730 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-21T10:23:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:30 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/761576899' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:30.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:30 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:31.109 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:30 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/761576899' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:31.109 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:30 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:31.109 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:30 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/761576899' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:31.109 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:30 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:23:31.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:23:31.778 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:31 smithi134 ceph-mon[90085]: pgmap v2430: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:31 smithi134 ceph-mon[90085]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:23:31.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:31 smithi134 ceph-mon[90085]: osdmap e1459: 8 total, 8 up, 8 in 2024-09-21T10:23:32.020 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:31 smithi017 ceph-mon[102913]: pgmap v2430: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:32.020 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:31 smithi017 ceph-mon[102913]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:23:32.020 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:31 smithi017 ceph-mon[102913]: osdmap e1459: 8 total, 8 up, 8 in 2024-09-21T10:23:32.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:31 smithi017 ceph-mon[105151]: pgmap v2430: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:32.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:31 smithi017 ceph-mon[105151]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:23:32.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:31 smithi017 ceph-mon[105151]: osdmap e1459: 8 total, 8 up, 8 in 2024-09-21T10:23:33.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:33 smithi017 ceph-mon[102913]: pgmap v2432: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:33.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:33 smithi017 ceph-mon[105151]: pgmap v2432: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:33.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:33 smithi134 ceph-mon[90085]: pgmap v2432: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:34.738 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:23:34.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:34 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1335379354' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:23:35.134 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:34 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1335379354' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:23:35.134 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:34 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1335379354' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:23:35.134 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:23:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:23:34] "GET /metrics HTTP/1.1" 200 34279 "" "Prometheus/2.51.0" 2024-09-21T10:23:35.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 1 1 2024-09-21T10:23:35.811 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:35.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:35 smithi017 ceph-mon[102913]: pgmap v2433: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:35.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:35 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1335379354' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:23:35.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:35 smithi017 ceph-mon[102913]: osdmap e1460: 8 total, 8 up, 8 in 2024-09-21T10:23:35.933 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:35 smithi017 ceph-mon[105151]: pgmap v2433: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:35.933 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:35 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1335379354' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:23:35.933 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:35 smithi017 ceph-mon[105151]: osdmap e1460: 8 total, 8 up, 8 in 2024-09-21T10:23:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:35 smithi134 ceph-mon[90085]: pgmap v2433: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:35 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1335379354' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:23:35.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:35 smithi134 ceph-mon[90085]: osdmap e1460: 8 total, 8 up, 8 in 2024-09-21T10:23:37.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:37 smithi017 ceph-mon[102913]: pgmap v2435: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:37.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:37 smithi017 ceph-mon[105151]: pgmap v2435: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:37.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:37 smithi134 ceph-mon[90085]: pgmap v2435: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:38.747 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-09-21T10:23:38.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:38 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2086829904' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:23:39.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:38 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2086829904' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:23:39.076 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:38 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2086829904' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:23:39.424 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-09-21T10:23:39.424 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-09-21T10:23:39.424 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-09-21T10:23:39.424 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-09-21T10:23:39.424 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T10:23:39.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:23:39.693 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:40.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:39 smithi017 ceph-mon[102913]: pgmap v2436: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:40.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:39 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2086829904' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:23:40.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:39 smithi017 ceph-mon[102913]: osdmap e1461: 8 total, 8 up, 8 in 2024-09-21T10:23:40.054 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:39 smithi017 ceph-mon[105151]: pgmap v2436: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:40.054 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:39 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2086829904' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:23:40.054 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:39 smithi017 ceph-mon[105151]: osdmap e1461: 8 total, 8 up, 8 in 2024-09-21T10:23:40.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:39 smithi134 ceph-mon[90085]: pgmap v2436: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:40.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:39 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2086829904' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:23:40.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:39 smithi134 ceph-mon[90085]: osdmap e1461: 8 total, 8 up, 8 in 2024-09-21T10:23:40.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:23:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:23:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:23:40.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:40 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:40.825 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:40 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:41.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:40 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:41.552 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:23:41.552 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":1461,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T10:23:38.739375+0000","last_up_change":"2024-09-21T10:21:22.081470+0000","last_in_change":"2024-09-21T10:22:32.010193+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1374","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1341","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:27.624125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:41.967182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:42.538769+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":466,"up_from":479,"up_thru":1448,"down_at":467,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6803","nonce":2105074120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6805","nonce":2105074120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6809","nonce":2105074120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6807","nonce":2105074120}]},"public_addr":"172.21.15.17:6803/2105074120","cluster_addr":"172.21.15.17:6805/2105074120","heartbeat_back_addr":"172.21.15.17:6809/2105074120","heartbeat_front_addr":"172.21.15.17:6807/2105074120","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":513,"last_clean_end":1368,"up_from":1413,"up_thru":1448,"down_at":1369,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":2519043258},{"type":"v1","addr":"172.21.15.17:6811","nonce":2519043258}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":2519043258},{"type":"v1","addr":"172.21.15.17:6813","nonce":2519043258}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":2519043258},{"type":"v1","addr":"172.21.15.17:6817","nonce":2519043258}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":2519043258},{"type":"v1","addr":"172.21.15.17:6815","nonce":2519043258}]},"public_addr":"172.21.15.17:6811/2519043258","cluster_addr":"172.21.15.17:6813/2519043258","heartbeat_back_addr":"172.21.15.17:6817/2519043258","heartbeat_front_addr":"172.21.15.17:6815/2519043258","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":534,"up_from":547,"up_thru":1448,"down_at":535,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6819","nonce":323142954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6821","nonce":323142954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6825","nonce":323142954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6823","nonce":323142954}]},"public_addr":"172.21.15.17:6819/323142954","cluster_addr":"172.21.15.17:6821/323142954","heartbeat_back_addr":"172.21.15.17:6825/323142954","heartbeat_front_addr":"172.21.15.17:6823/323142954","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":568,"up_from":581,"up_thru":1448,"down_at":569,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6827","nonce":3146888816}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6829","nonce":3146888816}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6833","nonce":3146888816}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6831","nonce":3146888816}]},"public_addr":"172.21.15.17:6827/3146888816","cluster_addr":"172.21.15.17:6829/3146888816","heartbeat_back_addr":"172.21.15.17:6833/3146888816","heartbeat_front_addr":"172.21.15.17:6831/3146888816","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":1206,"up_from":1224,"up_thru":1445,"down_at":1207,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1500230278},{"type":"v1","addr":"172.21.15.134:6801","nonce":1500230278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1500230278},{"type":"v1","addr":"172.21.15.134:6803","nonce":1500230278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1500230278},{"type":"v1","addr":"172.21.15.134:6807","nonce":1500230278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1500230278},{"type":"v1","addr":"172.21.15.134:6805","nonce":1500230278}]},"public_addr":"172.21.15.134:6801/1500230278","cluster_addr":"172.21.15.134:6803/1500230278","heartbeat_back_addr":"172.21.15.134:6807/1500230278","heartbeat_front_addr":"172.21.15.134:6805/1500230278","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":1450,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":1455,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":1455,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":466},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":1368},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":534},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":568},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":1206},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/2291861207":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/3530815435":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3512479442":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/3084797810":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/2599173714":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/381036987":"2024-09-22T09:41:39.758476+0000","172.21.15.134:6834/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3671582122":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6800/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.134:6835/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/758458220":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3206271349":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/3226197858":"2024-09-22T09:41:39.758476+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/2266250644":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/2350962326":"2024-09-22T09:40:59.077787+0000","172.21.15.134:0/3275227882":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6801/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/1454017192":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/877591523":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6801/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/1515484230":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6801/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3086297034":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/2023103508":"2024-09-22T09:41:39.758476+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:23:42.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:41 smithi017 ceph-mon[102913]: pgmap v2438: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:42.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:41 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/722828558' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:23:42.166 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:41 smithi017 ceph-mon[105151]: pgmap v2438: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:42.166 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:41 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/722828558' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:23:42.195 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T10:23:42.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:23:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:41 smithi134 ceph-mon[90085]: pgmap v2438: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:42.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:41 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/722828558' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:23:42.458 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:44.120 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:43 smithi017 ceph-mon[102913]: pgmap v2439: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:44.120 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:43 smithi017 ceph-mon[105151]: pgmap v2439: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:44.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:43 smithi134 ceph-mon[90085]: pgmap v2439: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:44.539 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:23:44.539 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":1461,"fsid":"ee55f8cc-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:47.760456+0000","modified":"2024-09-21T10:23:38.739375+0000","last_up_change":"2024-09-21T10:21:22.081470+0000","last_in_change":"2024-09-21T10:22:32.010193+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:54.362446+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1374","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1341","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:27.624125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"58","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":58,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:41.967182+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:42.538769+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bb258889-3128-4c2e-b5da-343281340eb4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":466,"up_from":479,"up_thru":1448,"down_at":467,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6803","nonce":2105074120}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6805","nonce":2105074120}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6809","nonce":2105074120}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2105074120},{"type":"v1","addr":"172.21.15.17:6807","nonce":2105074120}]},"public_addr":"172.21.15.17:6803/2105074120","cluster_addr":"172.21.15.17:6805/2105074120","heartbeat_back_addr":"172.21.15.17:6809/2105074120","heartbeat_front_addr":"172.21.15.17:6807/2105074120","state":["exists","up"]},{"osd":1,"uuid":"efe0af53-c763-4d08-8051-7f781782ab0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":513,"last_clean_end":1368,"up_from":1413,"up_thru":1448,"down_at":1369,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":2519043258},{"type":"v1","addr":"172.21.15.17:6811","nonce":2519043258}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":2519043258},{"type":"v1","addr":"172.21.15.17:6813","nonce":2519043258}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":2519043258},{"type":"v1","addr":"172.21.15.17:6817","nonce":2519043258}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":2519043258},{"type":"v1","addr":"172.21.15.17:6815","nonce":2519043258}]},"public_addr":"172.21.15.17:6811/2519043258","cluster_addr":"172.21.15.17:6813/2519043258","heartbeat_back_addr":"172.21.15.17:6817/2519043258","heartbeat_front_addr":"172.21.15.17:6815/2519043258","state":["exists","up"]},{"osd":2,"uuid":"59c1433f-c409-4616-a7b9-7a11d8fb28d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":534,"up_from":547,"up_thru":1448,"down_at":535,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6819","nonce":323142954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6821","nonce":323142954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6825","nonce":323142954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":323142954},{"type":"v1","addr":"172.21.15.17:6823","nonce":323142954}]},"public_addr":"172.21.15.17:6819/323142954","cluster_addr":"172.21.15.17:6821/323142954","heartbeat_back_addr":"172.21.15.17:6825/323142954","heartbeat_front_addr":"172.21.15.17:6823/323142954","state":["exists","up"]},{"osd":3,"uuid":"9fb2cdbe-3662-4ee4-b09d-96dffa74d226","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":568,"up_from":581,"up_thru":1448,"down_at":569,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6827","nonce":3146888816}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6829","nonce":3146888816}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6833","nonce":3146888816}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":3146888816},{"type":"v1","addr":"172.21.15.17:6831","nonce":3146888816}]},"public_addr":"172.21.15.17:6827/3146888816","cluster_addr":"172.21.15.17:6829/3146888816","heartbeat_back_addr":"172.21.15.17:6833/3146888816","heartbeat_front_addr":"172.21.15.17:6831/3146888816","state":["exists","up"]},{"osd":4,"uuid":"1d6fde21-8847-4673-8b7f-523f387faf50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":1206,"up_from":1224,"up_thru":1445,"down_at":1207,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1500230278},{"type":"v1","addr":"172.21.15.134:6801","nonce":1500230278}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1500230278},{"type":"v1","addr":"172.21.15.134:6803","nonce":1500230278}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1500230278},{"type":"v1","addr":"172.21.15.134:6807","nonce":1500230278}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1500230278},{"type":"v1","addr":"172.21.15.134:6805","nonce":1500230278}]},"public_addr":"172.21.15.134:6801/1500230278","cluster_addr":"172.21.15.134:6803/1500230278","heartbeat_back_addr":"172.21.15.134:6807/1500230278","heartbeat_front_addr":"172.21.15.134:6805/1500230278","state":["exists","up"]},{"osd":5,"uuid":"2bae1d15-c9ee-4f67-82ed-31fb25c11749","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":1450,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6809","nonce":852015742}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6811","nonce":852015742}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6815","nonce":852015742}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":852015742},{"type":"v1","addr":"172.21.15.134:6813","nonce":852015742}]},"public_addr":"172.21.15.134:6809/852015742","cluster_addr":"172.21.15.134:6811/852015742","heartbeat_back_addr":"172.21.15.134:6815/852015742","heartbeat_front_addr":"172.21.15.134:6813/852015742","state":["exists","up"]},{"osd":6,"uuid":"dc8f4f0e-887e-424a-bb0c-cb9cd88e6107","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":1455,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6817","nonce":2227687667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6819","nonce":2227687667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6823","nonce":2227687667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":2227687667},{"type":"v1","addr":"172.21.15.134:6821","nonce":2227687667}]},"public_addr":"172.21.15.134:6817/2227687667","cluster_addr":"172.21.15.134:6819/2227687667","heartbeat_back_addr":"172.21.15.134:6823/2227687667","heartbeat_front_addr":"172.21.15.134:6821/2227687667","state":["exists","up"]},{"osd":7,"uuid":"edff61f5-4a21-484b-9924-e15db0d909a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":1455,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6825","nonce":2365723708}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6827","nonce":2365723708}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6831","nonce":2365723708}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2365723708},{"type":"v1","addr":"172.21.15.134:6829","nonce":2365723708}]},"public_addr":"172.21.15.134:6825/2365723708","cluster_addr":"172.21.15.134:6827/2365723708","heartbeat_back_addr":"172.21.15.134:6831/2365723708","heartbeat_front_addr":"172.21.15.134:6829/2365723708","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:02.067157+0000","dead_epoch":466},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:26.012036+0000","dead_epoch":1368},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:49.994726+0000","dead_epoch":534},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:15.280147+0000","dead_epoch":568},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:37.327076+0000","dead_epoch":1206},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:01.341916+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:26.771108+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:51.216708+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/3863855841":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/2291861207":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/3530815435":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3512479442":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/3084797810":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/269146626":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/2599173714":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/510140310":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6800/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/381036987":"2024-09-22T09:41:39.758476+0000","172.21.15.134:6834/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/3671582122":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6800/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.134:6835/72443452":"2024-09-22T09:40:59.077787+0000","172.21.15.17:0/758458220":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3206271349":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/3226197858":"2024-09-22T09:41:39.758476+0000","172.21.15.17:6800/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/4015678849":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6800/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/1996088326":"2024-09-22T09:30:41.864498+0000","172.21.15.17:6801/1944945429":"2024-09-22T09:30:41.864498+0000","172.21.15.17:0/3490773646":"2024-09-22T09:30:18.002697+0000","172.21.15.17:0/2266250644":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/2350962326":"2024-09-22T09:40:59.077787+0000","172.21.15.134:0/3275227882":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6801/3745862344":"2024-09-22T09:30:18.002697+0000","172.21.15.17:6801/3625002403":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/1454017192":"2024-09-22T09:35:23.638701+0000","172.21.15.17:0/877591523":"2024-09-22T09:39:24.978166+0000","172.21.15.17:6801/2894617542":"2024-09-22T09:35:23.638701+0000","172.21.15.134:0/1515484230":"2024-09-22T09:40:59.077787+0000","172.21.15.17:6801/2576370885":"2024-09-22T09:41:39.758476+0000","172.21.15.17:0/3086297034":"2024-09-22T09:39:24.978166+0000","172.21.15.17:0/2023103508":"2024-09-22T09:41:39.758476+0000","172.21.15.17:6800/3745862344":"2024-09-22T09:30:18.002697+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:23:45.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:44 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/2948609388' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:23:45.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:23:44 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:23:44] "GET /metrics HTTP/1.1" 200 34273 "" "Prometheus/2.51.0" 2024-09-21T10:23:45.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:44 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/2948609388' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:23:45.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:44 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/2948609388' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:23:45.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T10:23:45.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T10:23:45.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T10:23:45.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T10:23:45.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T10:23:45.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T10:23:45.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T10:23:45.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T10:23:45.873 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:45.873 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:45.879 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:45.879 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:45.883 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:45.891 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:45.898 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:45.900 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:46.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:45 smithi017 ceph-mon[102913]: pgmap v2440: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:46.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:45 smithi017 ceph-mon[105151]: pgmap v2440: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:46.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:45 smithi134 ceph-mon[90085]: pgmap v2440: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:47.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:47 smithi017 ceph-mon[102913]: pgmap v2441: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:47.933 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:47 smithi017 ceph-mon[105151]: pgmap v2441: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:48.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:47 smithi134 ceph-mon[90085]: pgmap v2441: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:49.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:49 smithi017 ceph-mon[102913]: pgmap v2442: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:49.810 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:49 smithi017 ceph-mon[105151]: pgmap v2442: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:50.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:49 smithi134 ceph-mon[90085]: pgmap v2442: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:50.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:23:50 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:23:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:23:51.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:51 smithi017 ceph-mon[102913]: pgmap v2443: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:51.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:51 smithi017 ceph-mon[105151]: pgmap v2443: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:52.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:51 smithi134 ceph-mon[90085]: pgmap v2443: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:54.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:53 smithi017 ceph-mon[102913]: pgmap v2444: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:54.124 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:53 smithi017 ceph-mon[105151]: pgmap v2444: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:54.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:53 smithi134 ceph-mon[90085]: pgmap v2444: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:55.007 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:23:54 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:23:54] "GET /metrics HTTP/1.1" 200 34273 "" "Prometheus/2.51.0" 2024-09-21T10:23:56.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:55 smithi017 ceph-mon[105151]: pgmap v2445: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:56.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:55 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:56.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:55 smithi017 ceph-mon[102913]: pgmap v2445: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:56.157 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:55 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:55 smithi134 ceph-mon[90085]: pgmap v2445: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:56.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:55 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:57.398 INFO:teuthology.orchestra.run.smithi017.stdout:5257039970492 2024-09-21T10:23:57.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T10:23:57.921 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:57 smithi017 ceph-mon[102913]: pgmap v2446: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:57.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:57 smithi017 ceph-mon[105151]: pgmap v2446: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:58.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:57 smithi134 ceph-mon[90085]: pgmap v2446: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:58.283 INFO:teuthology.orchestra.run.smithi017.stdout:2349347111367 2024-09-21T10:23:58.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T10:23:59.606 INFO:teuthology.orchestra.run.smithi017.stdout:2495375999425 2024-09-21T10:23:59.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T10:23:59.835 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:23:59 smithi017 ceph-mon[102913]: pgmap v2447: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:23:59.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:23:59 smithi017 ceph-mon[105151]: pgmap v2447: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:00.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:23:59 smithi134 ceph-mon[90085]: pgmap v2447: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:00.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:24:00 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:24:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:24:01.877 INFO:teuthology.orchestra.run.smithi017.stdout:2057289335254 2024-09-21T10:24:01.877 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T10:24:02.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:01 smithi017 ceph-mon[102913]: pgmap v2448: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:02.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:01 smithi017 ceph-mon[105151]: pgmap v2448: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:02.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:01 smithi134 ceph-mon[90085]: pgmap v2448: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:02.395 INFO:teuthology.orchestra.run.smithi017.stdout:158913790556 2024-09-21T10:24:02.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T10:24:03.371 INFO:teuthology.orchestra.run.smithi017.stdout:210453398099 2024-09-21T10:24:03.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T10:24:03.836 INFO:teuthology.orchestra.run.smithi017.stdout:184683594327 2024-09-21T10:24:03.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T10:24:04.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:03 smithi017 ceph-mon[102913]: pgmap v2449: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:04.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:03 smithi017 ceph-mon[105151]: pgmap v2449: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:04.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:03 smithi134 ceph-mon[90085]: pgmap v2449: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:04.551 INFO:teuthology.orchestra.run.smithi017.stdout:6068788789282 2024-09-21T10:24:04.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T10:24:04.608 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:04.615 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:04.785 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:04 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:24:04] "GET /metrics HTTP/1.1" 200 34272 "" "Prometheus/2.51.0" 2024-09-21T10:24:05.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:05 smithi017 ceph-mon[105151]: pgmap v2450: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:05.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:05 smithi017 ceph-mon[102913]: pgmap v2450: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:05.996 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:05.996 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:06.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:05 smithi134 ceph-mon[90085]: pgmap v2450: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:07.258 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:08.046 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:07 smithi134 ceph-mon[90085]: pgmap v2451: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:08.046 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:24:08.046 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:08.046 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:07 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:08.066 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:07 smithi017 ceph-mon[102913]: pgmap v2451: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:08.066 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:24:08.066 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:08.066 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:07 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:08.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:07 smithi017 ceph-mon[105151]: pgmap v2451: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:08.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:24:08.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:08.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:07 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:10.045 INFO:teuthology.orchestra.run.smithi017.stdout:5257039970494 2024-09-21T10:24:10.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:09 smithi017 ceph-mon[102913]: pgmap v2452: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:10.046 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:09 smithi017 ceph-mon[105151]: pgmap v2452: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:10.196 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:09 smithi134 ceph-mon[90085]: pgmap v2452: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:10.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:24:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:24:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:24:10.857 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5257039970492 got 5257039970494 for osd.4 2024-09-21T10:24:10.857 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:11.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:10 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/614451538' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:24:11.053 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:10 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:11.053 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:10 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/614451538' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:24:11.054 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:10 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:10 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/614451538' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:24:11.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:10 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:11.613 INFO:teuthology.orchestra.run.smithi017.stdout:2349347111370 2024-09-21T10:24:11.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:11 smithi017 ceph-mon[102913]: pgmap v2453: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:11.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:11 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1724542142' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:24:11.910 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:11 smithi017 ceph-mon[105151]: pgmap v2453: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:11.910 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:11 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1724542142' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:24:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:11 smithi134 ceph-mon[90085]: pgmap v2453: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:11.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:11 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1724542142' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:24:12.433 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2349347111367 got 2349347111370 for osd.2 2024-09-21T10:24:12.433 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:13.317 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:13.318 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:13.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:13.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:13 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:13.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:13 smithi017 ceph-mon[102913]: pgmap v2454: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:13.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:13.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:13 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:13.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:13 smithi017 ceph-mon[105151]: pgmap v2454: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:13 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:13.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:13 smithi134 ceph-mon[90085]: pgmap v2454: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:13.804 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:13.984 INFO:teuthology.orchestra.run.smithi017.stdout:2495375999428 2024-09-21T10:24:14.326 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:14 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/1279099403' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:24:14.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:14 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/1279099403' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:24:14.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:14 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/1279099403' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:24:14.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:14 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:24:14] "GET /metrics HTTP/1.1" 200 34270 "" "Prometheus/2.51.0" 2024-09-21T10:24:15.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:15 smithi017 ceph-mon[102913]: pgmap v2455: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:15.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:15 smithi017 ceph-mon[105151]: pgmap v2455: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:15.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:15 smithi134 ceph-mon[90085]: pgmap v2455: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:15.697 INFO:teuthology.orchestra.run.smithi017.stdout:158913790559 2024-09-21T10:24:16.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:16 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3943321540' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:24:16.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:16 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3943321540' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:24:16.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:16 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3943321540' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:24:16.936 INFO:teuthology.orchestra.run.smithi017.stdout:210453398101 2024-09-21T10:24:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:17 smithi017 ceph-mon[102913]: pgmap v2456: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:17.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:17 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/566839631' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:24:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:17 smithi017 ceph-mon[105151]: pgmap v2456: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:17.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:17 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/566839631' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:24:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:17 smithi134 ceph-mon[90085]: pgmap v2456: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:17.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:17 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/566839631' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:24:17.563 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2495375999425 got 2495375999428 for osd.3 2024-09-21T10:24:17.563 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:17.976 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913790556 got 158913790559 for osd.5 2024-09-21T10:24:17.976 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:18.684 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453398099 got 210453398101 for osd.7 2024-09-21T10:24:18.684 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:19.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:19 smithi017 ceph-mon[102913]: pgmap v2457: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:19.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:19 smithi017 ceph-mon[105151]: pgmap v2457: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:19.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:19 smithi134 ceph-mon[90085]: pgmap v2457: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:20.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:24:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:24:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:24:21.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:21 smithi017 ceph-mon[102913]: pgmap v2458: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:21.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:21 smithi017 ceph-mon[105151]: pgmap v2458: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:21.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:21 smithi134 ceph-mon[90085]: pgmap v2458: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:22.108 INFO:teuthology.orchestra.run.smithi017.stdout:184683594331 2024-09-21T10:24:22.340 INFO:teuthology.orchestra.run.smithi017.stdout:6068788789285 2024-09-21T10:24:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:22 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3059959833' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:24:22.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:22 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/3564485285' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:24:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:22 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3059959833' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:24:22.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:22 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/3564485285' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:24:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:22 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3059959833' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:24:22.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:22 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/3564485285' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:24:23.101 INFO:tasks.cephadm.ceph_manager.ceph:need seq 6068788789282 got 6068788789285 for osd.1 2024-09-21T10:24:23.101 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:23.936 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683594327 got 184683594331 for osd.6 2024-09-21T10:24:23.937 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:23.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:23 smithi134 ceph-mon[90085]: pgmap v2459: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:24.056 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:23 smithi017 ceph-mon[102913]: pgmap v2459: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:24.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:23 smithi017 ceph-mon[105151]: pgmap v2459: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:24.362 INFO:teuthology.orchestra.run.smithi017.stdout:2057289335258 2024-09-21T10:24:24.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:24 smithi134 ceph-mon[90085]: from='client.? 172.21.15.17:0/912124552' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:24:25.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:24 smithi017 ceph-mon[102913]: from='client.? 172.21.15.17:0/912124552' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:24:25.154 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:24 smithi017 ceph-mon[105151]: from='client.? 172.21.15.17:0/912124552' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:24:25.154 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:24 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:24:24] "GET /metrics HTTP/1.1" 200 34270 "" "Prometheus/2.51.0" 2024-09-21T10:24:25.183 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2057289335254 got 2057289335258 for osd.0 2024-09-21T10:24:25.183 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:25.183 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:24:25.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:24:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:25 smithi017 ceph-mon[102913]: pgmap v2460: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:25.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:25 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:25 smithi017 ceph-mon[105151]: pgmap v2460: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:25.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:25 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:25.992 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:25 smithi134 ceph-mon[90085]: pgmap v2460: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:25.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:25 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:27.801 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:27 smithi017 ceph-mon[102913]: pgmap v2461: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:27.801 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:27 smithi017 ceph-mon[105151]: pgmap v2461: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:27.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:27 smithi134 ceph-mon[90085]: pgmap v2461: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:28.710 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:24:28.712 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:24:29.492 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":2462,"stamp":"2024-09-21T10:24:28.671033+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":25375,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3158,"ondisk_log_size":3158,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7983608,"kb_used_data":55512,"kb_used_omap":445,"kb_used_meta":7927554,"kb_avail":741977608,"statfs":{"total":767960285184,"available":759785070592,"internally_reserved":0,"allocated":56844288,"data_stored":38330007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":456154,"internal_metadata":8117815846},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003139"},"pg_stats":[{"pgid":"1.1d","version":"288'100","reported_seq":82,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081903+0000","last_change":"2024-09-21T10:23:07.539539+0000","last_active":"2024-09-21T10:23:39.081903+0000","last_peered":"2024-09-21T10:23:39.081903+0000","last_clean":"2024-09-21T10:23:39.081903+0000","last_became_active":"2024-09-21T10:22:58.042554+0000","last_became_peered":"2024-09-21T10:22:58.042554+0000","last_unstale":"2024-09-21T10:23:39.081903+0000","last_undegraded":"2024-09-21T10:23:39.081903+0000","last_fullsized":"2024-09-21T10:23:39.081903+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:07.539489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:23:07.539489+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:43.553992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":2088,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081053+0000","last_change":"2024-09-21T10:22:50.995219+0000","last_active":"2024-09-21T10:23:39.081053+0000","last_peered":"2024-09-21T10:23:39.081053+0000","last_clean":"2024-09-21T10:23:39.081053+0000","last_became_active":"2024-09-21T10:22:50.991290+0000","last_became_peered":"2024-09-21T10:22:50.991290+0000","last_unstale":"2024-09-21T10:23:39.081053+0000","last_undegraded":"2024-09-21T10:23:39.081053+0000","last_fullsized":"2024-09-21T10:23:39.081053+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:43.705105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1469,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737811+0000","last_change":"2024-09-21T10:22:33.042466+0000","last_active":"2024-09-21T10:23:34.737811+0000","last_peered":"2024-09-21T10:23:34.737811+0000","last_clean":"2024-09-21T10:23:34.737811+0000","last_became_active":"2024-09-21T10:22:33.041986+0000","last_became_peered":"2024-09-21T10:22:33.041986+0000","last_unstale":"2024-09-21T10:23:34.737811+0000","last_undegraded":"2024-09-21T10:23:34.737811+0000","last_fullsized":"2024-09-21T10:23:34.737811+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:29.767550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":2293,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083350+0000","last_change":"2024-09-21T10:10:13.651599+0000","last_active":"2024-09-21T10:23:39.083350+0000","last_peered":"2024-09-21T10:23:39.083350+0000","last_clean":"2024-09-21T10:23:39.083350+0000","last_became_active":"2024-09-21T10:10:13.651250+0000","last_became_peered":"2024-09-21T10:10:13.651250+0000","last_unstale":"2024-09-21T10:23:39.083350+0000","last_undegraded":"2024-09-21T10:23:39.083350+0000","last_fullsized":"2024-09-21T10:23:39.083350+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:34.100056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2277,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081906+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:23:39.081906+0000","last_peered":"2024-09-21T10:23:39.081906+0000","last_clean":"2024-09-21T10:23:39.081906+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:23:39.081906+0000","last_undegraded":"2024-09-21T10:23:39.081906+0000","last_fullsized":"2024-09-21T10:23:39.081906+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"288'100","reported_seq":135,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250092+0000","last_change":"2024-09-21T10:22:33.042504+0000","last_active":"2024-09-21T10:23:35.250092+0000","last_peered":"2024-09-21T10:23:35.250092+0000","last_clean":"2024-09-21T10:23:35.250092+0000","last_became_active":"2024-09-21T10:22:33.042111+0000","last_became_peered":"2024-09-21T10:22:33.042111+0000","last_unstale":"2024-09-21T10:23:35.250092+0000","last_undegraded":"2024-09-21T10:23:35.250092+0000","last_fullsized":"2024-09-21T10:23:35.250092+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:26.113747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:26.113747+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:12.293819+0000","scrub_duration":0.0042569799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":1051,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250837+0000","last_change":"2024-09-21T10:23:10.371047+0000","last_active":"2024-09-21T10:23:35.250837+0000","last_peered":"2024-09-21T10:23:35.250837+0000","last_clean":"2024-09-21T10:23:35.250837+0000","last_became_active":"2024-09-21T10:23:10.369734+0000","last_became_peered":"2024-09-21T10:23:10.369734+0000","last_unstale":"2024-09-21T10:23:35.250837+0000","last_undegraded":"2024-09-21T10:23:35.250837+0000","last_fullsized":"2024-09-21T10:23:35.250837+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:34.312103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1406,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737947+0000","last_change":"2024-09-21T10:08:25.430667+0000","last_active":"2024-09-21T10:23:34.737947+0000","last_peered":"2024-09-21T10:23:34.737947+0000","last_clean":"2024-09-21T10:23:34.737947+0000","last_became_active":"2024-09-21T10:08:25.430357+0000","last_became_peered":"2024-09-21T10:08:25.430357+0000","last_unstale":"2024-09-21T10:23:34.737947+0000","last_undegraded":"2024-09-21T10:23:34.737947+0000","last_fullsized":"2024-09-21T10:23:34.737947+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:19.972020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"288'100","reported_seq":578,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083311+0000","last_change":"2024-09-21T10:19:37.200842+0000","last_active":"2024-09-21T10:23:39.083311+0000","last_peered":"2024-09-21T10:23:39.083311+0000","last_clean":"2024-09-21T10:23:39.083311+0000","last_became_active":"2024-09-21T10:17:59.914846+0000","last_became_peered":"2024-09-21T10:17:59.914846+0000","last_unstale":"2024-09-21T10:23:39.083311+0000","last_undegraded":"2024-09-21T10:23:39.083311+0000","last_fullsized":"2024-09-21T10:23:39.083311+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:37.200788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:37.200788+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:26:30.373346+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":1069,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250320+0000","last_change":"2024-09-21T10:23:10.368894+0000","last_active":"2024-09-21T10:23:35.250320+0000","last_peered":"2024-09-21T10:23:35.250320+0000","last_clean":"2024-09-21T10:23:35.250320+0000","last_became_active":"2024-09-21T10:23:10.368487+0000","last_became_peered":"2024-09-21T10:23:10.368487+0000","last_unstale":"2024-09-21T10:23:35.250320+0000","last_undegraded":"2024-09-21T10:23:35.250320+0000","last_fullsized":"2024-09-21T10:23:35.250320+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:19.938321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1446,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738357+0000","last_change":"2024-09-21T10:22:33.044570+0000","last_active":"2024-09-21T10:23:34.738357+0000","last_peered":"2024-09-21T10:23:34.738357+0000","last_clean":"2024-09-21T10:23:34.738357+0000","last_became_active":"2024-09-21T10:22:33.044188+0000","last_became_peered":"2024-09-21T10:22:33.044188+0000","last_unstale":"2024-09-21T10:23:34.738357+0000","last_undegraded":"2024-09-21T10:23:34.738357+0000","last_fullsized":"2024-09-21T10:23:34.738357+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:30.781767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":219,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082108+0000","last_change":"2024-09-21T10:22:47.984002+0000","last_active":"2024-09-21T10:23:39.082108+0000","last_peered":"2024-09-21T10:23:39.082108+0000","last_clean":"2024-09-21T10:23:39.082108+0000","last_became_active":"2024-09-21T10:22:47.983130+0000","last_became_peered":"2024-09-21T10:22:47.983130+0000","last_unstale":"2024-09-21T10:23:39.082108+0000","last_undegraded":"2024-09-21T10:23:39.082108+0000","last_fullsized":"2024-09-21T10:23:39.082108+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:30.409448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":48,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747315+0000","last_change":"2024-09-21T10:22:51.763013+0000","last_active":"2024-09-21T10:23:38.747315+0000","last_peered":"2024-09-21T10:23:38.747315+0000","last_clean":"2024-09-21T10:23:38.747315+0000","last_became_active":"2024-09-21T10:22:51.762651+0000","last_became_peered":"2024-09-21T10:22:51.762651+0000","last_unstale":"2024-09-21T10:23:38.747315+0000","last_undegraded":"2024-09-21T10:23:38.747315+0000","last_fullsized":"2024-09-21T10:23:38.747315+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:55:09.969973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"288'100","reported_seq":1473,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081353+0000","last_change":"2024-09-21T10:22:44.958109+0000","last_active":"2024-09-21T10:23:39.081353+0000","last_peered":"2024-09-21T10:23:39.081353+0000","last_clean":"2024-09-21T10:23:39.081353+0000","last_became_active":"2024-09-21T10:22:44.376211+0000","last_became_peered":"2024-09-21T10:22:44.376211+0000","last_unstale":"2024-09-21T10:23:39.081353+0000","last_undegraded":"2024-09-21T10:23:39.081353+0000","last_fullsized":"2024-09-21T10:23:39.081353+0000","mapping_epoch":1438,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1439,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:44.958060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:22:44.958060+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:33:17.440014+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1448,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738311+0000","last_change":"2024-09-21T10:22:33.045346+0000","last_active":"2024-09-21T10:23:34.738311+0000","last_peered":"2024-09-21T10:23:34.738311+0000","last_clean":"2024-09-21T10:23:34.738311+0000","last_became_active":"2024-09-21T10:22:33.045008+0000","last_became_peered":"2024-09-21T10:22:33.045008+0000","last_unstale":"2024-09-21T10:23:34.738311+0000","last_undegraded":"2024-09-21T10:23:34.738311+0000","last_fullsized":"2024-09-21T10:23:34.738311+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:52.061863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2306,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082925+0000","last_change":"2024-09-21T10:22:33.036913+0000","last_active":"2024-09-21T10:23:39.082925+0000","last_peered":"2024-09-21T10:23:39.082925+0000","last_clean":"2024-09-21T10:23:39.082925+0000","last_became_active":"2024-09-21T10:22:33.036531+0000","last_became_peered":"2024-09-21T10:22:33.036531+0000","last_unstale":"2024-09-21T10:23:39.082925+0000","last_undegraded":"2024-09-21T10:23:39.082925+0000","last_fullsized":"2024-09-21T10:23:39.082925+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:12.311660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"288'100","reported_seq":776,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251063+0000","last_change":"2024-09-21T10:23:10.368389+0000","last_active":"2024-09-21T10:23:35.251063+0000","last_peered":"2024-09-21T10:23:35.251063+0000","last_clean":"2024-09-21T10:23:35.251063+0000","last_became_active":"2024-09-21T10:23:10.367413+0000","last_became_peered":"2024-09-21T10:23:10.367413+0000","last_unstale":"2024-09-21T10:23:35.251063+0000","last_undegraded":"2024-09-21T10:23:35.251063+0000","last_fullsized":"2024-09-21T10:23:35.251063+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:38.463672+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:22:38.463672+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:29:43.881870+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081462+0000","last_change":"2024-09-21T10:22:33.050426+0000","last_active":"2024-09-21T10:23:39.081462+0000","last_peered":"2024-09-21T10:23:39.081462+0000","last_clean":"2024-09-21T10:23:39.081462+0000","last_became_active":"2024-09-21T10:22:33.049560+0000","last_became_peered":"2024-09-21T10:22:33.049560+0000","last_unstale":"2024-09-21T10:23:39.081462+0000","last_undegraded":"2024-09-21T10:23:39.081462+0000","last_fullsized":"2024-09-21T10:23:39.081462+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:51.566146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2168,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082821+0000","last_change":"2024-09-21T10:22:58.043866+0000","last_active":"2024-09-21T10:23:39.082821+0000","last_peered":"2024-09-21T10:23:39.082821+0000","last_clean":"2024-09-21T10:23:39.082821+0000","last_became_active":"2024-09-21T10:22:58.043485+0000","last_became_peered":"2024-09-21T10:22:58.043485+0000","last_unstale":"2024-09-21T10:23:39.082821+0000","last_undegraded":"2024-09-21T10:23:39.082821+0000","last_fullsized":"2024-09-21T10:23:39.082821+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:10.855019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"288'100","reported_seq":105,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250034+0000","last_change":"2024-09-21T10:23:10.367814+0000","last_active":"2024-09-21T10:23:35.250034+0000","last_peered":"2024-09-21T10:23:35.250034+0000","last_clean":"2024-09-21T10:23:35.250034+0000","last_became_active":"2024-09-21T10:23:10.366849+0000","last_became_peered":"2024-09-21T10:23:10.366849+0000","last_unstale":"2024-09-21T10:23:35.250034+0000","last_undegraded":"2024-09-21T10:23:35.250034+0000","last_fullsized":"2024-09-21T10:23:35.250034+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:48.372952+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:18:48.372952+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:00:41.913631+0000","scrub_duration":0.024424405,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2327,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081735+0000","last_change":"2024-09-21T10:22:44.376279+0000","last_active":"2024-09-21T10:23:39.081735+0000","last_peered":"2024-09-21T10:23:39.081735+0000","last_clean":"2024-09-21T10:23:39.081735+0000","last_became_active":"2024-09-21T10:22:44.375345+0000","last_became_peered":"2024-09-21T10:22:44.375345+0000","last_unstale":"2024-09-21T10:23:39.081735+0000","last_undegraded":"2024-09-21T10:23:39.081735+0000","last_fullsized":"2024-09-21T10:23:39.081735+0000","mapping_epoch":1438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:37:46.243557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":238,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251515+0000","last_change":"2024-09-21T10:22:33.046026+0000","last_active":"2024-09-21T10:23:35.251515+0000","last_peered":"2024-09-21T10:23:35.251515+0000","last_clean":"2024-09-21T10:23:35.251515+0000","last_became_active":"2024-09-21T10:22:33.045576+0000","last_became_peered":"2024-09-21T10:22:33.045576+0000","last_unstale":"2024-09-21T10:23:35.251515+0000","last_undegraded":"2024-09-21T10:23:35.251515+0000","last_fullsized":"2024-09-21T10:23:35.251515+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:23.069579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"288'100","reported_seq":136,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737948+0000","last_change":"2024-09-21T10:22:33.042799+0000","last_active":"2024-09-21T10:23:34.737948+0000","last_peered":"2024-09-21T10:23:34.737948+0000","last_clean":"2024-09-21T10:23:34.737948+0000","last_became_active":"2024-09-21T10:22:33.042445+0000","last_became_peered":"2024-09-21T10:22:33.042445+0000","last_unstale":"2024-09-21T10:23:34.737948+0000","last_undegraded":"2024-09-21T10:23:34.737948+0000","last_fullsized":"2024-09-21T10:23:34.737948+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:25.436066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:25.436066+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:53.646658+0000","scrub_duration":0.0032626159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2342,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081672+0000","last_change":"2024-09-21T10:22:33.038765+0000","last_active":"2024-09-21T10:23:39.081672+0000","last_peered":"2024-09-21T10:23:39.081672+0000","last_clean":"2024-09-21T10:23:39.081672+0000","last_became_active":"2024-09-21T10:22:33.038047+0000","last_became_peered":"2024-09-21T10:22:33.038047+0000","last_unstale":"2024-09-21T10:23:39.081672+0000","last_undegraded":"2024-09-21T10:23:39.081672+0000","last_fullsized":"2024-09-21T10:23:39.081672+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:06.217075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2224,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082944+0000","last_change":"2024-09-21T10:22:47.986131+0000","last_active":"2024-09-21T10:23:39.082944+0000","last_peered":"2024-09-21T10:23:39.082944+0000","last_clean":"2024-09-21T10:23:39.082944+0000","last_became_active":"2024-09-21T10:22:47.985179+0000","last_became_peered":"2024-09-21T10:22:47.985179+0000","last_unstale":"2024-09-21T10:23:39.082944+0000","last_undegraded":"2024-09-21T10:23:39.082944+0000","last_fullsized":"2024-09-21T10:23:39.082944+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:06.515185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"288'100","reported_seq":132,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250286+0000","last_change":"2024-09-21T10:22:33.042197+0000","last_active":"2024-09-21T10:23:35.250286+0000","last_peered":"2024-09-21T10:23:35.250286+0000","last_clean":"2024-09-21T10:23:35.250286+0000","last_became_active":"2024-09-21T10:22:33.041839+0000","last_became_peered":"2024-09-21T10:22:33.041839+0000","last_unstale":"2024-09-21T10:23:35.250286+0000","last_undegraded":"2024-09-21T10:23:35.250286+0000","last_fullsized":"2024-09-21T10:23:35.250286+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:52.067480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:52.067480+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:45.030904+0000","scrub_duration":0.0041194669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2293,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081673+0000","last_change":"2024-09-21T10:12:25.707393+0000","last_active":"2024-09-21T10:23:39.081673+0000","last_peered":"2024-09-21T10:23:39.081673+0000","last_clean":"2024-09-21T10:23:39.081673+0000","last_became_active":"2024-09-21T10:12:25.706457+0000","last_became_peered":"2024-09-21T10:12:25.706457+0000","last_unstale":"2024-09-21T10:23:39.081673+0000","last_undegraded":"2024-09-21T10:23:39.081673+0000","last_fullsized":"2024-09-21T10:23:39.081673+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:56.520799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":263,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250481+0000","last_change":"2024-09-21T10:23:10.367127+0000","last_active":"2024-09-21T10:23:35.250481+0000","last_peered":"2024-09-21T10:23:35.250481+0000","last_clean":"2024-09-21T10:23:35.250481+0000","last_became_active":"2024-09-21T10:23:10.366087+0000","last_became_peered":"2024-09-21T10:23:10.366087+0000","last_unstale":"2024-09-21T10:23:35.250481+0000","last_undegraded":"2024-09-21T10:23:35.250481+0000","last_fullsized":"2024-09-21T10:23:35.250481+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:59.742948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"288'100","reported_seq":489,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747476+0000","last_change":"2024-09-21T10:12:25.698905+0000","last_active":"2024-09-21T10:23:38.747476+0000","last_peered":"2024-09-21T10:23:38.747476+0000","last_clean":"2024-09-21T10:23:38.747476+0000","last_became_active":"2024-09-21T10:12:25.698552+0000","last_became_peered":"2024-09-21T10:12:25.698552+0000","last_unstale":"2024-09-21T10:23:38.747476+0000","last_undegraded":"2024-09-21T10:23:38.747476+0000","last_fullsized":"2024-09-21T10:23:38.747476+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.389965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.389965+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:27.529104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":207,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250192+0000","last_change":"2024-09-21T10:12:25.717602+0000","last_active":"2024-09-21T10:23:35.250192+0000","last_peered":"2024-09-21T10:23:35.250192+0000","last_clean":"2024-09-21T10:23:35.250192+0000","last_became_active":"2024-09-21T10:12:25.705211+0000","last_became_peered":"2024-09-21T10:12:25.705211+0000","last_unstale":"2024-09-21T10:23:35.250192+0000","last_undegraded":"2024-09-21T10:23:35.250192+0000","last_fullsized":"2024-09-21T10:23:35.250192+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:30.822884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081994+0000","last_change":"2024-09-21T10:22:33.043194+0000","last_active":"2024-09-21T10:23:39.081994+0000","last_peered":"2024-09-21T10:23:39.081994+0000","last_clean":"2024-09-21T10:23:39.081994+0000","last_became_active":"2024-09-21T10:22:33.042881+0000","last_became_peered":"2024-09-21T10:22:33.042881+0000","last_unstale":"2024-09-21T10:23:39.081994+0000","last_undegraded":"2024-09-21T10:23:39.081994+0000","last_fullsized":"2024-09-21T10:23:39.081994+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:51.918905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"288'100","reported_seq":140,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250066+0000","last_change":"2024-09-21T10:23:10.370481+0000","last_active":"2024-09-21T10:23:35.250066+0000","last_peered":"2024-09-21T10:23:35.250066+0000","last_clean":"2024-09-21T10:23:35.250066+0000","last_became_active":"2024-09-21T10:23:10.370164+0000","last_became_peered":"2024-09-21T10:23:10.370164+0000","last_unstale":"2024-09-21T10:23:35.250066+0000","last_undegraded":"2024-09-21T10:23:35.250066+0000","last_fullsized":"2024-09-21T10:23:35.250066+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:52.363524+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:22:52.363524+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:02.523728+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1489,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738197+0000","last_change":"2024-09-21T10:12:25.713745+0000","last_active":"2024-09-21T10:23:34.738197+0000","last_peered":"2024-09-21T10:23:34.738197+0000","last_clean":"2024-09-21T10:23:34.738197+0000","last_became_active":"2024-09-21T10:12:25.698011+0000","last_became_peered":"2024-09-21T10:12:25.698011+0000","last_unstale":"2024-09-21T10:23:34.738197+0000","last_undegraded":"2024-09-21T10:23:34.738197+0000","last_fullsized":"2024-09-21T10:23:34.738197+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:45.276913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23720,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1469,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738267+0000","last_change":"2024-09-21T10:22:33.047352+0000","last_active":"2024-09-21T10:23:34.738267+0000","last_peered":"2024-09-21T10:23:34.738267+0000","last_clean":"2024-09-21T10:23:34.738267+0000","last_became_active":"2024-09-21T10:22:33.046962+0000","last_became_peered":"2024-09-21T10:22:33.046962+0000","last_unstale":"2024-09-21T10:23:34.738267+0000","last_undegraded":"2024-09-21T10:23:34.738267+0000","last_fullsized":"2024-09-21T10:23:34.738267+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:49.393019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"288'100","reported_seq":75,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082479+0000","last_change":"2024-09-21T10:22:36.689148+0000","last_active":"2024-09-21T10:23:39.082479+0000","last_peered":"2024-09-21T10:23:39.082479+0000","last_clean":"2024-09-21T10:23:39.082479+0000","last_became_active":"2024-09-21T10:22:33.048932+0000","last_became_peered":"2024-09-21T10:22:33.048932+0000","last_unstale":"2024-09-21T10:23:39.082479+0000","last_undegraded":"2024-09-21T10:23:39.082479+0000","last_fullsized":"2024-09-21T10:23:39.082479+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:36.689077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:22:36.689077+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:51:38.323411+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1406,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737858+0000","last_change":"2024-09-21T10:17:34.767553+0000","last_active":"2024-09-21T10:23:34.737858+0000","last_peered":"2024-09-21T10:23:34.737858+0000","last_clean":"2024-09-21T10:23:34.737858+0000","last_became_active":"2024-09-21T10:17:34.767246+0000","last_became_peered":"2024-09-21T10:17:34.767246+0000","last_unstale":"2024-09-21T10:23:34.737858+0000","last_undegraded":"2024-09-21T10:23:34.737858+0000","last_fullsized":"2024-09-21T10:23:34.737858+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:52.295023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":248,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251427+0000","last_change":"2024-09-21T10:23:20.637723+0000","last_active":"2024-09-21T10:23:35.251427+0000","last_peered":"2024-09-21T10:23:35.251427+0000","last_clean":"2024-09-21T10:23:35.251427+0000","last_became_active":"2024-09-21T10:23:20.637372+0000","last_became_peered":"2024-09-21T10:23:20.637372+0000","last_unstale":"2024-09-21T10:23:35.251427+0000","last_undegraded":"2024-09-21T10:23:35.251427+0000","last_fullsized":"2024-09-21T10:23:35.251427+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:13.603516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":2175,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083007+0000","last_change":"2024-09-21T10:22:47.986522+0000","last_active":"2024-09-21T10:23:39.083007+0000","last_peered":"2024-09-21T10:23:39.083007+0000","last_clean":"2024-09-21T10:23:39.083007+0000","last_became_active":"2024-09-21T10:22:47.986189+0000","last_became_peered":"2024-09-21T10:22:47.986189+0000","last_unstale":"2024-09-21T10:23:39.083007+0000","last_undegraded":"2024-09-21T10:23:39.083007+0000","last_fullsized":"2024-09-21T10:23:39.083007+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.311237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":781,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082603+0000","last_change":"2024-09-21T10:23:06.597098+0000","last_active":"2024-09-21T10:23:39.082603+0000","last_peered":"2024-09-21T10:23:39.082603+0000","last_clean":"2024-09-21T10:23:39.082603+0000","last_became_active":"2024-09-21T10:23:06.596712+0000","last_became_peered":"2024-09-21T10:23:06.596712+0000","last_unstale":"2024-09-21T10:23:39.082603+0000","last_undegraded":"2024-09-21T10:23:39.082603+0000","last_fullsized":"2024-09-21T10:23:39.082603+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.423664+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.423664+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:04:37.927592+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":42,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081257+0000","last_change":"2024-09-21T10:22:58.041707+0000","last_active":"2024-09-21T10:23:39.081257+0000","last_peered":"2024-09-21T10:23:39.081257+0000","last_clean":"2024-09-21T10:23:39.081257+0000","last_became_active":"2024-09-21T10:22:58.041384+0000","last_became_peered":"2024-09-21T10:22:58.041384+0000","last_unstale":"2024-09-21T10:23:39.081257+0000","last_undegraded":"2024-09-21T10:23:39.081257+0000","last_fullsized":"2024-09-21T10:23:39.081257+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:31:14.321936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082584+0000","last_change":"2024-09-21T10:22:33.052199+0000","last_active":"2024-09-21T10:23:39.082584+0000","last_peered":"2024-09-21T10:23:39.082584+0000","last_clean":"2024-09-21T10:23:39.082584+0000","last_became_active":"2024-09-21T10:22:33.051111+0000","last_became_peered":"2024-09-21T10:22:33.051111+0000","last_unstale":"2024-09-21T10:23:39.082584+0000","last_undegraded":"2024-09-21T10:23:39.082584+0000","last_fullsized":"2024-09-21T10:23:39.082584+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:00:20.814368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":2301,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083398+0000","last_change":"2024-09-21T10:12:25.718713+0000","last_active":"2024-09-21T10:23:39.083398+0000","last_peered":"2024-09-21T10:23:39.083398+0000","last_clean":"2024-09-21T10:23:39.083398+0000","last_became_active":"2024-09-21T10:12:25.713007+0000","last_became_peered":"2024-09-21T10:12:25.713007+0000","last_unstale":"2024-09-21T10:23:39.083398+0000","last_undegraded":"2024-09-21T10:23:39.083398+0000","last_fullsized":"2024-09-21T10:23:39.083398+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:09.753443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":384,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738381+0000","last_change":"2024-09-21T10:18:56.346749+0000","last_active":"2024-09-21T10:23:34.738381+0000","last_peered":"2024-09-21T10:23:34.738381+0000","last_clean":"2024-09-21T10:23:34.738381+0000","last_became_active":"2024-09-21T10:17:19.685397+0000","last_became_peered":"2024-09-21T10:17:19.685397+0000","last_unstale":"2024-09-21T10:23:34.738381+0000","last_undegraded":"2024-09-21T10:23:34.738381+0000","last_fullsized":"2024-09-21T10:23:34.738381+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:56.346587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:56.346587+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:24.457314+0000","scrub_duration":0.0039726459999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2172,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082319+0000","last_change":"2024-09-21T10:22:47.985208+0000","last_active":"2024-09-21T10:23:39.082319+0000","last_peered":"2024-09-21T10:23:39.082319+0000","last_clean":"2024-09-21T10:23:39.082319+0000","last_became_active":"2024-09-21T10:22:47.984756+0000","last_became_peered":"2024-09-21T10:22:47.984756+0000","last_unstale":"2024-09-21T10:23:39.082319+0000","last_undegraded":"2024-09-21T10:23:39.082319+0000","last_fullsized":"2024-09-21T10:23:39.082319+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:33.877389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":201,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250367+0000","last_change":"2024-09-21T10:12:25.709819+0000","last_active":"2024-09-21T10:23:35.250367+0000","last_peered":"2024-09-21T10:23:35.250367+0000","last_clean":"2024-09-21T10:23:35.250367+0000","last_became_active":"2024-09-21T10:12:25.709548+0000","last_became_peered":"2024-09-21T10:12:25.709548+0000","last_unstale":"2024-09-21T10:23:35.250367+0000","last_undegraded":"2024-09-21T10:23:35.250367+0000","last_fullsized":"2024-09-21T10:23:35.250367+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:35.986614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":74,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082457+0000","last_change":"2024-09-21T10:23:06.597237+0000","last_active":"2024-09-21T10:23:39.082457+0000","last_peered":"2024-09-21T10:23:39.082457+0000","last_clean":"2024-09-21T10:23:39.082457+0000","last_became_active":"2024-09-21T10:23:06.596904+0000","last_became_peered":"2024-09-21T10:23:06.596904+0000","last_unstale":"2024-09-21T10:23:39.082457+0000","last_undegraded":"2024-09-21T10:23:39.082457+0000","last_fullsized":"2024-09-21T10:23:39.082457+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:58:16.006665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":104,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250408+0000","last_change":"2024-09-21T10:23:17.784587+0000","last_active":"2024-09-21T10:23:35.250408+0000","last_peered":"2024-09-21T10:23:35.250408+0000","last_clean":"2024-09-21T10:23:35.250408+0000","last_became_active":"2024-09-21T10:23:16.613104+0000","last_became_peered":"2024-09-21T10:23:16.613104+0000","last_unstale":"2024-09-21T10:23:35.250408+0000","last_undegraded":"2024-09-21T10:23:35.250408+0000","last_fullsized":"2024-09-21T10:23:35.250408+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:17.784520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:23:17.784520+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:09:29.075918+0000","scrub_duration":0.001690001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747227+0000","last_change":"2024-09-21T10:23:06.599594+0000","last_active":"2024-09-21T10:23:38.747227+0000","last_peered":"2024-09-21T10:23:38.747227+0000","last_clean":"2024-09-21T10:23:38.747227+0000","last_became_active":"2024-09-21T10:23:06.599239+0000","last_became_peered":"2024-09-21T10:23:06.599239+0000","last_unstale":"2024-09-21T10:23:38.747227+0000","last_undegraded":"2024-09-21T10:23:38.747227+0000","last_fullsized":"2024-09-21T10:23:38.747227+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:18.384899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082350+0000","last_change":"2024-09-21T10:22:33.047721+0000","last_active":"2024-09-21T10:23:39.082350+0000","last_peered":"2024-09-21T10:23:39.082350+0000","last_clean":"2024-09-21T10:23:39.082350+0000","last_became_active":"2024-09-21T10:22:33.045293+0000","last_became_peered":"2024-09-21T10:22:33.045293+0000","last_unstale":"2024-09-21T10:23:39.082350+0000","last_undegraded":"2024-09-21T10:23:39.082350+0000","last_fullsized":"2024-09-21T10:23:39.082350+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:25.630200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1116,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250626+0000","last_change":"2024-09-21T10:23:10.367137+0000","last_active":"2024-09-21T10:23:35.250626+0000","last_peered":"2024-09-21T10:23:35.250626+0000","last_clean":"2024-09-21T10:23:35.250626+0000","last_became_active":"2024-09-21T10:23:10.366794+0000","last_became_peered":"2024-09-21T10:23:10.366794+0000","last_unstale":"2024-09-21T10:23:35.250626+0000","last_undegraded":"2024-09-21T10:23:35.250626+0000","last_fullsized":"2024-09-21T10:23:35.250626+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:10:26.572649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":168,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251418+0000","last_change":"2024-09-21T10:23:16.831798+0000","last_active":"2024-09-21T10:23:35.251418+0000","last_peered":"2024-09-21T10:23:35.251418+0000","last_clean":"2024-09-21T10:23:35.251418+0000","last_became_active":"2024-09-21T10:23:16.614393+0000","last_became_peered":"2024-09-21T10:23:16.614393+0000","last_unstale":"2024-09-21T10:23:35.251418+0000","last_undegraded":"2024-09-21T10:23:35.251418+0000","last_fullsized":"2024-09-21T10:23:35.251418+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:16.831734+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:23:16.831734+0000","last_clean_scrub_stamp":"2024-09-21T10:23:16.831734+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:47.540649+0000","scrub_duration":0.001899548,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082340+0000","last_change":"2024-09-21T10:22:33.042737+0000","last_active":"2024-09-21T10:23:39.082340+0000","last_peered":"2024-09-21T10:23:39.082340+0000","last_clean":"2024-09-21T10:23:39.082340+0000","last_became_active":"2024-09-21T10:22:33.042348+0000","last_became_peered":"2024-09-21T10:22:33.042348+0000","last_unstale":"2024-09-21T10:23:39.082340+0000","last_undegraded":"2024-09-21T10:23:39.082340+0000","last_fullsized":"2024-09-21T10:23:39.082340+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:00.941239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1404,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738606+0000","last_change":"2024-09-21T10:10:13.686243+0000","last_active":"2024-09-21T10:23:34.738606+0000","last_peered":"2024-09-21T10:23:34.738606+0000","last_clean":"2024-09-21T10:23:34.738606+0000","last_became_active":"2024-09-21T10:10:13.685894+0000","last_became_peered":"2024-09-21T10:10:13.685894+0000","last_unstale":"2024-09-21T10:23:34.738606+0000","last_undegraded":"2024-09-21T10:23:34.738606+0000","last_fullsized":"2024-09-21T10:23:34.738606+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:12.394732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1196,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250639+0000","last_change":"2024-09-21T10:23:10.370371+0000","last_active":"2024-09-21T10:23:35.250639+0000","last_peered":"2024-09-21T10:23:35.250639+0000","last_clean":"2024-09-21T10:23:35.250639+0000","last_became_active":"2024-09-21T10:23:10.369529+0000","last_became_peered":"2024-09-21T10:23:10.369529+0000","last_unstale":"2024-09-21T10:23:35.250639+0000","last_undegraded":"2024-09-21T10:23:35.250639+0000","last_fullsized":"2024-09-21T10:23:35.250639+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:02:23.128537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":52,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.249952+0000","last_change":"2024-09-21T10:23:10.365099+0000","last_active":"2024-09-21T10:23:35.249952+0000","last_peered":"2024-09-21T10:23:35.249952+0000","last_clean":"2024-09-21T10:23:35.249952+0000","last_became_active":"2024-09-21T10:23:10.364753+0000","last_became_peered":"2024-09-21T10:23:10.364753+0000","last_unstale":"2024-09-21T10:23:35.249952+0000","last_undegraded":"2024-09-21T10:23:35.249952+0000","last_fullsized":"2024-09-21T10:23:35.249952+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:19.901507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:19.901507+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:03.501890+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082107+0000","last_change":"2024-09-21T10:22:33.048943+0000","last_active":"2024-09-21T10:23:39.082107+0000","last_peered":"2024-09-21T10:23:39.082107+0000","last_clean":"2024-09-21T10:23:39.082107+0000","last_became_active":"2024-09-21T10:22:33.048176+0000","last_became_peered":"2024-09-21T10:22:33.048176+0000","last_unstale":"2024-09-21T10:23:39.082107+0000","last_undegraded":"2024-09-21T10:23:39.082107+0000","last_fullsized":"2024-09-21T10:23:39.082107+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:37.352591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082735+0000","last_change":"2024-09-21T10:22:33.046183+0000","last_active":"2024-09-21T10:23:39.082735+0000","last_peered":"2024-09-21T10:23:39.082735+0000","last_clean":"2024-09-21T10:23:39.082735+0000","last_became_active":"2024-09-21T10:22:33.045814+0000","last_became_peered":"2024-09-21T10:22:33.045814+0000","last_unstale":"2024-09-21T10:23:39.082735+0000","last_undegraded":"2024-09-21T10:23:39.082735+0000","last_fullsized":"2024-09-21T10:23:39.082735+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:34:18.457195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2328,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747189+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:23:38.747189+0000","last_peered":"2024-09-21T10:23:38.747189+0000","last_clean":"2024-09-21T10:23:38.747189+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:23:38.747189+0000","last_undegraded":"2024-09-21T10:23:38.747189+0000","last_fullsized":"2024-09-21T10:23:38.747189+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":15,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251244+0000","last_change":"2024-09-21T10:23:20.638309+0000","last_active":"2024-09-21T10:23:35.251244+0000","last_peered":"2024-09-21T10:23:35.251244+0000","last_clean":"2024-09-21T10:23:35.251244+0000","last_became_active":"2024-09-21T10:23:20.637944+0000","last_became_peered":"2024-09-21T10:23:20.637944+0000","last_unstale":"2024-09-21T10:23:35.251244+0000","last_undegraded":"2024-09-21T10:23:35.251244+0000","last_fullsized":"2024-09-21T10:23:35.251244+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:53.368905+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:22:08.152253+0000","last_clean_scrub_stamp":"2024-09-21T10:22:53.368905+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:54.785148+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":73,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082483+0000","last_change":"2024-09-21T10:22:47.986298+0000","last_active":"2024-09-21T10:23:39.082483+0000","last_peered":"2024-09-21T10:23:39.082483+0000","last_clean":"2024-09-21T10:23:39.082483+0000","last_became_active":"2024-09-21T10:22:47.985994+0000","last_became_peered":"2024-09-21T10:22:47.985994+0000","last_unstale":"2024-09-21T10:23:39.082483+0000","last_undegraded":"2024-09-21T10:23:39.082483+0000","last_fullsized":"2024-09-21T10:23:39.082483+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:00.932876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2145,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081448+0000","last_change":"2024-09-21T10:22:58.045948+0000","last_active":"2024-09-21T10:23:39.081448+0000","last_peered":"2024-09-21T10:23:39.081448+0000","last_clean":"2024-09-21T10:23:39.081448+0000","last_became_active":"2024-09-21T10:22:58.045494+0000","last_became_peered":"2024-09-21T10:22:58.045494+0000","last_unstale":"2024-09-21T10:23:39.081448+0000","last_undegraded":"2024-09-21T10:23:39.081448+0000","last_fullsized":"2024-09-21T10:23:39.081448+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:29.385283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":221,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250751+0000","last_change":"2024-09-21T10:22:33.041385+0000","last_active":"2024-09-21T10:23:35.250751+0000","last_peered":"2024-09-21T10:23:35.250751+0000","last_clean":"2024-09-21T10:23:35.250751+0000","last_became_active":"2024-09-21T10:22:33.041045+0000","last_became_peered":"2024-09-21T10:22:33.041045+0000","last_unstale":"2024-09-21T10:23:35.250751+0000","last_undegraded":"2024-09-21T10:23:35.250751+0000","last_fullsized":"2024-09-21T10:23:35.250751+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:11.937604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":75,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081908+0000","last_change":"2024-09-21T10:22:37.705040+0000","last_active":"2024-09-21T10:23:39.081908+0000","last_peered":"2024-09-21T10:23:39.081908+0000","last_clean":"2024-09-21T10:23:39.081908+0000","last_became_active":"2024-09-21T10:22:33.053060+0000","last_became_peered":"2024-09-21T10:22:33.053060+0000","last_unstale":"2024-09-21T10:23:39.081908+0000","last_undegraded":"2024-09-21T10:23:39.081908+0000","last_fullsized":"2024-09-21T10:23:39.081908+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:37.704977+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:22:37.704977+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:36:11.035210+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":180,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251009+0000","last_change":"2024-09-21T10:23:16.614752+0000","last_active":"2024-09-21T10:23:35.251009+0000","last_peered":"2024-09-21T10:23:35.251009+0000","last_clean":"2024-09-21T10:23:35.251009+0000","last_became_active":"2024-09-21T10:23:16.614428+0000","last_became_peered":"2024-09-21T10:23:16.614428+0000","last_unstale":"2024-09-21T10:23:35.251009+0000","last_undegraded":"2024-09-21T10:23:35.251009+0000","last_fullsized":"2024-09-21T10:23:35.251009+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:13.826655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2295,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081350+0000","last_change":"2024-09-21T10:23:06.599521+0000","last_active":"2024-09-21T10:23:39.081350+0000","last_peered":"2024-09-21T10:23:39.081350+0000","last_clean":"2024-09-21T10:23:39.081350+0000","last_became_active":"2024-09-21T10:23:06.599178+0000","last_became_peered":"2024-09-21T10:23:06.599178+0000","last_unstale":"2024-09-21T10:23:39.081350+0000","last_undegraded":"2024-09-21T10:23:39.081350+0000","last_fullsized":"2024-09-21T10:23:39.081350+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:59.325935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":587,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250119+0000","last_change":"2024-09-21T10:23:10.367465+0000","last_active":"2024-09-21T10:23:35.250119+0000","last_peered":"2024-09-21T10:23:35.250119+0000","last_clean":"2024-09-21T10:23:35.250119+0000","last_became_active":"2024-09-21T10:23:10.366025+0000","last_became_peered":"2024-09-21T10:23:10.366025+0000","last_unstale":"2024-09-21T10:23:35.250119+0000","last_undegraded":"2024-09-21T10:23:35.250119+0000","last_fullsized":"2024-09-21T10:23:35.250119+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:41.453486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":211,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250665+0000","last_change":"2024-09-21T10:13:20.647735+0000","last_active":"2024-09-21T10:23:35.250665+0000","last_peered":"2024-09-21T10:23:35.250665+0000","last_clean":"2024-09-21T10:23:35.250665+0000","last_became_active":"2024-09-21T10:12:25.708657+0000","last_became_peered":"2024-09-21T10:12:25.708657+0000","last_unstale":"2024-09-21T10:23:35.250665+0000","last_undegraded":"2024-09-21T10:23:35.250665+0000","last_fullsized":"2024-09-21T10:23:35.250665+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:20.647659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:20.647659+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:09.770331+0000","scrub_duration":0.0016005629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":60,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747645+0000","last_change":"2024-09-21T10:22:42.315636+0000","last_active":"2024-09-21T10:23:38.747645+0000","last_peered":"2024-09-21T10:23:38.747645+0000","last_clean":"2024-09-21T10:23:38.747645+0000","last_became_active":"2024-09-21T10:22:42.315339+0000","last_became_peered":"2024-09-21T10:22:42.315339+0000","last_unstale":"2024-09-21T10:23:38.747645+0000","last_undegraded":"2024-09-21T10:23:38.747645+0000","last_fullsized":"2024-09-21T10:23:38.747645+0000","mapping_epoch":1436,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:17:34.930301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":42,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081354+0000","last_change":"2024-09-21T10:22:58.041652+0000","last_active":"2024-09-21T10:23:39.081354+0000","last_peered":"2024-09-21T10:23:39.081354+0000","last_clean":"2024-09-21T10:23:39.081354+0000","last_became_active":"2024-09-21T10:22:58.041333+0000","last_became_peered":"2024-09-21T10:22:58.041333+0000","last_unstale":"2024-09-21T10:23:39.081354+0000","last_undegraded":"2024-09-21T10:23:39.081354+0000","last_fullsized":"2024-09-21T10:23:39.081354+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.224773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":34,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250560+0000","last_change":"2024-09-21T10:23:21.203033+0000","last_active":"2024-09-21T10:23:35.250560+0000","last_peered":"2024-09-21T10:23:35.250560+0000","last_clean":"2024-09-21T10:23:35.250560+0000","last_became_active":"2024-09-21T10:23:10.370050+0000","last_became_peered":"2024-09-21T10:23:10.370050+0000","last_unstale":"2024-09-21T10:23:35.250560+0000","last_undegraded":"2024-09-21T10:23:35.250560+0000","last_fullsized":"2024-09-21T10:23:35.250560+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:21.202969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:23:21.202969+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:08:57.680472+0000","scrub_duration":0.0022498879999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1469,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738625+0000","last_change":"2024-09-21T10:22:33.048246+0000","last_active":"2024-09-21T10:23:34.738625+0000","last_peered":"2024-09-21T10:23:34.738625+0000","last_clean":"2024-09-21T10:23:34.738625+0000","last_became_active":"2024-09-21T10:22:33.047853+0000","last_became_peered":"2024-09-21T10:22:33.047853+0000","last_unstale":"2024-09-21T10:23:34.738625+0000","last_undegraded":"2024-09-21T10:23:34.738625+0000","last_fullsized":"2024-09-21T10:23:34.738625+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:11:19.033632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1417,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737877+0000","last_change":"2024-09-21T10:12:25.697464+0000","last_active":"2024-09-21T10:23:34.737877+0000","last_peered":"2024-09-21T10:23:34.737877+0000","last_clean":"2024-09-21T10:23:34.737877+0000","last_became_active":"2024-09-21T10:12:25.697050+0000","last_became_peered":"2024-09-21T10:12:25.697050+0000","last_unstale":"2024-09-21T10:23:34.737877+0000","last_undegraded":"2024-09-21T10:23:34.737877+0000","last_fullsized":"2024-09-21T10:23:34.737877+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:50.687008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"288'100","reported_seq":82,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082126+0000","last_change":"2024-09-21T10:23:13.578911+0000","last_active":"2024-09-21T10:23:39.082126+0000","last_peered":"2024-09-21T10:23:39.082126+0000","last_clean":"2024-09-21T10:23:39.082126+0000","last_became_active":"2024-09-21T10:23:06.596989+0000","last_became_peered":"2024-09-21T10:23:06.596989+0000","last_unstale":"2024-09-21T10:23:39.082126+0000","last_undegraded":"2024-09-21T10:23:39.082126+0000","last_fullsized":"2024-09-21T10:23:39.082126+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:13.578856+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:23:13.578856+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:47.965583+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1446,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738609+0000","last_change":"2024-09-21T10:22:33.045553+0000","last_active":"2024-09-21T10:23:34.738609+0000","last_peered":"2024-09-21T10:23:34.738609+0000","last_clean":"2024-09-21T10:23:34.738609+0000","last_became_active":"2024-09-21T10:22:33.044637+0000","last_became_peered":"2024-09-21T10:22:33.044637+0000","last_unstale":"2024-09-21T10:23:34.738609+0000","last_undegraded":"2024-09-21T10:23:34.738609+0000","last_fullsized":"2024-09-21T10:23:34.738609+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:26.997458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":648,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250365+0000","last_change":"2024-09-21T10:23:16.612207+0000","last_active":"2024-09-21T10:23:35.250365+0000","last_peered":"2024-09-21T10:23:35.250365+0000","last_clean":"2024-09-21T10:23:35.250365+0000","last_became_active":"2024-09-21T10:23:16.611858+0000","last_became_peered":"2024-09-21T10:23:16.611858+0000","last_unstale":"2024-09-21T10:23:35.250365+0000","last_undegraded":"2024-09-21T10:23:35.250365+0000","last_fullsized":"2024-09-21T10:23:35.250365+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:50.839744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":751,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250893+0000","last_change":"2024-09-21T10:23:12.159934+0000","last_active":"2024-09-21T10:23:35.250893+0000","last_peered":"2024-09-21T10:23:35.250893+0000","last_clean":"2024-09-21T10:23:35.250893+0000","last_became_active":"2024-09-21T10:23:10.367850+0000","last_became_peered":"2024-09-21T10:23:10.367850+0000","last_unstale":"2024-09-21T10:23:35.250893+0000","last_undegraded":"2024-09-21T10:23:35.250893+0000","last_fullsized":"2024-09-21T10:23:35.250893+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:12.159856+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:23:12.159856+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:43.475443+0000","scrub_duration":0.001761882,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1403,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738381+0000","last_change":"2024-09-21T10:10:13.661328+0000","last_active":"2024-09-21T10:23:34.738381+0000","last_peered":"2024-09-21T10:23:34.738381+0000","last_clean":"2024-09-21T10:23:34.738381+0000","last_became_active":"2024-09-21T10:10:13.661028+0000","last_became_peered":"2024-09-21T10:10:13.661028+0000","last_unstale":"2024-09-21T10:23:34.738381+0000","last_undegraded":"2024-09-21T10:23:34.738381+0000","last_fullsized":"2024-09-21T10:23:34.738381+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:36.965214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":2289,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082916+0000","last_change":"2024-09-21T10:08:25.436371+0000","last_active":"2024-09-21T10:23:39.082916+0000","last_peered":"2024-09-21T10:23:39.082916+0000","last_clean":"2024-09-21T10:23:39.082916+0000","last_became_active":"2024-09-21T10:08:25.435921+0000","last_became_peered":"2024-09-21T10:08:25.435921+0000","last_unstale":"2024-09-21T10:23:39.082916+0000","last_undegraded":"2024-09-21T10:23:39.082916+0000","last_fullsized":"2024-09-21T10:23:39.082916+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:27.612386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":231,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.249923+0000","last_change":"2024-09-21T10:17:27.870379+0000","last_active":"2024-09-21T10:23:35.249923+0000","last_peered":"2024-09-21T10:23:35.249923+0000","last_clean":"2024-09-21T10:23:35.249923+0000","last_became_active":"2024-09-21T10:17:19.686196+0000","last_became_peered":"2024-09-21T10:17:19.686196+0000","last_unstale":"2024-09-21T10:23:35.249923+0000","last_undegraded":"2024-09-21T10:23:35.249923+0000","last_fullsized":"2024-09-21T10:23:35.249923+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:17:27.870303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:17:27.870303+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:22.680785+0000","scrub_duration":0.001791443,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2153,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082256+0000","last_change":"2024-09-21T10:22:47.984893+0000","last_active":"2024-09-21T10:23:39.082256+0000","last_peered":"2024-09-21T10:23:39.082256+0000","last_clean":"2024-09-21T10:23:39.082256+0000","last_became_active":"2024-09-21T10:22:47.983631+0000","last_became_peered":"2024-09-21T10:22:47.983631+0000","last_unstale":"2024-09-21T10:23:39.082256+0000","last_undegraded":"2024-09-21T10:23:39.082256+0000","last_fullsized":"2024-09-21T10:23:39.082256+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:15:16.137659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1067,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250396+0000","last_change":"2024-09-21T10:23:10.369746+0000","last_active":"2024-09-21T10:23:35.250396+0000","last_peered":"2024-09-21T10:23:35.250396+0000","last_clean":"2024-09-21T10:23:35.250396+0000","last_became_active":"2024-09-21T10:23:10.369418+0000","last_became_peered":"2024-09-21T10:23:10.369418+0000","last_unstale":"2024-09-21T10:23:35.250396+0000","last_undegraded":"2024-09-21T10:23:35.250396+0000","last_fullsized":"2024-09-21T10:23:35.250396+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.584132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"288'100","reported_seq":287,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738452+0000","last_change":"2024-09-21T10:18:31.396074+0000","last_active":"2024-09-21T10:23:34.738452+0000","last_peered":"2024-09-21T10:23:34.738452+0000","last_clean":"2024-09-21T10:23:34.738452+0000","last_became_active":"2024-09-21T10:17:19.681584+0000","last_became_peered":"2024-09-21T10:17:19.681584+0000","last_unstale":"2024-09-21T10:23:34.738452+0000","last_undegraded":"2024-09-21T10:23:34.738452+0000","last_fullsized":"2024-09-21T10:23:34.738452+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:31.396016+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:31.396016+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:22.138287+0000","scrub_duration":0.001381925,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":248,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250221+0000","last_change":"2024-09-21T10:22:33.047543+0000","last_active":"2024-09-21T10:23:35.250221+0000","last_peered":"2024-09-21T10:23:35.250221+0000","last_clean":"2024-09-21T10:23:35.250221+0000","last_became_active":"2024-09-21T10:22:33.046756+0000","last_became_peered":"2024-09-21T10:22:33.046756+0000","last_unstale":"2024-09-21T10:23:35.250221+0000","last_undegraded":"2024-09-21T10:23:35.250221+0000","last_fullsized":"2024-09-21T10:23:35.250221+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:01.549176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":233,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250091+0000","last_change":"2024-09-21T10:23:06.604259+0000","last_active":"2024-09-21T10:23:35.250091+0000","last_peered":"2024-09-21T10:23:35.250091+0000","last_clean":"2024-09-21T10:23:35.250091+0000","last_became_active":"2024-09-21T10:23:06.598107+0000","last_became_peered":"2024-09-21T10:23:06.598107+0000","last_unstale":"2024-09-21T10:23:35.250091+0000","last_undegraded":"2024-09-21T10:23:35.250091+0000","last_fullsized":"2024-09-21T10:23:35.250091+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:14.773082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"288'100","reported_seq":84,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251529+0000","last_change":"2024-09-21T10:23:16.612225+0000","last_active":"2024-09-21T10:23:35.251529+0000","last_peered":"2024-09-21T10:23:35.251529+0000","last_clean":"2024-09-21T10:23:35.251529+0000","last_became_active":"2024-09-21T10:23:16.611856+0000","last_became_peered":"2024-09-21T10:23:16.611856+0000","last_unstale":"2024-09-21T10:23:35.251529+0000","last_undegraded":"2024-09-21T10:23:35.251529+0000","last_fullsized":"2024-09-21T10:23:35.251529+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:23.746751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:23.746751+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:11.238986+0000","scrub_duration":0.0021986229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1466,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738119+0000","last_change":"2024-09-21T10:22:58.043427+0000","last_active":"2024-09-21T10:23:34.738119+0000","last_peered":"2024-09-21T10:23:34.738119+0000","last_clean":"2024-09-21T10:23:34.738119+0000","last_became_active":"2024-09-21T10:22:58.043145+0000","last_became_peered":"2024-09-21T10:22:58.043145+0000","last_unstale":"2024-09-21T10:23:34.738119+0000","last_undegraded":"2024-09-21T10:23:34.738119+0000","last_fullsized":"2024-09-21T10:23:34.738119+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:29:31.738371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":247,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251121+0000","last_change":"2024-09-21T10:22:33.055842+0000","last_active":"2024-09-21T10:23:35.251121+0000","last_peered":"2024-09-21T10:23:35.251121+0000","last_clean":"2024-09-21T10:23:35.251121+0000","last_became_active":"2024-09-21T10:22:33.043460+0000","last_became_peered":"2024-09-21T10:22:33.043460+0000","last_unstale":"2024-09-21T10:23:35.251121+0000","last_undegraded":"2024-09-21T10:23:35.251121+0000","last_fullsized":"2024-09-21T10:23:35.251121+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:45.942510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"288'100","reported_seq":777,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082786+0000","last_change":"2024-09-21T10:21:24.467648+0000","last_active":"2024-09-21T10:23:39.082786+0000","last_peered":"2024-09-21T10:23:39.082786+0000","last_clean":"2024-09-21T10:23:39.082786+0000","last_became_active":"2024-09-21T10:17:19.680813+0000","last_became_peered":"2024-09-21T10:17:19.680813+0000","last_unstale":"2024-09-21T10:23:39.082786+0000","last_undegraded":"2024-09-21T10:23:39.082786+0000","last_fullsized":"2024-09-21T10:23:39.082786+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:21:24.467561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:21:24.467561+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:08:02.729976+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":223,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250980+0000","last_change":"2024-09-21T10:23:10.371553+0000","last_active":"2024-09-21T10:23:35.250980+0000","last_peered":"2024-09-21T10:23:35.250980+0000","last_clean":"2024-09-21T10:23:35.250980+0000","last_became_active":"2024-09-21T10:23:10.371261+0000","last_became_peered":"2024-09-21T10:23:10.371261+0000","last_unstale":"2024-09-21T10:23:35.250980+0000","last_undegraded":"2024-09-21T10:23:35.250980+0000","last_fullsized":"2024-09-21T10:23:35.250980+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:40.760922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":205,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250840+0000","last_change":"2024-09-21T10:12:25.764496+0000","last_active":"2024-09-21T10:23:35.250840+0000","last_peered":"2024-09-21T10:23:35.250840+0000","last_clean":"2024-09-21T10:23:35.250840+0000","last_became_active":"2024-09-21T10:12:25.707263+0000","last_became_peered":"2024-09-21T10:12:25.707263+0000","last_unstale":"2024-09-21T10:23:35.250840+0000","last_undegraded":"2024-09-21T10:23:35.250840+0000","last_fullsized":"2024-09-21T10:23:35.250840+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:45.728577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"288'100","reported_seq":254,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250456+0000","last_change":"2024-09-21T10:22:33.041456+0000","last_active":"2024-09-21T10:23:35.250456+0000","last_peered":"2024-09-21T10:23:35.250456+0000","last_clean":"2024-09-21T10:23:35.250456+0000","last_became_active":"2024-09-21T10:22:33.041009+0000","last_became_peered":"2024-09-21T10:22:33.041009+0000","last_unstale":"2024-09-21T10:23:35.250456+0000","last_undegraded":"2024-09-21T10:23:35.250456+0000","last_fullsized":"2024-09-21T10:23:35.250456+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:36.066408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:36.066408+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:47:29.351359+0000","scrub_duration":0.006208028,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":308,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.249824+0000","last_change":"2024-09-21T10:23:06.608302+0000","last_active":"2024-09-21T10:23:35.249824+0000","last_peered":"2024-09-21T10:23:35.249824+0000","last_clean":"2024-09-21T10:23:35.249824+0000","last_became_active":"2024-09-21T10:23:06.601367+0000","last_became_peered":"2024-09-21T10:23:06.601367+0000","last_unstale":"2024-09-21T10:23:35.249824+0000","last_undegraded":"2024-09-21T10:23:35.249824+0000","last_fullsized":"2024-09-21T10:23:35.249824+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:22.111234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":42,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083202+0000","last_change":"2024-09-21T10:22:58.040371+0000","last_active":"2024-09-21T10:23:39.083202+0000","last_peered":"2024-09-21T10:23:39.083202+0000","last_clean":"2024-09-21T10:23:39.083202+0000","last_became_active":"2024-09-21T10:22:58.039759+0000","last_became_peered":"2024-09-21T10:22:58.039759+0000","last_unstale":"2024-09-21T10:23:39.083202+0000","last_undegraded":"2024-09-21T10:23:39.083202+0000","last_fullsized":"2024-09-21T10:23:39.083202+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:15.508750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"288'100","reported_seq":183,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250481+0000","last_change":"2024-09-21T10:23:10.371087+0000","last_active":"2024-09-21T10:23:35.250481+0000","last_peered":"2024-09-21T10:23:35.250481+0000","last_clean":"2024-09-21T10:23:35.250481+0000","last_became_active":"2024-09-21T10:23:10.370760+0000","last_became_peered":"2024-09-21T10:23:10.370760+0000","last_unstale":"2024-09-21T10:23:35.250481+0000","last_undegraded":"2024-09-21T10:23:35.250481+0000","last_fullsized":"2024-09-21T10:23:35.250481+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:31.567804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:31.567804+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:51.257478+0000","scrub_duration":0.001700088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":201,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.249706+0000","last_change":"2024-09-21T10:12:25.700080+0000","last_active":"2024-09-21T10:23:35.249706+0000","last_peered":"2024-09-21T10:23:35.249706+0000","last_clean":"2024-09-21T10:23:35.249706+0000","last_became_active":"2024-09-21T10:12:25.699762+0000","last_became_peered":"2024-09-21T10:12:25.699762+0000","last_unstale":"2024-09-21T10:23:35.249706+0000","last_undegraded":"2024-09-21T10:23:35.249706+0000","last_fullsized":"2024-09-21T10:23:35.249706+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:41.509175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2141,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082910+0000","last_change":"2024-09-21T10:22:47.985318+0000","last_active":"2024-09-21T10:23:39.082910+0000","last_peered":"2024-09-21T10:23:39.082910+0000","last_clean":"2024-09-21T10:23:39.082910+0000","last_became_active":"2024-09-21T10:22:47.984858+0000","last_became_peered":"2024-09-21T10:22:47.984858+0000","last_unstale":"2024-09-21T10:23:39.082910+0000","last_undegraded":"2024-09-21T10:23:39.082910+0000","last_fullsized":"2024-09-21T10:23:39.082910+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:50.554661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"288'100","reported_seq":82,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081739+0000","last_change":"2024-09-21T10:22:58.040184+0000","last_active":"2024-09-21T10:23:39.081739+0000","last_peered":"2024-09-21T10:23:39.081739+0000","last_clean":"2024-09-21T10:23:39.081739+0000","last_became_active":"2024-09-21T10:22:58.039792+0000","last_became_peered":"2024-09-21T10:22:58.039792+0000","last_unstale":"2024-09-21T10:23:39.081739+0000","last_undegraded":"2024-09-21T10:23:39.081739+0000","last_fullsized":"2024-09-21T10:23:39.081739+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:39.711091+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:22:39.711091+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:01.398455+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2154,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081960+0000","last_change":"2024-09-21T10:22:58.044039+0000","last_active":"2024-09-21T10:23:39.081960+0000","last_peered":"2024-09-21T10:23:39.081960+0000","last_clean":"2024-09-21T10:23:39.081960+0000","last_became_active":"2024-09-21T10:22:58.043669+0000","last_became_peered":"2024-09-21T10:22:58.043669+0000","last_unstale":"2024-09-21T10:23:39.081960+0000","last_undegraded":"2024-09-21T10:23:39.081960+0000","last_fullsized":"2024-09-21T10:23:39.081960+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:55:29.838109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":223,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250756+0000","last_change":"2024-09-21T10:23:10.369533+0000","last_active":"2024-09-21T10:23:35.250756+0000","last_peered":"2024-09-21T10:23:35.250756+0000","last_clean":"2024-09-21T10:23:35.250756+0000","last_became_active":"2024-09-21T10:23:10.369199+0000","last_became_peered":"2024-09-21T10:23:10.369199+0000","last_unstale":"2024-09-21T10:23:35.250756+0000","last_undegraded":"2024-09-21T10:23:35.250756+0000","last_fullsized":"2024-09-21T10:23:35.250756+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:55:10.565881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"288'100","reported_seq":233,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250732+0000","last_change":"2024-09-21T10:23:10.371714+0000","last_active":"2024-09-21T10:23:35.250732+0000","last_peered":"2024-09-21T10:23:35.250732+0000","last_clean":"2024-09-21T10:23:35.250732+0000","last_became_active":"2024-09-21T10:23:10.371370+0000","last_became_peered":"2024-09-21T10:23:10.371370+0000","last_unstale":"2024-09-21T10:23:35.250732+0000","last_undegraded":"2024-09-21T10:23:35.250732+0000","last_fullsized":"2024-09-21T10:23:35.250732+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:22.620446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:22.620446+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:21:41.741511+0000","scrub_duration":0.0014078199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":207,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251258+0000","last_change":"2024-09-21T10:12:25.720808+0000","last_active":"2024-09-21T10:23:35.251258+0000","last_peered":"2024-09-21T10:23:35.251258+0000","last_clean":"2024-09-21T10:23:35.251258+0000","last_became_active":"2024-09-21T10:12:25.706699+0000","last_became_peered":"2024-09-21T10:12:25.706699+0000","last_unstale":"2024-09-21T10:23:35.251258+0000","last_undegraded":"2024-09-21T10:23:35.251258+0000","last_fullsized":"2024-09-21T10:23:35.251258+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:29.044701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2317,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082956+0000","last_change":"2024-09-21T10:22:42.357135+0000","last_active":"2024-09-21T10:23:39.082956+0000","last_peered":"2024-09-21T10:23:39.082956+0000","last_clean":"2024-09-21T10:23:39.082956+0000","last_became_active":"2024-09-21T10:22:42.356786+0000","last_became_peered":"2024-09-21T10:22:42.356786+0000","last_unstale":"2024-09-21T10:23:39.082956+0000","last_undegraded":"2024-09-21T10:23:39.082956+0000","last_fullsized":"2024-09-21T10:23:39.082956+0000","mapping_epoch":1436,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:24:04.084804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"288'100","reported_seq":870,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081564+0000","last_change":"2024-09-21T10:23:06.598640+0000","last_active":"2024-09-21T10:23:39.081564+0000","last_peered":"2024-09-21T10:23:39.081564+0000","last_clean":"2024-09-21T10:23:39.081564+0000","last_became_active":"2024-09-21T10:23:06.597998+0000","last_became_peered":"2024-09-21T10:23:06.597998+0000","last_unstale":"2024-09-21T10:23:39.081564+0000","last_undegraded":"2024-09-21T10:23:39.081564+0000","last_fullsized":"2024-09-21T10:23:39.081564+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.888350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.888350+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:50.450004+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":71,"num_bytes_recovered":25356,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39860,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398104,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1512608,"kb_used_data":6996,"kb_used_omap":1,"kb_used_meta":1505598,"kb_avail":92232544,"statfs":{"total":95995035648,"available":94446125056,"internally_reserved":0,"allocated":7163904,"data_stored":4835429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1541732696},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594333,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898772,"kb_used_data":7048,"kb_used_omap":1,"kb_used_meta":891710,"kb_avail":92846380,"statfs":{"total":95995035648,"available":95074693120,"internally_reserved":0,"allocated":7217152,"data_stored":4900286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913111384},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790562,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":901136,"kb_used_data":6980,"kb_used_omap":1,"kb_used_meta":894142,"kb_avail":92844016,"statfs":{"total":95995035648,"available":95072272384,"internally_reserved":0,"allocated":7147520,"data_stored":4831120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915601752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1224,"seq":5257039970498,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":939800,"kb_used_data":6732,"kb_used_omap":290,"kb_used_meta":932765,"kb_avail":92805352,"statfs":{"total":95995035648,"available":95032680448,"internally_reserved":0,"allocated":6893568,"data_stored":4572663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":297377,"internal_metadata":955151967},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999431,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1235076,"kb_used_data":7032,"kb_used_omap":25,"kb_used_meta":1228006,"kb_avail":92510076,"statfs":{"total":95995035648,"available":94730317824,"internally_reserved":0,"allocated":7200768,"data_stored":4895563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":1257479111},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111374,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1264204,"kb_used_data":6976,"kb_used_omap":31,"kb_used_meta":1257184,"kb_avail":92480948,"statfs":{"total":95995035648,"available":94700490752,"internally_reserved":0,"allocated":7143424,"data_stored":4831077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":1287356648},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":1413,"seq":6068788789287,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":45508,"kb_used_data":6712,"kb_used_omap":61,"kb_used_meta":38722,"kb_avail":93699644,"statfs":{"total":95995035648,"available":95948435456,"internally_reserved":0,"allocated":6873088,"data_stored":4567812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62800,"internal_metadata":39652016},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335260,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1186504,"kb_used_data":7036,"kb_used_omap":31,"kb_used_meta":1179424,"kb_avail":92558648,"statfs":{"total":95995035648,"available":94780055552,"internally_reserved":0,"allocated":7204864,"data_stored":4896057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":1207730272},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12089,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:24:29.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:24:29.993 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:30.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:29 smithi017 ceph-mon[102913]: pgmap v2462: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:30.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:29 smithi017 ceph-mon[102913]: from='client.36975 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:30.117 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:29 smithi017 ceph-mon[105151]: pgmap v2462: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:30.118 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:29 smithi017 ceph-mon[105151]: from='client.36975 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:30.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:29 smithi134 ceph-mon[90085]: pgmap v2462: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:30.214 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:29 smithi134 ceph-mon[90085]: from='client.36975 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:24:30.494 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:24:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:24:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:24:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:24:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:24:31.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[102913]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:31.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:31.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:31.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:24:31.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:24:31.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:30 smithi017 ceph-mon[105151]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:24:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:24:31.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:30 smithi134 ceph-mon[90085]: from='mgr.25589 172.21.15.17:0/3796279320' entity='mgr.y' 2024-09-21T10:24:32.104 INFO:teuthology.orchestra.run.smithi017.stdout: 2024-09-21T10:24:32.105 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2024-09-21T10:24:32.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:31 smithi017 ceph-mon[102913]: pgmap v2463: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:32.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:31 smithi017 ceph-mon[105151]: pgmap v2463: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:32.244 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:31 smithi134 ceph-mon[90085]: pgmap v2463: 103 pgs: 103 active+clean; 582 KiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2024-09-21T10:24:33.024 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":2463,"stamp":"2024-09-21T10:24:30.671410+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":25375,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3158,"ondisk_log_size":3158,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7983608,"kb_used_data":55512,"kb_used_omap":445,"kb_used_meta":7927554,"kb_avail":741977608,"statfs":{"total":767960285184,"available":759785070592,"internally_reserved":0,"allocated":56844288,"data_stored":38330007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":456154,"internal_metadata":8117815846},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002920"},"pg_stats":[{"pgid":"1.1d","version":"288'100","reported_seq":82,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081903+0000","last_change":"2024-09-21T10:23:07.539539+0000","last_active":"2024-09-21T10:23:39.081903+0000","last_peered":"2024-09-21T10:23:39.081903+0000","last_clean":"2024-09-21T10:23:39.081903+0000","last_became_active":"2024-09-21T10:22:58.042554+0000","last_became_peered":"2024-09-21T10:22:58.042554+0000","last_unstale":"2024-09-21T10:23:39.081903+0000","last_undegraded":"2024-09-21T10:23:39.081903+0000","last_fullsized":"2024-09-21T10:23:39.081903+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:07.539489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:23:07.539489+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:43.553992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":2088,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081053+0000","last_change":"2024-09-21T10:22:50.995219+0000","last_active":"2024-09-21T10:23:39.081053+0000","last_peered":"2024-09-21T10:23:39.081053+0000","last_clean":"2024-09-21T10:23:39.081053+0000","last_became_active":"2024-09-21T10:22:50.991290+0000","last_became_peered":"2024-09-21T10:22:50.991290+0000","last_unstale":"2024-09-21T10:23:39.081053+0000","last_undegraded":"2024-09-21T10:23:39.081053+0000","last_fullsized":"2024-09-21T10:23:39.081053+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:43.705105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1469,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737811+0000","last_change":"2024-09-21T10:22:33.042466+0000","last_active":"2024-09-21T10:23:34.737811+0000","last_peered":"2024-09-21T10:23:34.737811+0000","last_clean":"2024-09-21T10:23:34.737811+0000","last_became_active":"2024-09-21T10:22:33.041986+0000","last_became_peered":"2024-09-21T10:22:33.041986+0000","last_unstale":"2024-09-21T10:23:34.737811+0000","last_undegraded":"2024-09-21T10:23:34.737811+0000","last_fullsized":"2024-09-21T10:23:34.737811+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:29.767550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":2293,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083350+0000","last_change":"2024-09-21T10:10:13.651599+0000","last_active":"2024-09-21T10:23:39.083350+0000","last_peered":"2024-09-21T10:23:39.083350+0000","last_clean":"2024-09-21T10:23:39.083350+0000","last_became_active":"2024-09-21T10:10:13.651250+0000","last_became_peered":"2024-09-21T10:10:13.651250+0000","last_unstale":"2024-09-21T10:23:39.083350+0000","last_undegraded":"2024-09-21T10:23:39.083350+0000","last_fullsized":"2024-09-21T10:23:39.083350+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:34.100056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2277,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081906+0000","last_change":"2024-09-21T09:46:47.120172+0000","last_active":"2024-09-21T10:23:39.081906+0000","last_peered":"2024-09-21T10:23:39.081906+0000","last_clean":"2024-09-21T10:23:39.081906+0000","last_became_active":"2024-09-21T09:46:47.119847+0000","last_became_peered":"2024-09-21T09:46:47.119847+0000","last_unstale":"2024-09-21T10:23:39.081906+0000","last_undegraded":"2024-09-21T10:23:39.081906+0000","last_fullsized":"2024-09-21T10:23:39.081906+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:04.716498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"288'100","reported_seq":135,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250092+0000","last_change":"2024-09-21T10:22:33.042504+0000","last_active":"2024-09-21T10:23:35.250092+0000","last_peered":"2024-09-21T10:23:35.250092+0000","last_clean":"2024-09-21T10:23:35.250092+0000","last_became_active":"2024-09-21T10:22:33.042111+0000","last_became_peered":"2024-09-21T10:22:33.042111+0000","last_unstale":"2024-09-21T10:23:35.250092+0000","last_undegraded":"2024-09-21T10:23:35.250092+0000","last_fullsized":"2024-09-21T10:23:35.250092+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:26.113747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:26.113747+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:29:12.293819+0000","scrub_duration":0.0042569799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":1051,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250837+0000","last_change":"2024-09-21T10:23:10.371047+0000","last_active":"2024-09-21T10:23:35.250837+0000","last_peered":"2024-09-21T10:23:35.250837+0000","last_clean":"2024-09-21T10:23:35.250837+0000","last_became_active":"2024-09-21T10:23:10.369734+0000","last_became_peered":"2024-09-21T10:23:10.369734+0000","last_unstale":"2024-09-21T10:23:35.250837+0000","last_undegraded":"2024-09-21T10:23:35.250837+0000","last_fullsized":"2024-09-21T10:23:35.250837+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:31:34.312103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1406,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737947+0000","last_change":"2024-09-21T10:08:25.430667+0000","last_active":"2024-09-21T10:23:34.737947+0000","last_peered":"2024-09-21T10:23:34.737947+0000","last_clean":"2024-09-21T10:23:34.737947+0000","last_became_active":"2024-09-21T10:08:25.430357+0000","last_became_peered":"2024-09-21T10:08:25.430357+0000","last_unstale":"2024-09-21T10:23:34.737947+0000","last_undegraded":"2024-09-21T10:23:34.737947+0000","last_fullsized":"2024-09-21T10:23:34.737947+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:19.972020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"288'100","reported_seq":578,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083311+0000","last_change":"2024-09-21T10:19:37.200842+0000","last_active":"2024-09-21T10:23:39.083311+0000","last_peered":"2024-09-21T10:23:39.083311+0000","last_clean":"2024-09-21T10:23:39.083311+0000","last_became_active":"2024-09-21T10:17:59.914846+0000","last_became_peered":"2024-09-21T10:17:59.914846+0000","last_unstale":"2024-09-21T10:23:39.083311+0000","last_undegraded":"2024-09-21T10:23:39.083311+0000","last_fullsized":"2024-09-21T10:23:39.083311+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:37.200788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:37.200788+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:26:30.373346+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":1069,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250320+0000","last_change":"2024-09-21T10:23:10.368894+0000","last_active":"2024-09-21T10:23:35.250320+0000","last_peered":"2024-09-21T10:23:35.250320+0000","last_clean":"2024-09-21T10:23:35.250320+0000","last_became_active":"2024-09-21T10:23:10.368487+0000","last_became_peered":"2024-09-21T10:23:10.368487+0000","last_unstale":"2024-09-21T10:23:35.250320+0000","last_undegraded":"2024-09-21T10:23:35.250320+0000","last_fullsized":"2024-09-21T10:23:35.250320+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:19.938321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1446,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738357+0000","last_change":"2024-09-21T10:22:33.044570+0000","last_active":"2024-09-21T10:23:34.738357+0000","last_peered":"2024-09-21T10:23:34.738357+0000","last_clean":"2024-09-21T10:23:34.738357+0000","last_became_active":"2024-09-21T10:22:33.044188+0000","last_became_peered":"2024-09-21T10:22:33.044188+0000","last_unstale":"2024-09-21T10:23:34.738357+0000","last_undegraded":"2024-09-21T10:23:34.738357+0000","last_fullsized":"2024-09-21T10:23:34.738357+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:30.781767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":219,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082108+0000","last_change":"2024-09-21T10:22:47.984002+0000","last_active":"2024-09-21T10:23:39.082108+0000","last_peered":"2024-09-21T10:23:39.082108+0000","last_clean":"2024-09-21T10:23:39.082108+0000","last_became_active":"2024-09-21T10:22:47.983130+0000","last_became_peered":"2024-09-21T10:22:47.983130+0000","last_unstale":"2024-09-21T10:23:39.082108+0000","last_undegraded":"2024-09-21T10:23:39.082108+0000","last_fullsized":"2024-09-21T10:23:39.082108+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:30.409448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":48,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747315+0000","last_change":"2024-09-21T10:22:51.763013+0000","last_active":"2024-09-21T10:23:38.747315+0000","last_peered":"2024-09-21T10:23:38.747315+0000","last_clean":"2024-09-21T10:23:38.747315+0000","last_became_active":"2024-09-21T10:22:51.762651+0000","last_became_peered":"2024-09-21T10:22:51.762651+0000","last_unstale":"2024-09-21T10:23:38.747315+0000","last_undegraded":"2024-09-21T10:23:38.747315+0000","last_fullsized":"2024-09-21T10:23:38.747315+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:55:09.969973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"288'100","reported_seq":1473,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081353+0000","last_change":"2024-09-21T10:22:44.958109+0000","last_active":"2024-09-21T10:23:39.081353+0000","last_peered":"2024-09-21T10:23:39.081353+0000","last_clean":"2024-09-21T10:23:39.081353+0000","last_became_active":"2024-09-21T10:22:44.376211+0000","last_became_peered":"2024-09-21T10:22:44.376211+0000","last_unstale":"2024-09-21T10:23:39.081353+0000","last_undegraded":"2024-09-21T10:23:39.081353+0000","last_fullsized":"2024-09-21T10:23:39.081353+0000","mapping_epoch":1438,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1439,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:44.958060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:22:44.958060+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:33:17.440014+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":1448,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738311+0000","last_change":"2024-09-21T10:22:33.045346+0000","last_active":"2024-09-21T10:23:34.738311+0000","last_peered":"2024-09-21T10:23:34.738311+0000","last_clean":"2024-09-21T10:23:34.738311+0000","last_became_active":"2024-09-21T10:22:33.045008+0000","last_became_peered":"2024-09-21T10:22:33.045008+0000","last_unstale":"2024-09-21T10:23:34.738311+0000","last_undegraded":"2024-09-21T10:23:34.738311+0000","last_fullsized":"2024-09-21T10:23:34.738311+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:52.061863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2306,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082925+0000","last_change":"2024-09-21T10:22:33.036913+0000","last_active":"2024-09-21T10:23:39.082925+0000","last_peered":"2024-09-21T10:23:39.082925+0000","last_clean":"2024-09-21T10:23:39.082925+0000","last_became_active":"2024-09-21T10:22:33.036531+0000","last_became_peered":"2024-09-21T10:22:33.036531+0000","last_unstale":"2024-09-21T10:23:39.082925+0000","last_undegraded":"2024-09-21T10:23:39.082925+0000","last_fullsized":"2024-09-21T10:23:39.082925+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:12.311660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"288'100","reported_seq":776,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251063+0000","last_change":"2024-09-21T10:23:10.368389+0000","last_active":"2024-09-21T10:23:35.251063+0000","last_peered":"2024-09-21T10:23:35.251063+0000","last_clean":"2024-09-21T10:23:35.251063+0000","last_became_active":"2024-09-21T10:23:10.367413+0000","last_became_peered":"2024-09-21T10:23:10.367413+0000","last_unstale":"2024-09-21T10:23:35.251063+0000","last_undegraded":"2024-09-21T10:23:35.251063+0000","last_fullsized":"2024-09-21T10:23:35.251063+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:38.463672+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:22:38.463672+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:29:43.881870+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081462+0000","last_change":"2024-09-21T10:22:33.050426+0000","last_active":"2024-09-21T10:23:39.081462+0000","last_peered":"2024-09-21T10:23:39.081462+0000","last_clean":"2024-09-21T10:23:39.081462+0000","last_became_active":"2024-09-21T10:22:33.049560+0000","last_became_peered":"2024-09-21T10:22:33.049560+0000","last_unstale":"2024-09-21T10:23:39.081462+0000","last_undegraded":"2024-09-21T10:23:39.081462+0000","last_fullsized":"2024-09-21T10:23:39.081462+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:52:51.566146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2168,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082821+0000","last_change":"2024-09-21T10:22:58.043866+0000","last_active":"2024-09-21T10:23:39.082821+0000","last_peered":"2024-09-21T10:23:39.082821+0000","last_clean":"2024-09-21T10:23:39.082821+0000","last_became_active":"2024-09-21T10:22:58.043485+0000","last_became_peered":"2024-09-21T10:22:58.043485+0000","last_unstale":"2024-09-21T10:23:39.082821+0000","last_undegraded":"2024-09-21T10:23:39.082821+0000","last_fullsized":"2024-09-21T10:23:39.082821+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:10.855019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"288'100","reported_seq":105,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250034+0000","last_change":"2024-09-21T10:23:10.367814+0000","last_active":"2024-09-21T10:23:35.250034+0000","last_peered":"2024-09-21T10:23:35.250034+0000","last_clean":"2024-09-21T10:23:35.250034+0000","last_became_active":"2024-09-21T10:23:10.366849+0000","last_became_peered":"2024-09-21T10:23:10.366849+0000","last_unstale":"2024-09-21T10:23:35.250034+0000","last_undegraded":"2024-09-21T10:23:35.250034+0000","last_fullsized":"2024-09-21T10:23:35.250034+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:48.372952+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:18:48.372952+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:00:41.913631+0000","scrub_duration":0.024424405,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2327,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081735+0000","last_change":"2024-09-21T10:22:44.376279+0000","last_active":"2024-09-21T10:23:39.081735+0000","last_peered":"2024-09-21T10:23:39.081735+0000","last_clean":"2024-09-21T10:23:39.081735+0000","last_became_active":"2024-09-21T10:22:44.375345+0000","last_became_peered":"2024-09-21T10:22:44.375345+0000","last_unstale":"2024-09-21T10:23:39.081735+0000","last_undegraded":"2024-09-21T10:23:39.081735+0000","last_fullsized":"2024-09-21T10:23:39.081735+0000","mapping_epoch":1438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:37:46.243557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":238,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251515+0000","last_change":"2024-09-21T10:22:33.046026+0000","last_active":"2024-09-21T10:23:35.251515+0000","last_peered":"2024-09-21T10:23:35.251515+0000","last_clean":"2024-09-21T10:23:35.251515+0000","last_became_active":"2024-09-21T10:22:33.045576+0000","last_became_peered":"2024-09-21T10:22:33.045576+0000","last_unstale":"2024-09-21T10:23:35.251515+0000","last_undegraded":"2024-09-21T10:23:35.251515+0000","last_fullsized":"2024-09-21T10:23:35.251515+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:27:23.069579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"288'100","reported_seq":136,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737948+0000","last_change":"2024-09-21T10:22:33.042799+0000","last_active":"2024-09-21T10:23:34.737948+0000","last_peered":"2024-09-21T10:23:34.737948+0000","last_clean":"2024-09-21T10:23:34.737948+0000","last_became_active":"2024-09-21T10:22:33.042445+0000","last_became_peered":"2024-09-21T10:22:33.042445+0000","last_unstale":"2024-09-21T10:23:34.737948+0000","last_undegraded":"2024-09-21T10:23:34.737948+0000","last_fullsized":"2024-09-21T10:23:34.737948+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:19:25.436066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:19:25.436066+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:53.646658+0000","scrub_duration":0.0032626159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2342,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081672+0000","last_change":"2024-09-21T10:22:33.038765+0000","last_active":"2024-09-21T10:23:39.081672+0000","last_peered":"2024-09-21T10:23:39.081672+0000","last_clean":"2024-09-21T10:23:39.081672+0000","last_became_active":"2024-09-21T10:22:33.038047+0000","last_became_peered":"2024-09-21T10:22:33.038047+0000","last_unstale":"2024-09-21T10:23:39.081672+0000","last_undegraded":"2024-09-21T10:23:39.081672+0000","last_fullsized":"2024-09-21T10:23:39.081672+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:06.217075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2224,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082944+0000","last_change":"2024-09-21T10:22:47.986131+0000","last_active":"2024-09-21T10:23:39.082944+0000","last_peered":"2024-09-21T10:23:39.082944+0000","last_clean":"2024-09-21T10:23:39.082944+0000","last_became_active":"2024-09-21T10:22:47.985179+0000","last_became_peered":"2024-09-21T10:22:47.985179+0000","last_unstale":"2024-09-21T10:23:39.082944+0000","last_undegraded":"2024-09-21T10:23:39.082944+0000","last_fullsized":"2024-09-21T10:23:39.082944+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:06.515185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"288'100","reported_seq":132,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250286+0000","last_change":"2024-09-21T10:22:33.042197+0000","last_active":"2024-09-21T10:23:35.250286+0000","last_peered":"2024-09-21T10:23:35.250286+0000","last_clean":"2024-09-21T10:23:35.250286+0000","last_became_active":"2024-09-21T10:22:33.041839+0000","last_became_peered":"2024-09-21T10:22:33.041839+0000","last_unstale":"2024-09-21T10:23:35.250286+0000","last_undegraded":"2024-09-21T10:23:35.250286+0000","last_fullsized":"2024-09-21T10:23:35.250286+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:52.067480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:52.067480+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:17:45.030904+0000","scrub_duration":0.0041194669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2293,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081673+0000","last_change":"2024-09-21T10:12:25.707393+0000","last_active":"2024-09-21T10:23:39.081673+0000","last_peered":"2024-09-21T10:23:39.081673+0000","last_clean":"2024-09-21T10:23:39.081673+0000","last_became_active":"2024-09-21T10:12:25.706457+0000","last_became_peered":"2024-09-21T10:12:25.706457+0000","last_unstale":"2024-09-21T10:23:39.081673+0000","last_undegraded":"2024-09-21T10:23:39.081673+0000","last_fullsized":"2024-09-21T10:23:39.081673+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:56.520799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":263,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250481+0000","last_change":"2024-09-21T10:23:10.367127+0000","last_active":"2024-09-21T10:23:35.250481+0000","last_peered":"2024-09-21T10:23:35.250481+0000","last_clean":"2024-09-21T10:23:35.250481+0000","last_became_active":"2024-09-21T10:23:10.366087+0000","last_became_peered":"2024-09-21T10:23:10.366087+0000","last_unstale":"2024-09-21T10:23:35.250481+0000","last_undegraded":"2024-09-21T10:23:35.250481+0000","last_fullsized":"2024-09-21T10:23:35.250481+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:59.742948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"288'100","reported_seq":489,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747476+0000","last_change":"2024-09-21T10:12:25.698905+0000","last_active":"2024-09-21T10:23:38.747476+0000","last_peered":"2024-09-21T10:23:38.747476+0000","last_clean":"2024-09-21T10:23:38.747476+0000","last_became_active":"2024-09-21T10:12:25.698552+0000","last_became_peered":"2024-09-21T10:12:25.698552+0000","last_unstale":"2024-09-21T10:23:38.747476+0000","last_undegraded":"2024-09-21T10:23:38.747476+0000","last_fullsized":"2024-09-21T10:23:38.747476+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.389965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.389965+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:27.529104+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":207,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250192+0000","last_change":"2024-09-21T10:12:25.717602+0000","last_active":"2024-09-21T10:23:35.250192+0000","last_peered":"2024-09-21T10:23:35.250192+0000","last_clean":"2024-09-21T10:23:35.250192+0000","last_became_active":"2024-09-21T10:12:25.705211+0000","last_became_peered":"2024-09-21T10:12:25.705211+0000","last_unstale":"2024-09-21T10:23:35.250192+0000","last_undegraded":"2024-09-21T10:23:35.250192+0000","last_fullsized":"2024-09-21T10:23:35.250192+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:30.822884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081994+0000","last_change":"2024-09-21T10:22:33.043194+0000","last_active":"2024-09-21T10:23:39.081994+0000","last_peered":"2024-09-21T10:23:39.081994+0000","last_clean":"2024-09-21T10:23:39.081994+0000","last_became_active":"2024-09-21T10:22:33.042881+0000","last_became_peered":"2024-09-21T10:22:33.042881+0000","last_unstale":"2024-09-21T10:23:39.081994+0000","last_undegraded":"2024-09-21T10:23:39.081994+0000","last_fullsized":"2024-09-21T10:23:39.081994+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:50:51.918905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"288'100","reported_seq":140,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250066+0000","last_change":"2024-09-21T10:23:10.370481+0000","last_active":"2024-09-21T10:23:35.250066+0000","last_peered":"2024-09-21T10:23:35.250066+0000","last_clean":"2024-09-21T10:23:35.250066+0000","last_became_active":"2024-09-21T10:23:10.370164+0000","last_became_peered":"2024-09-21T10:23:10.370164+0000","last_unstale":"2024-09-21T10:23:35.250066+0000","last_undegraded":"2024-09-21T10:23:35.250066+0000","last_fullsized":"2024-09-21T10:23:35.250066+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:52.363524+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:22:52.363524+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:02.523728+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"370'22","reported_seq":1489,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738197+0000","last_change":"2024-09-21T10:12:25.713745+0000","last_active":"2024-09-21T10:23:34.738197+0000","last_peered":"2024-09-21T10:23:34.738197+0000","last_clean":"2024-09-21T10:23:34.738197+0000","last_became_active":"2024-09-21T10:12:25.698011+0000","last_became_peered":"2024-09-21T10:12:25.698011+0000","last_unstale":"2024-09-21T10:23:34.738197+0000","last_undegraded":"2024-09-21T10:23:34.738197+0000","last_fullsized":"2024-09-21T10:23:34.738197+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:45.276913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23720,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1469,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738267+0000","last_change":"2024-09-21T10:22:33.047352+0000","last_active":"2024-09-21T10:23:34.738267+0000","last_peered":"2024-09-21T10:23:34.738267+0000","last_clean":"2024-09-21T10:23:34.738267+0000","last_became_active":"2024-09-21T10:22:33.046962+0000","last_became_peered":"2024-09-21T10:22:33.046962+0000","last_unstale":"2024-09-21T10:23:34.738267+0000","last_undegraded":"2024-09-21T10:23:34.738267+0000","last_fullsized":"2024-09-21T10:23:34.738267+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:49.393019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"288'100","reported_seq":75,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082479+0000","last_change":"2024-09-21T10:22:36.689148+0000","last_active":"2024-09-21T10:23:39.082479+0000","last_peered":"2024-09-21T10:23:39.082479+0000","last_clean":"2024-09-21T10:23:39.082479+0000","last_became_active":"2024-09-21T10:22:33.048932+0000","last_became_peered":"2024-09-21T10:22:33.048932+0000","last_unstale":"2024-09-21T10:23:39.082479+0000","last_undegraded":"2024-09-21T10:23:39.082479+0000","last_fullsized":"2024-09-21T10:23:39.082479+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:36.689077+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:22:36.689077+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:51:38.323411+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1406,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737858+0000","last_change":"2024-09-21T10:17:34.767553+0000","last_active":"2024-09-21T10:23:34.737858+0000","last_peered":"2024-09-21T10:23:34.737858+0000","last_clean":"2024-09-21T10:23:34.737858+0000","last_became_active":"2024-09-21T10:17:34.767246+0000","last_became_peered":"2024-09-21T10:17:34.767246+0000","last_unstale":"2024-09-21T10:23:34.737858+0000","last_undegraded":"2024-09-21T10:23:34.737858+0000","last_fullsized":"2024-09-21T10:23:34.737858+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:52.295023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":248,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251427+0000","last_change":"2024-09-21T10:23:20.637723+0000","last_active":"2024-09-21T10:23:35.251427+0000","last_peered":"2024-09-21T10:23:35.251427+0000","last_clean":"2024-09-21T10:23:35.251427+0000","last_became_active":"2024-09-21T10:23:20.637372+0000","last_became_peered":"2024-09-21T10:23:20.637372+0000","last_unstale":"2024-09-21T10:23:35.251427+0000","last_undegraded":"2024-09-21T10:23:35.251427+0000","last_fullsized":"2024-09-21T10:23:35.251427+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:13.603516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027559500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":2175,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083007+0000","last_change":"2024-09-21T10:22:47.986522+0000","last_active":"2024-09-21T10:23:39.083007+0000","last_peered":"2024-09-21T10:23:39.083007+0000","last_clean":"2024-09-21T10:23:39.083007+0000","last_became_active":"2024-09-21T10:22:47.986189+0000","last_became_peered":"2024-09-21T10:22:47.986189+0000","last_unstale":"2024-09-21T10:23:39.083007+0000","last_undegraded":"2024-09-21T10:23:39.083007+0000","last_fullsized":"2024-09-21T10:23:39.083007+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.311237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"288'100","reported_seq":781,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082603+0000","last_change":"2024-09-21T10:23:06.597098+0000","last_active":"2024-09-21T10:23:39.082603+0000","last_peered":"2024-09-21T10:23:39.082603+0000","last_clean":"2024-09-21T10:23:39.082603+0000","last_became_active":"2024-09-21T10:23:06.596712+0000","last_became_peered":"2024-09-21T10:23:06.596712+0000","last_unstale":"2024-09-21T10:23:39.082603+0000","last_undegraded":"2024-09-21T10:23:39.082603+0000","last_fullsized":"2024-09-21T10:23:39.082603+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:15.423664+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:15.423664+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:04:37.927592+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":42,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081257+0000","last_change":"2024-09-21T10:22:58.041707+0000","last_active":"2024-09-21T10:23:39.081257+0000","last_peered":"2024-09-21T10:23:39.081257+0000","last_clean":"2024-09-21T10:23:39.081257+0000","last_became_active":"2024-09-21T10:22:58.041384+0000","last_became_peered":"2024-09-21T10:22:58.041384+0000","last_unstale":"2024-09-21T10:23:39.081257+0000","last_undegraded":"2024-09-21T10:23:39.081257+0000","last_fullsized":"2024-09-21T10:23:39.081257+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:31:14.321936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082584+0000","last_change":"2024-09-21T10:22:33.052199+0000","last_active":"2024-09-21T10:23:39.082584+0000","last_peered":"2024-09-21T10:23:39.082584+0000","last_clean":"2024-09-21T10:23:39.082584+0000","last_became_active":"2024-09-21T10:22:33.051111+0000","last_became_peered":"2024-09-21T10:22:33.051111+0000","last_unstale":"2024-09-21T10:23:39.082584+0000","last_undegraded":"2024-09-21T10:23:39.082584+0000","last_fullsized":"2024-09-21T10:23:39.082584+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:00:20.814368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":2301,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083398+0000","last_change":"2024-09-21T10:12:25.718713+0000","last_active":"2024-09-21T10:23:39.083398+0000","last_peered":"2024-09-21T10:23:39.083398+0000","last_clean":"2024-09-21T10:23:39.083398+0000","last_became_active":"2024-09-21T10:12:25.713007+0000","last_became_peered":"2024-09-21T10:12:25.713007+0000","last_unstale":"2024-09-21T10:23:39.083398+0000","last_undegraded":"2024-09-21T10:23:39.083398+0000","last_fullsized":"2024-09-21T10:23:39.083398+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:09.753443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"288'100","reported_seq":384,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738381+0000","last_change":"2024-09-21T10:18:56.346749+0000","last_active":"2024-09-21T10:23:34.738381+0000","last_peered":"2024-09-21T10:23:34.738381+0000","last_clean":"2024-09-21T10:23:34.738381+0000","last_became_active":"2024-09-21T10:17:19.685397+0000","last_became_peered":"2024-09-21T10:17:19.685397+0000","last_unstale":"2024-09-21T10:23:34.738381+0000","last_undegraded":"2024-09-21T10:23:34.738381+0000","last_fullsized":"2024-09-21T10:23:34.738381+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:56.346587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:56.346587+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:24.457314+0000","scrub_duration":0.0039726459999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2172,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082319+0000","last_change":"2024-09-21T10:22:47.985208+0000","last_active":"2024-09-21T10:23:39.082319+0000","last_peered":"2024-09-21T10:23:39.082319+0000","last_clean":"2024-09-21T10:23:39.082319+0000","last_became_active":"2024-09-21T10:22:47.984756+0000","last_became_peered":"2024-09-21T10:22:47.984756+0000","last_unstale":"2024-09-21T10:23:39.082319+0000","last_undegraded":"2024-09-21T10:23:39.082319+0000","last_fullsized":"2024-09-21T10:23:39.082319+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:39:33.877389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":201,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250367+0000","last_change":"2024-09-21T10:12:25.709819+0000","last_active":"2024-09-21T10:23:35.250367+0000","last_peered":"2024-09-21T10:23:35.250367+0000","last_clean":"2024-09-21T10:23:35.250367+0000","last_became_active":"2024-09-21T10:12:25.709548+0000","last_became_peered":"2024-09-21T10:12:25.709548+0000","last_unstale":"2024-09-21T10:23:35.250367+0000","last_undegraded":"2024-09-21T10:23:35.250367+0000","last_fullsized":"2024-09-21T10:23:35.250367+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:35.986614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025919400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":74,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082457+0000","last_change":"2024-09-21T10:23:06.597237+0000","last_active":"2024-09-21T10:23:39.082457+0000","last_peered":"2024-09-21T10:23:39.082457+0000","last_clean":"2024-09-21T10:23:39.082457+0000","last_became_active":"2024-09-21T10:23:06.596904+0000","last_became_peered":"2024-09-21T10:23:06.596904+0000","last_unstale":"2024-09-21T10:23:39.082457+0000","last_undegraded":"2024-09-21T10:23:39.082457+0000","last_fullsized":"2024-09-21T10:23:39.082457+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:58:16.006665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"288'100","reported_seq":104,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250408+0000","last_change":"2024-09-21T10:23:17.784587+0000","last_active":"2024-09-21T10:23:35.250408+0000","last_peered":"2024-09-21T10:23:35.250408+0000","last_clean":"2024-09-21T10:23:35.250408+0000","last_became_active":"2024-09-21T10:23:16.613104+0000","last_became_peered":"2024-09-21T10:23:16.613104+0000","last_unstale":"2024-09-21T10:23:35.250408+0000","last_undegraded":"2024-09-21T10:23:35.250408+0000","last_fullsized":"2024-09-21T10:23:35.250408+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:17.784520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:23:17.784520+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:09:29.075918+0000","scrub_duration":0.001690001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747227+0000","last_change":"2024-09-21T10:23:06.599594+0000","last_active":"2024-09-21T10:23:38.747227+0000","last_peered":"2024-09-21T10:23:38.747227+0000","last_clean":"2024-09-21T10:23:38.747227+0000","last_became_active":"2024-09-21T10:23:06.599239+0000","last_became_peered":"2024-09-21T10:23:06.599239+0000","last_unstale":"2024-09-21T10:23:38.747227+0000","last_undegraded":"2024-09-21T10:23:38.747227+0000","last_fullsized":"2024-09-21T10:23:38.747227+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:18.384899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082350+0000","last_change":"2024-09-21T10:22:33.047721+0000","last_active":"2024-09-21T10:23:39.082350+0000","last_peered":"2024-09-21T10:23:39.082350+0000","last_clean":"2024-09-21T10:23:39.082350+0000","last_became_active":"2024-09-21T10:22:33.045293+0000","last_became_peered":"2024-09-21T10:22:33.045293+0000","last_unstale":"2024-09-21T10:23:39.082350+0000","last_undegraded":"2024-09-21T10:23:39.082350+0000","last_fullsized":"2024-09-21T10:23:39.082350+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:25.630200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048796799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1116,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250626+0000","last_change":"2024-09-21T10:23:10.367137+0000","last_active":"2024-09-21T10:23:35.250626+0000","last_peered":"2024-09-21T10:23:35.250626+0000","last_clean":"2024-09-21T10:23:35.250626+0000","last_became_active":"2024-09-21T10:23:10.366794+0000","last_became_peered":"2024-09-21T10:23:10.366794+0000","last_unstale":"2024-09-21T10:23:35.250626+0000","last_undegraded":"2024-09-21T10:23:35.250626+0000","last_fullsized":"2024-09-21T10:23:35.250626+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:10:26.572649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"288'100","reported_seq":168,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251418+0000","last_change":"2024-09-21T10:23:16.831798+0000","last_active":"2024-09-21T10:23:35.251418+0000","last_peered":"2024-09-21T10:23:35.251418+0000","last_clean":"2024-09-21T10:23:35.251418+0000","last_became_active":"2024-09-21T10:23:16.614393+0000","last_became_peered":"2024-09-21T10:23:16.614393+0000","last_unstale":"2024-09-21T10:23:35.251418+0000","last_undegraded":"2024-09-21T10:23:35.251418+0000","last_fullsized":"2024-09-21T10:23:35.251418+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:16.831734+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:23:16.831734+0000","last_clean_scrub_stamp":"2024-09-21T10:23:16.831734+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:47.540649+0000","scrub_duration":0.001899548,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082340+0000","last_change":"2024-09-21T10:22:33.042737+0000","last_active":"2024-09-21T10:23:39.082340+0000","last_peered":"2024-09-21T10:23:39.082340+0000","last_clean":"2024-09-21T10:23:39.082340+0000","last_became_active":"2024-09-21T10:22:33.042348+0000","last_became_peered":"2024-09-21T10:22:33.042348+0000","last_unstale":"2024-09-21T10:23:39.082340+0000","last_undegraded":"2024-09-21T10:23:39.082340+0000","last_fullsized":"2024-09-21T10:23:39.082340+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:00.941239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1404,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738606+0000","last_change":"2024-09-21T10:10:13.686243+0000","last_active":"2024-09-21T10:23:34.738606+0000","last_peered":"2024-09-21T10:23:34.738606+0000","last_clean":"2024-09-21T10:23:34.738606+0000","last_became_active":"2024-09-21T10:10:13.685894+0000","last_became_peered":"2024-09-21T10:10:13.685894+0000","last_unstale":"2024-09-21T10:23:34.738606+0000","last_undegraded":"2024-09-21T10:23:34.738606+0000","last_fullsized":"2024-09-21T10:23:34.738606+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:12.394732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"58'2","reported_seq":1196,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250639+0000","last_change":"2024-09-21T10:23:10.370371+0000","last_active":"2024-09-21T10:23:35.250639+0000","last_peered":"2024-09-21T10:23:35.250639+0000","last_clean":"2024-09-21T10:23:35.250639+0000","last_became_active":"2024-09-21T10:23:10.369529+0000","last_became_peered":"2024-09-21T10:23:10.369529+0000","last_unstale":"2024-09-21T10:23:35.250639+0000","last_undegraded":"2024-09-21T10:23:35.250639+0000","last_fullsized":"2024-09-21T10:23:35.250639+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:02:23.128537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075012699999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"288'100","reported_seq":52,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.249952+0000","last_change":"2024-09-21T10:23:10.365099+0000","last_active":"2024-09-21T10:23:35.249952+0000","last_peered":"2024-09-21T10:23:35.249952+0000","last_clean":"2024-09-21T10:23:35.249952+0000","last_became_active":"2024-09-21T10:23:10.364753+0000","last_became_peered":"2024-09-21T10:23:10.364753+0000","last_unstale":"2024-09-21T10:23:35.249952+0000","last_undegraded":"2024-09-21T10:23:35.249952+0000","last_fullsized":"2024-09-21T10:23:35.249952+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:19.901507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:19.901507+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:03.501890+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082107+0000","last_change":"2024-09-21T10:22:33.048943+0000","last_active":"2024-09-21T10:23:39.082107+0000","last_peered":"2024-09-21T10:23:39.082107+0000","last_clean":"2024-09-21T10:23:39.082107+0000","last_became_active":"2024-09-21T10:22:33.048176+0000","last_became_peered":"2024-09-21T10:22:33.048176+0000","last_unstale":"2024-09-21T10:23:39.082107+0000","last_undegraded":"2024-09-21T10:23:39.082107+0000","last_fullsized":"2024-09-21T10:23:39.082107+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:37.352591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":67,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082735+0000","last_change":"2024-09-21T10:22:33.046183+0000","last_active":"2024-09-21T10:23:39.082735+0000","last_peered":"2024-09-21T10:23:39.082735+0000","last_clean":"2024-09-21T10:23:39.082735+0000","last_became_active":"2024-09-21T10:22:33.045814+0000","last_became_peered":"2024-09-21T10:22:33.045814+0000","last_unstale":"2024-09-21T10:23:39.082735+0000","last_undegraded":"2024-09-21T10:23:39.082735+0000","last_fullsized":"2024-09-21T10:23:39.082735+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:34:18.457195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2328,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747189+0000","last_change":"2024-09-21T09:46:47.119379+0000","last_active":"2024-09-21T10:23:38.747189+0000","last_peered":"2024-09-21T10:23:38.747189+0000","last_clean":"2024-09-21T10:23:38.747189+0000","last_became_active":"2024-09-21T09:46:47.119057+0000","last_became_peered":"2024-09-21T09:46:47.119057+0000","last_unstale":"2024-09-21T10:23:38.747189+0000","last_undegraded":"2024-09-21T10:23:38.747189+0000","last_fullsized":"2024-09-21T10:23:38.747189+0000","mapping_epoch":581,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":582,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:39.903543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038870899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"288'100","reported_seq":15,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251244+0000","last_change":"2024-09-21T10:23:20.638309+0000","last_active":"2024-09-21T10:23:35.251244+0000","last_peered":"2024-09-21T10:23:35.251244+0000","last_clean":"2024-09-21T10:23:35.251244+0000","last_became_active":"2024-09-21T10:23:20.637944+0000","last_became_peered":"2024-09-21T10:23:20.637944+0000","last_unstale":"2024-09-21T10:23:35.251244+0000","last_undegraded":"2024-09-21T10:23:35.251244+0000","last_fullsized":"2024-09-21T10:23:35.251244+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:53.368905+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:22:08.152253+0000","last_clean_scrub_stamp":"2024-09-21T10:22:53.368905+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:56:54.785148+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":73,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082483+0000","last_change":"2024-09-21T10:22:47.986298+0000","last_active":"2024-09-21T10:23:39.082483+0000","last_peered":"2024-09-21T10:23:39.082483+0000","last_clean":"2024-09-21T10:23:39.082483+0000","last_became_active":"2024-09-21T10:22:47.985994+0000","last_became_peered":"2024-09-21T10:22:47.985994+0000","last_unstale":"2024-09-21T10:23:39.082483+0000","last_undegraded":"2024-09-21T10:23:39.082483+0000","last_fullsized":"2024-09-21T10:23:39.082483+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:00.932876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2145,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081448+0000","last_change":"2024-09-21T10:22:58.045948+0000","last_active":"2024-09-21T10:23:39.081448+0000","last_peered":"2024-09-21T10:23:39.081448+0000","last_clean":"2024-09-21T10:23:39.081448+0000","last_became_active":"2024-09-21T10:22:58.045494+0000","last_became_peered":"2024-09-21T10:22:58.045494+0000","last_unstale":"2024-09-21T10:23:39.081448+0000","last_undegraded":"2024-09-21T10:23:39.081448+0000","last_fullsized":"2024-09-21T10:23:39.081448+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:29.385283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":221,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250751+0000","last_change":"2024-09-21T10:22:33.041385+0000","last_active":"2024-09-21T10:23:35.250751+0000","last_peered":"2024-09-21T10:23:35.250751+0000","last_clean":"2024-09-21T10:23:35.250751+0000","last_became_active":"2024-09-21T10:22:33.041045+0000","last_became_peered":"2024-09-21T10:22:33.041045+0000","last_unstale":"2024-09-21T10:23:35.250751+0000","last_undegraded":"2024-09-21T10:23:35.250751+0000","last_fullsized":"2024-09-21T10:23:35.250751+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:11.937604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000195923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"288'100","reported_seq":75,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081908+0000","last_change":"2024-09-21T10:22:37.705040+0000","last_active":"2024-09-21T10:23:39.081908+0000","last_peered":"2024-09-21T10:23:39.081908+0000","last_clean":"2024-09-21T10:23:39.081908+0000","last_became_active":"2024-09-21T10:22:33.053060+0000","last_became_peered":"2024-09-21T10:22:33.053060+0000","last_unstale":"2024-09-21T10:23:39.081908+0000","last_undegraded":"2024-09-21T10:23:39.081908+0000","last_fullsized":"2024-09-21T10:23:39.081908+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":3,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:37.704977+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:22:37.704977+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:36:11.035210+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":180,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251009+0000","last_change":"2024-09-21T10:23:16.614752+0000","last_active":"2024-09-21T10:23:35.251009+0000","last_peered":"2024-09-21T10:23:35.251009+0000","last_clean":"2024-09-21T10:23:35.251009+0000","last_became_active":"2024-09-21T10:23:16.614428+0000","last_became_peered":"2024-09-21T10:23:16.614428+0000","last_unstale":"2024-09-21T10:23:35.251009+0000","last_undegraded":"2024-09-21T10:23:35.251009+0000","last_fullsized":"2024-09-21T10:23:35.251009+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:13.826655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2295,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081350+0000","last_change":"2024-09-21T10:23:06.599521+0000","last_active":"2024-09-21T10:23:39.081350+0000","last_peered":"2024-09-21T10:23:39.081350+0000","last_clean":"2024-09-21T10:23:39.081350+0000","last_became_active":"2024-09-21T10:23:06.599178+0000","last_became_peered":"2024-09-21T10:23:06.599178+0000","last_unstale":"2024-09-21T10:23:39.081350+0000","last_undegraded":"2024-09-21T10:23:39.081350+0000","last_fullsized":"2024-09-21T10:23:39.081350+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:59.325935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"56'1","reported_seq":587,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250119+0000","last_change":"2024-09-21T10:23:10.367465+0000","last_active":"2024-09-21T10:23:35.250119+0000","last_peered":"2024-09-21T10:23:35.250119+0000","last_clean":"2024-09-21T10:23:35.250119+0000","last_became_active":"2024-09-21T10:23:10.366025+0000","last_became_peered":"2024-09-21T10:23:10.366025+0000","last_unstale":"2024-09-21T10:23:35.250119+0000","last_undegraded":"2024-09-21T10:23:35.250119+0000","last_fullsized":"2024-09-21T10:23:35.250119+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:28.134145+0000","last_clean_scrub_stamp":"2024-09-21T09:35:28.134145+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:41.453486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"288'100","reported_seq":211,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250665+0000","last_change":"2024-09-21T10:13:20.647735+0000","last_active":"2024-09-21T10:23:35.250665+0000","last_peered":"2024-09-21T10:23:35.250665+0000","last_clean":"2024-09-21T10:23:35.250665+0000","last_became_active":"2024-09-21T10:12:25.708657+0000","last_became_peered":"2024-09-21T10:12:25.708657+0000","last_unstale":"2024-09-21T10:23:35.250665+0000","last_undegraded":"2024-09-21T10:23:35.250665+0000","last_fullsized":"2024-09-21T10:23:35.250665+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:20.647659+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:20.647659+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:09.770331+0000","scrub_duration":0.0016005629999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":60,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:38.747645+0000","last_change":"2024-09-21T10:22:42.315636+0000","last_active":"2024-09-21T10:23:38.747645+0000","last_peered":"2024-09-21T10:23:38.747645+0000","last_clean":"2024-09-21T10:23:38.747645+0000","last_became_active":"2024-09-21T10:22:42.315339+0000","last_became_peered":"2024-09-21T10:22:42.315339+0000","last_unstale":"2024-09-21T10:23:38.747645+0000","last_undegraded":"2024-09-21T10:23:38.747645+0000","last_fullsized":"2024-09-21T10:23:38.747645+0000","mapping_epoch":1436,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:17:34.930301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":42,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081354+0000","last_change":"2024-09-21T10:22:58.041652+0000","last_active":"2024-09-21T10:23:39.081354+0000","last_peered":"2024-09-21T10:23:39.081354+0000","last_clean":"2024-09-21T10:23:39.081354+0000","last_became_active":"2024-09-21T10:22:58.041333+0000","last_became_peered":"2024-09-21T10:22:58.041333+0000","last_unstale":"2024-09-21T10:23:39.081354+0000","last_undegraded":"2024-09-21T10:23:39.081354+0000","last_fullsized":"2024-09-21T10:23:39.081354+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:44:23.224773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"288'100","reported_seq":34,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250560+0000","last_change":"2024-09-21T10:23:21.203033+0000","last_active":"2024-09-21T10:23:35.250560+0000","last_peered":"2024-09-21T10:23:35.250560+0000","last_clean":"2024-09-21T10:23:35.250560+0000","last_became_active":"2024-09-21T10:23:10.370050+0000","last_became_peered":"2024-09-21T10:23:10.370050+0000","last_unstale":"2024-09-21T10:23:35.250560+0000","last_undegraded":"2024-09-21T10:23:35.250560+0000","last_fullsized":"2024-09-21T10:23:35.250560+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:21.202969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:23:21.202969+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:08:57.680472+0000","scrub_duration":0.0022498879999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1469,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738625+0000","last_change":"2024-09-21T10:22:33.048246+0000","last_active":"2024-09-21T10:23:34.738625+0000","last_peered":"2024-09-21T10:23:34.738625+0000","last_clean":"2024-09-21T10:23:34.738625+0000","last_became_active":"2024-09-21T10:22:33.047853+0000","last_became_peered":"2024-09-21T10:22:33.047853+0000","last_unstale":"2024-09-21T10:23:34.738625+0000","last_undegraded":"2024-09-21T10:23:34.738625+0000","last_fullsized":"2024-09-21T10:23:34.738625+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:11:19.033632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1417,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.737877+0000","last_change":"2024-09-21T10:12:25.697464+0000","last_active":"2024-09-21T10:23:34.737877+0000","last_peered":"2024-09-21T10:23:34.737877+0000","last_clean":"2024-09-21T10:23:34.737877+0000","last_became_active":"2024-09-21T10:12:25.697050+0000","last_became_peered":"2024-09-21T10:12:25.697050+0000","last_unstale":"2024-09-21T10:23:34.737877+0000","last_undegraded":"2024-09-21T10:23:34.737877+0000","last_fullsized":"2024-09-21T10:23:34.737877+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:50.687008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"288'100","reported_seq":82,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082126+0000","last_change":"2024-09-21T10:23:13.578911+0000","last_active":"2024-09-21T10:23:39.082126+0000","last_peered":"2024-09-21T10:23:39.082126+0000","last_clean":"2024-09-21T10:23:39.082126+0000","last_became_active":"2024-09-21T10:23:06.596989+0000","last_became_peered":"2024-09-21T10:23:06.596989+0000","last_unstale":"2024-09-21T10:23:39.082126+0000","last_undegraded":"2024-09-21T10:23:39.082126+0000","last_fullsized":"2024-09-21T10:23:39.082126+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:13.578856+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:23:13.578856+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:47.965583+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1446,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738609+0000","last_change":"2024-09-21T10:22:33.045553+0000","last_active":"2024-09-21T10:23:34.738609+0000","last_peered":"2024-09-21T10:23:34.738609+0000","last_clean":"2024-09-21T10:23:34.738609+0000","last_became_active":"2024-09-21T10:22:33.044637+0000","last_became_peered":"2024-09-21T10:22:33.044637+0000","last_unstale":"2024-09-21T10:23:34.738609+0000","last_undegraded":"2024-09-21T10:23:34.738609+0000","last_fullsized":"2024-09-21T10:23:34.738609+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:26.997458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":648,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250365+0000","last_change":"2024-09-21T10:23:16.612207+0000","last_active":"2024-09-21T10:23:35.250365+0000","last_peered":"2024-09-21T10:23:35.250365+0000","last_clean":"2024-09-21T10:23:35.250365+0000","last_became_active":"2024-09-21T10:23:16.611858+0000","last_became_peered":"2024-09-21T10:23:16.611858+0000","last_unstale":"2024-09-21T10:23:35.250365+0000","last_undegraded":"2024-09-21T10:23:35.250365+0000","last_fullsized":"2024-09-21T10:23:35.250365+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:50.839744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"288'100","reported_seq":751,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250893+0000","last_change":"2024-09-21T10:23:12.159934+0000","last_active":"2024-09-21T10:23:35.250893+0000","last_peered":"2024-09-21T10:23:35.250893+0000","last_clean":"2024-09-21T10:23:35.250893+0000","last_became_active":"2024-09-21T10:23:10.367850+0000","last_became_peered":"2024-09-21T10:23:10.367850+0000","last_unstale":"2024-09-21T10:23:35.250893+0000","last_undegraded":"2024-09-21T10:23:35.250893+0000","last_fullsized":"2024-09-21T10:23:35.250893+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:23:12.159856+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:23:12.159856+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:43.475443+0000","scrub_duration":0.001761882,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1403,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738381+0000","last_change":"2024-09-21T10:10:13.661328+0000","last_active":"2024-09-21T10:23:34.738381+0000","last_peered":"2024-09-21T10:23:34.738381+0000","last_clean":"2024-09-21T10:23:34.738381+0000","last_became_active":"2024-09-21T10:10:13.661028+0000","last_became_peered":"2024-09-21T10:10:13.661028+0000","last_unstale":"2024-09-21T10:23:34.738381+0000","last_undegraded":"2024-09-21T10:23:34.738381+0000","last_fullsized":"2024-09-21T10:23:34.738381+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:28:36.965214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":2289,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082916+0000","last_change":"2024-09-21T10:08:25.436371+0000","last_active":"2024-09-21T10:23:39.082916+0000","last_peered":"2024-09-21T10:23:39.082916+0000","last_clean":"2024-09-21T10:23:39.082916+0000","last_became_active":"2024-09-21T10:08:25.435921+0000","last_became_peered":"2024-09-21T10:08:25.435921+0000","last_unstale":"2024-09-21T10:23:39.082916+0000","last_undegraded":"2024-09-21T10:23:39.082916+0000","last_fullsized":"2024-09-21T10:23:39.082916+0000","mapping_epoch":1224,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:27.612386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"288'100","reported_seq":231,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.249923+0000","last_change":"2024-09-21T10:17:27.870379+0000","last_active":"2024-09-21T10:23:35.249923+0000","last_peered":"2024-09-21T10:23:35.249923+0000","last_clean":"2024-09-21T10:23:35.249923+0000","last_became_active":"2024-09-21T10:17:19.686196+0000","last_became_peered":"2024-09-21T10:17:19.686196+0000","last_unstale":"2024-09-21T10:23:35.249923+0000","last_undegraded":"2024-09-21T10:23:35.249923+0000","last_fullsized":"2024-09-21T10:23:35.249923+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1107,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:17:27.870303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:17:27.870303+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:28:22.680785+0000","scrub_duration":0.001791443,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2153,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082256+0000","last_change":"2024-09-21T10:22:47.984893+0000","last_active":"2024-09-21T10:23:39.082256+0000","last_peered":"2024-09-21T10:23:39.082256+0000","last_clean":"2024-09-21T10:23:39.082256+0000","last_became_active":"2024-09-21T10:22:47.983631+0000","last_became_peered":"2024-09-21T10:22:47.983631+0000","last_unstale":"2024-09-21T10:23:39.082256+0000","last_undegraded":"2024-09-21T10:23:39.082256+0000","last_fullsized":"2024-09-21T10:23:39.082256+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:15:16.137659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1067,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250396+0000","last_change":"2024-09-21T10:23:10.369746+0000","last_active":"2024-09-21T10:23:35.250396+0000","last_peered":"2024-09-21T10:23:35.250396+0000","last_clean":"2024-09-21T10:23:35.250396+0000","last_became_active":"2024-09-21T10:23:10.369418+0000","last_became_peered":"2024-09-21T10:23:10.369418+0000","last_unstale":"2024-09-21T10:23:35.250396+0000","last_undegraded":"2024-09-21T10:23:35.250396+0000","last_fullsized":"2024-09-21T10:23:35.250396+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:32:58.584132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"288'100","reported_seq":287,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738452+0000","last_change":"2024-09-21T10:18:31.396074+0000","last_active":"2024-09-21T10:23:34.738452+0000","last_peered":"2024-09-21T10:23:34.738452+0000","last_clean":"2024-09-21T10:23:34.738452+0000","last_became_active":"2024-09-21T10:17:19.681584+0000","last_became_peered":"2024-09-21T10:17:19.681584+0000","last_unstale":"2024-09-21T10:23:34.738452+0000","last_undegraded":"2024-09-21T10:23:34.738452+0000","last_fullsized":"2024-09-21T10:23:34.738452+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:31.396016+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:31.396016+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:22.138287+0000","scrub_duration":0.001381925,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":248,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250221+0000","last_change":"2024-09-21T10:22:33.047543+0000","last_active":"2024-09-21T10:23:35.250221+0000","last_peered":"2024-09-21T10:23:35.250221+0000","last_clean":"2024-09-21T10:23:35.250221+0000","last_became_active":"2024-09-21T10:22:33.046756+0000","last_became_peered":"2024-09-21T10:22:33.046756+0000","last_unstale":"2024-09-21T10:23:35.250221+0000","last_undegraded":"2024-09-21T10:23:35.250221+0000","last_fullsized":"2024-09-21T10:23:35.250221+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:01.549176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":233,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250091+0000","last_change":"2024-09-21T10:23:06.604259+0000","last_active":"2024-09-21T10:23:35.250091+0000","last_peered":"2024-09-21T10:23:35.250091+0000","last_clean":"2024-09-21T10:23:35.250091+0000","last_became_active":"2024-09-21T10:23:06.598107+0000","last_became_peered":"2024-09-21T10:23:06.598107+0000","last_unstale":"2024-09-21T10:23:35.250091+0000","last_undegraded":"2024-09-21T10:23:35.250091+0000","last_fullsized":"2024-09-21T10:23:35.250091+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:14.773082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"288'100","reported_seq":84,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251529+0000","last_change":"2024-09-21T10:23:16.612225+0000","last_active":"2024-09-21T10:23:35.251529+0000","last_peered":"2024-09-21T10:23:35.251529+0000","last_clean":"2024-09-21T10:23:35.251529+0000","last_became_active":"2024-09-21T10:23:16.611856+0000","last_became_peered":"2024-09-21T10:23:16.611856+0000","last_unstale":"2024-09-21T10:23:35.251529+0000","last_undegraded":"2024-09-21T10:23:35.251529+0000","last_fullsized":"2024-09-21T10:23:35.251529+0000","mapping_epoch":1453,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1454,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:23.746751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:23.746751+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:11.238986+0000","scrub_duration":0.0021986229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1466,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:34.738119+0000","last_change":"2024-09-21T10:22:58.043427+0000","last_active":"2024-09-21T10:23:34.738119+0000","last_peered":"2024-09-21T10:23:34.738119+0000","last_clean":"2024-09-21T10:23:34.738119+0000","last_became_active":"2024-09-21T10:22:58.043145+0000","last_became_peered":"2024-09-21T10:22:58.043145+0000","last_unstale":"2024-09-21T10:23:34.738119+0000","last_undegraded":"2024-09-21T10:23:34.738119+0000","last_fullsized":"2024-09-21T10:23:34.738119+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:29:31.738371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":247,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251121+0000","last_change":"2024-09-21T10:22:33.055842+0000","last_active":"2024-09-21T10:23:35.251121+0000","last_peered":"2024-09-21T10:23:35.251121+0000","last_clean":"2024-09-21T10:23:35.251121+0000","last_became_active":"2024-09-21T10:22:33.043460+0000","last_became_peered":"2024-09-21T10:22:33.043460+0000","last_unstale":"2024-09-21T10:23:35.251121+0000","last_undegraded":"2024-09-21T10:23:35.251121+0000","last_fullsized":"2024-09-21T10:23:35.251121+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:45.942510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"288'100","reported_seq":777,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082786+0000","last_change":"2024-09-21T10:21:24.467648+0000","last_active":"2024-09-21T10:23:39.082786+0000","last_peered":"2024-09-21T10:23:39.082786+0000","last_clean":"2024-09-21T10:23:39.082786+0000","last_became_active":"2024-09-21T10:17:19.680813+0000","last_became_peered":"2024-09-21T10:17:19.680813+0000","last_unstale":"2024-09-21T10:23:39.082786+0000","last_undegraded":"2024-09-21T10:23:39.082786+0000","last_fullsized":"2024-09-21T10:23:39.082786+0000","mapping_epoch":1252,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:21:24.467561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:21:24.467561+0000","objects_scrubbed":1,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:08:02.729976+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":223,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250980+0000","last_change":"2024-09-21T10:23:10.371553+0000","last_active":"2024-09-21T10:23:35.250980+0000","last_peered":"2024-09-21T10:23:35.250980+0000","last_clean":"2024-09-21T10:23:35.250980+0000","last_became_active":"2024-09-21T10:23:10.371261+0000","last_became_peered":"2024-09-21T10:23:10.371261+0000","last_unstale":"2024-09-21T10:23:35.250980+0000","last_undegraded":"2024-09-21T10:23:35.250980+0000","last_fullsized":"2024-09-21T10:23:35.250980+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:40.760922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":205,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250840+0000","last_change":"2024-09-21T10:12:25.764496+0000","last_active":"2024-09-21T10:23:35.250840+0000","last_peered":"2024-09-21T10:23:35.250840+0000","last_clean":"2024-09-21T10:23:35.250840+0000","last_became_active":"2024-09-21T10:12:25.707263+0000","last_became_peered":"2024-09-21T10:12:25.707263+0000","last_unstale":"2024-09-21T10:23:35.250840+0000","last_undegraded":"2024-09-21T10:23:35.250840+0000","last_fullsized":"2024-09-21T10:23:35.250840+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:48:45.728577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"288'100","reported_seq":254,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250456+0000","last_change":"2024-09-21T10:22:33.041456+0000","last_active":"2024-09-21T10:23:35.250456+0000","last_peered":"2024-09-21T10:23:35.250456+0000","last_clean":"2024-09-21T10:23:35.250456+0000","last_became_active":"2024-09-21T10:22:33.041009+0000","last_became_peered":"2024-09-21T10:22:33.041009+0000","last_unstale":"2024-09-21T10:23:35.250456+0000","last_undegraded":"2024-09-21T10:23:35.250456+0000","last_fullsized":"2024-09-21T10:23:35.250456+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":4,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:36.066408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:36.066408+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:47:29.351359+0000","scrub_duration":0.006208028,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":308,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.249824+0000","last_change":"2024-09-21T10:23:06.608302+0000","last_active":"2024-09-21T10:23:35.249824+0000","last_peered":"2024-09-21T10:23:35.249824+0000","last_clean":"2024-09-21T10:23:35.249824+0000","last_became_active":"2024-09-21T10:23:06.601367+0000","last_became_peered":"2024-09-21T10:23:06.601367+0000","last_unstale":"2024-09-21T10:23:35.249824+0000","last_undegraded":"2024-09-21T10:23:35.249824+0000","last_fullsized":"2024-09-21T10:23:35.249824+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:22.111234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":42,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.083202+0000","last_change":"2024-09-21T10:22:58.040371+0000","last_active":"2024-09-21T10:23:39.083202+0000","last_peered":"2024-09-21T10:23:39.083202+0000","last_clean":"2024-09-21T10:23:39.083202+0000","last_became_active":"2024-09-21T10:22:58.039759+0000","last_became_peered":"2024-09-21T10:22:58.039759+0000","last_unstale":"2024-09-21T10:23:39.083202+0000","last_undegraded":"2024-09-21T10:23:39.083202+0000","last_fullsized":"2024-09-21T10:23:39.083202+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:15.508750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"288'100","reported_seq":183,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250481+0000","last_change":"2024-09-21T10:23:10.371087+0000","last_active":"2024-09-21T10:23:35.250481+0000","last_peered":"2024-09-21T10:23:35.250481+0000","last_clean":"2024-09-21T10:23:35.250481+0000","last_became_active":"2024-09-21T10:23:10.370760+0000","last_became_peered":"2024-09-21T10:23:10.370760+0000","last_unstale":"2024-09-21T10:23:35.250481+0000","last_undegraded":"2024-09-21T10:23:35.250481+0000","last_fullsized":"2024-09-21T10:23:35.250481+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:18:31.567804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:18:31.567804+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:51.257478+0000","scrub_duration":0.001700088,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":201,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.249706+0000","last_change":"2024-09-21T10:12:25.700080+0000","last_active":"2024-09-21T10:23:35.249706+0000","last_peered":"2024-09-21T10:23:35.249706+0000","last_clean":"2024-09-21T10:23:35.249706+0000","last_became_active":"2024-09-21T10:12:25.699762+0000","last_became_peered":"2024-09-21T10:12:25.699762+0000","last_unstale":"2024-09-21T10:23:35.249706+0000","last_undegraded":"2024-09-21T10:23:35.249706+0000","last_fullsized":"2024-09-21T10:23:35.249706+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:41.509175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2141,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082910+0000","last_change":"2024-09-21T10:22:47.985318+0000","last_active":"2024-09-21T10:23:39.082910+0000","last_peered":"2024-09-21T10:23:39.082910+0000","last_clean":"2024-09-21T10:23:39.082910+0000","last_became_active":"2024-09-21T10:22:47.984858+0000","last_became_peered":"2024-09-21T10:22:47.984858+0000","last_unstale":"2024-09-21T10:23:39.082910+0000","last_undegraded":"2024-09-21T10:23:39.082910+0000","last_fullsized":"2024-09-21T10:23:39.082910+0000","mapping_epoch":1440,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:59:50.554661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"288'100","reported_seq":82,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081739+0000","last_change":"2024-09-21T10:22:58.040184+0000","last_active":"2024-09-21T10:23:39.081739+0000","last_peered":"2024-09-21T10:23:39.081739+0000","last_clean":"2024-09-21T10:23:39.081739+0000","last_became_active":"2024-09-21T10:22:58.039792+0000","last_became_peered":"2024-09-21T10:22:58.039792+0000","last_unstale":"2024-09-21T10:23:39.081739+0000","last_undegraded":"2024-09-21T10:23:39.081739+0000","last_fullsized":"2024-09-21T10:23:39.081739+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:22:39.711091+0000","last_deep_scrub":"288'100","last_deep_scrub_stamp":"2024-09-21T10:04:35.993739+0000","last_clean_scrub_stamp":"2024-09-21T10:22:39.711091+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:01.398455+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2154,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081960+0000","last_change":"2024-09-21T10:22:58.044039+0000","last_active":"2024-09-21T10:23:39.081960+0000","last_peered":"2024-09-21T10:23:39.081960+0000","last_clean":"2024-09-21T10:23:39.081960+0000","last_became_active":"2024-09-21T10:22:58.043669+0000","last_became_peered":"2024-09-21T10:22:58.043669+0000","last_unstale":"2024-09-21T10:23:39.081960+0000","last_undegraded":"2024-09-21T10:23:39.081960+0000","last_fullsized":"2024-09-21T10:23:39.081960+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:55:29.838109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":223,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250756+0000","last_change":"2024-09-21T10:23:10.369533+0000","last_active":"2024-09-21T10:23:35.250756+0000","last_peered":"2024-09-21T10:23:35.250756+0000","last_clean":"2024-09-21T10:23:35.250756+0000","last_became_active":"2024-09-21T10:23:10.369199+0000","last_became_peered":"2024-09-21T10:23:10.369199+0000","last_unstale":"2024-09-21T10:23:35.250756+0000","last_undegraded":"2024-09-21T10:23:35.250756+0000","last_fullsized":"2024-09-21T10:23:35.250756+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:55:10.565881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"288'100","reported_seq":233,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.250732+0000","last_change":"2024-09-21T10:23:10.371714+0000","last_active":"2024-09-21T10:23:35.250732+0000","last_peered":"2024-09-21T10:23:35.250732+0000","last_clean":"2024-09-21T10:23:35.250732+0000","last_became_active":"2024-09-21T10:23:10.371370+0000","last_became_peered":"2024-09-21T10:23:10.371370+0000","last_unstale":"2024-09-21T10:23:35.250732+0000","last_undegraded":"2024-09-21T10:23:35.250732+0000","last_fullsized":"2024-09-21T10:23:35.250732+0000","mapping_epoch":1450,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1451,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:13:22.620446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:13:22.620446+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:21:41.741511+0000","scrub_duration":0.0014078199999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":207,"reported_epoch":1460,"state":"active+clean","last_fresh":"2024-09-21T10:23:35.251258+0000","last_change":"2024-09-21T10:12:25.720808+0000","last_active":"2024-09-21T10:23:35.251258+0000","last_peered":"2024-09-21T10:23:35.251258+0000","last_clean":"2024-09-21T10:23:35.251258+0000","last_became_active":"2024-09-21T10:12:25.706699+0000","last_became_peered":"2024-09-21T10:12:25.706699+0000","last_unstale":"2024-09-21T10:23:35.251258+0000","last_undegraded":"2024-09-21T10:23:35.251258+0000","last_fullsized":"2024-09-21T10:23:35.251258+0000","mapping_epoch":1267,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:42.092105+0000","last_clean_scrub_stamp":"2024-09-21T09:36:42.092105+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:29.044701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2317,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.082956+0000","last_change":"2024-09-21T10:22:42.357135+0000","last_active":"2024-09-21T10:23:39.082956+0000","last_peered":"2024-09-21T10:23:39.082956+0000","last_clean":"2024-09-21T10:23:39.082956+0000","last_became_active":"2024-09-21T10:22:42.356786+0000","last_became_peered":"2024-09-21T10:22:42.356786+0000","last_unstale":"2024-09-21T10:23:39.082956+0000","last_undegraded":"2024-09-21T10:23:39.082956+0000","last_fullsized":"2024-09-21T10:23:39.082956+0000","mapping_epoch":1436,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:43.096207+0000","last_clean_scrub_stamp":"2024-09-21T09:36:43.096207+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:24:04.084804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"288'100","reported_seq":870,"reported_epoch":1461,"state":"active+clean","last_fresh":"2024-09-21T10:23:39.081564+0000","last_change":"2024-09-21T10:23:06.598640+0000","last_active":"2024-09-21T10:23:39.081564+0000","last_peered":"2024-09-21T10:23:39.081564+0000","last_clean":"2024-09-21T10:23:39.081564+0000","last_became_active":"2024-09-21T10:23:06.597998+0000","last_became_peered":"2024-09-21T10:23:06.597998+0000","last_unstale":"2024-09-21T10:23:39.081564+0000","last_undegraded":"2024-09-21T10:23:39.081564+0000","last_fullsized":"2024-09-21T10:23:39.081564+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":1212,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":5,"last_scrub":"288'100","last_scrub_stamp":"2024-09-21T10:06:17.888350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:54.834683+0000","last_clean_scrub_stamp":"2024-09-21T10:06:17.888350+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:04:50.450004+0000","scrub_duration":13,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":71,"num_bytes_recovered":25356,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39860,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3100,"ondisk_log_size":3100,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453398104,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1512608,"kb_used_data":6996,"kb_used_omap":1,"kb_used_meta":1505598,"kb_avail":92232544,"statfs":{"total":95995035648,"available":94446125056,"internally_reserved":0,"allocated":7163904,"data_stored":4835429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1541732696},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":43,"seq":184683594333,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898772,"kb_used_data":7048,"kb_used_omap":1,"kb_used_meta":891710,"kb_avail":92846380,"statfs":{"total":95995035648,"available":95074693120,"internally_reserved":0,"allocated":7217152,"data_stored":4900286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":913111384},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":37,"seq":158913790562,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":901136,"kb_used_data":6980,"kb_used_omap":1,"kb_used_meta":894142,"kb_avail":92844016,"statfs":{"total":95995035648,"available":95072272384,"internally_reserved":0,"allocated":7147520,"data_stored":4831120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915601752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1224,"seq":5257039970499,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":939800,"kb_used_data":6732,"kb_used_omap":290,"kb_used_meta":932765,"kb_avail":92805352,"statfs":{"total":95995035648,"available":95032680448,"internally_reserved":0,"allocated":6893568,"data_stored":4572663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":297377,"internal_metadata":955151967},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":581,"seq":2495375999432,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1235076,"kb_used_data":7032,"kb_used_omap":25,"kb_used_meta":1228006,"kb_avail":92510076,"statfs":{"total":95995035648,"available":94730317824,"internally_reserved":0,"allocated":7200768,"data_stored":4895563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25657,"internal_metadata":1257479111},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":547,"seq":2349347111374,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1264204,"kb_used_data":6976,"kb_used_omap":31,"kb_used_meta":1257184,"kb_avail":92480948,"statfs":{"total":95995035648,"available":94700490752,"internally_reserved":0,"allocated":7143424,"data_stored":4831077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32536,"internal_metadata":1287356648},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":1413,"seq":6068788789287,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":45508,"kb_used_data":6712,"kb_used_omap":61,"kb_used_meta":38722,"kb_avail":93699644,"statfs":{"total":95995035648,"available":95948435456,"internally_reserved":0,"allocated":6873088,"data_stored":4567812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62800,"internal_metadata":39652016},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":479,"seq":2057289335260,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1186504,"kb_used_data":7036,"kb_used_omap":31,"kb_used_meta":1179424,"kb_avail":92558648,"statfs":{"total":95995035648,"available":94780055552,"internally_reserved":0,"allocated":7204864,"data_stored":4896057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32672,"internal_metadata":1207730272},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6239,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12089,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1869,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5235,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:24:33.028 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:24:33.029 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi017 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T10:24:33.195 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=8f3e5253f07a427b927147ac0895f4a8 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi017 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T10:24:33.198 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T10:24:33.213 INFO:tasks.cephadm:Teardown begin 2024-09-21T10:24:33.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:24:33.259 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:24:33.306 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T10:24:33.306 DEBUG:teuthology.orchestra.run.smithi017:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:24:33.325 DEBUG:teuthology.orchestra.run.smithi134:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:24:33.342 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T10:24:33.343 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-21T10:24:33.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a 2024-09-21T10:24:33.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:33 smithi017 systemd[1]: Stopping Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:24:34.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:24:33.711+0000 7f67e8ef0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:24:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a[102890]: 2024-09-21T10:24:33.711+0000 7f67e8ef0640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-21T10:24:34.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:33 smithi017 podman[342402]: 2024-09-21 10:24:33.926943298 +0000 UTC m=+0.341198241 container died 0590143b06cf4e29de8e56942b7a39ee42652dc198eb59c58ac4ea25b7d618e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, RELEASE=main-62da54b) 2024-09-21T10:24:34.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:33] ENGINE Bus STOPPING 2024-09-21T10:24:34.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T10:24:34.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:33] ENGINE Bus STOPPED 2024-09-21T10:24:34.201 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:33] ENGINE Bus STARTING 2024-09-21T10:24:34.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:33] ENGINE Serving on http://:::9283 2024-09-21T10:24:34.202 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:33 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:33] ENGINE Bus STARTED 2024-09-21T10:24:34.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:34 smithi017 podman[342402]: 2024-09-21 10:24:34.444522961 +0000 UTC m=+0.858777943 container cleanup 0590143b06cf4e29de8e56942b7a39ee42652dc198eb59c58ac4ea25b7d618e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T10:24:34.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:34 smithi017 bash[342402]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a 2024-09-21T10:24:34.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service' 2024-09-21T10:24:34.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:34 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: ::ffff:172.21.15.134 - - [21/Sep/2024:10:24:34] "GET /metrics HTTP/1.1" 200 34269 "" "Prometheus/2.51.0" 2024-09-21T10:24:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:34 smithi017 podman[342430]: 2024-09-21 10:24:34.696111765 +0000 UTC m=+0.768571626 container remove 0590143b06cf4e29de8e56942b7a39ee42652dc198eb59c58ac4ea25b7d618e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T10:24:34.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:34 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T10:24:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:34 smithi017 systemd[1]: Stopped Ceph mon.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:24:34.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 21 10:24:34 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.a.service: Consumed 44.173s CPU time. 2024-09-21T10:24:34.977 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:24:34.977 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-21T10:24:34.978 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-21T10:24:34.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.c 2024-09-21T10:24:35.311 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:35 smithi017 systemd[1]: Stopping Ceph mon.c for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:24:35.700 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:35 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:35] ENGINE Bus STOPPING 2024-09-21T10:24:35.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:35 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c[105128]: 2024-09-21T10:24:35.308+0000 7fda5a909640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:24:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:35 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c[105128]: 2024-09-21T10:24:35.308+0000 7fda5a909640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-21T10:24:35.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:35 smithi017 podman[342544]: 2024-09-21 10:24:35.53449069 +0000 UTC m=+0.330237155 container died 013cf9450c578dd09408b41d25b12e9b5b3de35d41bd8a0cd0adc72bc322c210 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T10:24:36.169 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:35 smithi017 podman[342544]: 2024-09-21 10:24:35.916383925 +0000 UTC m=+0.712130375 container cleanup 013cf9450c578dd09408b41d25b12e9b5b3de35d41bd8a0cd0adc72bc322c210 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-09-21T10:24:36.169 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 21 10:24:35 smithi017 bash[342544]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-c 2024-09-21T10:24:36.170 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:35 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T10:24:36.170 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:35 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:35] ENGINE Bus STOPPED 2024-09-21T10:24:36.170 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:35 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:35] ENGINE Bus STARTING 2024-09-21T10:24:36.170 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:35 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:35] ENGINE Serving on http://:::9283 2024-09-21T10:24:36.170 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:35 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: [21/Sep/2024:10:24:35] ENGINE Bus STARTED 2024-09-21T10:24:36.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.c.service' 2024-09-21T10:24:36.358 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:24:36.358 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-21T10:24:36.359 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-21T10:24:36.359 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.b 2024-09-21T10:24:36.673 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:36 smithi134 systemd[1]: Stopping Ceph mon.b for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:24:36.994 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b[90062]: 2024-09-21T10:24:36.670+0000 7f6476986640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:24:36.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:36 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b[90062]: 2024-09-21T10:24:36.670+0000 7f6476986640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-21T10:24:36.995 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:36 smithi134 podman[235204]: 2024-09-21 10:24:36.743104868 +0000 UTC m=+0.178912868 container died 10638b01de94ace450167417aa0e40829016d1580c224b86af361cb1241dcb26 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-21T10:24:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:37 smithi134 podman[235204]: 2024-09-21 10:24:37.243330149 +0000 UTC m=+0.679138181 container cleanup 10638b01de94ace450167417aa0e40829016d1580c224b86af361cb1241dcb26 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T10:24:37.494 INFO:journalctl@ceph.mon.b.smithi134.stdout:Sep 21 10:24:37 smithi134 bash[235204]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mon-b 2024-09-21T10:24:37.628 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mon.b.service' 2024-09-21T10:24:37.670 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:24:37.670 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-21T10:24:37.670 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-21T10:24:37.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y 2024-09-21T10:24:37.950 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:37 smithi017 systemd[1]: Stopping Ceph mgr.y for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:24:38.332 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T10:24:37.997+0000 7f318f5e7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:24:38.333 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:37 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y[93872]: 2024-09-21T10:24:37.997+0000 7f318f5e7640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-21T10:24:38.333 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:38 smithi017 podman[342686]: 2024-09-21 10:24:38.039700588 +0000 UTC m=+0.166133939 container died 14c61243c535dfa35de3c16d7ee0023566ab65b7e2800bf0a5f95d597cfb0f88 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True) 2024-09-21T10:24:38.591 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:38 smithi017 podman[342686]: 2024-09-21 10:24:38.413514468 +0000 UTC m=+0.539947780 container cleanup 14c61243c535dfa35de3c16d7ee0023566ab65b7e2800bf0a5f95d597cfb0f88 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T10:24:38.592 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:38 smithi017 bash[342686]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y 2024-09-21T10:24:38.842 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 21 10:24:38 smithi017 podman[342706]: 2024-09-21 10:24:38.590011893 +0000 UTC m=+0.542267725 container remove 14c61243c535dfa35de3c16d7ee0023566ab65b7e2800bf0a5f95d597cfb0f88 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-y, RELEASE=main-62da54b, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:24:38.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.y.service' 2024-09-21T10:24:38.882 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:24:38.882 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-21T10:24:38.882 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-21T10:24:38.882 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.x 2024-09-21T10:24:39.240 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 10:24:38 smithi134 systemd[1]: Stopping Ceph mgr.x for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:24:39.494 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 10:24:39 smithi134 podman[235344]: 2024-09-21 10:24:39.239269246 +0000 UTC m=+0.158337544 container died 4feb5ade269b0d0fcb1733c61dcc9fec3a695df503a649ddaf1a5dab4357b473 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T10:24:39.984 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 10:24:39 smithi134 podman[235344]: 2024-09-21 10:24:39.655724901 +0000 UTC m=+0.574793197 container cleanup 4feb5ade269b0d0fcb1733c61dcc9fec3a695df503a649ddaf1a5dab4357b473 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T10:24:39.984 INFO:journalctl@ceph.mgr.x.smithi134.stdout:Sep 21 10:24:39 smithi134 bash[235344]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-mgr-x 2024-09-21T10:24:40.111 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@mgr.x.service' 2024-09-21T10:24:40.148 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:24:40.148 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-21T10:24:40.148 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T10:24:40.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:24:40.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:24:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:24:40.211Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"ee55f8cc-77fb-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:24:40.450 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:40 smithi017 systemd[1]: Stopping Ceph osd.0 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:24:40.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[114123]: 2024-09-21T10:24:40.489+0000 7f7028781640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:24:40.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[114123]: 2024-09-21T10:24:40.489+0000 7f7028781640 -1 osd.0 1461 *** Got signal Terminated *** 2024-09-21T10:24:40.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:40 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0[114123]: 2024-09-21T10:24:40.489+0000 7f7028781640 -1 osd.0 1461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:24:46.280 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:45 smithi017 podman[342818]: 2024-09-21 10:24:45.987867946 +0000 UTC m=+5.639516460 container died 68ddc8c6bab9fda157e6fbb6414dc80ef7af1ef689ce76d50d3709d23f9500a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T10:24:46.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:46 smithi017 podman[342818]: 2024-09-21 10:24:46.476985356 +0000 UTC m=+6.128633866 container cleanup 68ddc8c6bab9fda157e6fbb6414dc80ef7af1ef689ce76d50d3709d23f9500a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:24:46.700 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:46 smithi017 bash[342818]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:24:47.054 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:46 smithi017 podman[342840]: 2024-09-21 10:24:46.737919099 +0000 UTC m=+0.750277654 container remove 68ddc8c6bab9fda157e6fbb6414dc80ef7af1ef689ce76d50d3709d23f9500a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:24:47.450 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:47 smithi017 podman[342937]: 2024-09-21 10:24:46.955806741 +0000 UTC m=+0.016436896 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:24:47.451 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:47 smithi017 podman[342937]: 2024-09-21 10:24:47.071588767 +0000 UTC m=+0.132218917 container create 1efb299f52128b42e0d00cdcfea257289609a4b0c3a7a24f01f050f00923a8ef (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T10:24:47.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:47 smithi017 podman[342937]: 2024-09-21 10:24:47.633332064 +0000 UTC m=+0.693962226 container init 1efb299f52128b42e0d00cdcfea257289609a4b0c3a7a24f01f050f00923a8ef (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T10:24:47.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:47 smithi017 podman[342937]: 2024-09-21 10:24:47.638949986 +0000 UTC m=+0.699580135 container start 1efb299f52128b42e0d00cdcfea257289609a4b0c3a7a24f01f050f00923a8ef (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T10:24:47.951 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:47 smithi017 podman[342937]: 2024-09-21 10:24:47.799847122 +0000 UTC m=+0.860477343 container attach 1efb299f52128b42e0d00cdcfea257289609a4b0c3a7a24f01f050f00923a8ef (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T10:24:48.950 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 21 10:24:48 smithi017 podman[342937]: 2024-09-21 10:24:48.621596374 +0000 UTC m=+1.682226532 container died 1efb299f52128b42e0d00cdcfea257289609a4b0c3a7a24f01f050f00923a8ef (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, ceph=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T10:24:49.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T10:24:49.423 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:24:49.423 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T10:24:49.423 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T10:24:49.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.1 2024-09-21T10:24:49.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:49 smithi017 systemd[1]: Stopping Ceph osd.1 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:24:50.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:49 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[322267]: 2024-09-21T10:24:49.768+0000 7f28bda10640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:24:50.201 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:49 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[322267]: 2024-09-21T10:24:49.768+0000 7f28bda10640 -1 osd.1 1461 *** Got signal Terminated *** 2024-09-21T10:24:50.201 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:49 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1[322267]: 2024-09-21T10:24:49.768+0000 7f28bda10640 -1 osd.1 1461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:24:55.421 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:55 smithi017 podman[343067]: 2024-09-21 10:24:55.147412316 +0000 UTC m=+5.501954045 container died d5f81ff1b44049ee3d44701860af266987c3b1046776f34aa11549b17cfe1fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T10:24:55.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:55 smithi017 podman[343067]: 2024-09-21 10:24:55.618108714 +0000 UTC m=+5.972650489 container cleanup d5f81ff1b44049ee3d44701860af266987c3b1046776f34aa11549b17cfe1fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T10:24:55.700 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:55 smithi017 bash[343067]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1 2024-09-21T10:24:56.200 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:55 smithi017 podman[343087]: 2024-09-21 10:24:55.936463042 +0000 UTC m=+0.785332147 container remove d5f81ff1b44049ee3d44701860af266987c3b1046776f34aa11549b17cfe1fa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T10:24:56.555 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:56 smithi017 podman[343182]: 2024-09-21 10:24:56.203104624 +0000 UTC m=+0.016427734 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:24:56.555 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:56 smithi017 podman[343182]: 2024-09-21 10:24:56.312265442 +0000 UTC m=+0.125588597 container create 2cd150d9975ab37fe0354a3483fa0a0d922fd029b73731660161a2cd8394f6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:24:56.555 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:56 smithi017 podman[343182]: 2024-09-21 10:24:56.547654777 +0000 UTC m=+0.360977887 container init 2cd150d9975ab37fe0354a3483fa0a0d922fd029b73731660161a2cd8394f6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T10:24:56.555 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:56 smithi017 podman[343182]: 2024-09-21 10:24:56.553016953 +0000 UTC m=+0.366340063 container start 2cd150d9975ab37fe0354a3483fa0a0d922fd029b73731660161a2cd8394f6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2) 2024-09-21T10:24:56.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:56 smithi017 podman[343182]: 2024-09-21 10:24:56.639350732 +0000 UTC m=+0.452673852 container attach 2cd150d9975ab37fe0354a3483fa0a0d922fd029b73731660161a2cd8394f6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-62da54b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:24:56.950 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 21 10:24:56 smithi017 podman[343182]: 2024-09-21 10:24:56.747002081 +0000 UTC m=+0.560325191 container died 2cd150d9975ab37fe0354a3483fa0a0d922fd029b73731660161a2cd8394f6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:24:57.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.1.service' 2024-09-21T10:24:57.444 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:24:57.445 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T10:24:57.445 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T10:24:57.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.2 2024-09-21T10:24:57.700 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:24:57 smithi017 systemd[1]: Stopping Ceph osd.2 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:24:58.200 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:24:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[125118]: 2024-09-21T10:24:57.807+0000 7f3b8495a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:24:58.201 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:24:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[125118]: 2024-09-21T10:24:57.807+0000 7f3b8495a640 -1 osd.2 1461 *** Got signal Terminated *** 2024-09-21T10:24:58.201 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:24:57 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2[125118]: 2024-09-21T10:24:57.807+0000 7f3b8495a640 -1 osd.2 1461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:25:03.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:03 smithi017 podman[343309]: 2024-09-21 10:25:03.368434215 +0000 UTC m=+5.689748255 container died f98642c0dccf82c1d91dc8c52471560903a50f90e833c2404f0333d7f67078c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b) 2024-09-21T10:25:03.950 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:03 smithi017 podman[343309]: 2024-09-21 10:25:03.896941598 +0000 UTC m=+6.218255625 container cleanup f98642c0dccf82c1d91dc8c52471560903a50f90e833c2404f0333d7f67078c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T10:25:03.951 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:03 smithi017 bash[343309]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2 2024-09-21T10:25:04.450 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:04 smithi017 podman[343329]: 2024-09-21 10:25:04.1318464 +0000 UTC m=+0.759896233 container remove f98642c0dccf82c1d91dc8c52471560903a50f90e833c2404f0333d7f67078c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:25:04.820 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:04 smithi017 podman[343422]: 2024-09-21 10:25:04.540989925 +0000 UTC m=+0.104294272 container create 3f1261898cd9a2a6c563646c4ce81431443f00e7f03f21a34051c0f14489f4ec (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T10:25:04.820 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:04 smithi017 podman[343422]: 2024-09-21 10:25:04.452762846 +0000 UTC m=+0.016067165 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:25:04.820 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:04 smithi017 podman[343422]: 2024-09-21 10:25:04.734898811 +0000 UTC m=+0.298203125 container init 3f1261898cd9a2a6c563646c4ce81431443f00e7f03f21a34051c0f14489f4ec (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:25:04.820 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:04 smithi017 podman[343422]: 2024-09-21 10:25:04.739875193 +0000 UTC m=+0.303179502 container start 3f1261898cd9a2a6c563646c4ce81431443f00e7f03f21a34051c0f14489f4ec (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:25:04.821 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:04 smithi017 podman[343422]: 2024-09-21 10:25:04.818020009 +0000 UTC m=+0.381324339 container attach 3f1261898cd9a2a6c563646c4ce81431443f00e7f03f21a34051c0f14489f4ec (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T10:25:05.200 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 21 10:25:04 smithi017 podman[343422]: 2024-09-21 10:25:04.941785872 +0000 UTC m=+0.505090182 container died 3f1261898cd9a2a6c563646c4ce81431443f00e7f03f21a34051c0f14489f4ec (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T10:25:05.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.2.service' 2024-09-21T10:25:05.636 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:25:05.636 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T10:25:05.636 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T10:25:05.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.3 2024-09-21T10:25:05.863 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:05 smithi017 systemd[1]: Stopping Ceph osd.3 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:25:06.201 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:06 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[130497]: 2024-09-21T10:25:06.001+0000 7faa16cb5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:25:06.201 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:06 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[130497]: 2024-09-21T10:25:06.001+0000 7faa16cb5640 -1 osd.3 1461 *** Got signal Terminated *** 2024-09-21T10:25:06.201 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:06 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[130497]: 2024-09-21T10:25:06.001+0000 7faa16cb5640 -1 osd.3 1461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:25:07.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:06 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:06.854+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:08.145 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:07 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:07.852+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:08.146 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:07 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:07.989+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:09.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:08 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:08.957+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:09.245 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:09.035+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:09.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:08 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:08.890+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:09.994 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:09.950+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:09.994 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:09.924+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:09.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:09 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:09.708+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:10.494 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:10.075+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:10.700 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:10 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3[130497]: 2024-09-21T10:25:10.394+0000 7faa132ca640 -1 osd.3 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:44.418217+0000 front 2024-09-21T10:24:44.418406+0000 (oldest deadline 2024-09-21T10:25:10.318406+0000) 2024-09-21T10:25:10.974 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:10.705+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:10.974 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:10.894+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:11.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:10 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:10.971+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:11.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:11.078+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:11.950 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:11 smithi017 podman[343551]: 2024-09-21 10:25:11.692027405 +0000 UTC m=+5.828374241 container died 673e3e8b7c0aed588d30d7e1ff542e1466f68f77c086e9c3afd1db81c8934664 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T10:25:12.092 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:11.944+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:12.093 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:11.752+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:12.093 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:11 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:11.930+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:12.444 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:12 smithi017 podman[343551]: 2024-09-21 10:25:12.149619635 +0000 UTC m=+6.285966456 container cleanup 673e3e8b7c0aed588d30d7e1ff542e1466f68f77c086e9c3afd1db81c8934664 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-21T10:25:12.444 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:12 smithi017 bash[343551]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3 2024-09-21T10:25:12.494 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:12.090+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:12.700 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:12 smithi017 podman[343584]: 2024-09-21 10:25:12.442707064 +0000 UTC m=+0.751493419 container remove 673e3e8b7c0aed588d30d7e1ff542e1466f68f77c086e9c3afd1db81c8934664 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T10:25:13.097 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:12 smithi017 podman[343675]: 2024-09-21 10:25:12.675875191 +0000 UTC m=+0.016240645 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:25:13.097 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:12 smithi017 podman[343675]: 2024-09-21 10:25:12.801777874 +0000 UTC m=+0.142143319 container create 6e67648eadd46c3e55a82ad21f154f666109f39d7151e65c3373f3feb1b5fe87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:25:13.097 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:13 smithi017 podman[343675]: 2024-09-21 10:25:13.012445369 +0000 UTC m=+0.352810822 container init 6e67648eadd46c3e55a82ad21f154f666109f39d7151e65c3373f3feb1b5fe87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, RELEASE=main-62da54b, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T10:25:13.097 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:13 smithi017 podman[343675]: 2024-09-21 10:25:13.017602976 +0000 UTC m=+0.357968421 container start 6e67648eadd46c3e55a82ad21f154f666109f39d7151e65c3373f3feb1b5fe87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:25:13.097 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:13 smithi017 podman[343675]: 2024-09-21 10:25:13.095455966 +0000 UTC m=+0.435821426 container attach 6e67648eadd46c3e55a82ad21f154f666109f39d7151e65c3373f3feb1b5fe87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:25:13.106 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:12.940+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:13.106 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:12.753+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:13.107 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:12 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:12.917+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:13.450 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:13 smithi017 podman[343675]: 2024-09-21 10:25:13.21424288 +0000 UTC m=+0.554608327 container died 6e67648eadd46c3e55a82ad21f154f666109f39d7151e65c3373f3feb1b5fe87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:25:13.494 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:13.104+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:13.752 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 21 10:25:13 smithi017 podman[343718]: 2024-09-21 10:25:13.748688287 +0000 UTC m=+0.528095005 container remove 6e67648eadd46c3e55a82ad21f154f666109f39d7151e65c3373f3feb1b5fe87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T10:25:13.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T10:25:13.814 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:25:13.815 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T10:25:13.815 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T10:25:13.815 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.4 2024-09-21T10:25:13.994 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:13.931+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:13.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:13.738+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:13.995 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:13 smithi134 systemd[1]: Stopping Ceph osd.4 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:25:13.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:13.912+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:13.728Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-21T10:25:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:13.729Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-21T10:25:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:13.730Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-21T10:25:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:13.734Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-21T10:25:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:13.734Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-21T10:25:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:13.735Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-21T10:25:13.996 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:13 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:13.735Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.17:8765: connect: connection refused" 2024-09-21T10:25:14.494 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:14.083+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:14.494 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:14.133+0000 7f9d614ca640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:25:14.494 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:14.133+0000 7f9d614ca640 -1 osd.4 1461 *** Got signal Terminated *** 2024-09-21T10:25:14.495 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:14.133+0000 7f9d614ca640 -1 osd.4 1461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:25:14.994 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:14.916+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:14.994 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:14.899+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:14.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:14 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:14.727+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:15.494 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:15.038+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:16.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:15.764+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:16.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:15.930+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:16.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:16.009+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:16.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:16.009+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:51.592140+0000 front 2024-09-21T10:24:51.592009+0000 (oldest deadline 2024-09-21T10:25:15.692058+0000) 2024-09-21T10:25:16.246 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:15.922+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:16.247 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:15 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:15.922+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:16.994 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:16.945+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:16.994 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:16.945+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:16.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:16.717+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:16.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:16.717+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:16.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:16.953+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:16.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:16.953+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:16.995 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:16.963+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:16.996 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:16 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:16.963+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:51.592140+0000 front 2024-09-21T10:24:51.592009+0000 (oldest deadline 2024-09-21T10:25:15.692058+0000) 2024-09-21T10:25:17.991 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:17.718+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:17.992 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:17.718+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:17.992 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:17.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:17.992 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:17.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:18.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:17.991+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:18.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:17.991+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:51.592140+0000 front 2024-09-21T10:24:51.592009+0000 (oldest deadline 2024-09-21T10:25:15.692058+0000) 2024-09-21T10:25:18.245 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:17.989+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:18.245 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:17 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:17.989+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:18.955 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:18 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:18.700+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:18.955 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:18 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:18.700+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:19.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:19.028+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:45.390285+0000 front 2024-09-21T10:24:45.390166+0000 (oldest deadline 2024-09-21T10:25:07.090092+0000) 2024-09-21T10:25:19.246 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4[158201]: 2024-09-21T10:25:19.028+0000 7f9d5bade640 -1 osd.4 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:51.592140+0000 front 2024-09-21T10:24:51.592009+0000 (oldest deadline 2024-09-21T10:25:15.692058+0000) 2024-09-21T10:25:19.246 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:18 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:18.952+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:19.247 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:18 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:18.952+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:19.247 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:19.024+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:19.247 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:19.024+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:19.976 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:19.709+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:19.976 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:19.709+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:19.977 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:19 smithi134 podman[235493]: 2024-09-21 10:25:19.765217751 +0000 UTC m=+5.749153009 container died f5ea068205091e9651900bcbecc0fd263ea3b54e86c7837d784fcdca32b3a7e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T10:25:20.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:20 smithi134 podman[235493]: 2024-09-21 10:25:20.148491802 +0000 UTC m=+6.132427065 container cleanup f5ea068205091e9651900bcbecc0fd263ea3b54e86c7837d784fcdca32b3a7e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T10:25:20.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:20 smithi134 bash[235493]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4 2024-09-21T10:25:20.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:20.000+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:20.245 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:20.000+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:20.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:19.973+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:20.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:19 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:19.973+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:20.733 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:20.673+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:20.734 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:20.673+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:20.734 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:20 smithi134 podman[235514]: 2024-09-21 10:25:20.424988787 +0000 UTC m=+0.662082465 container remove f5ea068205091e9651900bcbecc0fd263ea3b54e86c7837d784fcdca32b3a7e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:25:20.988 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:20.956+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:20.988 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:20.956+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:20.988 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:20.926+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:20.989 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:20 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:20.926+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:20.989 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:20 smithi134 podman[235610]: 2024-09-21 10:25:20.63528744 +0000 UTC m=+0.016183046 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T10:25:20.989 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:20 smithi134 podman[235610]: 2024-09-21 10:25:20.775893933 +0000 UTC m=+0.156789538 container create 8c77b37267ac38a7b890d6c1e044733e6f9e58c94dd32d7225986fee073c746c (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a) 2024-09-21T10:25:20.989 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:20 smithi134 podman[235610]: 2024-09-21 10:25:20.986416114 +0000 UTC m=+0.367311718 container init 8c77b37267ac38a7b890d6c1e044733e6f9e58c94dd32d7225986fee073c746c (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T10:25:21.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:20 smithi134 podman[235610]: 2024-09-21 10:25:20.9916575 +0000 UTC m=+0.372553104 container start 8c77b37267ac38a7b890d6c1e044733e6f9e58c94dd32d7225986fee073c746c (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:25:21.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:21 smithi134 podman[235610]: 2024-09-21 10:25:21.08650441 +0000 UTC m=+0.467400021 container attach 8c77b37267ac38a7b890d6c1e044733e6f9e58c94dd32d7225986fee073c746c (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:25:21.244 INFO:journalctl@ceph.osd.4.smithi134.stdout:Sep 21 10:25:21 smithi134 podman[235610]: 2024-09-21 10:25:21.177983295 +0000 UTC m=+0.558878906 container died 8c77b37267ac38a7b890d6c1e044733e6f9e58c94dd32d7225986fee073c746c (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T10:25:21.808 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:21.629+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:21.808 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:21.629+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:21.823 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.4.service' 2024-09-21T10:25:21.863 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:25:21.863 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T10:25:21.864 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T10:25:21.864 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.5 2024-09-21T10:25:22.180 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:21.917+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:22.180 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:21.917+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:22.181 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:21 smithi134 systemd[1]: Stopping Ceph osd.5 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:25:22.181 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:21.985+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:22.181 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:21 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:21.985+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:22.181 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:22.178+0000 7f4e9772b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:25:22.181 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:22.178+0000 7f4e9772b640 -1 osd.5 1461 *** Got signal Terminated *** 2024-09-21T10:25:22.181 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:22.178+0000 7f4e9772b640 -1 osd.5 1461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:25:22.898 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:22.643+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:22.899 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:22.643+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:23.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:22.989+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:23.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:22.989+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:23.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:22.896+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:23.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:22 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:22.896+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:23.902 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:23.648+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:23.903 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:23.648+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:24.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:24.012+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:24.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:24.012+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:24.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:23.900+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:24.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:23.900+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:24.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:23 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:23.900+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:24.994 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:24.876+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:24.994 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:24.876+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:24.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:24.876+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:24.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:24.642+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:24.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:24.642+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:24.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:24 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:24.642+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:25.494 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:25.029+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:25.494 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:25.029+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:25.896 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:25.621+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:25.896 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:25.621+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:25.896 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:25.621+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:26.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:25.893+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:26.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:25.893+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:26.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:25.893+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:26.245 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:25.992+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:26.245 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:25.992+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:26.245 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:25 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:25.992+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.307177+0000 front 2024-09-21T10:24:59.307139+0000 (oldest deadline 2024-09-21T10:25:25.207024+0000) 2024-09-21T10:25:26.890 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:26.631+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:26.890 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:26.631+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:26.890 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:26.631+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:27.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:26.961+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:42.305519+0000 front 2024-09-21T10:24:42.305593+0000 (oldest deadline 2024-09-21T10:25:08.205417+0000) 2024-09-21T10:25:27.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:26.961+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:49.306233+0000 front 2024-09-21T10:24:49.306291+0000 (oldest deadline 2024-09-21T10:25:15.206201+0000) 2024-09-21T10:25:27.244 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5[44919]: 2024-09-21T10:25:26.961+0000 7f4e94d45640 -1 osd.5 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.307177+0000 front 2024-09-21T10:24:59.307139+0000 (oldest deadline 2024-09-21T10:25:25.207024+0000) 2024-09-21T10:25:27.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:26.888+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:27.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:26.888+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:27.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:26 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:26.888+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:27.994 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:27 smithi134 podman[235739]: 2024-09-21 10:25:27.735014104 +0000 UTC m=+5.653781405 container died 5dec2255fcd71b94988054b5a6bef04764e2d63813e6fa119cc016212b64bf84 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2) 2024-09-21T10:25:27.994 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:27 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:27.633+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:27.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:27 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:27.633+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:27.995 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:27 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:27.633+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:27.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:27 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:27.879+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:27.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:27 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:27.879+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:27.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:27 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:27.879+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:28.463 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:28 smithi134 podman[235739]: 2024-09-21 10:25:28.17658034 +0000 UTC m=+6.095347633 container cleanup 5dec2255fcd71b94988054b5a6bef04764e2d63813e6fa119cc016212b64bf84 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2) 2024-09-21T10:25:28.463 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:28 smithi134 bash[235739]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5 2024-09-21T10:25:28.744 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:28 smithi134 podman[235760]: 2024-09-21 10:25:28.461055773 +0000 UTC m=+0.662989786 container remove 5dec2255fcd71b94988054b5a6bef04764e2d63813e6fa119cc016212b64bf84 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T10:25:28.744 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:28.675+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:28.745 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:28.675+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:28.745 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:28.675+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:29.116 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:28 smithi134 podman[235853]: 2024-09-21 10:25:28.674507067 +0000 UTC m=+0.015801944 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T10:25:29.116 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:28 smithi134 podman[235853]: 2024-09-21 10:25:28.787010916 +0000 UTC m=+0.128305776 container create 4c7ac1a25fe778e5cc8f792ba63f7a0598e2859eb70525330e2ce74c2c8bb67e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, ceph=True, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T10:25:29.116 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:29 smithi134 podman[235853]: 2024-09-21 10:25:29.005669209 +0000 UTC m=+0.346964081 container init 4c7ac1a25fe778e5cc8f792ba63f7a0598e2859eb70525330e2ce74c2c8bb67e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a) 2024-09-21T10:25:29.117 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:29 smithi134 podman[235853]: 2024-09-21 10:25:29.010751178 +0000 UTC m=+0.352046038 container start 4c7ac1a25fe778e5cc8f792ba63f7a0598e2859eb70525330e2ce74c2c8bb67e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-21T10:25:29.117 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:29 smithi134 podman[235853]: 2024-09-21 10:25:29.114217319 +0000 UTC m=+0.455512191 container attach 4c7ac1a25fe778e5cc8f792ba63f7a0598e2859eb70525330e2ce74c2c8bb67e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2) 2024-09-21T10:25:29.117 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:28.866+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:29.117 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:28.866+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:29.117 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:28 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:28.866+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:29.494 INFO:journalctl@ceph.osd.5.smithi134.stdout:Sep 21 10:25:29 smithi134 podman[235853]: 2024-09-21 10:25:29.181167729 +0000 UTC m=+0.522462605 container died 4c7ac1a25fe778e5cc8f792ba63f7a0598e2859eb70525330e2ce74c2c8bb67e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T10:25:29.777 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:29 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:29.689+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:29.777 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:29 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:29.689+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:29.777 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:29 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:29.689+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:29.801 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.5.service' 2024-09-21T10:25:29.840 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:25:29.840 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T10:25:29.841 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T10:25:29.841 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.6 2024-09-21T10:25:30.166 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:29 smithi134 systemd[1]: Stopping Ceph osd.6 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:25:30.166 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:30.163+0000 7fd9f3113640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:25:30.167 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:29 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:29.890+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:30.167 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:29 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:29.890+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:30.167 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:29 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:29.890+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:30.494 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:30.163+0000 7fd9f3113640 -1 osd.6 1461 *** Got signal Terminated *** 2024-09-21T10:25:30.494 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:30.163+0000 7fd9f3113640 -1 osd.6 1461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:25:30.994 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:30.726+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:30.994 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:30.726+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:30.994 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:30.726+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:30.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:30.881+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:30.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:30.881+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:30.995 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:30 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:30.881+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:32.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:31 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:31.773+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:32.245 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:31 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:31.773+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:32.245 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:31 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:31.773+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:32.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:31 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:31.921+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:32.246 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:31 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:31.921+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:32.246 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:31 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:31.921+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:33.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:32.793+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:33.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:32.793+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:33.245 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:32.793+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:33.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:32.935+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:33.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:32.935+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:33.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:32.935+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:33.246 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:32 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:32.935+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:34.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:33 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:33.844+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:34.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:33 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:33.844+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:34.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:33 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:33.844+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:34.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:33 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:33.905+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:34.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:33 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:33.905+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:34.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:33 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:33.905+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:34.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:33 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:33.905+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:35.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:34.827+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:43.519278+0000 front 2024-09-21T10:24:43.519457+0000 (oldest deadline 2024-09-21T10:25:08.819464+0000) 2024-09-21T10:25:35.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:34.827+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:54.120741+0000 front 2024-09-21T10:24:54.120607+0000 (oldest deadline 2024-09-21T10:25:15.820549+0000) 2024-09-21T10:25:35.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:34.827+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:59.321446+0000 front 2024-09-21T10:24:59.321407+0000 (oldest deadline 2024-09-21T10:25:24.621320+0000) 2024-09-21T10:25:35.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6[50848]: 2024-09-21T10:25:34.827+0000 7fd9ef72b640 -1 osd.6 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:09.822475+0000 front 2024-09-21T10:25:09.822482+0000 (oldest deadline 2024-09-21T10:25:34.522495+0000) 2024-09-21T10:25:35.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:34.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:35.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:34.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:35.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:34.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:35.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:34 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:34.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:35.987 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:35 smithi134 podman[235984]: 2024-09-21 10:25:35.725065063 +0000 UTC m=+5.665128311 container died 80d4436a0b8aecc032c83327a0dd4639829285bc81526c46969d87138d74dc6e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T10:25:36.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:36 smithi134 podman[235984]: 2024-09-21 10:25:36.094350048 +0000 UTC m=+6.034413294 container cleanup 80d4436a0b8aecc032c83327a0dd4639829285bc81526c46969d87138d74dc6e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:25:36.244 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:36 smithi134 bash[235984]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6 2024-09-21T10:25:36.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:35.993+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:36.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:35.993+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:36.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:35.993+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:36.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:35 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:35.993+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:36.702 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:36 smithi134 podman[236006]: 2024-09-21 10:25:36.36225392 +0000 UTC m=+0.637050482 container remove 80d4436a0b8aecc032c83327a0dd4639829285bc81526c46969d87138d74dc6e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T10:25:36.956 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:36 smithi134 podman[236100]: 2024-09-21 10:25:36.603408268 +0000 UTC m=+0.015606946 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T10:25:36.956 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:36 smithi134 podman[236100]: 2024-09-21 10:25:36.721650324 +0000 UTC m=+0.133848983 container create 5dca20c72a5f4ae0c43f757b9ce2f6f176bdd0de97d0c6200c9494ba3afbe196 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, RELEASE=quincy-fbd1a6a, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:25:36.956 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:36 smithi134 podman[236100]: 2024-09-21 10:25:36.949032103 +0000 UTC m=+0.361230772 container init 5dca20c72a5f4ae0c43f757b9ce2f6f176bdd0de97d0c6200c9494ba3afbe196 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T10:25:36.956 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:36 smithi134 podman[236100]: 2024-09-21 10:25:36.954339497 +0000 UTC m=+0.366538156 container start 5dca20c72a5f4ae0c43f757b9ce2f6f176bdd0de97d0c6200c9494ba3afbe196 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731) 2024-09-21T10:25:37.245 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:37 smithi134 podman[236100]: 2024-09-21 10:25:37.057374502 +0000 UTC m=+0.469573164 container attach 5dca20c72a5f4ae0c43f757b9ce2f6f176bdd0de97d0c6200c9494ba3afbe196 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:25:37.245 INFO:journalctl@ceph.osd.6.smithi134.stdout:Sep 21 10:25:37 smithi134 podman[236100]: 2024-09-21 10:25:37.13180026 +0000 UTC m=+0.543998934 container died 5dca20c72a5f4ae0c43f757b9ce2f6f176bdd0de97d0c6200c9494ba3afbe196 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T10:25:37.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:37 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:37.029+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:37.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:37 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:37.029+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:37.246 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:37 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:37.029+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:37.246 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:37 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:37.029+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:37.774 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.6.service' 2024-09-21T10:25:37.813 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:25:37.813 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T10:25:37.813 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T10:25:37.814 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.7 2024-09-21T10:25:38.018 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:37 smithi134 systemd[1]: Stopping Ceph osd.7 for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:25:38.494 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.014+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:38.494 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.014+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:38.494 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.014+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:38.495 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.014+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:38.495 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.131+0000 7f6d441ab640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:25:38.495 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.131+0000 7f6d441ab640 -1 osd.7 1461 *** Got signal Terminated *** 2024-09-21T10:25:38.495 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.131+0000 7f6d441ab640 -1 osd.7 1461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:25:39.284 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.995+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:39.284 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.995+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:39.284 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.995+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:39.284 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:38 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:38.995+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:40.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:39.990+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:40.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:39.990+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:40.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:39.990+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:40.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:39 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:39.990+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:41.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:40.968+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:41.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:40.968+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:41.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:40.968+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:41.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:40 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:40.968+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:42.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:41.972+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:42.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:41.972+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:42.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:41.972+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:42.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:41 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:41.972+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:43.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:42.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2024-09-21T10:24:41.534946+0000 front 2024-09-21T10:24:41.535081+0000 (oldest deadline 2024-09-21T10:25:06.234989+0000) 2024-09-21T10:25:43.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:42.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2024-09-21T10:24:50.935853+0000 front 2024-09-21T10:24:50.935926+0000 (oldest deadline 2024-09-21T10:25:16.835790+0000) 2024-09-21T10:25:43.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:42.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2024-09-21T10:24:58.437390+0000 front 2024-09-21T10:24:58.437325+0000 (oldest deadline 2024-09-21T10:25:23.737266+0000) 2024-09-21T10:25:43.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:42.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2024-09-21T10:25:06.638080+0000 front 2024-09-21T10:25:06.638108+0000 (oldest deadline 2024-09-21T10:25:32.538181+0000) 2024-09-21T10:25:43.245 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:42 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7[57024]: 2024-09-21T10:25:42.945+0000 7f6d3f7c1640 -1 osd.7 1461 heartbeat_check: no reply from 172.21.15.134:6804 osd.4 since back 2024-09-21T10:25:17.839107+0000 front 2024-09-21T10:25:17.839149+0000 (oldest deadline 2024-09-21T10:25:42.539171+0000) 2024-09-21T10:25:43.958 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:43 smithi134 podman[236228]: 2024-09-21 10:25:43.656041666 +0000 UTC m=+5.621124423 container died ed2e26ef569898cb893ca6477ba3480e2ee7fbf612f34474f540feacf1c91650 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=) 2024-09-21T10:25:44.232 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:44 smithi134 podman[236228]: 2024-09-21 10:25:44.054447557 +0000 UTC m=+6.019530306 container cleanup ed2e26ef569898cb893ca6477ba3480e2ee7fbf612f34474f540feacf1c91650 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T10:25:44.232 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:44 smithi134 bash[236228]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7 2024-09-21T10:25:44.494 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:44 smithi134 podman[236262]: 2024-09-21 10:25:44.230664701 +0000 UTC m=+0.569185774 container remove ed2e26ef569898cb893ca6477ba3480e2ee7fbf612f34474f540feacf1c91650 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:25:44.927 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:44 smithi134 podman[236361]: 2024-09-21 10:25:44.566899146 +0000 UTC m=+0.015686099 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T10:25:44.927 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:44 smithi134 podman[236361]: 2024-09-21 10:25:44.681353831 +0000 UTC m=+0.130140773 container create 01411e067ceffb014134056d3b606a4ad766109d63b13829ff0fe07d44dd54fb (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a) 2024-09-21T10:25:44.927 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:44 smithi134 podman[236361]: 2024-09-21 10:25:44.925459682 +0000 UTC m=+0.374246637 container init 01411e067ceffb014134056d3b606a4ad766109d63b13829ff0fe07d44dd54fb (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T10:25:45.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:44 smithi134 podman[236361]: 2024-09-21 10:25:44.930550006 +0000 UTC m=+0.379336945 container start 01411e067ceffb014134056d3b606a4ad766109d63b13829ff0fe07d44dd54fb (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T10:25:45.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:45 smithi134 podman[236361]: 2024-09-21 10:25:45.016964618 +0000 UTC m=+0.465751556 container attach 01411e067ceffb014134056d3b606a4ad766109d63b13829ff0fe07d44dd54fb (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:25:45.244 INFO:journalctl@ceph.osd.7.smithi134.stdout:Sep 21 10:25:45 smithi134 podman[236361]: 2024-09-21 10:25:45.109189259 +0000 UTC m=+0.557976206 container died 01411e067ceffb014134056d3b606a4ad766109d63b13829ff0fe07d44dd54fb (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T10:25:45.901 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@osd.7.service' 2024-09-21T10:25:45.943 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:25:45.943 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T10:25:45.943 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-21T10:25:45.944 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@prometheus.a 2024-09-21T10:25:46.244 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 systemd[1]: Stopping Ceph prometheus.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:25:46.585 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.308Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T10:25:46.585 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.308Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-21T10:25:46.585 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.308Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-21T10:25:46.585 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.308Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T10:25:46.585 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.308Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-21T10:25:46.585 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.308Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-21T10:25:46.585 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.308Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T10:25:46.586 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.308Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-21T10:25:46.586 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.308Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-21T10:25:46.586 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.313Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T10:25:46.586 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.313Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-21T10:25:46.586 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a[81204]: ts=2024-09-21T10:25:46.313Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-21T10:25:46.586 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 podman[236490]: 2024-09-21 10:25:46.318963558 +0000 UTC m=+0.142747002 container died 061a3294723d9bba519864e40145c126bd16d472e0cd54dc709160e191fbfd2e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T10:25:46.953 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 podman[236490]: 2024-09-21 10:25:46.681102406 +0000 UTC m=+0.504885854 container cleanup 061a3294723d9bba519864e40145c126bd16d472e0cd54dc709160e191fbfd2e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T10:25:46.953 INFO:journalctl@ceph.prometheus.a.smithi134.stdout:Sep 21 10:25:46 smithi134 bash[236490]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T10:25:47.086 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@prometheus.a.service' 2024-09-21T10:25:47.119 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:25:47.119 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-21T10:25:47.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:25:48.950 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:48 smithi017 systemd[1]: Stopping Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:25:49.369 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:49 smithi017 ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a[91292]: ts=2024-09-21T10:25:49.043Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T10:25:49.369 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:49 smithi017 podman[344101]: 2024-09-21 10:25:49.094698106 +0000 UTC m=+0.179328185 container died 0a4d8256f8d7818423b62865207ade6ae97fa0fdff7283789ad1763bd54affb3 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:25:49.700 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:49 smithi017 podman[344101]: 2024-09-21 10:25:49.558232576 +0000 UTC m=+0.642862655 container cleanup 0a4d8256f8d7818423b62865207ade6ae97fa0fdff7283789ad1763bd54affb3 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:25:49.700 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:49 smithi017 bash[344101]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T10:25:50.200 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:49 smithi017 podman[344121]: 2024-09-21 10:25:49.842913146 +0000 UTC m=+0.744354218 container remove 0a4d8256f8d7818423b62865207ade6ae97fa0fdff7283789ad1763bd54affb3 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:25:50.200 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:49 smithi017 podman[344121]: 2024-09-21 10:25:49.91062448 +0000 UTC m=+0.812065509 volume remove c3af648b4c0a7b19a59d0a7955985fa50e7c12b73f49f92dc9c6b58bab72f555 2024-09-21T10:25:50.200 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:50 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T10:25:50.201 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:50 smithi017 systemd[1]: Stopped Ceph alertmanager.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:25:50.201 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 21 10:25:50 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 4.077s CPU time. 2024-09-21T10:25:50.486 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 10:25:50 smithi017 systemd[1]: Stopping Ceph node-exporter.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:25:50.814 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 10:25:50 smithi017 podman[344233]: 2024-09-21 10:25:50.48442159 +0000 UTC m=+0.108831756 container died 057b63034a9e6600976eb2e71da4800bc610f1fbbb790a2399666c1da3353819 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:25:50.814 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 10:25:50 smithi017 podman[344233]: 2024-09-21 10:25:50.812759578 +0000 UTC m=+0.437169744 container cleanup 057b63034a9e6600976eb2e71da4800bc610f1fbbb790a2399666c1da3353819 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:25:51.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 10:25:50 smithi017 bash[344233]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T10:25:51.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 10:25:51 smithi017 podman[344251]: 2024-09-21 10:25:51.022291047 +0000 UTC m=+0.537444048 container remove 057b63034a9e6600976eb2e71da4800bc610f1fbbb790a2399666c1da3353819 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:25:51.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 10:25:51 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:25:51.687 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 10:25:51 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T10:25:51.688 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 10:25:51 smithi017 systemd[1]: Stopped Ceph node-exporter.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:25:51.688 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 21 10:25:51 smithi017 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 8.924s CPU time. 2024-09-21T10:26:02.562 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:26:04.244 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:26:04 smithi134 systemd[1]: Stopping Ceph grafana.a for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:26:04.631 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:26:04 smithi134 podman[236914]: 2024-09-21 10:26:04.359101377 +0000 UTC m=+0.120586370 container died 8d2c807b030864794a54fcbc34d81c4d1e0bb6e5dc2c9bc5295e10fd163e6d27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, io.openshift.expose-services=, vendor=Red Hat, Inc., description=Ceph Grafana Container, vcs-type=git, version=8.5, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-09-21T10:26:04.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:26:04 smithi134 podman[236914]: 2024-09-21 10:26:04.712015555 +0000 UTC m=+0.473500539 container cleanup 8d2c807b030864794a54fcbc34d81c4d1e0bb6e5dc2c9bc5295e10fd163e6d27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, maintainer=Paul Cuzner , name=ubi8, io.openshift.tags=base rhel8, io.openshift.expose-services=, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., architecture=x86_64, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-09-21T10:26:04.994 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:26:04 smithi134 bash[236914]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a 2024-09-21T10:26:05.302 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:26:04 smithi134 podman[236933]: 2024-09-21 10:26:04.996259226 +0000 UTC m=+0.635513336 container remove 8d2c807b030864794a54fcbc34d81c4d1e0bb6e5dc2c9bc5295e10fd163e6d27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-grafana-a, vcs-type=git, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, maintainer=Paul Cuzner , io.buildah.version=1.32.0, name=ubi8, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8) 2024-09-21T10:26:05.303 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:26:04 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:26:05.303 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:26:05 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@grafana.a.service: Failed with result 'exit-code'. 2024-09-21T10:26:05.303 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:26:05 smithi134 systemd[1]: Stopped Ceph grafana.a for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:26:05.303 INFO:journalctl@ceph.grafana.a.smithi134.stdout:Sep 21 10:26:05 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@grafana.a.service: Consumed 29.977s CPU time. 2024-09-21T10:26:05.563 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 10:26:05 smithi134 systemd[1]: Stopping Ceph node-exporter.b for ee55f8cc-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:26:05.834 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 10:26:05 smithi134 podman[237044]: 2024-09-21 10:26:05.561769697 +0000 UTC m=+0.119286498 container died c6b046d44374f131cfaa2c1a1e02d202770926863880de056dd93546758311fd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:26:06.184 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 10:26:05 smithi134 podman[237044]: 2024-09-21 10:26:05.923649102 +0000 UTC m=+0.481165893 container cleanup c6b046d44374f131cfaa2c1a1e02d202770926863880de056dd93546758311fd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:26:06.184 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 10:26:05 smithi134 bash[237044]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T10:26:06.494 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 10:26:06 smithi134 podman[237063]: 2024-09-21 10:26:06.183067805 +0000 UTC m=+0.625335265 container remove c6b046d44374f131cfaa2c1a1e02d202770926863880de056dd93546758311fd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:26:06.494 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 10:26:06 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:26:06.494 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 10:26:06 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T10:26:06.494 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 10:26:06 smithi134 systemd[1]: Stopped Ceph node-exporter.b for ee55f8cc-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:26:06.494 INFO:journalctl@ceph.node-exporter.b.smithi134.stdout:Sep 21 10:26:06 smithi134 systemd[1]: ceph-ee55f8cc-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 7.658s CPU time. 2024-09-21T10:26:10.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:26:10.783 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:26:10.812 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T10:26:10.814 DEBUG:teuthology.misc:Transferring archived files from smithi017:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913647/remote/smithi017/crash 2024-09-21T10:26:10.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo tar c -f - -C /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:26:10.865 INFO:teuthology.orchestra.run.smithi017.stderr:tar: /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:26:10.866 INFO:teuthology.orchestra.run.smithi017.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:26:10.868 DEBUG:teuthology.misc:Transferring archived files from smithi134:/var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913647/remote/smithi134/crash 2024-09-21T10:26:10.869 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:26:10.909 INFO:teuthology.orchestra.run.smithi134.stderr:tar: /var/lib/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:26:10.909 INFO:teuthology.orchestra.run.smithi134.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:26:10.910 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T10:26:10.910 DEBUG:teuthology.orchestra.run.smithi017:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v PG_AVAILABILITY | head -n 1 2024-09-21T10:26:11.015 INFO:teuthology.orchestra.run.smithi017.stdout:2024-09-21T10:00:00.000178+0000 mon.a (mon.0) 1613 : cluster [WRN] pg 6.1 is active+recovering+undersized+degraded+remapped, acting [1,2] 2024-09-21T10:26:11.016 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-21T10:26:11.016 INFO:tasks.cephadm:Compressing logs... 2024-09-21T10:26:11.017 DEBUG:teuthology.orchestra.run.smithi017:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:26:11.019 DEBUG:teuthology.orchestra.run.smithi134:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:26:11.065 INFO:teuthology.orchestra.run.smithi017.stderr:find: ‘/var/log/rbd-target-api’gzip: No such file or directory -5 2024-09-21T10:26:11.065 INFO:teuthology.orchestra.run.smithi017.stderr: --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.0.log 2024-09-21T10:26:11.066 INFO:teuthology.orchestra.run.smithi017.stderr:gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.1.log 2024-09-21T10:26:11.069 INFO:teuthology.orchestra.run.smithi017.stderr:gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:26:11.069 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.0.log: /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:26:11.069 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mon.c.log 2024-09-21T10:26:11.069 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:26:11.069 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mon.a.log 2024-09-21T10:26:11.069 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.2.log 2024-09-21T10:26:11.070 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mgr.y.log 2024-09-21T10:26:11.070 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:26:11.070 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi017.coodep.log 2024-09-21T10:26:11.070 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.3.log 2024-09-21T10:26:11.070 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi017.coodep.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:26:11.070 INFO:teuthology.orchestra.run.smithi017.stderr: 81.9% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi017.coodep.log.gz 2024-09-21T10:26:11.082 INFO:teuthology.orchestra.run.smithi017.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.2% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:26:11.130 INFO:teuthology.orchestra.run.smithi134.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.7.log 2024-09-21T10:26:11.130 INFO:teuthology.orchestra.run.smithi134.stderr:: No such file or directory 2024-09-21T10:26:11.130 INFO:teuthology.orchestra.run.smithi134.stderr:gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:26:11.130 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.4.log 2024-09-21T10:26:11.131 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:26:11.132 INFO:teuthology.orchestra.run.smithi134.stderr:gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:26:11.132 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.4.log: /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.6.log 2024-09-21T10:26:11.134 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.5.log 2024-09-21T10:26:11.134 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:26:11.134 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mgr.x.log 2024-09-21T10:26:11.134 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mon.b.log 2024-09-21T10:26:11.135 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi134.bwovvb.log 2024-09-21T10:26:11.135 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mon.b.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:26:11.135 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi134.bwovvb.log: 62.0% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi134.bwovvb.log.gz 2024-09-21T10:26:11.221 INFO:teuthology.orchestra.run.smithi017.stderr: 93.9% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:26:11.225 INFO:teuthology.orchestra.run.smithi017.stderr: 91.0% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:26:11.307 INFO:teuthology.orchestra.run.smithi017.stderr: 91.0% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:26:11.381 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/cephadm.log: 84.2% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:26:11.426 INFO:teuthology.orchestra.run.smithi017.stderr: 95.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:26:11.474 INFO:teuthology.orchestra.run.smithi134.stderr: 86.1% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:26:11.476 INFO:teuthology.orchestra.run.smithi134.stderr: 89.9% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:26:11.531 INFO:teuthology.orchestra.run.smithi134.stderr: 91.2% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:26:11.633 INFO:teuthology.orchestra.run.smithi134.stderr: 92.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:26:11.900 INFO:teuthology.orchestra.run.smithi134.stderr: 90.1% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mgr.x.log.gz 2024-09-21T10:26:17.477 INFO:teuthology.orchestra.run.smithi134.stderr: 92.2% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mon.b.log.gz 2024-09-21T10:26:23.169 INFO:teuthology.orchestra.run.smithi017.stderr: 89.4% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mgr.y.log.gz 2024-09-21T10:26:28.876 INFO:teuthology.orchestra.run.smithi017.stderr: 91.8% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mon.c.log.gz 2024-09-21T10:26:58.138 INFO:teuthology.orchestra.run.smithi017.stderr: 90.2% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-mon.a.log.gz 2024-09-21T10:34:00.941 INFO:teuthology.orchestra.run.smithi134.stderr: 94.3% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.5.log.gz 2024-09-21T10:36:22.823 INFO:teuthology.orchestra.run.smithi134.stderr: 92.8% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.4.log.gz 2024-09-21T10:39:20.572 INFO:teuthology.orchestra.run.smithi134.stderr: 93.3% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.7.log.gz 2024-09-21T10:39:51.052 INFO:teuthology.orchestra.run.smithi134.stderr: 92.9% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.6.log.gz 2024-09-21T10:39:51.055 INFO:teuthology.orchestra.run.smithi134.stderr: 2024-09-21T10:39:51.055 INFO:teuthology.orchestra.run.smithi134.stderr:real 13m40.020s 2024-09-21T10:39:51.055 INFO:teuthology.orchestra.run.smithi134.stderr:user 5m23.021s 2024-09-21T10:39:51.055 INFO:teuthology.orchestra.run.smithi134.stderr:sys 0m16.106s 2024-09-21T10:46:19.048 INFO:teuthology.orchestra.run.smithi017.stderr: 92.3% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.1.log.gz 2024-09-21T10:51:18.939 INFO:teuthology.orchestra.run.smithi017.stderr: 93.0% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.2.log.gz 2024-09-21T10:52:12.629 INFO:teuthology.orchestra.run.smithi017.stderr: 92.9% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.0.log.gz 2024-09-21T10:53:21.265 INFO:teuthology.orchestra.run.smithi017.stderr: 92.9% -- replaced with /var/log/ceph/ee55f8cc-77fb-11ef-baf6-efdc52797490/ceph-osd.3.log.gz 2024-09-21T10:53:21.267 INFO:teuthology.orchestra.run.smithi017.stderr: 2024-09-21T10:53:21.268 INFO:teuthology.orchestra.run.smithi017.stderr:real 27m10.234s 2024-09-21T10:53:21.268 INFO:teuthology.orchestra.run.smithi017.stderr:user 10m26.732s 2024-09-21T10:53:21.268 INFO:teuthology.orchestra.run.smithi017.stderr:sys 0m29.915s 2024-09-21T10:53:21.268 INFO:tasks.cephadm:Archiving logs... 2024-09-21T10:53:21.268 DEBUG:teuthology.misc:Transferring archived files from smithi017:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913647/remote/smithi017/log 2024-09-21T10:53:21.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:54:49.024 DEBUG:teuthology.misc:Transferring archived files from smithi134:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913647/remote/smithi134/log 2024-09-21T10:54:49.025 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:55:36.764 INFO:tasks.cephadm:Removing cluster... 2024-09-21T10:55:36.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 --force 2024-09-21T10:55:37.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ee55f8cc-77fb-11ef-baf6-efdc52797490 --force 2024-09-21T10:55:38.802 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T10:55:38.802 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:55:38.823 DEBUG:teuthology.orchestra.run.smithi134:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:55:38.840 INFO:tasks.cephadm:Teardown complete 2024-09-21T10:55:38.840 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-21T10:55:38.853 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-21T10:55:38.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T10:55:38.869 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T10:55:38.972 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T10:55:38.987 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T10:55:38.987 DEBUG:teuthology.orchestra.run.smithi017:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:55:38.991 DEBUG:teuthology.orchestra.run.smithi134:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:55:39.006 INFO:teuthology.orchestra.run.smithi134.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:55:39.006 INFO:teuthology.orchestra.run.smithi017.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:55:39.026 INFO:teuthology.orchestra.run.smithi134.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:55:39.026 INFO:teuthology.orchestra.run.smithi134.stdout:=============================================================================== 2024-09-21T10:55:39.027 INFO:teuthology.orchestra.run.smithi134.stdout:^* hv01.front.sepia.ceph.com 3 6 377 23 -189us[ -194us] +/- 52ms 2024-09-21T10:55:39.027 INFO:teuthology.orchestra.run.smithi134.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 28 +238us[ +233us] +/- 61ms 2024-09-21T10:55:39.027 INFO:teuthology.orchestra.run.smithi134.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 26 +17us[ +12us] +/- 63ms 2024-09-21T10:55:39.028 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:55:39.039 INFO:teuthology.orchestra.run.smithi017.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:55:39.039 INFO:teuthology.orchestra.run.smithi017.stdout:=============================================================================== 2024-09-21T10:55:39.040 INFO:teuthology.orchestra.run.smithi017.stdout:^* hv01.front.sepia.ceph.com 3 6 377 29 -199us[ -202us] +/- 52ms 2024-09-21T10:55:39.040 INFO:teuthology.orchestra.run.smithi017.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 26 +211us[ +211us] +/- 61ms 2024-09-21T10:55:39.040 INFO:teuthology.orchestra.run.smithi017.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 27 -11us[ -11us] +/- 63ms 2024-09-21T10:55:39.040 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:55:39.041 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T10:55:39.051 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T10:55:39.051 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T10:55:39.062 DEBUG:teuthology.orchestra.run.smithi017:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:55:39.160 DEBUG:teuthology.orchestra.run.smithi134:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:55:39.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:55:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:39.281 DEBUG:teuthology.orchestra.run.smithi134:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:55:39.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:39.308 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T10:55:39.319 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T10:55:39.427 INFO:teuthology.task.internal:Duration was 5565.143523 seconds 2024-09-21T10:55:39.428 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T10:55:39.437 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T10:55:39.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:55:39.440 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:55:39.499 INFO:teuthology.orchestra.run.smithi134.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:55:39.534 INFO:teuthology.orchestra.run.smithi017.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:55:40.005 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T10:55:40.006 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi017.front.sepia.ceph.com 2024-09-21T10:55:40.006 DEBUG:teuthology.orchestra.run.smithi017:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:55:40.032 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi134.front.sepia.ceph.com 2024-09-21T10:55:40.033 DEBUG:teuthology.orchestra.run.smithi134:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:55:40.079 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T10:55:40.080 DEBUG:teuthology.orchestra.run.smithi017:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:55:40.082 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:55:40.111 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T10:55:40.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:55:40.154 DEBUG:teuthology.orchestra.run.smithi134:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:55:41.821 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T10:55:41.832 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T10:55:41.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:55:41.858 DEBUG:teuthology.orchestra.run.smithi134:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:55:41.889 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T10:55:41.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:55:41.932 DEBUG:teuthology.orchestra.run.smithi134:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:55:41.964 INFO:teuthology.orchestra.run.smithi017.stdout:kernel.core_pattern = core 2024-09-21T10:55:41.966 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = core 2024-09-21T10:55:41.988 DEBUG:teuthology.orchestra.run.smithi017:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:55:42.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:42.039 DEBUG:teuthology.orchestra.run.smithi134:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:55:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:42.055 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T10:55:42.066 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T10:55:42.068 DEBUG:teuthology.misc:Transferring archived files from smithi017:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913647/remote/smithi017 2024-09-21T10:55:42.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:55:42.146 DEBUG:teuthology.misc:Transferring archived files from smithi134:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913647/remote/smithi134 2024-09-21T10:55:42.147 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:55:42.193 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T10:55:42.193 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:55:42.196 DEBUG:teuthology.orchestra.run.smithi134:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:55:42.241 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T10:55:42.252 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T10:55:42.252 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T10:55:42.261 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T10:55:42.262 DEBUG:teuthology.orchestra.run.smithi017:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:55:42.264 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:55:42.277 INFO:teuthology.orchestra.run.smithi017.stdout: 262169 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:55 /home/ubuntu/cephtest 2024-09-21T10:55:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: 524289 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:22 /home/ubuntu/cephtest/mnt.0 2024-09-21T10:55:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: 524290 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:22 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T10:55:42.278 INFO:teuthology.orchestra.run.smithi017.stdout: 665657 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:22 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:55:42.279 INFO:teuthology.orchestra.run.smithi017.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-21T10:55:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:55:42.284 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi017 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T10:55:42.291 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T10:55:42.301 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T10:55:42.318 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi017 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T10:55:42.322 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 5565.14352273941 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi017 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=8f3e5253f07a427b927147ac0895f4a8 status: fail success: false 2024-09-21T10:55:42.322 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T10:55:42.328 INFO:teuthology.orchestra.run.smithi134.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:55 /home/ubuntu/cephtest 2024-09-21T10:55:42.406 INFO:teuthology.run:FAIL